using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security.Cryptography.X509Certificates;
using System.Threading;
using Deltares.Dam.Data;
using Deltares.Dam.TestHelper;
using Deltares.Standard.IO;
using NUnit.Framework;
namespace Deltares.Dam.Tests
{
///
/// Run comparison tests for selected kernels
///
public class ComparisonTestRunner
{
private static object sync = new object();
private const int maxCalculationCores = 8;
private const string cmstabProgramPath = ".\\DGeoStability.exe";
public static string DataPath { get; set; }
public static DamIntegrationTestTolerance Tolerances { get; set; }
public static string[] PropertiesToCheck { get; set; }
public static string DamWtiTestReportsDir { get; set; }
public static int projectCount = 0;
///
/// Gets the path to the data directory in the Deltares development space for DelftGeoSystems
///
///
public static string GetDataPath()
{
string basePath = AppDomain.CurrentDomain.BaseDirectory;
while (!basePath.ToLower().EndsWith("src"))
{
basePath = Directory.GetParent(basePath).ToString();
}
basePath = Directory.GetParent(basePath).ToString();
string dataPath = System.IO.Path.Combine(basePath, "data");
return dataPath;
}
///
/// Gets the path to the data directory in the Deltares development space for DelftGeoSystems
///
///
public static string GetBinReleasePath()
{
string basePath = AppDomain.CurrentDomain.BaseDirectory;
while (!basePath.ToLower().EndsWith("src"))
{
basePath = Directory.GetParent(basePath).ToString();
}
string binReleasePath = System.IO.Path.Combine(basePath, "bin", "Release");
return binReleasePath;
}
///
/// Runs the comparison test.
///
/// Name of the dir.
/// The locations to calculate. If null then all locations are selected
/// The selected stability kernel types.
/// The text writer.
/// if set to true [is data from data dir].
///
public List RunStabilityComparisonTest(string damProjectFilename, IList locationsToCalculate, IList selectedStabilityKernelTypes,
TextWriter textWriter, bool isDataFromDataDir = true)
{
lock (sync)
{
projectCount++;
//prepare summary text for artifacts on buildserver
var testMessagesNotOk = new List();
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
var allMessages = new List();
DateTime time = DateTime.Now;
allMessages.Add(String.Format("Test start: {0}", time.ToShortDateString() + " " + time.ToShortTimeString()));
var actualResults = new Dictionary>();
foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes)
{
List> stabilityResultsActual = CalculateStabilityProject(selectedStabilityKernelType, damProjectFilename, locationsToCalculate);
List actual = CreateResultsList(stabilityResultsActual);
actualResults[selectedStabilityKernelType] = actual;
allMessages.Add("Stability kernel type: " + selectedStabilityKernelType.ToString());
//compare the calculation results of the original stability kernel to the new kernel
//foreach (var parameter in PropertiesToCheck)
//{
// for (int i = 0; i < actual.Count; i++)
// {
// CompareCalculationResults(expected[i], actual[i], parameter, testMessagesNotOk, allMessages);
// }
//}
//write failed results to console
foreach (var message in testMessagesNotOk)
{
System.Console.WriteLine(@"Test failed: " + message);
}
//when this assertion is carried out, some tests will fail and the results will not be persisted to files
//Assert.AreEqual(0, testMessagesNotOk.Count, "One or more comparison tests failed.");
}
//write all results to txt file
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
time = DateTime.Now;
allMessages.Add(String.Format("Test end: {0}", time.ToShortDateString() + " " + time.ToShortTimeString()));
WriteToTextFile(damProjectFilename, allMessages, textWriter);
OutputReport(damProjectFilename, selectedStabilityKernelTypes, actualResults);
return testMessagesNotOk;
}
}
///
/// Outputs the report.
///
/// The dam project filename.
/// The selected stability kernel types.
/// The actual results.
private void OutputReport(string damProjectFilename, IList selectedStabilityKernelTypes,
Dictionary> actualResults)
{
foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes)
{
// Export actual data to csv
var exportName = CreateExportName(damProjectFilename, selectedStabilityKernelType);
ExportDataToCsv(exportName, actualResults[selectedStabilityKernelType], ".csv");
}
OutputReportPerProperty(damProjectFilename, selectedStabilityKernelTypes, actualResults);
OutputReportPerKernel(damProjectFilename, selectedStabilityKernelTypes, actualResults);
}
///
/// Outputs the report per kernel.
///
/// The dam project filename.
/// The selected stability kernel types.
/// The actual results.
private void OutputReportPerKernel(string damProjectFilename, IList selectedStabilityKernelTypes,
Dictionary> actualResults)
{
string header;
string reportFilename;
var allResultsString = new List();
// Write header
header = "Location;";
foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes)
{
header = header + HeadersForKernelType(selectedStabilityKernelType);
}
allResultsString.Add(header);
// Write results
IList firstKernelResults = actualResults[selectedStabilityKernelTypes.First()];
for (int i = 0; i < firstKernelResults.Count; i++)
{
string resultsString = firstKernelResults[i].LocationName + ";";
foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes)
{
resultsString = resultsString + ResultsForKernelType(selectedStabilityKernelType, actualResults[selectedStabilityKernelType][i]);
}
allResultsString.Add(resultsString);
}
reportFilename = CreateReportName(damProjectFilename) + " - Comparison per kernel.csv";
using (var tw = new StreamWriter((string) reportFilename))
{
foreach (var line in allResultsString)
{
tw.WriteLine(line);
}
tw.Close();
}
}
///
/// Outputs the report per property.
///
/// The dam project filename.
/// The selected stability kernel types.
/// The actual results.
private void OutputReportPerProperty(string damProjectFilename, IList selectedStabilityKernelTypes,
Dictionary> actualResults)
{
string reportFilename;
// Write overview of results
var allResultsString = new List();
// Write header
string header = "Location;";
foreach (var property in PropertiesToCheck)
{
header = header + HeadersForProperty(property, selectedStabilityKernelTypes);
}
allResultsString.Add(header);
// Write results
IList firstKernelResults = actualResults[selectedStabilityKernelTypes.First()];
for (int i = 0; i < firstKernelResults.Count; i++)
{
string resultsString = firstKernelResults[i].LocationName + ";";
foreach (var property in PropertiesToCheck)
{
resultsString = resultsString + ResultsForProperty(property, selectedStabilityKernelTypes, actualResults, i);
}
allResultsString.Add(resultsString);
}
reportFilename = CreateReportName(damProjectFilename) + " - Comparison per property.csv";
using (var tw = new StreamWriter((string) reportFilename))
{
foreach (var line in allResultsString)
{
tw.WriteLine(line);
}
tw.Close();
}
}
///
/// Results for property.
///
/// The property.
/// The selected stability kernel types.
/// The actual results.
/// The index.
///
private string ResultsForProperty(string property , IList selectedStabilityKernelTypes,
Dictionary> actualResults, int index)
{
string resultString = "";
foreach (var stabilityKernelType in selectedStabilityKernelTypes)
{
resultString = resultString + String.Format("{0:0.0000}", GetPropertyValue(actualResults[stabilityKernelType][index], property))+ ";";
}
return resultString;
}
///
/// Headerses for property.
///
/// The property.
/// The selected stability kernel types.
///
private string HeadersForProperty(string property, IList selectedStabilityKernelTypes)
{
string header = "";
foreach (var stabilityKernelType in selectedStabilityKernelTypes)
{
header = header + stabilityKernelType + "-" + property + ";";
}
return header;
}
///
/// Headerses the type of for kernel.
///
/// Type of the stability kernel.
///
private string HeadersForKernelType(StabilityKernelType stabilityKernelType)
{
string header = "";
foreach (var property in PropertiesToCheck)
{
header = header + stabilityKernelType + "-" + property + ";";
}
return header;
}
///
/// Resultses the type of for kernel.
///
/// Type of the stability kernel.
/// The actual results.
///
private string ResultsForKernelType(StabilityKernelType stabilityKernelType, CsvExportData results)
{
string resultString = "";
foreach (var property in PropertiesToCheck)
{
var result = String.Format("{0:0.0000}", GetPropertyValue(results, property));
resultString = resultString + result + ";";
}
return resultString;
}
///
/// Creates the results list.
///
/// The calculation results.
///
private List CreateResultsList(List> calculationResults)
{
List resultList = new List();
foreach (var result in calculationResults)
{
resultList.AddRange(result);
}
return resultList;
}
///
/// Creates the name of the export.
///
/// Name of the project
/// Type of the selected stability kernel.
///
private static string CreateExportName(string projectFilename, StabilityKernelType selectedStabilityKernelType)
{
string filename = String.Format("{0:00} - {1} - {2}", projectCount, Path.GetFileNameWithoutExtension(projectFilename), selectedStabilityKernelType);
string exportName = Path.Combine(DamWtiTestReportsDir, filename);
return exportName;
}
///
/// Creates the name of the report.
///
/// The project filename.
///
private static string CreateReportName(string projectFilename)
{
string filename = String.Format("{0:00} - {1}", projectCount, Path.GetFileNameWithoutExtension(projectFilename));
string exportName = Path.Combine(DamWtiTestReportsDir, filename);
return exportName;
}
///
/// Gets the maximum allowed calculation cores for selected stability kernel.
///
/// Type of the selected stability kernel.
///
private static int GetMaxAllowedCalculationCoresForKernel(StabilityKernelType selectedStabilityKernelType)
{
switch (selectedStabilityKernelType)
{
case StabilityKernelType.DamClassicDotNet:
return 1;
case StabilityKernelType.AdvancedDotNet:
return 1;
default:
return maxCalculationCores;
}
}
///
/// Calculates the stability using specified kernels.
///
/// Type of the selected stability kernel.
/// Name of the directory.
/// The locations to calculate. If null then all locations are selected
///
private static List> CalculateStabilityProject(StabilityKernelType selectedStabilityKernelType, string projectFilename, IList locationsToCalculate)
{
DateTime timeStart = DateTime.Now;
DamProject.ProjectWorkingPathLocation = ProjectPathLocation.InProjectMap;
using (DamProjectData damProjectData = ReadDamProjectData(projectFilename))
{
if (damProjectData == null)
{
Assert.IsNotNull(damProjectData, "DAM project not loaded correct");
}
// setup calculation options
damProjectData.DamProjectType = DamProjectType.Design;
Dike dike = damProjectData.WaterBoard.Dikes[0];
// specify calculation
if (damProjectData.DamProjectCalculationSpecification.DamCalculationSpecifications.Count < 1)
{
damProjectData.DamProjectCalculationSpecification.DamCalculationSpecifications.Clear();
DamIntegrationTestHelper.AddCalculationSpecificationsToDamProjectData(damProjectData);
}
damProjectData.DamProjectCalculationSpecification.CurrentSpecification =
damProjectData.DamProjectCalculationSpecification.DamCalculationSpecifications[0];
// If no locations specified, select all locations
if (locationsToCalculate == null)
{
locationsToCalculate = damProjectData.Locations.Select(location => location.Name).ToList();
}
// Select the specificied locations
DamIntegrationTestHelper.SpecifyLocationsToCalculate(damProjectData, locationsToCalculate);
// Create scenarios for selected locations
List scenarios = new List();
foreach (var locationJob in damProjectData.LocationJobs)
{
if (locationJob.Run.HasValue && locationJob.Run.Value)
{
scenarios.AddRange(locationJob.Location.Scenarios);
}
}
// specify stability kernel type and waternet creator type
System.Console.WriteLine(@"Selected calculation kernel type: " + selectedStabilityKernelType.ToString());
damProjectData.DamProjectCalculationSpecification.SelectedStabilityKernelType = selectedStabilityKernelType;
// setup project calculator with other stability kernel
var damProjectCalculator = new DamProjectCalculator(damProjectData);
damProjectCalculator.ProjectDataDirectory = Path.GetDirectoryName(projectFilename);
if (!String.IsNullOrEmpty(damProjectCalculator.ProjectDataDirectory))
{
damProjectCalculator.ProjectDataDirectory += Path.DirectorySeparatorChar;
}
damProjectCalculator.CalculationBaseDirectory = Path.GetFullPath(Path.Combine(projectFilename, "CalculationFiles"));
damProjectCalculator.MStabProgramPath = cmstabProgramPath;
damProjectCalculator.MaxCalculationCores = GetMaxAllowedCalculationCoresForKernel(selectedStabilityKernelType);
//call
List> allCalculationresults = damProjectCalculator.Calculate(damProjectData, scenarios);
//calculation time
var timeSpan = DateTime.Now.Subtract(timeStart);
string newKernelTime = String.Format(@"calculation time kernel ({0}): {1:0.0} seconds", selectedStabilityKernelType, timeSpan.TotalSeconds);
System.Console.WriteLine(newKernelTime);
return allCalculationresults;
}
}
///
/// Reads the dam project data.
///
/// The dam file location.
///
private static DamProjectData ReadDamProjectData(string projectFilename)
{
Assert.AreEqual(true, File.Exists(projectFilename));
// Read dikering
var damProjectData = ProjectLoader.LoadProjectData(projectFilename);
return damProjectData;
}
///
/// Exports the data to CSV.
///
/// Name of the dir.
/// The CSV export data.
/// Type of the result.
private void ExportDataToCsv(string projectFilename, IList csvExportData, string resultType)
{
new TestResultsWriter().ExportDataToCsv(DamWtiTestReportsDir, projectFilename, csvExportData, resultType);
}
///
/// Writes to text file.
///
/// Name of the dir.
/// All messages.
/// The text writer.
private void WriteToTextFile(string projectFilename, List allMessages, TextWriter textWriter)
{
new TestResultsWriter().WriteToTextFile(projectFilename, allMessages, textWriter);
}
///
/// Compares the calculation results.
///
/// The calculation results original.
/// The calculation results new kernel.
/// The property to check.
/// The messages not ok.
/// All messages.
private void CompareCalculationResults(CsvExportData calculationResultsOriginal,
CsvExportData calculationResultsNewKernel, string propertyToCheck, List messagesNotOk,
List allMessages)
{
var valueActual = GetPropertyValue(calculationResultsNewKernel, propertyToCheck);
var valueExpected = GetPropertyValue(calculationResultsOriginal, propertyToCheck);
var toleranceValue = GetToleranceValue(Tolerances, propertyToCheck);
if (toleranceValue == null)
{
toleranceValue = 0.01;
}
if (valueExpected == null || valueActual == null)
{
string message = "Property " + propertyToCheck + " that was compared has a null value as result.";
messagesNotOk.Add(message);
}
else
{
double expectedValue = (double) valueExpected;
double actualValue = (double) valueActual;
bool ok = Math.Abs((double) (expectedValue - actualValue)) <
toleranceValue;
string message = propertyToCheck + " - Expected: " + expectedValue.ToString("F3") + " " +
propertyToCheck + " - Actual: " + actualValue.ToString("F3");
allMessages.Add(message);
System.Console.WriteLine(message);
if (!ok)
{
messagesNotOk.Add(message);
}
}
}
///
/// Gets the property value.
///
/// The calculation results.
/// The property to check.
///
private static double? GetPropertyValue(object calculationResults, string propertyToCheck)
{
double? valueExpected = null;
Type type = calculationResults.GetType();
BindingFlags flags = BindingFlags.Public | BindingFlags.Instance;
PropertyInfo[] properties = type.GetProperties(flags);
foreach (var propertyInfo in properties)
{
if (propertyInfo.Name == propertyToCheck)
{
valueExpected = (double?) propertyInfo.GetValue(calculationResults, null);
}
}
return valueExpected;
}
///
/// Gets the tolerance value.
///
/// The tolerances.
/// The property to check.
///
private static double? GetToleranceValue(object tolerances, string propertyToCheck)
{
double? toleranceValue = null;
Type type = tolerances.GetType();
BindingFlags flags = BindingFlags.Public | BindingFlags.Instance;
PropertyInfo[] properties = type.GetProperties(flags);
foreach (var propertyInfo in properties)
{
if (propertyInfo.Name.Contains(propertyToCheck))
{
toleranceValue = (double?) propertyInfo.GetValue(tolerances, null);
}
}
return toleranceValue;
}
}
}