using System; using System.Collections; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using System.Security.Cryptography.X509Certificates; using System.Threading; using Deltares.Dam.Data; using Deltares.Dam.TestHelper; using Deltares.Standard.IO; using NUnit.Framework; namespace Deltares.Dam.Tests { /// /// Run comparison tests for selected kernels /// public class ComparisonTestRunner { private static object sync = new object(); private const int maxCalculationCores = 8; private const string cmstabProgramPath = ".\\DGeoStability.exe"; public static string DataPath { get; set; } public static DamIntegrationTestTolerance Tolerances { get; set; } public static string[] PropertiesToCheck { get; set; } public static string DamWtiTestReportsDir { get; set; } public static int projectCount = 0; /// /// Gets the path to the data directory in the Deltares development space for DelftGeoSystems /// /// public static string GetDataPath() { string basePath = AppDomain.CurrentDomain.BaseDirectory; while (!basePath.ToLower().EndsWith("src")) { basePath = Directory.GetParent(basePath).ToString(); } basePath = Directory.GetParent(basePath).ToString(); string dataPath = System.IO.Path.Combine(basePath, "data"); return dataPath; } /// /// Gets the path to the data directory in the Deltares development space for DelftGeoSystems /// /// public static string GetBinReleasePath() { string basePath = AppDomain.CurrentDomain.BaseDirectory; while (!basePath.ToLower().EndsWith("src")) { basePath = Directory.GetParent(basePath).ToString(); } string binReleasePath = System.IO.Path.Combine(basePath, "bin", "Release"); return binReleasePath; } /// /// Runs the comparison test. /// /// Name of the dir. /// The locations to calculate. If null then all locations are selected /// The selected stability kernel types. /// The text writer. /// if set to true [is data from data dir]. /// public List RunStabilityComparisonTest(string damProjectFilename, IList locationsToCalculate, IList selectedStabilityKernelTypes, TextWriter textWriter, bool isDataFromDataDir = true) { lock (sync) { projectCount++; //prepare summary text for artifacts on buildserver var testMessagesNotOk = new List(); Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; var allMessages = new List(); DateTime time = DateTime.Now; allMessages.Add(String.Format("Test start: {0}", time.ToShortDateString() + " " + time.ToShortTimeString())); var actualResults = new Dictionary>(); foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes) { List> stabilityResultsActual = CalculateStabilityProject(selectedStabilityKernelType, damProjectFilename, locationsToCalculate); List actual = CreateResultsList(stabilityResultsActual); actualResults[selectedStabilityKernelType] = actual; allMessages.Add("Stability kernel type: " + selectedStabilityKernelType.ToString()); //compare the calculation results of the original stability kernel to the new kernel //foreach (var parameter in PropertiesToCheck) //{ // for (int i = 0; i < actual.Count; i++) // { // CompareCalculationResults(expected[i], actual[i], parameter, testMessagesNotOk, allMessages); // } //} //write failed results to console foreach (var message in testMessagesNotOk) { System.Console.WriteLine(@"Test failed: " + message); } //when this assertion is carried out, some tests will fail and the results will not be persisted to files //Assert.AreEqual(0, testMessagesNotOk.Count, "One or more comparison tests failed."); } //write all results to txt file Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; time = DateTime.Now; allMessages.Add(String.Format("Test end: {0}", time.ToShortDateString() + " " + time.ToShortTimeString())); WriteToTextFile(damProjectFilename, allMessages, textWriter); OutputReport(damProjectFilename, selectedStabilityKernelTypes, actualResults); return testMessagesNotOk; } } /// /// Outputs the report. /// /// The dam project filename. /// The selected stability kernel types. /// The actual results. private void OutputReport(string damProjectFilename, IList selectedStabilityKernelTypes, Dictionary> actualResults) { foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes) { // Export actual data to csv var exportName = CreateExportName(damProjectFilename, selectedStabilityKernelType); ExportDataToCsv(exportName, actualResults[selectedStabilityKernelType], ".csv"); } OutputReportPerProperty(damProjectFilename, selectedStabilityKernelTypes, actualResults); OutputReportPerKernel(damProjectFilename, selectedStabilityKernelTypes, actualResults); } /// /// Outputs the report per kernel. /// /// The dam project filename. /// The selected stability kernel types. /// The actual results. private void OutputReportPerKernel(string damProjectFilename, IList selectedStabilityKernelTypes, Dictionary> actualResults) { string header; string reportFilename; var allResultsString = new List(); // Write header header = "Location;"; foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes) { header = header + HeadersForKernelType(selectedStabilityKernelType); } allResultsString.Add(header); // Write results IList firstKernelResults = actualResults[selectedStabilityKernelTypes.First()]; for (int i = 0; i < firstKernelResults.Count; i++) { string resultsString = firstKernelResults[i].LocationName + ";"; foreach (var selectedStabilityKernelType in selectedStabilityKernelTypes) { resultsString = resultsString + ResultsForKernelType(selectedStabilityKernelType, actualResults[selectedStabilityKernelType][i]); } allResultsString.Add(resultsString); } reportFilename = CreateReportName(damProjectFilename) + " - Comparison per kernel.csv"; using (var tw = new StreamWriter((string) reportFilename)) { foreach (var line in allResultsString) { tw.WriteLine(line); } tw.Close(); } } /// /// Outputs the report per property. /// /// The dam project filename. /// The selected stability kernel types. /// The actual results. private void OutputReportPerProperty(string damProjectFilename, IList selectedStabilityKernelTypes, Dictionary> actualResults) { string reportFilename; // Write overview of results var allResultsString = new List(); // Write header string header = "Location;"; foreach (var property in PropertiesToCheck) { header = header + HeadersForProperty(property, selectedStabilityKernelTypes); } allResultsString.Add(header); // Write results IList firstKernelResults = actualResults[selectedStabilityKernelTypes.First()]; for (int i = 0; i < firstKernelResults.Count; i++) { string resultsString = firstKernelResults[i].LocationName + ";"; foreach (var property in PropertiesToCheck) { resultsString = resultsString + ResultsForProperty(property, selectedStabilityKernelTypes, actualResults, i); } allResultsString.Add(resultsString); } reportFilename = CreateReportName(damProjectFilename) + " - Comparison per property.csv"; using (var tw = new StreamWriter((string) reportFilename)) { foreach (var line in allResultsString) { tw.WriteLine(line); } tw.Close(); } } /// /// Results for property. /// /// The property. /// The selected stability kernel types. /// The actual results. /// The index. /// private string ResultsForProperty(string property , IList selectedStabilityKernelTypes, Dictionary> actualResults, int index) { string resultString = ""; foreach (var stabilityKernelType in selectedStabilityKernelTypes) { resultString = resultString + String.Format("{0:0.0000}", GetPropertyValue(actualResults[stabilityKernelType][index], property))+ ";"; } return resultString; } /// /// Headerses for property. /// /// The property. /// The selected stability kernel types. /// private string HeadersForProperty(string property, IList selectedStabilityKernelTypes) { string header = ""; foreach (var stabilityKernelType in selectedStabilityKernelTypes) { header = header + stabilityKernelType + "-" + property + ";"; } return header; } /// /// Headerses the type of for kernel. /// /// Type of the stability kernel. /// private string HeadersForKernelType(StabilityKernelType stabilityKernelType) { string header = ""; foreach (var property in PropertiesToCheck) { header = header + stabilityKernelType + "-" + property + ";"; } return header; } /// /// Resultses the type of for kernel. /// /// Type of the stability kernel. /// The actual results. /// private string ResultsForKernelType(StabilityKernelType stabilityKernelType, CsvExportData results) { string resultString = ""; foreach (var property in PropertiesToCheck) { var result = String.Format("{0:0.0000}", GetPropertyValue(results, property)); resultString = resultString + result + ";"; } return resultString; } /// /// Creates the results list. /// /// The calculation results. /// private List CreateResultsList(List> calculationResults) { List resultList = new List(); foreach (var result in calculationResults) { resultList.AddRange(result); } return resultList; } /// /// Creates the name of the export. /// /// Name of the project /// Type of the selected stability kernel. /// private static string CreateExportName(string projectFilename, StabilityKernelType selectedStabilityKernelType) { string filename = String.Format("{0:00} - {1} - {2}", projectCount, Path.GetFileNameWithoutExtension(projectFilename), selectedStabilityKernelType); string exportName = Path.Combine(DamWtiTestReportsDir, filename); return exportName; } /// /// Creates the name of the report. /// /// The project filename. /// private static string CreateReportName(string projectFilename) { string filename = String.Format("{0:00} - {1}", projectCount, Path.GetFileNameWithoutExtension(projectFilename)); string exportName = Path.Combine(DamWtiTestReportsDir, filename); return exportName; } /// /// Gets the maximum allowed calculation cores for selected stability kernel. /// /// Type of the selected stability kernel. /// private static int GetMaxAllowedCalculationCoresForKernel(StabilityKernelType selectedStabilityKernelType) { switch (selectedStabilityKernelType) { case StabilityKernelType.DamClassicDotNet: return 1; case StabilityKernelType.AdvancedDotNet: return 1; default: return maxCalculationCores; } } /// /// Calculates the stability using specified kernels. /// /// Type of the selected stability kernel. /// Name of the directory. /// The locations to calculate. If null then all locations are selected /// private static List> CalculateStabilityProject(StabilityKernelType selectedStabilityKernelType, string projectFilename, IList locationsToCalculate) { DateTime timeStart = DateTime.Now; DamProject.ProjectWorkingPathLocation = ProjectPathLocation.InProjectMap; using (DamProjectData damProjectData = ReadDamProjectData(projectFilename)) { if (damProjectData == null) { Assert.IsNotNull(damProjectData, "DAM project not loaded correct"); } // setup calculation options damProjectData.DamProjectType = DamProjectType.Design; Dike dike = damProjectData.WaterBoard.Dikes[0]; // specify calculation if (damProjectData.DamProjectCalculationSpecification.DamCalculationSpecifications.Count < 1) { damProjectData.DamProjectCalculationSpecification.DamCalculationSpecifications.Clear(); DamIntegrationTestHelper.AddCalculationSpecificationsToDamProjectData(damProjectData); } damProjectData.DamProjectCalculationSpecification.CurrentSpecification = damProjectData.DamProjectCalculationSpecification.DamCalculationSpecifications[0]; // If no locations specified, select all locations if (locationsToCalculate == null) { locationsToCalculate = damProjectData.Locations.Select(location => location.Name).ToList(); } // Select the specificied locations DamIntegrationTestHelper.SpecifyLocationsToCalculate(damProjectData, locationsToCalculate); // Create scenarios for selected locations List scenarios = new List(); foreach (var locationJob in damProjectData.LocationJobs) { if (locationJob.Run.HasValue && locationJob.Run.Value) { scenarios.AddRange(locationJob.Location.Scenarios); } } // specify stability kernel type and waternet creator type System.Console.WriteLine(@"Selected calculation kernel type: " + selectedStabilityKernelType.ToString()); damProjectData.DamProjectCalculationSpecification.SelectedStabilityKernelType = selectedStabilityKernelType; // setup project calculator with other stability kernel var damProjectCalculator = new DamProjectCalculator(damProjectData); damProjectCalculator.ProjectDataDirectory = Path.GetDirectoryName(projectFilename); if (!String.IsNullOrEmpty(damProjectCalculator.ProjectDataDirectory)) { damProjectCalculator.ProjectDataDirectory += Path.DirectorySeparatorChar; } damProjectCalculator.CalculationBaseDirectory = Path.GetFullPath(Path.Combine(projectFilename, "CalculationFiles")); damProjectCalculator.MStabProgramPath = cmstabProgramPath; damProjectCalculator.MaxCalculationCores = GetMaxAllowedCalculationCoresForKernel(selectedStabilityKernelType); //call List> allCalculationresults = damProjectCalculator.Calculate(damProjectData, scenarios); //calculation time var timeSpan = DateTime.Now.Subtract(timeStart); string newKernelTime = String.Format(@"calculation time kernel ({0}): {1:0.0} seconds", selectedStabilityKernelType, timeSpan.TotalSeconds); System.Console.WriteLine(newKernelTime); return allCalculationresults; } } /// /// Reads the dam project data. /// /// The dam file location. /// private static DamProjectData ReadDamProjectData(string projectFilename) { Assert.AreEqual(true, File.Exists(projectFilename)); // Read dikering var damProjectData = ProjectLoader.LoadProjectData(projectFilename); return damProjectData; } /// /// Exports the data to CSV. /// /// Name of the dir. /// The CSV export data. /// Type of the result. private void ExportDataToCsv(string projectFilename, IList csvExportData, string resultType) { new TestResultsWriter().ExportDataToCsv(DamWtiTestReportsDir, projectFilename, csvExportData, resultType); } /// /// Writes to text file. /// /// Name of the dir. /// All messages. /// The text writer. private void WriteToTextFile(string projectFilename, List allMessages, TextWriter textWriter) { new TestResultsWriter().WriteToTextFile(projectFilename, allMessages, textWriter); } /// /// Compares the calculation results. /// /// The calculation results original. /// The calculation results new kernel. /// The property to check. /// The messages not ok. /// All messages. private void CompareCalculationResults(CsvExportData calculationResultsOriginal, CsvExportData calculationResultsNewKernel, string propertyToCheck, List messagesNotOk, List allMessages) { var valueActual = GetPropertyValue(calculationResultsNewKernel, propertyToCheck); var valueExpected = GetPropertyValue(calculationResultsOriginal, propertyToCheck); var toleranceValue = GetToleranceValue(Tolerances, propertyToCheck); if (toleranceValue == null) { toleranceValue = 0.01; } if (valueExpected == null || valueActual == null) { string message = "Property " + propertyToCheck + " that was compared has a null value as result."; messagesNotOk.Add(message); } else { double expectedValue = (double) valueExpected; double actualValue = (double) valueActual; bool ok = Math.Abs((double) (expectedValue - actualValue)) < toleranceValue; string message = propertyToCheck + " - Expected: " + expectedValue.ToString("F3") + " " + propertyToCheck + " - Actual: " + actualValue.ToString("F3"); allMessages.Add(message); System.Console.WriteLine(message); if (!ok) { messagesNotOk.Add(message); } } } /// /// Gets the property value. /// /// The calculation results. /// The property to check. /// private static double? GetPropertyValue(object calculationResults, string propertyToCheck) { double? valueExpected = null; Type type = calculationResults.GetType(); BindingFlags flags = BindingFlags.Public | BindingFlags.Instance; PropertyInfo[] properties = type.GetProperties(flags); foreach (var propertyInfo in properties) { if (propertyInfo.Name == propertyToCheck) { valueExpected = (double?) propertyInfo.GetValue(calculationResults, null); } } return valueExpected; } /// /// Gets the tolerance value. /// /// The tolerances. /// The property to check. /// private static double? GetToleranceValue(object tolerances, string propertyToCheck) { double? toleranceValue = null; Type type = tolerances.GetType(); BindingFlags flags = BindingFlags.Public | BindingFlags.Instance; PropertyInfo[] properties = type.GetProperties(flags); foreach (var propertyInfo in properties) { if (propertyInfo.Name.Contains(propertyToCheck)) { toleranceValue = (double?) propertyInfo.GetValue(tolerances, null); } } return toleranceValue; } } }