// Copyright (C) Stichting Deltares 2025. All rights reserved.
//
// This file is part of the application DAM - UI.
//
// DAM - UI is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Deltares.Dam.Data;
using Deltares.Dam.Data.CsvImporters;
using Deltares.Dam.Data.Importers;
using Deltares.Dam.Data.IO;
using Deltares.DamEngine.Data.Standard.Calculation;
using Deltares.Geotechnics.GeotechnicalGeometry;
using Deltares.Geotechnics.Soils;
using Deltares.Geotechnics.SurfaceLines;
using Deltares.Standard.EventPublisher;
using Deltares.Standard.Logging;
using NUnit.Framework;
using DataAttribute = Deltares.Dam.Data.DataPlugins.Configuration.DataAttribute;
namespace Deltares.Dam.Tests.IO
{
public class CombineImportedDataTest
{
[Test]
[Category("Slow")]
public void TestCombiningData()
{
DataEventPublisher.InvokeWithoutPublishingEvents(() =>
{
LogManager.Messages.Clear();
DateTime start = DateTime.Now;
ProgressDelegate progress;
using (var damProjectData = new DamProjectData())
{
const string testDataFolder = @"TestData\CsvData\ImporterTest";
damProjectData.Dike = new Dike();
var configuredAttributes = new List
{
new DataAttribute
{
AttributeId = "location_id",
AttributeName = "LOCATIONID",
DataSource = "LocationID.shp"
}
};
var locationRecords = new List();
var locationImporter = LocationShapeFileImporter.Create(
locationRecords, configuredAttributes, testDataFolder);
if (locationImporter != null)
{
locationImporter.Import();
locationRecords = locationImporter.ImportedItems.ToList();
}
if (locationRecords.Count > 0)
{
progress = null;
var csvImporter = new CsvImporter();
foreach (CsvImporterLocations.LocationRecord locationRecord in locationRecords)
{
locationRecord.SegmentId = "1";
locationRecord.SurfaceLineId = "1";
}
csvImporter.LocationRecords = locationRecords;
csvImporter.ImportCsvDataFromDirectory(testDataFolder, progress, DamProjectType.Design);
foreach (LogMessage errormessage in csvImporter.ErrorMessages)
{
LogManager.Messages.Add(errormessage);
}
var combineImportedData = new CombineImportedData
{
Dike = damProjectData.Dike,
CharacteristicPointsRecords = csvImporter.CharacteristicPointsRecords,
SurfaceLineRecords = csvImporter.SurfaceLinesRecords,
SegmentRecords = csvImporter.SegmentRecords,
SoilProfilerecords = csvImporter.SoilProfilesRecords,
AquiferRecords = csvImporter.AquifersRecords,
ScenarioRecords = csvImporter.ScenariosRecords,
LocationRecords = csvImporter.LocationRecords
};
combineImportedData.AddCsvDataToDike();
combineImportedData.AddScenarioDataToDike();
foreach (LogMessage errormessage in combineImportedData.ErrorMessages)
{
LogManager.Messages.Add(errormessage);
}
}
DateTime end = DateTime.Now;
TimeSpan elapsed = end - start;
using (var file = new StreamWriter(testDataFolder + "\\Import Error log.txt"))
{
foreach (LogMessage logMessage in LogManager.Messages)
{
file.WriteLine(logMessage.Message + " " + logMessage.MessageType);
}
}
Assert.That(damProjectData.Dike.Locations.Count, Is.EqualTo(5547));
Assert.That(damProjectData.Dike.Segments.Count, Is.EqualTo(306));
Assert.That(damProjectData.Dike.SoilProfiles.Count, Is.EqualTo(3874)); // ca. 6 layers per profile. kan dus kloppen!
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers.Count, Is.EqualTo(8));
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers[0].Soil.Name, Is.EqualTo("HHNK1_kade"));
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers[0].Name, Is.EqualTo("Layer0"));
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers[0].TopLevel, Is.EqualTo(30.0));
Assert.That(damProjectData.Dike.SurfaceLines2.Count, Is.EqualTo(3));
// Test import of scenarios
Location loc = damProjectData.Dike.Locations.FirstOrDefault(x => x.Name.Equals("TPL_BALGZANDKANAAL_0110"));
Assert.That(loc.Scenarios.Count, Is.EqualTo(8));
loc = damProjectData.Dike.Locations.FirstOrDefault(x => x.Name.Equals("TPL_BALGZANDKANAAL_0180"));
Assert.That(loc.Scenarios.Count, Is.EqualTo(6));
// Following location has double entries in CSV file; check if this is handled correctly
loc = damProjectData.Dike.Locations.FirstOrDefault(x => x.Name.Equals("TPL_SCHERMER_BUITEN_WEST_0490"));
Assert.That(loc.Scenarios.Count, Is.EqualTo(6));
// Check aquifer data
IList aquifers = damProjectData.Dike.Aquifers;
Assert.That(aquifers.Count, Is.EqualTo(4));
Assert.That(aquifers[0].StixFileName, Is.EqualTo("DWP_1.stix"));
Assert.That(aquifers[0].LayerName, Is.EqualTo("L1"));
Assert.That(aquifers[1].StixFileName, Is.EqualTo("DWP_1.stix"));
Assert.That(aquifers[1].LayerName, Is.EqualTo("L5"));
Assert.That(aquifers[2].StixFileName, Is.EqualTo("DWP_2.stix"));
Assert.That(aquifers[2].LayerName, Is.EqualTo("L1"));
Assert.That(aquifers[3].StixFileName, Is.EqualTo("DWP_3.stix"));
Assert.That(aquifers[3].LayerName, Is.EqualTo("L2"));
// Check if all soilprobabbilites are read
Segment segment = damProjectData.Dike.Segments.FirstOrDefault(s => s.Name.Equals("2"));
var stabilityProbabilities = new List(segment.SoilProfileProbabilities.Where(
p => ((p.SegmentFailureMechanismType == FailureMechanismSystemType.StabilityInside) ||
(p.SegmentFailureMechanismType == FailureMechanismSystemType.StabilityOutside) ||
(p.SegmentFailureMechanismType == null))));
Assert.That(stabilityProbabilities.Count, Is.EqualTo(3));
var pipingProbabilities =
new List(segment.SoilProfileProbabilities.Where(
p => ((p.SegmentFailureMechanismType == FailureMechanismSystemType.Piping) ||
(p.SegmentFailureMechanismType == null))));
Assert.That(pipingProbabilities.Count, Is.EqualTo(3));
Assert.That(LogManager.Messages.Count, Is.EqualTo(3530));
// Check time needed to read
TimeSpan maxTime = TimeSpan.FromSeconds(80); // This is the time on the buildserver; local time was 25 seconds
Assert.That(elapsed, Is.LessThan(maxTime), string.Format("maximum time of {0} exceeded: {1}", maxTime, elapsed));
}
});
}
[Test]
public void TestCombiningDataFull1DProjectCsv()
{
LogManager.Messages.Clear();
const double tolerance = 0.001;
using (var damProjectData = new DamProjectData())
{
const string testDataFolder = @"TestData\CsvData\Full1DProject\csvfiles";
var csvImporter = new CsvImporter();
csvImporter.ImportCsvDataFromDirectory(testDataFolder, null, DamProjectType.Design);
foreach (LogMessage errorMessage in csvImporter.ErrorMessages)
{
LogManager.Messages.Add(errorMessage);
}
var combineImportedData = new CombineImportedData
{
Dike = damProjectData.Dike,
LocationRecords = csvImporter.LocationRecords,
CharacteristicPointsRecords = csvImporter.CharacteristicPointsRecords,
SurfaceLineRecords = csvImporter.SurfaceLinesRecords,
SegmentRecords = csvImporter.SegmentRecords,
SoilProfilerecords = csvImporter.SoilProfilesRecords,
AquiferRecords = csvImporter.AquifersRecords,
ScenarioRecords = csvImporter.ScenariosRecords,
SoilRecords = csvImporter.SoilsRecords,
SigmaTauCurveRecords = csvImporter.SigmaTauCurvesRecords,
SuTableRecords = csvImporter.SuTablesRecords
};
combineImportedData.AddCsvDataToDike();
combineImportedData.AddScenarioDataToDike();
foreach (LogMessage errorMessage in combineImportedData.ErrorMessages)
{
LogManager.Messages.Add(errorMessage);
}
using (var file = new StreamWriter(testDataFolder + "\\Import full csv error log.txt"))
{
foreach (LogMessage logMessage in LogManager.Messages)
{
file.WriteLine(logMessage.Message + " " + logMessage.MessageType);
}
}
Assert.Multiple(() =>
{
Assert.That(damProjectData.Dike.Locations, Has.Count.EqualTo(2));
Assert.That(damProjectData.Dike.Segments, Has.Count.EqualTo(2));
Assert.That(damProjectData.Dike.SoilProfiles, Has.Count.EqualTo(23));
});
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers, Has.Count.EqualTo(4));
Assert.Multiple(() =>
{
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers[0].Soil.Name, Is.EqualTo("kade"));
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers[0].Name, Is.EqualTo("Layer0"));
Assert.That(damProjectData.Dike.SoilProfiles[0].Layers[0].TopLevel, Is.EqualTo(60.0));
Assert.That(damProjectData.Dike.SurfaceLines2, Has.Count.EqualTo(2));
});
// Check soil data
Assert.That(damProjectData.Dike.SoilList.Soils, Has.Count.EqualTo(14));
Soil soil = damProjectData.Dike.SoilList.Soils.FirstOrDefault(s => s.Name == "CCC");
Assert.That(soil, Is.Not.Null);
Assert.Multiple(() =>
{
Assert.That(soil.ShearStrengthModel, Is.EqualTo(ShearStrengthModel.StressTable));
Assert.That(soil.StressTable.Name, Is.EqualTo("CurveKlei"));
});
soil = damProjectData.Dike.SoilList.Soils.FirstOrDefault(s => s.Name == "FFO");
Assert.That(soil, Is.Not.Null);
Assert.Multiple(() =>
{
Assert.That(soil.ShearStrengthModel, Is.EqualTo(ShearStrengthModel.SuTable));
Assert.That(soil.SuTable.Name, Is.EqualTo("SuKlei"));
});
// Check degree of consolidation data
Assert.That(damProjectData.Dike.DamSoils, Has.Count.EqualTo(14));
DamSoil damSoil = damProjectData.Dike.DamSoils.FirstOrDefault(s => s.Name == "CCC");
Assert.That(damSoil, Is.Not.Null);
Assert.That(damSoil.TrafficLoadDegreeOfConsolidation, Is.EqualTo(25));
damSoil = damProjectData.Dike.DamSoils.FirstOrDefault(s => s.Name == "BAS");
Assert.That(damSoil, Is.Not.Null);
Assert.That(damSoil.TrafficLoadDegreeOfConsolidation, Is.EqualTo(100));
// Check location specific data
Location location = damProjectData.Dike.Locations.FirstOrDefault(s => s.Name.Equals("25-2-2-A-1-A"));
Assert.That(location, Is.Not.Null);
Assert.Multiple(() =>
{
Assert.That(location.Name, Is.EqualTo("25-2-2-A-1-A"));
Assert.That(location.Segment.Name, Is.EqualTo("106"));
Assert.That(location.XRd, Is.EqualTo(66586.0).Within(tolerance));
Assert.That(location.YRd, Is.EqualTo(424173.0).Within(tolerance));
Assert.That(location.XSoilGeometry2DOrigin, Is.EqualTo(2.0).Within(tolerance));
Assert.That(location.DikeEmbankmentMaterial, Is.EqualTo("klei"));
Assert.That(location.ShoulderEmbankmentMaterial, Is.EqualTo("klei2"));
Assert.That(location.PenetrationLength, Is.EqualTo(1.3).Within(tolerance));
Assert.That(location.TrafficLoad, Is.EqualTo(10).Within(tolerance));
Assert.That(location.TrafficLoadDistributionAngle, Is.EqualTo(22.44).Within(tolerance));
Assert.That(location.MinimalCircleDepth, Is.EqualTo(1.5).Within(tolerance));
Assert.That(location.DampingFactorPL3, Is.EqualTo(0.3).Within(tolerance));
Assert.That(location.DampingFactorPL4, Is.EqualTo(0.4).Within(tolerance));
Assert.That(location.PLLineCreationMethod, Is.EqualTo(PLLineCreationMethod.ExpertKnowledgeRRD));
Assert.That(location.DistanceToEntryPoint, Is.EqualTo(2.1).Within(tolerance));
Assert.That(location.StabilityShoulderGrowSlope, Is.EqualTo(2.0).Within(tolerance));
Assert.That(location.StabilityShoulderGrowDeltaX, Is.EqualTo(0.2).Within(tolerance));
Assert.That(location.StabilitySlopeAdaptionDeltaX, Is.EqualTo(0.5).Within(tolerance));
});
// Assert that the scenarios are set correctly for this location
List scenarios = location.Scenarios;
Assert.That(scenarios.Count, Is.EqualTo(2));
Assert.That(scenarios.Select(sc => sc.LocationScenarioID), Is.EqualTo(new[]
{
"1",
"10"
}).AsCollection);
Assert.That(scenarios.Select(sc => sc.DikeTableHeight), Is.EqualTo(new[]
{
4.01,
7.02
}).AsCollection);
Assert.That(scenarios.Select(sc => sc.RequiredSafetyFactorStabilityInnerSlope), Is.EqualTo(new[]
{
1.08,
1.45
}).AsCollection);
Assert.That(scenarios.All(sc => Math.Abs(sc.PolderLevel + 0.25) < tolerance), Is.True);
Assert.That(scenarios.All(sc => Math.Abs(sc.HeadPl2.Value - 0.8727) < tolerance), Is.True);
}
}
[Test]
public void TestCheckCharacteristicPointsForCoincidingLocationsValid()
{
SurfaceLine2 surfaceLine = CreateValidSurfaceLine();
var combineImportedData = new CombineImportedData();
bool doNotCoincide = combineImportedData.CheckCharacteristicPointsForCoincidingLocations(surfaceLine, out _, out _);
Assert.That(doNotCoincide, Is.True);
}
[Test]
public void TestCheckCharacteristicPointsForCoincidingLocationsInValid()
{
SurfaceLine2 surfaceLine = CreateInvalidSurfaceLine();
var combineImportedData = new CombineImportedData();
CharacteristicPointType point1;
CharacteristicPointType point2;
bool doNotCoincide = combineImportedData.CheckCharacteristicPointsForCoincidingLocations(surfaceLine, out point1, out point2);
Assert.That(doNotCoincide, Is.False);
Assert.That(point1, Is.EqualTo(CharacteristicPointType.DikeTopAtPolder));
Assert.That(point2, Is.EqualTo(CharacteristicPointType.ShoulderBaseInside));
}
[Test]
public void TestCheckCharacteristicPointsForCoincidingLocationsValidTrafficLoad()
{
SurfaceLine2 surfaceLine = CreateValidSurfaceLineTrafficLoad();
var combineImportedData = new CombineImportedData();
bool doNotCoincide = combineImportedData.CheckCharacteristicPointsForCoincidingLocations(surfaceLine, out _, out _);
Assert.That(doNotCoincide, Is.True);
}
[Test]
public void TestProperMessageForCheckCharacteristicPointsForCoincidingLocationsInValid()
{
SurfaceLine2 surfaceLine = CreateInvalidSurfaceLine();
var combineImportedData = new CombineImportedData();
combineImportedData.ErrorMessages.Clear();
bool doNotCoincide = combineImportedData.CheckOnCoincidingPoints(surfaceLine);
Assert.That(doNotCoincide, Is.False);
Assert.That(combineImportedData.ErrorMessages.Count, Is.EqualTo(1));
Assert.That(combineImportedData.ErrorMessages[0].Message.Contains(CharacteristicPointType.DikeTopAtPolder.ToString()), Is.True);
Assert.That(combineImportedData.ErrorMessages[0].Message.Contains(CharacteristicPointType.ShoulderBaseInside.ToString()), Is.True);
}
private SurfaceLine2 CreateValidSurfaceLine()
{
var surfaceLine = new SurfaceLine2
{
Geometry = new LocalizedGeometryPointString(),
CharacteristicPoints =
{
GeometryMustContainPoint = true
}
};
surfaceLine.EnsurePointOfType(0, 0, 0, CharacteristicPointType.SurfaceLevelOutside);
surfaceLine.EnsurePointOfType(10, 0, 0, CharacteristicPointType.DikeToeAtRiver);
surfaceLine.EnsurePointOfType(12, 0, 2, CharacteristicPointType.DikeTopAtRiver);
surfaceLine.EnsurePointOfType(15, 0, 2, CharacteristicPointType.DikeTopAtPolder);
surfaceLine.EnsurePointOfType(20, 0, -1, CharacteristicPointType.DikeToeAtPolder);
surfaceLine.EnsurePointOfType(80, 0, -1, CharacteristicPointType.SurfaceLevelInside);
return surfaceLine;
}
private SurfaceLine2 CreateInvalidSurfaceLine()
{
SurfaceLine2 surfaceLine = CreateValidSurfaceLine();
surfaceLine.EnsurePointOfType(15, 0, 2, CharacteristicPointType.ShoulderBaseInside);
return surfaceLine;
}
private SurfaceLine2 CreateValidSurfaceLineTrafficLoad()
{
SurfaceLine2 surfaceLine = CreateValidSurfaceLine();
surfaceLine.EnsurePointOfType(15, 0, 2, CharacteristicPointType.TrafficLoadOutside);
return surfaceLine;
}
}
}