// Copyright (C) Stichting Deltares 2023. All rights reserved.
//
// This file is part of the application DAM - UI.
//
// DAM - UI is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Deltares.Dam.Data.DataPlugins;
using Deltares.Dam.Data.DataPlugins.Configuration;
using Deltares.Geometry;
using Deltares.Geotechnics.GeotechnicalGeometry;
using Deltares.Geotechnics.IO;
using Deltares.Geotechnics.Soils;
using Deltares.Geotechnics.SurfaceLines;
using Deltares.Maps;
using Deltares.Standard;
using Deltares.Standard.EventPublisher;
using Deltares.Standard.Logging;
using ProgressDelegate = Deltares.DamEngine.Data.Standard.Calculation.ProgressDelegate;
namespace Deltares.Dam.Data;
public class WaterBoardImporterException : Exception
{
public WaterBoardImporterException(string message)
: base(message) {}
}
///
/// Import all data into Waterboard
///
public class WaterBoardImporter
{
//
// Members & Properties
//
private readonly IDataPlugin importer;
private readonly WaterBoard waterBoard = new ();
///
/// Initializes a new instance of the class.
///
/// The data folder.
/// The data source container.
/// Type of the dam project.
private WaterBoardImporter(string dataFolder, DataSourceContainer dataSourceContainer, DamProjectType damProjectType)
{
importer = CreateDataPlugin(dataFolder, dataSourceContainer);
importer.DamProjectType = damProjectType;
}
///
/// Gets the import CSV log messages.
///
///
/// The import CSV log messages.
///
private List ImportCsvLogMessages { get; } = new ();
///
/// Imports the dike ring ids.
///
/// The data folder.
/// The data source container.
/// Type of the dam project.
/// The import log messages.
///
public static IEnumerable ImportDikeRingIds(
string dataFolder, DataSourceContainer dataSourceContainer, DamProjectType damProjectType, out List importLogMessages)
{
var importer = new WaterBoardImporter(dataFolder, dataSourceContainer, damProjectType);
IEnumerable allDikeRingIds = importer.AllDikeRingIDs;
importLogMessages = new List(importer.ImportCsvLogMessages);
return allDikeRingIds;
}
///
/// Imports all data.
///
/// The data folder.
/// The data source container.
/// Type of the dam project.
/// The progress.
/// The log messages.
///
public static WaterBoard ImportAllData(string dataFolder,
DataSourceContainer dataSourceContainer, DamProjectType damProjectType, ProgressDelegate progress,
out List logMessages)
{
DataPluginImporter importer = CreateDataPlugin(dataFolder, dataSourceContainer);
IEnumerable dikeRingIds = importer.GetDikeRingIdList();
WaterBoard waterboard = ImportDataForDikeRings("", dataFolder, dataSourceContainer,
dikeRingIds, damProjectType, progress, out logMessages);
return waterboard;
}
///
/// Imports the data for dike rings.
///
/// The dam project folder.
/// The data folder.
/// The data source container.
/// The dike ring ids.
/// Type of the dam project.
/// The progress.
/// The log messages.
///
public static WaterBoard ImportDataForDikeRings(string damProjectFolder, string dataFolder,
DataSourceContainer dataSourceContainer, IEnumerable dikeRingIds,
DamProjectType damProjectType, ProgressDelegate progress, out List logMessages)
{
var importer = new WaterBoardImporter(dataFolder, dataSourceContainer, damProjectType);
logMessages = new List();
var waterBoard = new WaterBoard();
try
{
waterBoard = importer.ImportDataForDikeRings(damProjectFolder, dikeRingIds, progress);
foreach (Dike dike in waterBoard.Dikes)
{
if (string.IsNullOrEmpty(dike.MapForSoilGeometries2D) && !string.IsNullOrEmpty(dataSourceContainer.MapSoilProfile2D))
{
dike.MapForSoilGeometries2D = dataSourceContainer.MapSoilProfile2D;
}
}
}
catch (Exception e)
{
string errorMessage = e.Message;
Exception innerException = e.InnerException;
while (innerException != null)
{
errorMessage = errorMessage + ";" + innerException.Message;
innerException = innerException.InnerException;
}
logMessages.Add(new LogMessage(LogMessageType.FatalError, null, errorMessage));
}
logMessages.AddRange(importer.ImportCsvLogMessages);
return waterBoard;
}
///
/// Gets all dike ring ids.
///
///
/// All dike ring i ds.
///
private IEnumerable AllDikeRingIDs
{
get
{
ImportCsvLogMessages.Clear();
List dikeRingIdList = importer.GetDikeRingIdList().ToList();
ImportCsvLogMessages.AddRange(importer.ImportLogMessages);
return dikeRingIdList;
}
}
///
/// Creates the data plugin.
///
/// The data folder.
/// The data source container.
///
private static DataPluginImporter CreateDataPlugin(string dataFolder, DataSourceContainer dataSourceContainer)
{
var plugin = new DataPluginImporter();
plugin.SetDataSources(dataFolder, dataSourceContainer.DataSourceList);
plugin.Attributes = dataSourceContainer.DataAttributes;
return plugin;
}
///
/// Imports the data for dike rings.
///
/// The dam project folder.
/// The dike ring ids.
/// The progress.
///
private WaterBoard ImportDataForDikeRings(string damProjectFolder, IEnumerable dikeRingIds, ProgressDelegate progress)
{
List dikeRingList = ValidateDikeRingIDs(dikeRingIds);
importer.DamProjectFolder = damProjectFolder;
// Import dikes
bool importOk = ImportDikes(progress);
if (!importOk)
{
return waterBoard;
}
// import data for dike rings
foreach (string dikeRingId in dikeRingList)
{
// Add dikes to waterboard
List importedCsvSoils = importer.WaterBoard.Dikes.First(d => d.Name == dikeRingId).ImportedCsvSoils;
Dike dike = ImportDataForDikeRing(importedCsvSoils, dikeRingId);
waterBoard.Dikes.Add(dike);
}
ImportBackground();
return waterBoard;
}
///
/// Validates the dike ring i ds.
///
/// The dike ring ids.
///
private List ValidateDikeRingIDs(IEnumerable dikeRingIds)
{
// if no dike ring IDs specified, take all
return dikeRingIds == null
? importer.GetDikeRingIdList().ToList()
: (dikeRingIds as List ?? dikeRingIds.ToList());
}
///
/// Imports the soils.
///
/// The dike ring identifier.
/// The dike.
private void ImportSoils(string dikeRingID, Dike dike)
{
dike.SoilList = GetSoils(dikeRingID);
}
///
/// Gets the soils.
///
/// The dike ring identifier.
///
private SoilList GetSoils(string dikeRingID)
{
IEnumerable soilIdList = importer.GetSoilIdList(dikeRingID);
List soils = soilIdList.Select(soilID => ImportSoil(dikeRingID, soilID)).ToList();
return new SoilList
{
Soils = soils
};
}
///
/// Imports a soil.
///
/// The dike ring identifier.
/// The soil identifier.
///
private Soil ImportSoil(string dikeRingID, string soilID)
{
var soil = new Soil
{
Name = soilID
};
IEnumerable soilDetails = importer.GetSoilDetails(dikeRingID, soilID);
foreach (NameValueParameter soilDetail in soilDetails)
{
SoilUtils.SetParameterFromNameValuePair(soil, soilDetail.ParameterName, soilDetail.ParameterValue);
DataSourceManager.SetSource(soil, soilDetail.ParameterName, soilDetail.Source);
}
return soil;
}
///
/// Imports the data for dike ring.
///
/// The soils as imported from the csv file
/// The dike ring identifier.
///
private Dike ImportDataForDikeRing(List importedCsvSoils, string dikeRingId)
{
var dike = new Dike
{
Name = dikeRingId
};
ImportSoils(dikeRingId, dike);
ImportSoilProfiles1D(dikeRingId, dike);
ImportSegments(dikeRingId, dike);
ImportSurfaceLines(dikeRingId, dike);
ImportModelParameters(dikeRingId, dike);
// TODO: Import PL1-lines
ImportLocations(dikeRingId, dike);
//ImportScenarios(importer, dikeRingId, dike, waterBoard);
ImportScenarios(dikeRingId, dike);
// Use InvokeWithoutPublishingEvents for performance reasons after introducing SurfaceLine2
DataEventPublisher.InvokeWithoutPublishingEvents(() =>
{
var dikeCoordinateSystemConverter = new DikeCoordinateSystemConverter();
if (dike.Locations.Count > 0)
{
dikeCoordinateSystemConverter.CreateLocalXZObjects(dike);
}
});
ImportCsvLogMessages.AddRange(dike.MakeDataConsistent(importedCsvSoils));
DisposeImportedItem(dikeRingId);
return dike;
}
private void DisposeImportedItem(string dikeRingId)
{
importer.DisposeImportedItem(dikeRingId);
}
///
/// Imports all specified dikes.
///
/// The progress.
///
private bool ImportDikes(ProgressDelegate progress)
{
// Import dikes
importer.ImportDataForDikeRings(progress);
// add log messages
ImportCsvLogMessages.AddRange(importer.ImportLogMessages);
// import is ok if no fatal messages
return ImportCsvLogMessages.All(message => message.MessageType != LogMessageType.FatalError);
}
///
/// Imports the background layer.
///
private void ImportBackground()
{
IEnumerable geometryFeatures = importer.MapGeometryIdList
.Select(id => importer.GetMapGeometry(id))
.Select(Feature.Create);
foreach (Feature geometry in geometryFeatures)
{
waterBoard.AddBackgroundGeometry(geometry);
}
waterBoard.FillFeatureList();
}
///
/// Imports the scenarios.
///
/// The dike ring identifier.
/// The dike.
private void ImportScenarios(string dikeRingId, Dike dike)
{
AddScenariosToLocations(dike, dikeRingId);
}
///
/// Adds the scenarios to locations.
///
/// The dike.
/// The dike ring identifier.
private void AddScenariosToLocations(Dike dike, string dikeRingId)
{
foreach (Location location in dike.Locations)
{
IEnumerable scenarioList = importer.GetScenarioList(dikeRingId, location.Name);
foreach (string scenarioId in scenarioList)
{
IEnumerable nameValues =
importer.GetScenarioDetails(dikeRingId, location.Name, scenarioId);
var scenario = new Scenario
{
LocationScenarioID = scenarioId
};
foreach (NameValueParameter nameValueParameter in nameValues)
{
scenario.SetParameterFromNameValuePair(
nameValueParameter.ParameterName,
nameValueParameter.ParameterValue);
}
scenario.Location = location;
location.Scenarios.Add(scenario);
}
}
}
///
/// Imports the locations.
///
/// The dike ring identifier.
/// The dike.
private void ImportLocations(string dikeRingId, Dike dike)
{
IEnumerable locationIdList = importer.GetLocationIdList(dikeRingId);
foreach (string locationId in locationIdList)
{
var location = new Location
{
Name = locationId
};
Info locationInfo = importer.GetLocationInfo(dikeRingId, locationId);
location.Name = locationInfo.Name;
DataSourceManager.SetSource(location, "Name", locationInfo.Source);
location.Description = locationInfo.Description;
DataSourceManager.SetSource(location, "Description", locationInfo.Source);
IEnumerable locationDetails = importer.GetLocationDetails(dikeRingId, locationId);
foreach (NameValueParameter locationDetail in locationDetails)
{
location.SetParameterFromNameValuePair(locationDetail.ParameterName, locationDetail.ParameterValue);
DataSourceManager.SetSource(location, locationDetail.ParameterName, locationDetail.Source);
if (locationDetail.ParameterName.Equals(LocationParameterNames.SegmentId))
{
Segment segment = waterBoard.Segments.FirstOrDefault(x => x.Name.Equals(locationDetail.ParameterValue));
location.Segment = segment;
}
// See comment below: the test WaterBoardImportedWithCsvFilesHasValidData using Groot Salland\Binnenwaarts proves Erik wrong
if (locationDetail.ParameterName.Equals(LocationParameterNames.SurfaceLineId))
{
SurfaceLine2 surfaceLine = dike.SurfaceLines2.FirstOrDefault(x => x.Name.Equals(locationDetail.ParameterValue));
location.SurfaceLine2 = surfaceLine;
}
// TODO: Assign PL1-line
}
// According to Erik Vastenburg, locations and surfacelines must match for matching pairs! So to find proper surfaceline
// for location, just search for it by name in surfacelines. SurfaceLineId can and should be removed entirely! But see above!
if (location.SurfaceLine2 == null)
{
location.SurfaceLine2 = dike.SurfaceLines2.FirstOrDefault(s => s.Name == location.Name);
}
location.SoilList = dike.SoilList;
dike.Locations.Add(location);
}
dike.SortLocations();
}
///
/// Imports the model parameters.
///
/// The dike ring identifier.
/// The dike.
private void ImportModelParameters(string dikeRingId, Dike dike)
{
IEnumerable modelParameters = importer.GetDikeParameters(dikeRingId);
foreach (NameValueParameter modelParameter in modelParameters)
{
dike.SetParameterFromNameValuePair(modelParameter.ParameterName, modelParameter.ParameterValue);
DataSourceManager.SetSource(dike, modelParameter.ParameterName, modelParameter.Source);
}
}
///
/// Imports the surface lines.
///
/// The dike ring identifier.
/// The dike.
private void ImportSurfaceLines(string dikeRingId, Dike dike)
{
// Use InvokeWithoutPublishingEvents for performance reasons after introducint SurfaceLine2
DataEventPublisher.InvokeWithoutPublishingEvents(() =>
{
IEnumerable surfaceLineIdList = importer.GetSurfaceLineIdList(dikeRingId);
foreach (string surfaceLineId in surfaceLineIdList)
{
var surfaceLine = new SurfaceLine2
{
Name = surfaceLineId,
Geometry = new LocalizedGeometryPointString(),
CharacteristicPoints =
{
GeometryMustContainPoint = true
}
};
// load the surface line points
IList surfaceLinePoints = importer.GetSurfaceLinePoints(dikeRingId, surfaceLineId);
foreach (GeometryPoint surfaceLinePoint in surfaceLinePoints)
{
surfaceLine.AddCharacteristicPoint(surfaceLinePoint);
}
// load the characteristic points
IEnumerable characteristicPoints = importer.GetSurfaceLineCharacteristicPoints(dikeRingId, surfaceLineId);
foreach (DpCharacteristicPoint cp in characteristicPoints)
{
var cpType = (CharacteristicPointType) Enum.Parse(
typeof(CharacteristicPointType), cp.Id);
surfaceLine.EnsurePointOfType(cp.X, cp.Y, cp.Z, cpType);
}
// add the surface line to the dike
dike.SurfaceLines2.Add(surfaceLine);
}
});
}
///
/// Imports the segments.
///
/// The dike ring identifier.
/// The dike.
private void ImportSegments(string dikeRingID, Dike dike)
{
IEnumerable segmentIdList = importer.GetSegmentIdList(dikeRingID);
foreach (string segmentID in segmentIdList)
{
var segment = new Segment
{
Name = segmentID
};
foreach (FailureMechanismSystemType failureMechanismSystemType in Enum.GetValues(typeof(FailureMechanismSystemType)))
{
AddSoilProfiles(segment, dikeRingID, segmentID, failureMechanismSystemType, dike);
AddSoilGeometry(segment, dikeRingID, segmentID, failureMechanismSystemType);
}
waterBoard.Segments.Add(segment);
}
}
///
/// Adds the soil profiles.
///
/// The segment.
/// The dike ring identifier.
/// The segment identifier.
/// Type of the failure mechanism system.
/// The dike.
private void AddSoilProfiles(Segment segment, string dikeRingID, string segmentID, FailureMechanismSystemType failureMechanismSystemType, Dike dike)
{
IEnumerable profile1DIdListForSegment = importer.GetProfile1DIdListForSegment(dikeRingID, segmentID, failureMechanismSystemType);
foreach (string soilProfile1DId in profile1DIdListForSegment)
{
var numberFormatInfo = new NumberFormatInfo
{
NumberDecimalSeparator = "."
};
// get soil profile
IEnumerable soilProfile1DDetails = importer.GetSegmentProfile1DDetails(dikeRingID, segmentID, soilProfile1DId, failureMechanismSystemType);
SoilProfile1D soilProfile1D = dike.SoilProfiles.FirstOrDefault(x => x.Name.Equals(soilProfile1DId));
double probability = 0;
foreach (NameValueParameter nameValueparameter in soilProfile1DDetails)
{
if (nameValueparameter.ParameterName.Equals("Probability"))
{
probability = Convert.ToDouble(nameValueparameter.ParameterValue, numberFormatInfo);
}
}
segment.AddSoilProfileProbability(soilProfile1D, probability, failureMechanismSystemType);
}
}
///
/// Adds the soil geometry.
///
/// The segment.
/// The dike ring identifier.
/// The segment identifier.
/// Type of the failure mechanism system.
private void AddSoilGeometry(Segment segment, string dikeRingID, string segmentID, FailureMechanismSystemType failureMechanismSystemType)
{
IEnumerable profile2DIdListForSegment = importer.GetProfile2DIdListForSegment(dikeRingID, segmentID, failureMechanismSystemType);
foreach (string soilGeometry2DName in profile2DIdListForSegment)
{
var numberFormatInfo = new NumberFormatInfo
{
NumberDecimalSeparator = "."
};
IEnumerable soilProfile2DDetails = importer.GetSegmentProfile2DDetails(dikeRingID, segmentID, soilGeometry2DName, failureMechanismSystemType);
var probability = 0.0;
foreach (NameValueParameter nameValueparameter in soilProfile2DDetails)
{
if (nameValueparameter.ParameterName.Equals("Probability"))
{
probability = Convert.ToDouble(nameValueparameter.ParameterValue, numberFormatInfo);
}
}
segment.AddSoilGeometry2DProbability(soilGeometry2DName, probability, failureMechanismSystemType);
}
}
///
/// Imports the soil profiles1 d.
///
/// The dike ring identifier.
/// The dike.
private void ImportSoilProfiles1D(string dikeRingId, Dike dike)
{
IEnumerable soilProfile1DIdList = importer.GetSoilProfile1DIdList(dikeRingId);
foreach (string soilProfile1DId in soilProfile1DIdList)
{
dike.SoilProfiles.Add(ImportProfile(soilProfile1DId, dikeRingId, dike));
}
}
///
/// Imports a profile.
///
/// The soil profile1 d identifier.
/// The dike ring identifier.
/// The dike.
///
///
///
private SoilProfile1D ImportProfile(string soilProfile1DId, string dikeRingId, Dike dike)
{
var soilProfile = new SoilProfile1D
{
Name = soilProfile1DId
};
var bottomLevel = 0.0;
DpSoilProfile dpSoilProfile = importer.GetSoilProfile1DDetails(dikeRingId, soilProfile1DId);
for (var layerIndex = 0; layerIndex < dpSoilProfile.Layers.Count; layerIndex++)
{
DpLayer layer = dpSoilProfile.Layers[layerIndex];
if (layer.SoilName == SoilProfile1D.SoilProfileBottomLevelId)
{
// This is a hack to communicate the bottomlevel via a layer
bottomLevel = layer.TopLevel;
}
else
{
int soilIndex = dike.SoilList.GetSoilIndexByName(layer.SoilName);
if (soilIndex < 0)
{
throw new WaterBoardImporterException($"Soil '{dpSoilProfile.Layers[layerIndex].SoilName}' not found in 1d-profile '{soilProfile1DId}'");
}
soilProfile.Layers.Add(new SoilLayer1D
{
TopLevel = layer.TopLevel,
Soil = dike.SoilList.Soils[soilIndex],
IsAquifer = layer.IsAquifer
});
}
}
soilProfile.EnsureUniqueLayerIds();
soilProfile.BottomLevel = bottomLevel;
if (soilProfile.LayerCount > 0)
{
soilProfile.EnsureLastLayerHasHeight();
}
else
{
throw new WaterBoardImporterException($"Soil 1d-profile '{soilProfile.Name}' has no layers");
}
return soilProfile;
}
}