// Copyright (C) Stichting Deltares 2018. All rights reserved.
//
// This file is part of the application DAM - Clients Library.
//
// DAM - UI is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Deltares.Standard.EventPublisher;
using Deltares.Geotechnics.GeotechnicalGeometry;
using Deltares.Geotechnics.IO;
using Deltares.Geotechnics.Soils;
using Deltares.Geotechnics.SurfaceLines;
using Deltares.Standard.Logging;
namespace Deltares.Dam.Data
{
using DataPlugins.Configuration;
using DataPlugins;
using Standard;
using Maps;
public class WaterBoardImporterException : Exception
{
public WaterBoardImporterException(string message)
: base(message)
{
}
}
///
/// Import all data into Waterboard
///
public class WaterBoardImporter
{
//
// Members & Properties
//
private readonly IDataPlugin importer;
private readonly WaterBoard waterBoard = new WaterBoard();
private readonly DamType damType;
private readonly DamProjectType damProjectType;
private readonly List importCsvLogMessages = new List();
///
/// Gets the import CSV log messages.
///
///
/// The import CSV log messages.
///
public List ImportCsvLogMessages
{
get { return importCsvLogMessages; }
}
///
/// Gets all dike ring ids.
///
///
/// All dike ring i ds.
///
private IEnumerable AllDikeRingIDs
{
get
{
importCsvLogMessages.Clear();
var dikeRingIdList = importer.GetDikeRingIdList(damType).ToList();
importCsvLogMessages.AddRange(importer.ImportLogMessages);
return dikeRingIdList;
}
}
///
/// Gets the data source list.
///
///
/// The data source list.
///
private IEnumerable DataSourceList
{
get { return importer.DataSources; }
}
///
/// Gets the data folder.
///
///
/// The data folder.
///
private string DataFolder
{
get { return importer.DataFolder; }
}
///
/// Initializes a new instance of the class.
///
public WaterBoardImporter()
{
}
///
/// Initializes a new instance of the class.
///
/// The data folder.
/// The data source container.
/// Type of the dam.
/// Type of the dam project.
private WaterBoardImporter(string dataFolder, DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType)
{
this.damProjectType = damProjectType;
importer = CreateDataPlugin(dataFolder, dataSourceContainer);
importer.DamProjectType = damProjectType;
this.damType = damType;
}
///
/// Creates the data plugin.
///
/// The data folder.
/// The data source container.
///
private static DataPluginImporter CreateDataPlugin(string dataFolder, DataSourceContainer dataSourceContainer)
{
var plugin = new DataPluginImporter();
plugin.SetDataSources(dataFolder, dataSourceContainer.DataSourceList);
plugin.Attributes = dataSourceContainer.DataAttributes;
return plugin;
}
///
/// Imports the dike ring ids.
///
/// The data folder.
/// The data source container.
/// Type of the dam.
/// Type of the dam project.
/// The import log messages.
///
public static IEnumerable ImportDikeRingIds(
string dataFolder, DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType, out List importLogMessages)
{
var importer = new WaterBoardImporter(dataFolder, dataSourceContainer, damType, damProjectType);
var allDikeRingIds = importer.AllDikeRingIDs;
importLogMessages = new List(importer.ImportCsvLogMessages);
return allDikeRingIds;
}
///
/// Imports all data.
///
/// The data folder.
/// The data source container.
/// Type of the dam.
/// Type of the dam project.
/// The progress.
/// The log messages.
///
public static WaterBoard ImportAllData(string dataFolder,
DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress,
out List logMessages)
{
var importer = CreateDataPlugin(dataFolder, dataSourceContainer);
var dikeRingIds = importer.GetDikeRingIdList(damType);
WaterBoard waterboard = ImportDataForDikeRings("", dataFolder, dataSourceContainer,
dikeRingIds, damType, damProjectType, progress, out logMessages);
return waterboard;
}
///
/// Imports all data.
///
/// The data folder.
/// The data source container.
/// Type of the dam.
/// Type of the dam project.
/// The progress.
///
public static WaterBoard ImportAllData(string dataFolder,
DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress)
{
List dummy;
return ImportAllData(dataFolder, dataSourceContainer, damType, damProjectType, progress, out dummy);
}
///
/// Imports the data for dike rings.
///
/// The dam project folder.
/// The data folder.
/// The data source container.
/// The dike ring ids.
/// Type of the dam.
/// Type of the dam project.
/// The progress.
///
public static WaterBoard ImportDataForDikeRings(string damProjectFolder, string dataFolder,
DataSourceContainer dataSourceContainer, IEnumerable dikeRingIds,
DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress)
{
List dummy;
return ImportDataForDikeRings(damProjectFolder, dataFolder, dataSourceContainer,
dikeRingIds, damType, damProjectType, progress, out dummy);
}
///
/// Imports the data for dike rings.
///
/// The dam project folder.
/// The data folder.
/// The data source container.
/// The dike ring ids.
/// Type of the dam.
/// Type of the dam project.
/// The progress.
/// The log messages.
///
public static WaterBoard ImportDataForDikeRings(string damProjectFolder, string dataFolder,
DataSourceContainer dataSourceContainer, IEnumerable dikeRingIds,
DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress, out List logMessages)
{
var importer = new WaterBoardImporter(dataFolder, dataSourceContainer, damType, damProjectType);
logMessages = new List();
WaterBoard waterBoard = new WaterBoard();
try
{
waterBoard = importer.ImportDataForDikeRings(damProjectFolder, dikeRingIds, progress);
foreach (Dike dike in waterBoard.Dikes)
{
if (String.IsNullOrEmpty(dike.MapForSoilGeometries2D) && !String.IsNullOrEmpty(dataSourceContainer.MapSoilProfile2D))
{
dike.MapForSoilGeometries2D = dataSourceContainer.MapSoilProfile2D;
}
}
}
catch (Exception e)
{
string errorMessage = e.Message;
Exception innerException = e.InnerException;
while (innerException != null)
{
errorMessage = errorMessage + ";" + innerException.Message;
innerException = innerException.InnerException;
}
logMessages.Add(new LogMessage(LogMessageType.FatalError, null, errorMessage));
}
logMessages.AddRange(importer.ImportCsvLogMessages);
return waterBoard;
}
///
/// Imports the data for dike rings.
///
/// The dam project folder.
/// The dike ring ids.
/// The progress.
///
private WaterBoard ImportDataForDikeRings(string damProjectFolder, IEnumerable dikeRingIds, DamEngine.Data.Standard.Calculation.ProgressDelegate progress)
{
var dikeRingList = ValidateDikeRingIDs(dikeRingIds);
importer.DamProjectFolder = damProjectFolder;
// Import dikes
var importOk = ImportDikes(dikeRingList, progress);
if (!importOk) return waterBoard;
// import data for dike rings
foreach (var dikeRingId in dikeRingList)
{
// Add dikes to waterboard
var dike = ImportDataForDikeRing(dikeRingId);
waterBoard.Dikes.Add(dike);
}
ImportBackground();
return waterBoard;
}
///
/// Validates the dike ring i ds.
///
/// The dike ring ids.
///
private List ValidateDikeRingIDs(IEnumerable dikeRingIds)
{
// if no dike ring IDs specified, take all
return dikeRingIds == null
? importer.GetDikeRingIdList(damType).ToList()
: (dikeRingIds as List ?? dikeRingIds.ToList());
}
///
/// Creates the soil base for dike.
///
/// The dike.
private void CreateSoilBaseForDike(Dike dike)
{
// TODO: soildatabase should be created. Now we make a direct link
// must be based on dike.SoilDatabaseName
}
///
/// Determines the database name from data sources.
///
/// The dike.
private void DetermineDatabaseNameFromDataSources(Dike dike)
{
var sources = DataSourceList.ToList();
// get the data base name from the data sources
var dataSource =
sources.FirstOrDefault(ds => ds.DataSourceType == DataSourceType.MSoilBase) ??
sources.FirstOrDefault(ds => ds.DataSourceType == DataSourceType.CsvFiles);
if (dataSource != null)
{
if (dataSource.DataSourceType == DataSourceType.MSoilBase)
{
dike.SoilDatabaseName = Path.Combine(DataFolder, dataSource.DataLocation);
}
else
{
var subFolder = Path.Combine(DataFolder, dataSource.DataLocation);
dike.SoilDatabaseName = Path.Combine(subFolder, "soilmaterials.mdb");
}
}
}
///
/// Imports the soils.
///
/// The dike ring identifier.
/// The dike.
private void ImportSoils(string dikeRingID, Dike dike)
{
dike.SoilList = GetSoils(dikeRingID);
DetermineDatabaseNameFromDataSources(dike);
dike.FillDataBaseSoilListFromSoilBase();
}
///
/// Gets the soils.
///
/// The dike ring identifier.
///
private SoilList GetSoils(string dikeRingID)
{
var soilIdList = importer.GetSoilIdList(dikeRingID);
var soils = soilIdList
.Select(soilID => ImportSoil(dikeRingID, soilID))
.ToList();
var aquifers = importer.GetSoilAquiferValues(dikeRingID);
var soilList = new SoilList();
soilList.Soils = soils;
foreach (var nameValueParameter in aquifers)
{
bool vala;
bool.TryParse(nameValueParameter.ParameterValue, out vala);
soilList.AquiferDictionary.Add(soilList.GetSoilByName(nameValueParameter.ParameterName), vala);
}
return soilList;
}
///
/// Imports a soil.
///
/// The dike ring identifier.
/// The soil identifier.
///
private Soil ImportSoil(string dikeRingID, string soilID)
{
var soil = new Soil { Name = soilID };
var soilDetails = importer.GetSoilDetails(dikeRingID, soilID);
foreach (var soilDetail in soilDetails)
{
SoilUtils.SetParameterFromNameValuePair(soil, soilDetail.ParameterName, soilDetail.ParameterValue);
DataSourceManager.SetSource(soil, soilDetail.ParameterName, soilDetail.Source);
}
return soil;
}
///
/// Imports the data for dike ring.
///
/// The dike ring identifier.
///
private Dike ImportDataForDikeRing(string dikeRingId)
{
var dike = new Dike { Name = dikeRingId };
ImportSoils(dikeRingId, dike);
ImportSoilProfiles1D(dikeRingId, dike);
ImportSegments(dikeRingId, dike);
ImportSurfaceLines(dikeRingId, dike);
ImportModelParameters(dikeRingId, dike);
// TODO: Import PL1-lines
ImportLocations(dikeRingId, dike);
//ImportScenarios(importer, dikeRingId, dike, waterBoard);
ImportScenarios(dikeRingId, dike);
// Use InvokeWithoutPublishingEvents for performance reasons after introducint SurfaceLine2
DataEventPublisher.InvokeWithoutPublishingEvents(() =>
{
var dikeCoordinateSystemConverter = new DikeCoordinateSystemConverter();
if (dike.Locations.Count > 0)
{
dikeCoordinateSystemConverter.CreateLocalXZObjects(dike);
}
});
ImportCsvLogMessages.AddRange(dike.MakeDataConsistent());
CreateSoilBaseForDike(dike);
DisposeImportedItem(dikeRingId);
return dike;
}
private void DisposeImportedItem(string dikeRingId)
{
importer.DisposeImportedItem(dikeRingId);
}
///
/// Imports all specified dikes.
///
/// The dike ring ids.
/// The progress.
///
private bool ImportDikes(IEnumerable dikeRingIds, DamEngine.Data.Standard.Calculation.ProgressDelegate progress)
{
// Import dikes
importer.ImportDataForDikeRings(dikeRingIds, damType, progress);
// add log messages
ImportCsvLogMessages.AddRange(importer.ImportLogMessages);
// import is ok if no fatal messages
return ImportCsvLogMessages.All(message => message.MessageType != LogMessageType.FatalError);
}
///
/// Imports the background layer.
///
private void ImportBackground()
{
var geometryFeatures = importer.MapGeometryIdList
.Select(id => importer.GetMapGeometry(id))
.Select(Feature.Create);
foreach (var geometry in geometryFeatures)
{
waterBoard.AddBackgroundGeometry(geometry);
}
waterBoard.FillFeatureList();
}
///
/// Imports the scenarios.
///
/// The dike ring identifier.
/// The dike.
private void ImportScenarios(string dikeRingId, Dike dike)
{
AddScenariosToLocations(dike, dikeRingId);
}
///
/// Adds the scenarios to locations.
///
/// The dike.
/// The dike ring identifier.
private void AddScenariosToLocations(Dike dike, string dikeRingId)
{
foreach (var location in dike.Locations)
{
var scenarioList = importer.GetScenarioList(dikeRingId, location.Name);
foreach (var scenarioId in scenarioList)
{
var nameValues =
importer.GetScenarioDetails(dikeRingId, location.Name, scenarioId);
var scenario = new Scenario { LocationScenarioID = scenarioId };
foreach (var nameValueParameter in nameValues)
{
scenario.SetParameterFromNameValuePair(
nameValueParameter.ParameterName,
nameValueParameter.ParameterValue);
}
scenario.Location = location;
location.Scenarios.Add(scenario);
}
}
}
///
/// Imports the locations.
///
/// The dike ring identifier.
/// The dike.
private void ImportLocations(string dikeRingId, Dike dike)
{
var locationIdList = importer.GetLocationIdList(dikeRingId, damType);
foreach (string locationId in locationIdList)
{
var location = new Location
{
Name = locationId
};
Info locationInfo = importer.GetLocationInfo(dikeRingId, locationId);
location.Name = locationInfo.Name;
DataSourceManager.SetSource(location, "Name", locationInfo.Source);
location.Description = locationInfo.Description;
DataSourceManager.SetSource(location, "Description", locationInfo.Source);
IEnumerable locationDetails = importer.GetLocationDetails(dikeRingId, locationId);
foreach (NameValueParameter locationDetail in locationDetails)
{
location.SetParameterFromNameValuePair(locationDetail.ParameterName, locationDetail.ParameterValue);
DataSourceManager.SetSource(location, locationDetail.ParameterName, locationDetail.Source);
if (locationDetail.ParameterName.Equals(LocationParameterNames.SegmentId))
{
Segment segment = waterBoard.Segments.FirstOrDefault(x => x.Name.Equals(locationDetail.ParameterValue));
location.Segment = segment;
}
// See comment below: the test WaterBoardImportedWithCsvFilesHasValidData using Groot Salland\Binnenwaarts proves Erik wrong
if (locationDetail.ParameterName.Equals(LocationParameterNames.SurfaceLineId))
{
SurfaceLine2 surfaceLine = dike.SurfaceLines2.FirstOrDefault(x => x.Name.Equals(locationDetail.ParameterValue));
location.SurfaceLine2 = surfaceLine;
}
// TODO: Assign PL1-line
}
// According to Erik Vastenburg, locations and surfacelines must match for matching pairs! So to find proper surfaceline
// for location, just search for it by name in surfacelines. SurfaceLineId can and should be removed entirely! But see above!
if (location.SurfaceLine2 == null)
{
location.SurfaceLine2 = dike.SurfaceLines2.FirstOrDefault(s => s.Name == location.Name);
}
location.SoilList = dike.SoilList;
CheckModelFactorsForLocation(location);
dike.Locations.Add(location);
}
dike.SortLocations();
}
///
/// Checks the model factors for location.
///
/// The location.
private static void CheckModelFactorsForLocation(Location location)
{
ModelParameters.AssignDefaultToNullValues(location.ModelFactors);
}
///
/// Imports the model parameters.
///
/// The dike ring identifier.
/// The dike.
private void ImportModelParameters(string dikeRingId, Dike dike)
{
var modelParameters = importer.GetDikeParameters(dikeRingId);
foreach (NameValueParameter modelParameter in modelParameters)
{
dike.SetParameterFromNameValuePair(modelParameter.ParameterName, modelParameter.ParameterValue);
DataSourceManager.SetSource(dike, modelParameter.ParameterName, modelParameter.Source);
}
}
///
/// Imports the surface lines.
///
/// The dike ring identifier.
/// The dike.
private void ImportSurfaceLines(string dikeRingId, Dike dike)
{
// Use InvokeWithoutPublishingEvents for performance reasons after introducint SurfaceLine2
DataEventPublisher.InvokeWithoutPublishingEvents(() =>
{
var surfaceLineIdList = importer.GetSurfaceLineIdList(dikeRingId);
foreach (var surfaceLineId in surfaceLineIdList)
{
var surfaceLine = new SurfaceLine2
{
Name = surfaceLineId,
Geometry = new LocalizedGeometryPointString(),
CharacteristicPoints = { GeometryMustContainPoint = true }
};
// load the surface line points
var surfaceLinePoints = importer.GetSurfaceLinePoints(dikeRingId, surfaceLineId);
foreach (var surfaceLinePoint in surfaceLinePoints)
{
surfaceLine.AddCharacteristicPoint(surfaceLinePoint);
}
// load the characteristic points
var characteristicPoints = importer.GetSurfaceLineCharacteristicPoints(dikeRingId, surfaceLineId);
foreach (var cp in characteristicPoints)
{
var cpType = (CharacteristicPointType)Enum.Parse(
typeof(CharacteristicPointType), cp.Id);
surfaceLine.EnsurePointOfType(cp.X, cp.Y, cp.Z, cpType);
}
// add the surface line to the dike
dike.SurfaceLines2.Add(surfaceLine);
}
});
}
///
/// Imports the segments.
///
/// The dike ring identifier.
/// The dike.
private void ImportSegments(string dikeRingID, Dike dike)
{
var segmentIdList = importer.GetSegmentIdList(dikeRingID);
foreach (var segmentID in segmentIdList)
{
var segment = new Segment { Name = segmentID };
foreach (FailureMechanismSystemType failureMechanismSystemType in Enum.GetValues(typeof(FailureMechanismSystemType)))
{
AddSoilProfiles(segment, dikeRingID, segmentID, failureMechanismSystemType, dike);
AddSoilGeometry(segment, dikeRingID, segmentID, failureMechanismSystemType);
}
waterBoard.Segments.Add(segment);
}
}
///
/// Adds the soil profiles.
///
/// The segment.
/// The dike ring identifier.
/// The segment identifier.
/// Type of the failure mechanism system.
/// The dike.
private void AddSoilProfiles(Segment segment, string dikeRingID, string segmentID, FailureMechanismSystemType failureMechanismSystemType, Dike dike)
{
var profile1DIdListForSegment = importer.GetProfile1DIdListForSegment(dikeRingID, segmentID, failureMechanismSystemType);
foreach (var soilProfile1DId in profile1DIdListForSegment)
{
var numberFormatInfo = new NumberFormatInfo { NumberDecimalSeparator = "." };
// get soil profile
var soilProfile1DDetails = importer.GetSegmentProfile1DDetails(dikeRingID, segmentID, soilProfile1DId, failureMechanismSystemType);
var soilProfile1D = dike.SoilProfiles.FirstOrDefault(x => x.Name.Equals(soilProfile1DId));
double probability = 0;
foreach (var nameValueparameter in soilProfile1DDetails)
{
if (nameValueparameter.ParameterName.Equals("Probability"))
probability = Convert.ToDouble(nameValueparameter.ParameterValue, numberFormatInfo);
}
segment.AddSoilProfileProbability(soilProfile1D, probability, failureMechanismSystemType);
}
}
///
/// Adds the soil geometry.
///
/// The segment.
/// The dike ring identifier.
/// The segment identifier.
/// Type of the failure mechanism system.
private void AddSoilGeometry(Segment segment, string dikeRingID, string segmentID, FailureMechanismSystemType failureMechanismSystemType)
{
var profile2DIdListForSegment = importer.GetProfile2DIdListForSegment(dikeRingID, segmentID, failureMechanismSystemType);
foreach (string soilGeometry2DName in profile2DIdListForSegment)
{
var numberFormatInfo = new NumberFormatInfo { NumberDecimalSeparator = "." };
var soilProfile2DDetails = importer.GetSegmentProfile2DDetails(dikeRingID, segmentID, soilGeometry2DName, failureMechanismSystemType);
double probability = 0.0;
foreach (var nameValueparameter in soilProfile2DDetails)
{
if (nameValueparameter.ParameterName.Equals("Probability"))
probability = Convert.ToDouble(nameValueparameter.ParameterValue, numberFormatInfo);
}
segment.AddSoilGeometry2DProbability(soilGeometry2DName, probability, failureMechanismSystemType);
}
}
///
/// Imports the soil profiles1 d.
///
/// The dike ring identifier.
/// The dike.
private void ImportSoilProfiles1D(string dikeRingId, Dike dike)
{
var soilProfile1DIdList = importer.GetSoilProfile1DIdList(dikeRingId);
foreach (var soilProfile1DId in soilProfile1DIdList)
{
dike.SoilProfiles.Add(ImportProfile(soilProfile1DId, dikeRingId, dike));
}
}
///
/// Imports a profile.
///
/// The soil profile1 d identifier.
/// The dike ring identifier.
/// The dike.
///
///
///
private SoilProfile1D ImportProfile(string soilProfile1DId, string dikeRingId, Dike dike)
{
var soilProfile = new SoilProfile1D { Name = soilProfile1DId };
double bottomLevel = 0.0;
var dpSoilProfile = importer.GetSoilProfile1DDetails(dikeRingId, soilProfile1DId);
for (int layerIndex = 0; layerIndex < dpSoilProfile.Layers.Count; layerIndex++)
{
var layer = dpSoilProfile.Layers[layerIndex];
if (layer.SoilName == SoilProfile1D.SoilProfileBottomLevelId)
{
// This is a hack to communicate the bottomlevel via a layer
bottomLevel = layer.TopLevel;
}
else
{
int soilIndex = dike.SoilList.GetSoilIndexByName(layer.SoilName);
if (soilIndex < 0)
{
throw new WaterBoardImporterException(String.Format("Soil '{0}' not found in 1d-profile '{1}'",
dpSoilProfile.Layers[layerIndex].SoilName, soilProfile1DId));
}
soilProfile.Layers.Add(new SoilLayer1D
{
TopLevel = layer.TopLevel,
Soil = dike.SoilList.Soils[soilIndex]
});
}
}
soilProfile.EnsureUniqueLayerIds();
soilProfile.BottomLevel = bottomLevel;
if (soilProfile.LayerCount > 0)
{
soilProfile.EnsureLastLayerHasHeight();
}
else
{
throw new WaterBoardImporterException(String.Format("Soil 1d-profile '{0}' has no layers", soilProfile.Name));
}
return soilProfile;
}
}
}