// Copyright (C) Stichting Deltares 2018. All rights reserved. // // This file is part of the application DAM - Clients Library. // // DAM - UI is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // // All names, logos, and references to "Deltares" are registered trademarks of // Stichting Deltares and remain full property of Stichting Deltares at all times. // All rights reserved. using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using Deltares.Standard.EventPublisher; using Deltares.Geotechnics.GeotechnicalGeometry; using Deltares.Geotechnics.IO; using Deltares.Geotechnics.Soils; using Deltares.Geotechnics.SurfaceLines; using Deltares.Standard.Logging; namespace Deltares.Dam.Data { using DataPlugins.Configuration; using DataPlugins; using Standard; using Maps; public class WaterBoardImporterException : Exception { public WaterBoardImporterException(string message) : base(message) { } } /// /// Import all data into Waterboard /// public class WaterBoardImporter { // // Members & Properties // private readonly IDataPlugin importer; private readonly WaterBoard waterBoard = new WaterBoard(); private readonly DamType damType; private readonly DamProjectType damProjectType; private readonly List importCsvLogMessages = new List(); /// /// Gets the import CSV log messages. /// /// /// The import CSV log messages. /// public List ImportCsvLogMessages { get { return importCsvLogMessages; } } /// /// Gets all dike ring ids. /// /// /// All dike ring i ds. /// private IEnumerable AllDikeRingIDs { get { importCsvLogMessages.Clear(); var dikeRingIdList = importer.GetDikeRingIdList(damType).ToList(); importCsvLogMessages.AddRange(importer.ImportLogMessages); return dikeRingIdList; } } /// /// Gets the data source list. /// /// /// The data source list. /// private IEnumerable DataSourceList { get { return importer.DataSources; } } /// /// Gets the data folder. /// /// /// The data folder. /// private string DataFolder { get { return importer.DataFolder; } } /// /// Initializes a new instance of the class. /// public WaterBoardImporter() { } /// /// Initializes a new instance of the class. /// /// The data folder. /// The data source container. /// Type of the dam. /// Type of the dam project. private WaterBoardImporter(string dataFolder, DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType) { this.damProjectType = damProjectType; importer = CreateDataPlugin(dataFolder, dataSourceContainer); importer.DamProjectType = damProjectType; this.damType = damType; } /// /// Creates the data plugin. /// /// The data folder. /// The data source container. /// private static DataPluginImporter CreateDataPlugin(string dataFolder, DataSourceContainer dataSourceContainer) { var plugin = new DataPluginImporter(); plugin.SetDataSources(dataFolder, dataSourceContainer.DataSourceList); plugin.Attributes = dataSourceContainer.DataAttributes; return plugin; } /// /// Imports the dike ring ids. /// /// The data folder. /// The data source container. /// Type of the dam. /// Type of the dam project. /// The import log messages. /// public static IEnumerable ImportDikeRingIds( string dataFolder, DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType, out List importLogMessages) { var importer = new WaterBoardImporter(dataFolder, dataSourceContainer, damType, damProjectType); var allDikeRingIds = importer.AllDikeRingIDs; importLogMessages = new List(importer.ImportCsvLogMessages); return allDikeRingIds; } /// /// Imports all data. /// /// The data folder. /// The data source container. /// Type of the dam. /// Type of the dam project. /// The progress. /// The log messages. /// public static WaterBoard ImportAllData(string dataFolder, DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress, out List logMessages) { var importer = CreateDataPlugin(dataFolder, dataSourceContainer); var dikeRingIds = importer.GetDikeRingIdList(damType); WaterBoard waterboard = ImportDataForDikeRings("", dataFolder, dataSourceContainer, dikeRingIds, damType, damProjectType, progress, out logMessages); return waterboard; } /// /// Imports all data. /// /// The data folder. /// The data source container. /// Type of the dam. /// Type of the dam project. /// The progress. /// public static WaterBoard ImportAllData(string dataFolder, DataSourceContainer dataSourceContainer, DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress) { List dummy; return ImportAllData(dataFolder, dataSourceContainer, damType, damProjectType, progress, out dummy); } /// /// Imports the data for dike rings. /// /// The dam project folder. /// The data folder. /// The data source container. /// The dike ring ids. /// Type of the dam. /// Type of the dam project. /// The progress. /// public static WaterBoard ImportDataForDikeRings(string damProjectFolder, string dataFolder, DataSourceContainer dataSourceContainer, IEnumerable dikeRingIds, DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress) { List dummy; return ImportDataForDikeRings(damProjectFolder, dataFolder, dataSourceContainer, dikeRingIds, damType, damProjectType, progress, out dummy); } /// /// Imports the data for dike rings. /// /// The dam project folder. /// The data folder. /// The data source container. /// The dike ring ids. /// Type of the dam. /// Type of the dam project. /// The progress. /// The log messages. /// public static WaterBoard ImportDataForDikeRings(string damProjectFolder, string dataFolder, DataSourceContainer dataSourceContainer, IEnumerable dikeRingIds, DamType damType, DamProjectType damProjectType, DamEngine.Data.Standard.Calculation.ProgressDelegate progress, out List logMessages) { var importer = new WaterBoardImporter(dataFolder, dataSourceContainer, damType, damProjectType); logMessages = new List(); WaterBoard waterBoard = new WaterBoard(); try { waterBoard = importer.ImportDataForDikeRings(damProjectFolder, dikeRingIds, progress); foreach (Dike dike in waterBoard.Dikes) { if (String.IsNullOrEmpty(dike.MapForSoilGeometries2D) && !String.IsNullOrEmpty(dataSourceContainer.MapSoilProfile2D)) { dike.MapForSoilGeometries2D = dataSourceContainer.MapSoilProfile2D; } } } catch (Exception e) { string errorMessage = e.Message; Exception innerException = e.InnerException; while (innerException != null) { errorMessage = errorMessage + ";" + innerException.Message; innerException = innerException.InnerException; } logMessages.Add(new LogMessage(LogMessageType.FatalError, null, errorMessage)); } logMessages.AddRange(importer.ImportCsvLogMessages); return waterBoard; } /// /// Imports the data for dike rings. /// /// The dam project folder. /// The dike ring ids. /// The progress. /// private WaterBoard ImportDataForDikeRings(string damProjectFolder, IEnumerable dikeRingIds, DamEngine.Data.Standard.Calculation.ProgressDelegate progress) { var dikeRingList = ValidateDikeRingIDs(dikeRingIds); importer.DamProjectFolder = damProjectFolder; // Import dikes var importOk = ImportDikes(dikeRingList, progress); if (!importOk) return waterBoard; // import data for dike rings foreach (var dikeRingId in dikeRingList) { // Add dikes to waterboard var dike = ImportDataForDikeRing(dikeRingId); waterBoard.Dikes.Add(dike); } ImportBackground(); return waterBoard; } /// /// Validates the dike ring i ds. /// /// The dike ring ids. /// private List ValidateDikeRingIDs(IEnumerable dikeRingIds) { // if no dike ring IDs specified, take all return dikeRingIds == null ? importer.GetDikeRingIdList(damType).ToList() : (dikeRingIds as List ?? dikeRingIds.ToList()); } /// /// Creates the soil base for dike. /// /// The dike. private void CreateSoilBaseForDike(Dike dike) { // TODO: soildatabase should be created. Now we make a direct link // must be based on dike.SoilDatabaseName } /// /// Determines the database name from data sources. /// /// The dike. private void DetermineDatabaseNameFromDataSources(Dike dike) { var sources = DataSourceList.ToList(); // get the data base name from the data sources var dataSource = sources.FirstOrDefault(ds => ds.DataSourceType == DataSourceType.MSoilBase) ?? sources.FirstOrDefault(ds => ds.DataSourceType == DataSourceType.CsvFiles); if (dataSource != null) { if (dataSource.DataSourceType == DataSourceType.MSoilBase) { dike.SoilDatabaseName = Path.Combine(DataFolder, dataSource.DataLocation); } else { var subFolder = Path.Combine(DataFolder, dataSource.DataLocation); dike.SoilDatabaseName = Path.Combine(subFolder, "soilmaterials.mdb"); } } } /// /// Imports the soils. /// /// The dike ring identifier. /// The dike. private void ImportSoils(string dikeRingID, Dike dike) { dike.SoilList = GetSoils(dikeRingID); DetermineDatabaseNameFromDataSources(dike); dike.FillDataBaseSoilListFromSoilBase(); } /// /// Gets the soils. /// /// The dike ring identifier. /// private SoilList GetSoils(string dikeRingID) { var soilIdList = importer.GetSoilIdList(dikeRingID); var soils = soilIdList .Select(soilID => ImportSoil(dikeRingID, soilID)) .ToList(); var aquifers = importer.GetSoilAquiferValues(dikeRingID); var soilList = new SoilList(); soilList.Soils = soils; foreach (var nameValueParameter in aquifers) { bool vala; bool.TryParse(nameValueParameter.ParameterValue, out vala); soilList.AquiferDictionary.Add(soilList.GetSoilByName(nameValueParameter.ParameterName), vala); } return soilList; } /// /// Imports a soil. /// /// The dike ring identifier. /// The soil identifier. /// private Soil ImportSoil(string dikeRingID, string soilID) { var soil = new Soil { Name = soilID }; var soilDetails = importer.GetSoilDetails(dikeRingID, soilID); foreach (var soilDetail in soilDetails) { SoilUtils.SetParameterFromNameValuePair(soil, soilDetail.ParameterName, soilDetail.ParameterValue); DataSourceManager.SetSource(soil, soilDetail.ParameterName, soilDetail.Source); } return soil; } /// /// Imports the data for dike ring. /// /// The dike ring identifier. /// private Dike ImportDataForDikeRing(string dikeRingId) { var dike = new Dike { Name = dikeRingId }; ImportSoils(dikeRingId, dike); ImportSoilProfiles1D(dikeRingId, dike); ImportSegments(dikeRingId, dike); ImportSurfaceLines(dikeRingId, dike); ImportModelParameters(dikeRingId, dike); // TODO: Import PL1-lines ImportLocations(dikeRingId, dike); //ImportScenarios(importer, dikeRingId, dike, waterBoard); ImportScenarios(dikeRingId, dike); // Use InvokeWithoutPublishingEvents for performance reasons after introducint SurfaceLine2 DataEventPublisher.InvokeWithoutPublishingEvents(() => { var dikeCoordinateSystemConverter = new DikeCoordinateSystemConverter(); if (dike.Locations.Count > 0) { dikeCoordinateSystemConverter.CreateLocalXZObjects(dike); } }); ImportCsvLogMessages.AddRange(dike.MakeDataConsistent()); CreateSoilBaseForDike(dike); DisposeImportedItem(dikeRingId); return dike; } private void DisposeImportedItem(string dikeRingId) { importer.DisposeImportedItem(dikeRingId); } /// /// Imports all specified dikes. /// /// The dike ring ids. /// The progress. /// private bool ImportDikes(IEnumerable dikeRingIds, DamEngine.Data.Standard.Calculation.ProgressDelegate progress) { // Import dikes importer.ImportDataForDikeRings(dikeRingIds, damType, progress); // add log messages ImportCsvLogMessages.AddRange(importer.ImportLogMessages); // import is ok if no fatal messages return ImportCsvLogMessages.All(message => message.MessageType != LogMessageType.FatalError); } /// /// Imports the background layer. /// private void ImportBackground() { var geometryFeatures = importer.MapGeometryIdList .Select(id => importer.GetMapGeometry(id)) .Select(Feature.Create); foreach (var geometry in geometryFeatures) { waterBoard.AddBackgroundGeometry(geometry); } waterBoard.FillFeatureList(); } /// /// Imports the scenarios. /// /// The dike ring identifier. /// The dike. private void ImportScenarios(string dikeRingId, Dike dike) { AddScenariosToLocations(dike, dikeRingId); } /// /// Adds the scenarios to locations. /// /// The dike. /// The dike ring identifier. private void AddScenariosToLocations(Dike dike, string dikeRingId) { foreach (var location in dike.Locations) { var scenarioList = importer.GetScenarioList(dikeRingId, location.Name); foreach (var scenarioId in scenarioList) { var nameValues = importer.GetScenarioDetails(dikeRingId, location.Name, scenarioId); var scenario = new Scenario { LocationScenarioID = scenarioId }; foreach (var nameValueParameter in nameValues) { scenario.SetParameterFromNameValuePair( nameValueParameter.ParameterName, nameValueParameter.ParameterValue); } scenario.Location = location; location.Scenarios.Add(scenario); } } } /// /// Imports the locations. /// /// The dike ring identifier. /// The dike. private void ImportLocations(string dikeRingId, Dike dike) { var locationIdList = importer.GetLocationIdList(dikeRingId, damType); foreach (string locationId in locationIdList) { var location = new Location { Name = locationId }; Info locationInfo = importer.GetLocationInfo(dikeRingId, locationId); location.Name = locationInfo.Name; DataSourceManager.SetSource(location, "Name", locationInfo.Source); location.Description = locationInfo.Description; DataSourceManager.SetSource(location, "Description", locationInfo.Source); IEnumerable locationDetails = importer.GetLocationDetails(dikeRingId, locationId); foreach (NameValueParameter locationDetail in locationDetails) { location.SetParameterFromNameValuePair(locationDetail.ParameterName, locationDetail.ParameterValue); DataSourceManager.SetSource(location, locationDetail.ParameterName, locationDetail.Source); if (locationDetail.ParameterName.Equals(LocationParameterNames.SegmentId)) { Segment segment = waterBoard.Segments.FirstOrDefault(x => x.Name.Equals(locationDetail.ParameterValue)); location.Segment = segment; } // See comment below: the test WaterBoardImportedWithCsvFilesHasValidData using Groot Salland\Binnenwaarts proves Erik wrong if (locationDetail.ParameterName.Equals(LocationParameterNames.SurfaceLineId)) { SurfaceLine2 surfaceLine = dike.SurfaceLines2.FirstOrDefault(x => x.Name.Equals(locationDetail.ParameterValue)); location.SurfaceLine2 = surfaceLine; } // TODO: Assign PL1-line } // According to Erik Vastenburg, locations and surfacelines must match for matching pairs! So to find proper surfaceline // for location, just search for it by name in surfacelines. SurfaceLineId can and should be removed entirely! But see above! if (location.SurfaceLine2 == null) { location.SurfaceLine2 = dike.SurfaceLines2.FirstOrDefault(s => s.Name == location.Name); } location.SoilList = dike.SoilList; CheckModelFactorsForLocation(location); dike.Locations.Add(location); } dike.SortLocations(); } /// /// Checks the model factors for location. /// /// The location. private static void CheckModelFactorsForLocation(Location location) { ModelParameters.AssignDefaultToNullValues(location.ModelFactors); } /// /// Imports the model parameters. /// /// The dike ring identifier. /// The dike. private void ImportModelParameters(string dikeRingId, Dike dike) { var modelParameters = importer.GetDikeParameters(dikeRingId); foreach (NameValueParameter modelParameter in modelParameters) { dike.SetParameterFromNameValuePair(modelParameter.ParameterName, modelParameter.ParameterValue); DataSourceManager.SetSource(dike, modelParameter.ParameterName, modelParameter.Source); } } /// /// Imports the surface lines. /// /// The dike ring identifier. /// The dike. private void ImportSurfaceLines(string dikeRingId, Dike dike) { // Use InvokeWithoutPublishingEvents for performance reasons after introducint SurfaceLine2 DataEventPublisher.InvokeWithoutPublishingEvents(() => { var surfaceLineIdList = importer.GetSurfaceLineIdList(dikeRingId); foreach (var surfaceLineId in surfaceLineIdList) { var surfaceLine = new SurfaceLine2 { Name = surfaceLineId, Geometry = new LocalizedGeometryPointString(), CharacteristicPoints = { GeometryMustContainPoint = true } }; // load the surface line points var surfaceLinePoints = importer.GetSurfaceLinePoints(dikeRingId, surfaceLineId); foreach (var surfaceLinePoint in surfaceLinePoints) { surfaceLine.AddCharacteristicPoint(surfaceLinePoint); } // load the characteristic points var characteristicPoints = importer.GetSurfaceLineCharacteristicPoints(dikeRingId, surfaceLineId); foreach (var cp in characteristicPoints) { var cpType = (CharacteristicPointType)Enum.Parse( typeof(CharacteristicPointType), cp.Id); surfaceLine.EnsurePointOfType(cp.X, cp.Y, cp.Z, cpType); } // add the surface line to the dike dike.SurfaceLines2.Add(surfaceLine); } }); } /// /// Imports the segments. /// /// The dike ring identifier. /// The dike. private void ImportSegments(string dikeRingID, Dike dike) { var segmentIdList = importer.GetSegmentIdList(dikeRingID); foreach (var segmentID in segmentIdList) { var segment = new Segment { Name = segmentID }; foreach (FailureMechanismSystemType failureMechanismSystemType in Enum.GetValues(typeof(FailureMechanismSystemType))) { AddSoilProfiles(segment, dikeRingID, segmentID, failureMechanismSystemType, dike); AddSoilGeometry(segment, dikeRingID, segmentID, failureMechanismSystemType); } waterBoard.Segments.Add(segment); } } /// /// Adds the soil profiles. /// /// The segment. /// The dike ring identifier. /// The segment identifier. /// Type of the failure mechanism system. /// The dike. private void AddSoilProfiles(Segment segment, string dikeRingID, string segmentID, FailureMechanismSystemType failureMechanismSystemType, Dike dike) { var profile1DIdListForSegment = importer.GetProfile1DIdListForSegment(dikeRingID, segmentID, failureMechanismSystemType); foreach (var soilProfile1DId in profile1DIdListForSegment) { var numberFormatInfo = new NumberFormatInfo { NumberDecimalSeparator = "." }; // get soil profile var soilProfile1DDetails = importer.GetSegmentProfile1DDetails(dikeRingID, segmentID, soilProfile1DId, failureMechanismSystemType); var soilProfile1D = dike.SoilProfiles.FirstOrDefault(x => x.Name.Equals(soilProfile1DId)); double probability = 0; foreach (var nameValueparameter in soilProfile1DDetails) { if (nameValueparameter.ParameterName.Equals("Probability")) probability = Convert.ToDouble(nameValueparameter.ParameterValue, numberFormatInfo); } segment.AddSoilProfileProbability(soilProfile1D, probability, failureMechanismSystemType); } } /// /// Adds the soil geometry. /// /// The segment. /// The dike ring identifier. /// The segment identifier. /// Type of the failure mechanism system. private void AddSoilGeometry(Segment segment, string dikeRingID, string segmentID, FailureMechanismSystemType failureMechanismSystemType) { var profile2DIdListForSegment = importer.GetProfile2DIdListForSegment(dikeRingID, segmentID, failureMechanismSystemType); foreach (string soilGeometry2DName in profile2DIdListForSegment) { var numberFormatInfo = new NumberFormatInfo { NumberDecimalSeparator = "." }; var soilProfile2DDetails = importer.GetSegmentProfile2DDetails(dikeRingID, segmentID, soilGeometry2DName, failureMechanismSystemType); double probability = 0.0; foreach (var nameValueparameter in soilProfile2DDetails) { if (nameValueparameter.ParameterName.Equals("Probability")) probability = Convert.ToDouble(nameValueparameter.ParameterValue, numberFormatInfo); } segment.AddSoilGeometry2DProbability(soilGeometry2DName, probability, failureMechanismSystemType); } } /// /// Imports the soil profiles1 d. /// /// The dike ring identifier. /// The dike. private void ImportSoilProfiles1D(string dikeRingId, Dike dike) { var soilProfile1DIdList = importer.GetSoilProfile1DIdList(dikeRingId); foreach (var soilProfile1DId in soilProfile1DIdList) { dike.SoilProfiles.Add(ImportProfile(soilProfile1DId, dikeRingId, dike)); } } /// /// Imports a profile. /// /// The soil profile1 d identifier. /// The dike ring identifier. /// The dike. /// /// /// private SoilProfile1D ImportProfile(string soilProfile1DId, string dikeRingId, Dike dike) { var soilProfile = new SoilProfile1D { Name = soilProfile1DId }; double bottomLevel = 0.0; var dpSoilProfile = importer.GetSoilProfile1DDetails(dikeRingId, soilProfile1DId); for (int layerIndex = 0; layerIndex < dpSoilProfile.Layers.Count; layerIndex++) { var layer = dpSoilProfile.Layers[layerIndex]; if (layer.SoilName == SoilProfile1D.SoilProfileBottomLevelId) { // This is a hack to communicate the bottomlevel via a layer bottomLevel = layer.TopLevel; } else { int soilIndex = dike.SoilList.GetSoilIndexByName(layer.SoilName); if (soilIndex < 0) { throw new WaterBoardImporterException(String.Format("Soil '{0}' not found in 1d-profile '{1}'", dpSoilProfile.Layers[layerIndex].SoilName, soilProfile1DId)); } soilProfile.Layers.Add(new SoilLayer1D { TopLevel = layer.TopLevel, Soil = dike.SoilList.Soils[soilIndex] }); } } soilProfile.EnsureUniqueLayerIds(); soilProfile.BottomLevel = bottomLevel; if (soilProfile.LayerCount > 0) { soilProfile.EnsureLastLayerHasHeight(); } else { throw new WaterBoardImporterException(String.Format("Soil 1d-profile '{0}' has no layers", soilProfile.Name)); } return soilProfile; } } }