// Copyright (C) Stichting Deltares 2016. All rights reserved.
//
// This file is part of Ringtoets.
//
// Ringtoets is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Core.Common.Base.Geometry;
using Core.Common.Base.IO;
using Core.Common.IO.Exceptions;
using Core.Common.IO.Readers;
using Core.Common.Utils;
using Core.Common.Utils.Builders;
using Ringtoets.Piping.IO.Properties;
using UtilsResources = Core.Common.Utils.Properties.Resources;
namespace Ringtoets.Piping.IO.SurfaceLines
{
///
/// File reader for a plain text file in comma-separated values format (*.csv), containing
/// data specifying characteristic points.
/// Expects data to be specified in the following format:
///
/// LocationID;X_Maaiveld binnenwaarts;Y_Maaiveld binnenwaarts;Z_Maaiveld binnenwaarts;X_Insteek sloot polderzijde;Y_Insteek sloot polderzijde;Z_Insteek sloot polderzijde;X_Slootbodem polderzijde;Y_Slootbodem polderzijde;Z_Slootbodem polderzijde;X_Slootbodem dijkzijde;Y_Slootbodem dijkzijde;Z_Slootbodem dijkzijde;X_Insteek sloot dijkzijde;Y_Insteek_sloot dijkzijde;Z_Insteek sloot dijkzijde;X_Teen dijk binnenwaarts;Y_Teen dijk binnenwaarts;Z_Teen dijk binnenwaarts;X_Kruin binnenberm;Y_Kruin binnenberm;Z_Kruin binnenberm;X_Insteek binnenberm;Y_Insteek binnenberm;Z_Insteek binnenberm;X_Kruin binnentalud;Y_Kruin binnentalud;Z_Kruin binnentalud;X_Verkeersbelasting kant binnenwaarts;Y_Verkeersbelasting kant binnenwaarts;Z_Verkeersbelasting kant binnenwaarts;X_Verkeersbelasting kant buitenwaarts;Y_Verkeersbelasting kant buitenwaarts;Z_Verkeersbelasting kant buitenwaarts;X_Kruin buitentalud;Y_Kruin buitentalud;Z_Kruin buitentalud;X_Insteek buitenberm;Y_Insteek buitenberm;Z_Insteek buitenberm;X_Kruin buitenberm;Y_Kruin buitenberm;Z_Kruin buitenberm;X_Teen dijk buitenwaarts;Y_Teen dijk buitenwaarts;Z_Teen dijk buitenwaarts;X_Maaiveld buitenwaarts;Y_Maaiveld buitenwaarts;Z_Maaiveld buitenwaarts;X_Dijktafelhoogte;Y_Dijktafelhoogte;Z_Dijktafelhoogte;Volgnummer
///
/// Where {LocationID} has to be a particular accepted text, {Volgnummer} is ignored, and n triplets of doubles form the
/// 3D coordinates defining each characteristic point.
///
public class CharacteristicPointsCsvReader : IDisposable
{
private const char separator = ';';
private const string xPrefix = "x_";
private const string yPrefix = "y_";
private const string zPrefix = "z_";
private static readonly Point3D undefinedPoint = new Point3D(-1, -1, -1);
private readonly string filePath;
///
/// Lower case string representations of the known characteristic point types.
///
private readonly Dictionary columnsInFile = new Dictionary();
private StreamReader fileReader;
private int lineNumber;
///
/// Initializes a new instance of using
/// the given .
///
/// The path to use for reading characteristic points.
/// Thrown when is invalid.
public CharacteristicPointsCsvReader(string path)
{
IOUtils.ValidateFilePath(path);
filePath = path;
}
///
/// Reads the file to determine the number of available
/// data rows.
///
/// A value greater than or equal to 0.
/// Thrown when a critical error has occurred, which may be caused by:
///
/// - The file cannot be found at specified path.
/// - The specified path is invalid, such as being on an unmapped drive.
/// - Some other I/O related issue occurred, such as: path includes an incorrect
/// or invalid syntax for file name, directory name, or volume label.
/// - There is insufficient memory to allocate a buffer for the returned string.
/// - The file incompatible for importing characteristic points locations.
///
///
public int GetLocationsCount()
{
using (var reader = StreamReaderHelper.InitializeStreamReader(filePath))
{
ValidateHeader(reader);
return CountNonEmptyLines(reader, 2);
}
}
///
/// Reads and parses the next data row to create a new instance of ,
/// which will contain all the declared characteristic points for some surfaceline.
///
/// Return the parsed characteristic points location, or null when at the end of the file.
/// Thrown when a critical error has occurred, which may be caused by:
///
/// - The file cannot be found at specified path.
/// - The specified path is invalid, such as being on an unmapped drive.
/// - Some other I/O related issue occurred, such as: path includes an incorrect
/// or invalid syntax for file name, directory name, or volume label.
/// - There is insufficient memory to allocate a buffer for the returned string.
/// - The file incompatible for importing characteristic points.
///
///
/// Thrown when a parse error has occurred for the current row, which may be caused by:
///
/// - The row doesn't use ';' as separator character.
/// - The row contains a coordinate value that cannot be parsed as a double.
/// - The row contains a number that is too big or too small to be represented with a double.
/// - The row is missing an identifier value.
/// - The row is missing values to form a characteristic point.
///
///
public CharacteristicPoints ReadCharacteristicPointsLocation()
{
if (fileReader == null)
{
fileReader = StreamReaderHelper.InitializeStreamReader(filePath);
ValidateHeader(fileReader);
lineNumber = 2;
}
var readText = ReadNextNonEmptyLine();
if (readText != null)
{
try
{
return CreatePipingCharacteristicPointsLocation(readText);
}
finally
{
lineNumber++;
}
}
return null;
}
///
/// Disposes the current .
///
public void Dispose()
{
if (fileReader != null)
{
fileReader.Dispose();
fileReader = null;
}
}
///
/// Reads lines from file until the first non-white line is hit.
///
/// The next line which is not a white line, or null when no non-white
/// line could be found before the end of file.
private string ReadNextNonEmptyLine()
{
string readText;
while ((readText = ReadLineAndHandleIOExceptions(fileReader, lineNumber)) != null)
{
if (string.IsNullOrWhiteSpace(readText))
{
lineNumber++;
}
else
{
break;
}
}
return readText;
}
///
/// Validates the header of the file.
///
/// The reader, which is currently at the header row.
/// Thrown when the header is not in the
/// required format or the file is empty.
private void ValidateHeader(TextReader reader)
{
var currentLine = 1;
var header = ReadLineAndHandleIOExceptions(reader, currentLine);
if (header != null)
{
if (!IsHeaderValid(header))
{
throw CreateCriticalFileReadException(currentLine, Resources.CharacteristicPointsCsvReader_File_invalid_header);
}
}
else
{
throw CreateCriticalFileReadException(currentLine, UtilsResources.Error_File_empty);
}
}
///
/// Reads the next line and handles I/O exceptions.
///
/// The opened text file reader.
/// Row number for error messaging.
/// The read line, or null when at the end of the file.
/// Thrown when a critical I/O exception occurred.
private string ReadLineAndHandleIOExceptions(TextReader reader, int currentLine)
{
try
{
return reader.ReadLine();
}
catch (OutOfMemoryException e)
{
throw CreateCriticalFileReadException(currentLine, UtilsResources.Error_Line_too_big_for_RAM, e);
}
catch (IOException e)
{
var message = new FileReaderErrorMessageBuilder(filePath).Build(string.Format(UtilsResources.Error_General_IO_ErrorMessage_0_, e.Message));
throw new CriticalFileReadException(message, e);
}
}
///
/// Checks whether the given is valid. A valid header has a locationid column and is followed by triplets
/// of x_{characteristic_point_type};y_{characteristic_point_type};z_{characteristic_point_type} triplets.
///
/// The line which should represent the header of a characteristic point file.
/// true if the is valid, false otherwise.
private bool IsHeaderValid(string header)
{
columnsInFile.Clear();
lineNumber = 1;
string[] tokenizedHeader = TokenizeString(header.ToLowerInvariant().Replace(' ', '_'));
if (!DetermineIdColumn(tokenizedHeader))
{
return false;
}
if (tokenizedHeader.Where(c => c.StartsWith(xPrefix))
.Any(c => !DetermineRequiredYZColumns(c, tokenizedHeader)))
{
return false;
}
DetermineOrderColumn(tokenizedHeader);
return tokenizedHeader.Length == columnsInFile.Count;
}
private void DetermineOrderColumn(string[] tokenizedHeader)
{
var orderColumnIndex = Array.IndexOf(tokenizedHeader, orderNumberKey);
if (orderColumnIndex > -1)
{
columnsInFile[orderNumberKey] = orderColumnIndex;
}
}
private bool DetermineRequiredYZColumns(string column, string[] tokenizedHeader)
{
var key = column.Substring(2);
var xColumnIdentifier = xPrefix + key;
var yColumnIdentifier = yPrefix + key;
var zColumnIdentifier = zPrefix + key;
var xColumnIndex = Array.IndexOf(tokenizedHeader, xColumnIdentifier);
var yColumnIndex = Array.IndexOf(tokenizedHeader, yColumnIdentifier);
var zColumnIndex = Array.IndexOf(tokenizedHeader, zColumnIdentifier);
if (yColumnIndex == -1 || zColumnIndex == -1)
{
return false;
}
columnsInFile[xColumnIdentifier] = xColumnIndex;
columnsInFile[yColumnIdentifier] = yColumnIndex;
columnsInFile[zColumnIdentifier] = zColumnIndex;
return true;
}
private bool DetermineIdColumn(string[] tokenizedHeader)
{
var locationIdColumnIndex = Array.IndexOf(tokenizedHeader, locationIdKey);
if (locationIdColumnIndex == -1)
{
locationIdColumnIndex = Array.IndexOf(tokenizedHeader, surfaceLineKey);
}
if (locationIdColumnIndex > -1)
{
columnsInFile[locationIdKey] = locationIdColumnIndex;
}
else
{
return false;
}
return true;
}
///
/// Counts the remaining non-empty lines.
///
/// The reader at the row from which counting should start.
/// The current line, used for error messaging.
/// An integer greater than or equal to 0, being the number of characteristic points location rows.
/// Thrown when an I/O exception occurred.
private int CountNonEmptyLines(TextReader reader, int currentLine)
{
int count = 0;
int lineNumberForMessage = currentLine;
string line;
while ((line = ReadLineAndHandleIOExceptions(reader, lineNumberForMessage)) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
count++;
}
lineNumberForMessage++;
}
return count;
}
///
/// Creates a new from the .
///
/// A single line read from file.
/// A new with name and characteristic points set.
/// Thrown when:
///
/// - has too many or few columns.
/// - contains a coordinate value which could not be parsed to .
///
private CharacteristicPoints CreatePipingCharacteristicPointsLocation(string readText)
{
string[] tokenizedString = TokenizeString(readText);
string locationName = GetLocationName(tokenizedString);
if (tokenizedString.Length != columnsInFile.Count)
{
throw CreateLineParseException(lineNumber, locationName, Resources.CharacteristicPointsCsvReader_ReadCharacteristicPointsLocation_Location_lacks_values_for_characteristic_points);
}
var location = new CharacteristicPoints(locationName);
SetCharacteristicPoints(tokenizedString, location);
return location;
}
///
/// Sets the characteristic points from the given to the given .
///
/// The string read from file.
/// The to set the characteristic points for.
/// Thrown when
/// contains a coordinate value which could not be parsed to .
private void SetCharacteristicPoints(string[] tokenizedString, CharacteristicPoints location)
{
location.SurfaceLevelInside = GetPoint3D(tokenizedString, surfaceLevelInsideKey, location.Name);
location.DitchPolderSide = GetPoint3D(tokenizedString, ditchPolderSideKey, location.Name);
location.BottomDitchPolderSide = GetPoint3D(tokenizedString, bottomDitchPolderSideKey, location.Name);
location.BottomDitchDikeSide = GetPoint3D(tokenizedString, bottomDitchDikeSideKey, location.Name);
location.DitchDikeSide = GetPoint3D(tokenizedString, ditchDikeSideKey, location.Name);
location.DikeToeAtPolder = GetPoint3D(tokenizedString, dikeToeAtPolderKey, location.Name);
location.TopShoulderInside = GetPoint3D(tokenizedString, topShoulderInsideKey, location.Name);
location.ShoulderInside = GetPoint3D(tokenizedString, shoulderInsideKey, location.Name);
location.DikeTopAtPolder = GetPoint3D(tokenizedString, dikeTopAtPolderKey, location.Name);
location.TrafficLoadInside = GetPoint3D(tokenizedString, trafficLoadInsideKey, location.Name);
location.TrafficLoadOutside = GetPoint3D(tokenizedString, trafficLoadOutsideKey, location.Name);
location.DikeTopAtRiver = GetPoint3D(tokenizedString, dikeTopAtRiverKey, location.Name);
location.ShoulderOutisde = GetPoint3D(tokenizedString, shoulderOutsideKey, location.Name);
location.TopShoulderOutside = GetPoint3D(tokenizedString, topShoulderOutsideKey, location.Name);
location.DikeToeAtRiver = GetPoint3D(tokenizedString, dikeToeAtRiverKey, location.Name);
location.SurfaceLevelOutside = GetPoint3D(tokenizedString, surfaceLevelOutsideKey, location.Name);
location.DikeTableHeight = GetPoint3D(tokenizedString, dikeTableHeightKey, location.Name);
location.InsertRiverChannel = GetPoint3D(tokenizedString, insertRiverChannelKey, location.Name);
location.BottomRiverChannel = GetPoint3D(tokenizedString, bottomRiverChannelKey, location.Name);
}
///
/// Creates a new from the collection of , by using
/// the to create the column identifiers to read from.
///
/// The collection of read data.
/// The key for the type of characteristic point.
/// The name of the location used for creating descriptive errors.
/// A new with values for x,y,z set.
/// Thrown when
/// contains a value which could not be parsed to a double in the column that had to be read for creating
/// the .
private Point3D GetPoint3D(string[] valuesRead, string typeKey, string locationName)
{
try
{
Point3D point = null;
var xColumnKey = xPrefix + typeKey;
if (columnsInFile.ContainsKey(xColumnKey))
{
var xColumnIndex = columnsInFile[xColumnKey];
var yColumnIndex = columnsInFile[yPrefix + typeKey];
var zColumnIndex = columnsInFile[zPrefix + typeKey];
point = new Point3D(
double.Parse(valuesRead[xColumnIndex], CultureInfo.InvariantCulture),
double.Parse(valuesRead[yColumnIndex], CultureInfo.InvariantCulture),
double.Parse(valuesRead[zColumnIndex], CultureInfo.InvariantCulture)
);
if (point.Equals(undefinedPoint))
{
point = null;
}
}
return point;
}
catch (FormatException e)
{
throw CreateLineParseException(lineNumber, locationName, Resources.Error_CharacteristicPoint_has_not_double, e);
}
catch (OverflowException e)
{
throw CreateLineParseException(lineNumber, locationName, Resources.Error_CharacteristicPoint_parsing_causes_overflow, e);
}
}
///
/// Gets the name of the location for which the defines the
/// characteristic points.
///
/// The tokenized string from which the name should be extracted.
/// The name of the location.
/// Thrown when id value is null or empty.
private string GetLocationName(IList tokenizedString)
{
string name = tokenizedString.Any() ? tokenizedString[columnsInFile[locationIdKey]].Trim() : null;
if (string.IsNullOrEmpty(name))
{
throw CreateLineParseException(lineNumber, Resources.PipingSurfaceLinesCsvReader_ReadLine_Line_lacks_ID);
}
return name;
}
///
/// Tokenizes a string using a separator character up to the first empty field.
///
/// The text.
/// The tokenized parts.
/// Thrown when lacks the separator character.
private string[] TokenizeString(string readText)
{
if (!readText.Contains(separator))
{
throw CreateLineParseException(lineNumber, string.Format(Resources.CharacteristicPointsCsvReader_ReadCharacteristicPointsLocation_Line_lacks_separator_0_,
separator));
}
return readText.Split(separator)
.TakeWhile(text => !string.IsNullOrEmpty(text))
.ToArray();
}
///
/// Throws a configured instance of .
///
/// The line number being read.
/// The critical error message.
/// Optional: exception that caused this exception to be thrown.
/// New with message and inner exception set.
private CriticalFileReadException CreateCriticalFileReadException(int currentLine, string criticalErrorMessage, Exception innerException = null)
{
string locationDescription = string.Format(UtilsResources.TextFile_On_LineNumber_0_, currentLine);
var message = new FileReaderErrorMessageBuilder(filePath).WithLocation(locationDescription)
.Build(criticalErrorMessage);
return new CriticalFileReadException(message, innerException);
}
///
/// Throws a configured instance of .
///
/// The line number being read.
/// The critical error message.
/// New with message set.
private LineParseException CreateLineParseException(int currentLine, string lineParseErrorMessage)
{
string locationDescription = string.Format(UtilsResources.TextFile_On_LineNumber_0_, currentLine);
var message = new FileReaderErrorMessageBuilder(filePath).WithLocation(locationDescription)
.Build(lineParseErrorMessage);
return new LineParseException(message);
}
///
/// Throws a configured instance of .
///
/// The line number being read.
/// The name of the location being read.
/// The critical error message.
/// Optional: exception that caused this exception to be thrown.
/// New with message and inner exceptions set.
private LineParseException CreateLineParseException(int currentLine, string locationName, string lineParseErrorMessage, Exception innerException = null)
{
string locationDescription = string.Format(UtilsResources.TextFile_On_LineNumber_0_, currentLine);
string subjectDescription = string.Format(Resources.CharacteristicPointsCsvReader_LocationName_0_, locationName);
var message = new FileReaderErrorMessageBuilder(filePath).WithLocation(locationDescription)
.WithSubject(subjectDescription)
.Build(lineParseErrorMessage);
return new LineParseException(message, innerException);
}
#region csv column names
private const string locationIdKey = "locationid";
private const string surfaceLineKey = "profielnaam";
private const string surfaceLevelInsideKey = "maaiveld_binnenwaarts";
private const string ditchPolderSideKey = "insteek_sloot_polderzijde";
private const string bottomDitchPolderSideKey = "slootbodem_polderzijde";
private const string bottomDitchDikeSideKey = "slootbodem_dijkzijde";
private const string ditchDikeSideKey = "insteek_sloot_dijkzijde";
private const string dikeToeAtPolderKey = "teen_dijk_binnenwaarts";
private const string topShoulderInsideKey = "kruin_binnenberm";
private const string shoulderInsideKey = "insteek_binnenberm";
private const string dikeTopAtPolderKey = "kruin_binnentalud";
private const string trafficLoadInsideKey = "verkeersbelasting_kant_binnenwaarts";
private const string trafficLoadOutsideKey = "verkeersbelasting_kant_buitenwaarts";
private const string dikeTopAtRiverKey = "kruin_buitentalud";
private const string shoulderOutsideKey = "insteek_buitenberm";
private const string topShoulderOutsideKey = "kruin_buitenberm";
private const string dikeToeAtRiverKey = "teen_dijk_buitenwaarts";
private const string surfaceLevelOutsideKey = "maaiveld_buitenwaarts";
private const string dikeTableHeightKey = "dijktafelhoogte";
private const string insertRiverChannelKey = "insteek_geul";
private const string bottomRiverChannelKey = "teen_geul";
private const string orderNumberKey = "volgnummer";
#endregion
}
}