Index: src/Common/SharpMap/Converters/WellKnownText/StreamTokenizer.cs
===================================================================
diff -u -r8f6ae890fed8e8eae3a32f9c0498a10f82e0ddf9 -r5fc71a385897af92ccb092f2f969b5709afab85a
--- src/Common/SharpMap/Converters/WellKnownText/StreamTokenizer.cs (.../StreamTokenizer.cs) (revision 8f6ae890fed8e8eae3a32f9c0498a10f82e0ddf9)
+++ src/Common/SharpMap/Converters/WellKnownText/StreamTokenizer.cs (.../StreamTokenizer.cs) (revision 5fc71a385897af92ccb092f2f969b5709afab85a)
@@ -36,6 +36,7 @@
*/
#region Using
+
using System;
using System.IO;
using System.Text;
@@ -50,277 +51,277 @@
namespace SharpMap.Converters.WellKnownText.IO
{
+ ///
+ ///The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment style
+ ///
+ ///
+ ///This is a crude c# implementation of Java's StreamTokenizer class.
+ ///
+ internal class StreamTokenizer
+ {
+ private TokenType _currentTokenType;
+ private readonly TextReader _reader;
+ private string _currentToken;
+ private readonly bool _ignoreWhitespace = false;
+ private int _lineNumber = 1;
+ private int _colNumber = 1;
- ///
- ///The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment style
- ///
- ///
- ///This is a crude c# implementation of Java's StreamTokenizer class.
- ///
- internal class StreamTokenizer
- {
- TokenType _currentTokenType;
- TextReader _reader;
- string _currentToken;
- bool _ignoreWhitespace = false;
- int _lineNumber=1;
- int _colNumber=1;
-
- #region Constructors
-
-
- ///
- /// Initializes a new instance of the StreamTokenizer class.
- ///
- /// A TextReader with some text to read.
- /// Flag indicating whether whitespace should be ignored.
- public StreamTokenizer(TextReader reader, bool ignoreWhitespace)
- {
- if (reader==null)
- {
- throw new ArgumentNullException("reader");
- }
- _reader = reader;
- _ignoreWhitespace = ignoreWhitespace;
- }
- #endregion
+ #region Constructors
- #region Properties
- ///
- /// The current line number of the stream being read.
- ///
- public int LineNumber
- {
- get
- {
- return _lineNumber;
- }
- }
- ///
- /// The current column number of the stream being read.
- ///
- public int Column
- {
- get
- {
- return _colNumber;
- }
- }
-
-
- #endregion
+ ///
+ /// Initializes a new instance of the StreamTokenizer class.
+ ///
+ /// A TextReader with some text to read.
+ /// Flag indicating whether whitespace should be ignored.
+ public StreamTokenizer(TextReader reader, bool ignoreWhitespace)
+ {
+ if (reader == null)
+ {
+ throw new ArgumentNullException("reader");
+ }
+ _reader = reader;
+ _ignoreWhitespace = ignoreWhitespace;
+ }
- #region Methods
- ///
- /// If the current token is a number, this field contains the value of that number.
- ///
- ///
- /// If the current token is a number, this field contains the value of that number. The current token is a number when the value of the ttype field is TT_NUMBER.
- ///
- /// Current token is not a number in a valid format.
- public double GetNumericValue()
- {
- string number = this.GetStringValue();
- if (this.GetTokenType()==TokenType.Number)
- {
- return double.Parse(number,SharpMap.Map.numberFormat_EnUS);
- }
- throw new Exception(String.Format(SharpMap.Map.numberFormat_EnUS, "The token '{0}' is not a number at line {1} column {2}.", number, this.LineNumber, this.Column)); ;
+ #endregion
- }
- ///
- /// If the current token is a word token, this field contains a string giving the characters of the word token.
- ///
- public string GetStringValue()
- {
- return _currentToken;
- }
+ #region Properties
- ///
- /// Gets the token type of the current token.
- ///
- ///
- public TokenType GetTokenType()
- {
- return _currentTokenType;
- }
-
- ///
- /// Returns the next token.
- ///
- /// Determines is whitespace is ignored. True if whitespace is to be ignored.
- /// The TokenType of the next token.
- public TokenType NextToken(bool ignoreWhitespace)
- {
- TokenType nextTokenType;
- if (ignoreWhitespace)
- {
- nextTokenType= NextNonWhitespaceToken();
- }
- else
- {
- nextTokenType=NextTokenAny();
- }
- return nextTokenType;
- }
+ ///
+ /// The current line number of the stream being read.
+ ///
+ public int LineNumber
+ {
+ get
+ {
+ return _lineNumber;
+ }
+ }
- ///
- /// Returns the next token.
- ///
- /// The TokenType of the next token.
- public TokenType NextToken()
- {
- return NextToken(_ignoreWhitespace);
- }
- private TokenType NextTokenAny()
- {
- TokenType nextTokenType = TokenType.Eof;
- char[] chars = new char[1];
- _currentToken="";
- _currentTokenType = TokenType.Eof;
- int finished = _reader.Read(chars,0,1);
+ ///
+ /// The current column number of the stream being read.
+ ///
+ public int Column
+ {
+ get
+ {
+ return _colNumber;
+ }
+ }
- bool isNumber=false;
- bool isWord=false;
- byte[] ba=null;
- ASCIIEncoding AE = new ASCIIEncoding();
- char[] ascii=null;
- Char currentCharacter;
- Char nextCharacter;
- while (finished != 0 )
- {
- // convert int to char
- ba = new Byte[]{(byte)_reader.Peek()};
-
- ascii = AE.GetChars(ba);
+ #endregion
- currentCharacter = chars[0];
- nextCharacter = ascii[0];
- _currentTokenType = GetType(currentCharacter);
- nextTokenType = GetType(nextCharacter);
+ #region Methods
- // handling of words with _
- if (isWord && currentCharacter=='_')
- {
- _currentTokenType= TokenType.Word;
- }
- // handing of words ending in numbers
- if (isWord && _currentTokenType==TokenType.Number)
- {
- _currentTokenType= TokenType.Word;
- }
-
- if (_currentTokenType==TokenType.Word && nextCharacter=='_')
- {
- //enable words with _ inbetween
- nextTokenType = TokenType.Word;
- isWord=true;
- }
- if (_currentTokenType==TokenType.Word && nextTokenType==TokenType.Number)
- {
- //enable words ending with numbers
- nextTokenType = TokenType.Word;
- isWord=true;
- }
+ ///
+ /// If the current token is a number, this field contains the value of that number.
+ ///
+ ///
+ /// If the current token is a number, this field contains the value of that number. The current token is a number when the value of the ttype field is TT_NUMBER.
+ ///
+ /// Current token is not a number in a valid format.
+ public double GetNumericValue()
+ {
+ string number = GetStringValue();
+ if (GetTokenType() == TokenType.Number)
+ {
+ return double.Parse(number, Map.numberFormat_EnUS);
+ }
+ throw new Exception(String.Format(Map.numberFormat_EnUS, "The token '{0}' is not a number at line {1} column {2}.", number, LineNumber, Column));
+ ;
+ }
- // handle negative numbers
- if (currentCharacter=='-' && nextTokenType==TokenType.Number && isNumber ==false)
- {
- _currentTokenType= TokenType.Number;
- nextTokenType = TokenType.Number;
- //isNumber = true;
- }
+ ///
+ /// If the current token is a word token, this field contains a string giving the characters of the word token.
+ ///
+ public string GetStringValue()
+ {
+ return _currentToken;
+ }
+ ///
+ /// Gets the token type of the current token.
+ ///
+ ///
+ public TokenType GetTokenType()
+ {
+ return _currentTokenType;
+ }
- // this handles numbers with a decimal point
- if (isNumber && nextTokenType == TokenType.Number && currentCharacter=='.' )
- {
- _currentTokenType = TokenType.Number;
- }
- if (_currentTokenType == TokenType.Number && nextCharacter=='.' && isNumber ==false)
- {
- nextTokenType = TokenType.Number;
- isNumber = true;
- }
-
-
+ ///
+ /// Returns the next token.
+ ///
+ /// Determines is whitespace is ignored. True if whitespace is to be ignored.
+ /// The TokenType of the next token.
+ public TokenType NextToken(bool ignoreWhitespace)
+ {
+ TokenType nextTokenType;
+ if (ignoreWhitespace)
+ {
+ nextTokenType = NextNonWhitespaceToken();
+ }
+ else
+ {
+ nextTokenType = NextTokenAny();
+ }
+ return nextTokenType;
+ }
-
- _colNumber++;
- if (_currentTokenType==TokenType.Eol)
- {
- _lineNumber++;
- _colNumber=1;
- }
-
- _currentToken = _currentToken + currentCharacter;
- //if (_currentTokenType==TokenType.Word && nextCharacter=='_')
- //{
- // enable words with _ inbetween
- // finished = _reader.Read(chars,0,1);
- //}
- if (_currentTokenType!=nextTokenType)
- {
- finished = 0;
- }
- else if (_currentTokenType==TokenType.Symbol && currentCharacter!='-')
- {
- finished = 0;
- }
- else
- {
- finished = _reader.Read(chars,0,1);
- }
- }
- return _currentTokenType;
- }
+ ///
+ /// Returns the next token.
+ ///
+ /// The TokenType of the next token.
+ public TokenType NextToken()
+ {
+ return NextToken(_ignoreWhitespace);
+ }
- ///
- /// Determines a characters type (e.g. number, symbols, character).
- ///
- /// The character to determine.
- /// The TokenType the character is.
- private TokenType GetType(char character)
- {
- if (Char.IsDigit(character))
- {
- return TokenType.Number;
- }
- else if (Char.IsLetter(character))
- {
- return TokenType.Word;
- }
- else if (character=='\n')
- {
- return TokenType.Eol;
- }
- else if (Char.IsWhiteSpace(character) || Char.IsControl(character))
- {
- return TokenType.Whitespace;
- }
- else //(Char.IsSymbol(character))
- {
- return TokenType.Symbol;
- }
-
- }
+ private TokenType NextTokenAny()
+ {
+ TokenType nextTokenType = TokenType.Eof;
+ char[] chars = new char[1];
+ _currentToken = "";
+ _currentTokenType = TokenType.Eof;
+ int finished = _reader.Read(chars, 0, 1);
- ///
- /// Returns next token that is not whitespace.
- ///
- ///
- private TokenType NextNonWhitespaceToken()
- {
+ bool isNumber = false;
+ bool isWord = false;
+ byte[] ba = null;
+ ASCIIEncoding AE = new ASCIIEncoding();
+ char[] ascii = null;
+ Char currentCharacter;
+ Char nextCharacter;
+ while (finished != 0)
+ {
+ // convert int to char
+ ba = new Byte[]
+ {
+ (byte) _reader.Peek()
+ };
- TokenType tokentype = this.NextTokenAny();
- while (tokentype== TokenType.Whitespace || tokentype== TokenType.Eol)
- {
- tokentype = this.NextTokenAny();
- }
-
- return tokentype;
- }
- #endregion
+ ascii = AE.GetChars(ba);
- }
-}
+ currentCharacter = chars[0];
+ nextCharacter = ascii[0];
+ _currentTokenType = GetType(currentCharacter);
+ nextTokenType = GetType(nextCharacter);
+
+ // handling of words with _
+ if (isWord && currentCharacter == '_')
+ {
+ _currentTokenType = TokenType.Word;
+ }
+ // handing of words ending in numbers
+ if (isWord && _currentTokenType == TokenType.Number)
+ {
+ _currentTokenType = TokenType.Word;
+ }
+
+ if (_currentTokenType == TokenType.Word && nextCharacter == '_')
+ {
+ //enable words with _ inbetween
+ nextTokenType = TokenType.Word;
+ isWord = true;
+ }
+ if (_currentTokenType == TokenType.Word && nextTokenType == TokenType.Number)
+ {
+ //enable words ending with numbers
+ nextTokenType = TokenType.Word;
+ isWord = true;
+ }
+
+ // handle negative numbers
+ if (currentCharacter == '-' && nextTokenType == TokenType.Number && isNumber == false)
+ {
+ _currentTokenType = TokenType.Number;
+ nextTokenType = TokenType.Number;
+ //isNumber = true;
+ }
+
+ // this handles numbers with a decimal point
+ if (isNumber && nextTokenType == TokenType.Number && currentCharacter == '.')
+ {
+ _currentTokenType = TokenType.Number;
+ }
+ if (_currentTokenType == TokenType.Number && nextCharacter == '.' && isNumber == false)
+ {
+ nextTokenType = TokenType.Number;
+ isNumber = true;
+ }
+
+ _colNumber++;
+ if (_currentTokenType == TokenType.Eol)
+ {
+ _lineNumber++;
+ _colNumber = 1;
+ }
+
+ _currentToken = _currentToken + currentCharacter;
+ //if (_currentTokenType==TokenType.Word && nextCharacter=='_')
+ //{
+ // enable words with _ inbetween
+ // finished = _reader.Read(chars,0,1);
+ //}
+ if (_currentTokenType != nextTokenType)
+ {
+ finished = 0;
+ }
+ else if (_currentTokenType == TokenType.Symbol && currentCharacter != '-')
+ {
+ finished = 0;
+ }
+ else
+ {
+ finished = _reader.Read(chars, 0, 1);
+ }
+ }
+ return _currentTokenType;
+ }
+
+ ///
+ /// Determines a characters type (e.g. number, symbols, character).
+ ///
+ /// The character to determine.
+ /// The TokenType the character is.
+ private TokenType GetType(char character)
+ {
+ if (Char.IsDigit(character))
+ {
+ return TokenType.Number;
+ }
+ else if (Char.IsLetter(character))
+ {
+ return TokenType.Word;
+ }
+ else if (character == '\n')
+ {
+ return TokenType.Eol;
+ }
+ else if (Char.IsWhiteSpace(character) || Char.IsControl(character))
+ {
+ return TokenType.Whitespace;
+ }
+ else //(Char.IsSymbol(character))
+ {
+ return TokenType.Symbol;
+ }
+ }
+
+ ///
+ /// Returns next token that is not whitespace.
+ ///
+ ///
+ private TokenType NextNonWhitespaceToken()
+ {
+ TokenType tokentype = NextTokenAny();
+ while (tokentype == TokenType.Whitespace || tokentype == TokenType.Eol)
+ {
+ tokentype = NextTokenAny();
+ }
+
+ return tokentype;
+ }
+
+ #endregion
+ }
+}
\ No newline at end of file