aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/sdlang/lexer.d
diff options
context:
space:
mode:
Diffstat (limited to 'src/sdlang/lexer.d')
-rw-r--r--src/sdlang/lexer.d2068
1 files changed, 2068 insertions, 0 deletions
diff --git a/src/sdlang/lexer.d b/src/sdlang/lexer.d
new file mode 100644
index 0000000..6eeeac2
--- /dev/null
+++ b/src/sdlang/lexer.d
@@ -0,0 +1,2068 @@
+// SDLang-D
+// Written in the D programming language.
+
+module sdlang.lexer;
+
+import std.algorithm;
+import std.array;
+import std.base64;
+import std.bigint;
+import std.conv;
+import std.datetime;
+import std.file;
+import std.stream : ByteOrderMarks, BOM;
+import std.traits;
+import std.typecons;
+import std.uni;
+import std.utf;
+import std.variant;
+
+import sdlang.exception;
+import sdlang.symbol;
+import sdlang.token;
+import sdlang.util;
+
+alias sdlang.util.startsWith startsWith;
+
+Token[] lexFile(string filename)
+{
+ auto source = cast(string)read(filename);
+ return lexSource(source, filename);
+}
+
+Token[] lexSource(string source, string filename=null)
+{
+ auto lexer = scoped!Lexer(source, filename);
+
+ // Can't use 'std.array.array(Range)' because 'lexer' is scoped
+ // and therefore cannot have its reference copied.
+ Appender!(Token[]) tokens;
+ foreach(tok; lexer)
+ tokens.put(tok);
+
+ return tokens.data;
+}
+
+// Kind of a poor-man's yield, but fast.
+// Only to be used inside Lexer.popFront (and Lexer.this).
+private template accept(string symbolName)
+{
+ static assert(symbolName != "Value", "Value symbols must also take a value.");
+ enum accept = acceptImpl!(symbolName, "null");
+}
+private template accept(string symbolName, string value)
+{
+ static assert(symbolName == "Value", "Only a Value symbol can take a value.");
+ enum accept = acceptImpl!(symbolName, value);
+}
+private template accept(string symbolName, string value, string startLocation, string endLocation)
+{
+ static assert(symbolName == "Value", "Only a Value symbol can take a value.");
+ enum accept = ("
+ {
+ _front = makeToken!"~symbolName.stringof~";
+ _front.value = "~value~";
+ _front.location = "~(startLocation==""? "tokenStart" : startLocation)~";
+ _front.data = source[
+ "~(startLocation==""? "tokenStart.index" : startLocation)~"
+ ..
+ "~(endLocation==""? "location.index" : endLocation)~"
+ ];
+ return;
+ }
+ ").replace("\n", "");
+}
+private template acceptImpl(string symbolName, string value)
+{
+ enum acceptImpl = ("
+ {
+ _front = makeToken!"~symbolName.stringof~";
+ _front.value = "~value~";
+ return;
+ }
+ ").replace("\n", "");
+}
+
+class Lexer
+{
+ string source;
+ string filename;
+ Location location; /// Location of current character in source
+
+ private dchar ch; // Current character
+ private dchar nextCh; // Lookahead character
+ private size_t nextPos; // Position of lookahead character (an index into source)
+ private bool hasNextCh; // If false, then there's no more lookahead, just EOF
+ private size_t posAfterLookahead; // Position after lookahead character (an index into source)
+
+ private Location tokenStart; // The starting location of the token being lexed
+
+ // Length so far of the token being lexed, not including current char
+ private size_t tokenLength; // Length in UTF-8 code units
+ private size_t tokenLength32; // Length in UTF-32 code units
+
+ // Slight kludge:
+ // If a numeric fragment is found after a Date (separated by arbitrary
+ // whitespace), it could be the "hours" part of a DateTime, or it could
+ // be a separate numeric literal that simply follows a plain Date. If the
+ // latter, then the Date must be emitted, but numeric fragment that was
+ // found after it needs to be saved for the the lexer's next iteration.
+ //
+ // It's a slight kludge, and could instead be implemented as a slightly
+ // kludgey parser hack, but it's the only situation where SDL's lexing
+ // needs to lookahead more than one character, so this is good enough.
+ private struct LookaheadTokenInfo
+ {
+ bool exists = false;
+ string numericFragment = "";
+ bool isNegative = false;
+ Location tokenStart;
+ }
+ private LookaheadTokenInfo lookaheadTokenInfo;
+
+ this(string source=null, string filename=null)
+ {
+ this.filename = filename;
+ this.source = source;
+
+ _front = Token(symbol!"Error", Location());
+ lookaheadTokenInfo = LookaheadTokenInfo.init;
+
+ if( source.startsWith( ByteOrderMarks[BOM.UTF8] ) )
+ {
+ source = source[ ByteOrderMarks[BOM.UTF8].length .. $ ];
+ this.source = source;
+ }
+
+ foreach(bom; ByteOrderMarks)
+ if( source.startsWith(bom) )
+ error(Location(filename,0,0,0), "SDL spec only supports UTF-8, not UTF-16 or UTF-32");
+
+ if(source == "")
+ mixin(accept!"EOF");
+
+ // Prime everything
+ hasNextCh = true;
+ nextCh = source.decode(posAfterLookahead);
+ advanceChar(ErrorOnEOF.Yes);
+ location = Location(filename, 0, 0, 0);
+ popFront();
+ }
+
+ @property bool empty()
+ {
+ return _front.symbol == symbol!"EOF";
+ }
+
+ Token _front;
+ @property Token front()
+ {
+ return _front;
+ }
+
+ @property bool isEOF()
+ {
+ return location.index == source.length && !lookaheadTokenInfo.exists;
+ }
+
+ private void error(string msg)
+ {
+ error(location, msg);
+ }
+
+ private void error(Location loc, string msg)
+ {
+ throw new SDLangParseException(loc, "Error: "~msg);
+ }
+
+ private Token makeToken(string symbolName)()
+ {
+ auto tok = Token(symbol!symbolName, tokenStart);
+ tok.data = tokenData;
+ return tok;
+ }
+
+ private @property string tokenData()
+ {
+ return source[ tokenStart.index .. location.index ];
+ }
+
+ /// Check the lookahead character
+ private bool lookahead(dchar ch)
+ {
+ return hasNextCh && nextCh == ch;
+ }
+
+ private bool lookahead(bool function(dchar) condition)
+ {
+ return hasNextCh && condition(nextCh);
+ }
+
+ private static bool isNewline(dchar ch)
+ {
+ return ch == '\n' || ch == '\r' || ch == lineSep || ch == paraSep;
+ }
+
+ /// Returns the length of the newline sequence, or zero if the current
+ /// character is not a newline
+ ///
+ /// Note that there are only single character sequences and the two
+ /// character sequence `\r\n` as used on Windows.
+ private size_t isAtNewline()
+ {
+ if(ch == '\n' || ch == lineSep || ch == paraSep) return 1;
+ else if(ch == '\r') return lookahead('\n') ? 2 : 1;
+ else return 0;
+ }
+
+ /// Is 'ch' a valid base 64 character?
+ private bool isBase64(dchar ch)
+ {
+ if(ch >= 'A' && ch <= 'Z')
+ return true;
+
+ if(ch >= 'a' && ch <= 'z')
+ return true;
+
+ if(ch >= '0' && ch <= '9')
+ return true;
+
+ return ch == '+' || ch == '/' || ch == '=';
+ }
+
+ /// Is the current character one that's allowed
+ /// immediately *after* an int/float literal?
+ private bool isEndOfNumber()
+ {
+ if(isEOF)
+ return true;
+
+ return !isDigit(ch) && ch != ':' && ch != '_' && !isAlpha(ch);
+ }
+
+ /// Is current character the last one in an ident?
+ private bool isEndOfIdentCached = false;
+ private bool _isEndOfIdent;
+ private bool isEndOfIdent()
+ {
+ if(!isEndOfIdentCached)
+ {
+ if(!hasNextCh)
+ _isEndOfIdent = true;
+ else
+ _isEndOfIdent = !isIdentChar(nextCh);
+
+ isEndOfIdentCached = true;
+ }
+
+ return _isEndOfIdent;
+ }
+
+ /// Is 'ch' a character that's allowed *somewhere* in an identifier?
+ private bool isIdentChar(dchar ch)
+ {
+ if(isAlpha(ch))
+ return true;
+
+ else if(isNumber(ch))
+ return true;
+
+ else
+ return
+ ch == '-' ||
+ ch == '_' ||
+ ch == '.' ||
+ ch == '$';
+ }
+
+ private bool isDigit(dchar ch)
+ {
+ return ch >= '0' && ch <= '9';
+ }
+
+ private enum KeywordResult
+ {
+ Accept, // Keyword is matched
+ Continue, // Keyword is not matched *yet*
+ Failed, // Keyword doesn't match
+ }
+ private KeywordResult checkKeyword(dstring keyword32)
+ {
+ // Still within length of keyword
+ if(tokenLength32 < keyword32.length)
+ {
+ if(ch == keyword32[tokenLength32])
+ return KeywordResult.Continue;
+ else
+ return KeywordResult.Failed;
+ }
+
+ // At position after keyword
+ else if(tokenLength32 == keyword32.length)
+ {
+ if(isEOF || !isIdentChar(ch))
+ {
+ debug assert(tokenData == to!string(keyword32));
+ return KeywordResult.Accept;
+ }
+ else
+ return KeywordResult.Failed;
+ }
+
+ assert(0, "Fell off end of keyword to check");
+ }
+
+ enum ErrorOnEOF { No, Yes }
+
+ /// Advance one code point.
+ private void advanceChar(ErrorOnEOF errorOnEOF)
+ {
+ if(auto cnt = isAtNewline())
+ {
+ if (cnt == 1)
+ location.line++;
+ location.col = 0;
+ }
+ else
+ location.col++;
+
+ location.index = nextPos;
+
+ nextPos = posAfterLookahead;
+ ch = nextCh;
+
+ if(!hasNextCh)
+ {
+ if(errorOnEOF == ErrorOnEOF.Yes)
+ error("Unexpected end of file");
+
+ return;
+ }
+
+ tokenLength32++;
+ tokenLength = location.index - tokenStart.index;
+
+ if(nextPos == source.length)
+ {
+ nextCh = dchar.init;
+ hasNextCh = false;
+ return;
+ }
+
+ nextCh = source.decode(posAfterLookahead);
+ isEndOfIdentCached = false;
+ }
+
+ /// Advances the specified amount of characters
+ private void advanceChar(size_t count, ErrorOnEOF errorOnEOF)
+ {
+ while(count-- > 0)
+ advanceChar(errorOnEOF);
+ }
+
+ void popFront()
+ {
+ // -- Main Lexer -------------
+
+ eatWhite();
+
+ if(isEOF)
+ mixin(accept!"EOF");
+
+ tokenStart = location;
+ tokenLength = 0;
+ tokenLength32 = 0;
+ isEndOfIdentCached = false;
+
+ if(lookaheadTokenInfo.exists)
+ {
+ tokenStart = lookaheadTokenInfo.tokenStart;
+
+ auto prevLATokenInfo = lookaheadTokenInfo;
+ lookaheadTokenInfo = LookaheadTokenInfo.init;
+ lexNumeric(prevLATokenInfo);
+ return;
+ }
+
+ if(ch == '=')
+ {
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!"=");
+ }
+
+ else if(ch == '{')
+ {
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!"{");
+ }
+
+ else if(ch == '}')
+ {
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!"}");
+ }
+
+ else if(ch == ':')
+ {
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!":");
+ }
+
+ else if(ch == ';')
+ {
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!"EOL");
+ }
+
+ else if(auto cnt = isAtNewline())
+ {
+ advanceChar(cnt, ErrorOnEOF.No);
+ mixin(accept!"EOL");
+ }
+
+ else if(isAlpha(ch) || ch == '_')
+ lexIdentKeyword();
+
+ else if(ch == '"')
+ lexRegularString();
+
+ else if(ch == '`')
+ lexRawString();
+
+ else if(ch == '\'')
+ lexCharacter();
+
+ else if(ch == '[')
+ lexBinary();
+
+ else if(ch == '-' || ch == '.' || isDigit(ch))
+ lexNumeric();
+
+ else
+ {
+ advanceChar(ErrorOnEOF.No);
+ error("Syntax error");
+ }
+ }
+
+ /// Lex Ident or Keyword
+ private void lexIdentKeyword()
+ {
+ assert(isAlpha(ch) || ch == '_');
+
+ // Keyword
+ struct Key
+ {
+ dstring name;
+ Value value;
+ bool failed = false;
+ }
+ static Key[5] keywords;
+ static keywordsInited = false;
+ if(!keywordsInited)
+ {
+ // Value (as a std.variant-based type) can't be statically inited
+ keywords[0] = Key("true", Value(true ));
+ keywords[1] = Key("false", Value(false));
+ keywords[2] = Key("on", Value(true ));
+ keywords[3] = Key("off", Value(false));
+ keywords[4] = Key("null", Value(null ));
+ keywordsInited = true;
+ }
+
+ foreach(ref key; keywords)
+ key.failed = false;
+
+ auto numKeys = keywords.length;
+
+ do
+ {
+ foreach(ref key; keywords)
+ if(!key.failed)
+ {
+ final switch(checkKeyword(key.name))
+ {
+ case KeywordResult.Accept:
+ mixin(accept!("Value", "key.value"));
+
+ case KeywordResult.Continue:
+ break;
+
+ case KeywordResult.Failed:
+ key.failed = true;
+ numKeys--;
+ break;
+ }
+ }
+
+ if(numKeys == 0)
+ {
+ lexIdent();
+ return;
+ }
+
+ advanceChar(ErrorOnEOF.No);
+
+ } while(!isEOF);
+
+ foreach(ref key; keywords)
+ if(!key.failed)
+ if(key.name.length == tokenLength32+1)
+ mixin(accept!("Value", "key.value"));
+
+ mixin(accept!"Ident");
+ }
+
+ /// Lex Ident
+ private void lexIdent()
+ {
+ if(tokenLength == 0)
+ assert(isAlpha(ch) || ch == '_');
+
+ while(!isEOF && isIdentChar(ch))
+ advanceChar(ErrorOnEOF.No);
+
+ mixin(accept!"Ident");
+ }
+
+ /// Lex regular string
+ private void lexRegularString()
+ {
+ assert(ch == '"');
+
+ Appender!string buf;
+ size_t spanStart = nextPos;
+
+ // Doesn't include current character
+ void updateBuf()
+ {
+ if(location.index == spanStart)
+ return;
+
+ buf.put( source[spanStart..location.index] );
+ }
+
+ advanceChar(ErrorOnEOF.Yes);
+ while(ch != '"')
+ {
+ if(ch == '\\')
+ {
+ updateBuf();
+
+ bool wasEscSequence = true;
+ if(hasNextCh)
+ {
+ switch(nextCh)
+ {
+ case 'n': buf.put('\n'); break;
+ case 'r': buf.put('\r'); break;
+ case 't': buf.put('\t'); break;
+ case '"': buf.put('\"'); break;
+ case '\\': buf.put('\\'); break;
+ default: wasEscSequence = false; break;
+ }
+ }
+
+ if(wasEscSequence)
+ {
+ advanceChar(ErrorOnEOF.Yes);
+ spanStart = nextPos;
+ }
+ else
+ {
+ eatWhite(false);
+ spanStart = location.index;
+ }
+ }
+
+ else if(isNewline(ch))
+ error("Unescaped newlines are only allowed in raw strings, not regular strings.");
+
+ advanceChar(ErrorOnEOF.Yes);
+ }
+
+ updateBuf();
+ advanceChar(ErrorOnEOF.No); // Skip closing double-quote
+ mixin(accept!("Value", "buf.data"));
+ }
+
+ /// Lex raw string
+ private void lexRawString()
+ {
+ assert(ch == '`');
+
+ do
+ advanceChar(ErrorOnEOF.Yes);
+ while(ch != '`');
+
+ advanceChar(ErrorOnEOF.No); // Skip closing back-tick
+ mixin(accept!("Value", "tokenData[1..$-1]"));
+ }
+
+ /// Lex character literal
+ private void lexCharacter()
+ {
+ assert(ch == '\'');
+ advanceChar(ErrorOnEOF.Yes); // Skip opening single-quote
+
+ dchar value;
+ if(ch == '\\')
+ {
+ advanceChar(ErrorOnEOF.Yes); // Skip escape backslash
+ switch(ch)
+ {
+ case 'n': value = '\n'; break;
+ case 'r': value = '\r'; break;
+ case 't': value = '\t'; break;
+ case '\'': value = '\''; break;
+ case '\\': value = '\\'; break;
+ default: error("Invalid escape sequence.");
+ }
+ }
+ else if(isNewline(ch))
+ error("Newline not alowed in character literal.");
+ else
+ value = ch;
+ advanceChar(ErrorOnEOF.Yes); // Skip the character itself
+
+ if(ch == '\'')
+ advanceChar(ErrorOnEOF.No); // Skip closing single-quote
+ else
+ error("Expected closing single-quote.");
+
+ mixin(accept!("Value", "value"));
+ }
+
+ /// Lex base64 binary literal
+ private void lexBinary()
+ {
+ assert(ch == '[');
+ advanceChar(ErrorOnEOF.Yes);
+
+ void eatBase64Whitespace()
+ {
+ while(!isEOF && isWhite(ch))
+ {
+ if(isNewline(ch))
+ advanceChar(ErrorOnEOF.Yes);
+
+ if(!isEOF && isWhite(ch))
+ eatWhite();
+ }
+ }
+
+ eatBase64Whitespace();
+
+ // Iterates all valid base64 characters, ending at ']'.
+ // Skips all whitespace. Throws on invalid chars.
+ struct Base64InputRange
+ {
+ Lexer lexer;
+ private bool isInited = false;
+ private int numInputCharsMod4 = 0;
+
+ @property bool empty()
+ {
+ if(lexer.ch == ']')
+ {
+ if(numInputCharsMod4 != 0)
+ lexer.error("Length of Base64 encoding must be a multiple of 4. ("~to!string(numInputCharsMod4)~")");
+
+ return true;
+ }
+
+ return false;
+ }
+
+ @property dchar front()
+ {
+ return lexer.ch;
+ }
+
+ void popFront()
+ {
+ auto lex = lexer;
+
+ if(!isInited)
+ {
+ if(lexer.isBase64(lexer.ch))
+ {
+ numInputCharsMod4++;
+ numInputCharsMod4 %= 4;
+ }
+
+ isInited = true;
+ }
+
+ lex.advanceChar(lex.ErrorOnEOF.Yes);
+
+ eatBase64Whitespace();
+
+ if(lex.isEOF)
+ lex.error("Unexpected end of file.");
+
+ if(lex.ch != ']')
+ {
+ if(!lex.isBase64(lex.ch))
+ lex.error("Invalid character in base64 binary literal.");
+
+ numInputCharsMod4++;
+ numInputCharsMod4 %= 4;
+ }
+ }
+ }
+
+ // This is a slow ugly hack. It's necessary because Base64.decode
+ // currently requires the source to have known length.
+ //TODO: Remove this when DMD issue #9543 is fixed.
+ dchar[] tmpBuf = array(Base64InputRange(this));
+
+ Appender!(ubyte[]) outputBuf;
+ // Ugly workaround for DMD issue #9102
+ //TODO: Remove this when DMD #9102 is fixed
+ struct OutputBuf
+ {
+ void put(ubyte ch)
+ {
+ outputBuf.put(ch);
+ }
+ }
+
+ try
+ //Base64.decode(Base64InputRange(this), OutputBuf());
+ Base64.decode(tmpBuf, OutputBuf());
+
+ //TODO: Starting with dmd 2.062, this should be a Base64Exception
+ catch(Exception e)
+ error("Invalid character in base64 binary literal.");
+
+ advanceChar(ErrorOnEOF.No); // Skip ']'
+ mixin(accept!("Value", "outputBuf.data"));
+ }
+
+ private BigInt toBigInt(bool isNegative, string absValue)
+ {
+ auto num = BigInt(absValue);
+ assert(num >= 0);
+
+ if(isNegative)
+ num = -num;
+
+ return num;
+ }
+
+ /// Lex [0-9]+, but without emitting a token.
+ /// This is used by the other numeric parsing functions.
+ private string lexNumericFragment()
+ {
+ if(!isDigit(ch))
+ error("Expected a digit 0-9.");
+
+ auto spanStart = location.index;
+
+ do
+ {
+ advanceChar(ErrorOnEOF.No);
+ } while(!isEOF && isDigit(ch));
+
+ return source[spanStart..location.index];
+ }
+
+ /// Lex anything that starts with 0-9 or '-'. Ints, floats, dates, etc.
+ private void lexNumeric(LookaheadTokenInfo laTokenInfo = LookaheadTokenInfo.init)
+ {
+ bool isNegative;
+ string firstFragment;
+ if(laTokenInfo.exists)
+ {
+ firstFragment = laTokenInfo.numericFragment;
+ isNegative = laTokenInfo.isNegative;
+ }
+ else
+ {
+ assert(ch == '-' || ch == '.' || isDigit(ch));
+
+ // Check for negative
+ isNegative = ch == '-';
+ if(isNegative)
+ advanceChar(ErrorOnEOF.Yes);
+
+ // Some floating point with omitted leading zero?
+ if(ch == '.')
+ {
+ lexFloatingPoint("");
+ return;
+ }
+
+ firstFragment = lexNumericFragment();
+ }
+
+ // Long integer (64-bit signed)?
+ if(ch == 'L' || ch == 'l')
+ {
+ advanceChar(ErrorOnEOF.No);
+
+ // BigInt(long.min) is a workaround for DMD issue #9548
+ auto num = toBigInt(isNegative, firstFragment);
+ if(num < BigInt(long.min) || num > long.max)
+ error(tokenStart, "Value doesn't fit in 64-bit signed long integer: "~to!string(num));
+
+ mixin(accept!("Value", "num.toLong()"));
+ }
+
+ // Float (32-bit signed)?
+ else if(ch == 'F' || ch == 'f')
+ {
+ auto value = to!float(tokenData);
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!("Value", "value"));
+ }
+
+ // Double float (64-bit signed) with suffix?
+ else if((ch == 'D' || ch == 'd') && !lookahead(':')
+ )
+ {
+ auto value = to!double(tokenData);
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!("Value", "value"));
+ }
+
+ // Decimal (128+ bits signed)?
+ else if(
+ (ch == 'B' || ch == 'b') &&
+ (lookahead('D') || lookahead('d'))
+ )
+ {
+ auto value = to!real(tokenData);
+ advanceChar(ErrorOnEOF.No);
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!("Value", "value"));
+ }
+
+ // Some floating point?
+ else if(ch == '.')
+ lexFloatingPoint(firstFragment);
+
+ // Some date?
+ else if(ch == '/' && hasNextCh && isDigit(nextCh))
+ lexDate(isNegative, firstFragment);
+
+ // Some time span?
+ else if(ch == ':' || ch == 'd')
+ lexTimeSpan(isNegative, firstFragment);
+
+ // Integer (32-bit signed)?
+ else if(isEndOfNumber())
+ {
+ auto num = toBigInt(isNegative, firstFragment);
+ if(num < int.min || num > int.max)
+ error(tokenStart, "Value doesn't fit in 32-bit signed integer: "~to!string(num));
+
+ mixin(accept!("Value", "num.toInt()"));
+ }
+
+ // Invalid suffix
+ else
+ error("Invalid integer suffix.");
+ }
+
+ /// Lex any floating-point literal (after the initial numeric fragment was lexed)
+ private void lexFloatingPoint(string firstPart)
+ {
+ assert(ch == '.');
+ advanceChar(ErrorOnEOF.No);
+
+ auto secondPart = lexNumericFragment();
+
+ try
+ {
+ // Double float (64-bit signed) with suffix?
+ if(ch == 'D' || ch == 'd')
+ {
+ auto value = to!double(tokenData);
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!("Value", "value"));
+ }
+
+ // Float (32-bit signed)?
+ else if(ch == 'F' || ch == 'f')
+ {
+ auto value = to!float(tokenData);
+ advanceChar(ErrorOnEOF.No);
+ mixin(accept!("Value", "value"));
+ }
+
+ // Decimal (128+ bits signed)?
+ else if(ch == 'B' || ch == 'b')
+ {
+ auto value = to!real(tokenData);
+ advanceChar(ErrorOnEOF.Yes);
+
+ if(!isEOF && (ch == 'D' || ch == 'd'))
+ {
+ advanceChar(ErrorOnEOF.No);
+ if(isEndOfNumber())
+ mixin(accept!("Value", "value"));
+ }
+
+ error("Invalid floating point suffix.");
+ }
+
+ // Double float (64-bit signed) without suffix?
+ else if(isEOF || !isIdentChar(ch))
+ {
+ auto value = to!double(tokenData);
+ mixin(accept!("Value", "value"));
+ }
+
+ // Invalid suffix
+ else
+ error("Invalid floating point suffix.");
+ }
+ catch(ConvException e)
+ error("Invalid floating point literal.");
+ }
+
+ private Date makeDate(bool isNegative, string yearStr, string monthStr, string dayStr)
+ {
+ BigInt biTmp;
+
+ biTmp = BigInt(yearStr);
+ if(isNegative)
+ biTmp = -biTmp;
+ if(biTmp < int.min || biTmp > int.max)
+ error(tokenStart, "Date's year is out of range. (Must fit within a 32-bit signed int.)");
+ auto year = biTmp.toInt();
+
+ biTmp = BigInt(monthStr);
+ if(biTmp < 1 || biTmp > 12)
+ error(tokenStart, "Date's month is out of range.");
+ auto month = biTmp.toInt();
+
+ biTmp = BigInt(dayStr);
+ if(biTmp < 1 || biTmp > 31)
+ error(tokenStart, "Date's month is out of range.");
+ auto day = biTmp.toInt();
+
+ return Date(year, month, day);
+ }
+
+ private DateTimeFrac makeDateTimeFrac(
+ bool isNegative, Date date, string hourStr, string minuteStr,
+ string secondStr, string millisecondStr
+ )
+ {
+ BigInt biTmp;
+
+ biTmp = BigInt(hourStr);
+ if(biTmp < int.min || biTmp > int.max)
+ error(tokenStart, "Datetime's hour is out of range.");
+ auto numHours = biTmp.toInt();
+
+ biTmp = BigInt(minuteStr);
+ if(biTmp < 0 || biTmp > int.max)
+ error(tokenStart, "Datetime's minute is out of range.");
+ auto numMinutes = biTmp.toInt();
+
+ int numSeconds = 0;
+ if(secondStr != "")
+ {
+ biTmp = BigInt(secondStr);
+ if(biTmp < 0 || biTmp > int.max)
+ error(tokenStart, "Datetime's second is out of range.");
+ numSeconds = biTmp.toInt();
+ }
+
+ int millisecond = 0;
+ if(millisecondStr != "")
+ {
+ biTmp = BigInt(millisecondStr);
+ if(biTmp < 0 || biTmp > int.max)
+ error(tokenStart, "Datetime's millisecond is out of range.");
+ millisecond = biTmp.toInt();
+
+ if(millisecondStr.length == 1)
+ millisecond *= 100;
+ else if(millisecondStr.length == 2)
+ millisecond *= 10;
+ }
+
+ Duration fracSecs = millisecond.msecs;
+
+ auto offset = hours(numHours) + minutes(numMinutes) + seconds(numSeconds);
+
+ if(isNegative)
+ {
+ offset = -offset;
+ fracSecs = -fracSecs;
+ }
+
+ return DateTimeFrac(DateTime(date) + offset, fracSecs);
+ }
+
+ private Duration makeDuration(
+ bool isNegative, string dayStr,
+ string hourStr, string minuteStr, string secondStr,
+ string millisecondStr
+ )
+ {
+ BigInt biTmp;
+
+ long day = 0;
+ if(dayStr != "")
+ {
+ biTmp = BigInt(dayStr);
+ if(biTmp < long.min || biTmp > long.max)
+ error(tokenStart, "Time span's day is out of range.");
+ day = biTmp.toLong();
+ }
+
+ biTmp = BigInt(hourStr);
+ if(biTmp < long.min || biTmp > long.max)
+ error(tokenStart, "Time span's hour is out of range.");
+ auto hour = biTmp.toLong();
+
+ biTmp = BigInt(minuteStr);
+ if(biTmp < long.min || biTmp > long.max)
+ error(tokenStart, "Time span's minute is out of range.");
+ auto minute = biTmp.toLong();
+
+ biTmp = BigInt(secondStr);
+ if(biTmp < long.min || biTmp > long.max)
+ error(tokenStart, "Time span's second is out of range.");
+ auto second = biTmp.toLong();
+
+ long millisecond = 0;
+ if(millisecondStr != "")
+ {
+ biTmp = BigInt(millisecondStr);
+ if(biTmp < long.min || biTmp > long.max)
+ error(tokenStart, "Time span's millisecond is out of range.");
+ millisecond = biTmp.toLong();
+
+ if(millisecondStr.length == 1)
+ millisecond *= 100;
+ else if(millisecondStr.length == 2)
+ millisecond *= 10;
+ }
+
+ auto duration =
+ dur!"days" (day) +
+ dur!"hours" (hour) +
+ dur!"minutes"(minute) +
+ dur!"seconds"(second) +
+ dur!"msecs" (millisecond);
+
+ if(isNegative)
+ duration = -duration;
+
+ return duration;
+ }
+
+ // This has to reproduce some weird corner case behaviors from the
+ // original Java version of SDL. So some of this may seem weird.
+ private Nullable!Duration getTimeZoneOffset(string str)
+ {
+ if(str.length < 2)
+ return Nullable!Duration(); // Unknown timezone
+
+ if(str[0] != '+' && str[0] != '-')
+ return Nullable!Duration(); // Unknown timezone
+
+ auto isNegative = str[0] == '-';
+
+ string numHoursStr;
+ string numMinutesStr;
+ if(str[1] == ':')
+ {
+ numMinutesStr = str[1..$];
+ numHoursStr = "";
+ }
+ else
+ {
+ numMinutesStr = str.find(':');
+ numHoursStr = str[1 .. $-numMinutesStr.length];
+ }
+
+ long numHours = 0;
+ long numMinutes = 0;
+ bool isUnknown = false;
+ try
+ {
+ switch(numHoursStr.length)
+ {
+ case 0:
+ if(numMinutesStr.length == 3)
+ {
+ numHours = 0;
+ numMinutes = to!long(numMinutesStr[1..$]);
+ }
+ else
+ isUnknown = true;
+ break;
+
+ case 1:
+ case 2:
+ if(numMinutesStr.length == 0)
+ {
+ numHours = to!long(numHoursStr);
+ numMinutes = 0;
+ }
+ else if(numMinutesStr.length == 3)
+ {
+ numHours = to!long(numHoursStr);
+ numMinutes = to!long(numMinutesStr[1..$]);
+ }
+ else
+ isUnknown = true;
+ break;
+
+ default:
+ if(numMinutesStr.length == 0)
+ {
+ // Yes, this is correct
+ numHours = 0;
+ numMinutes = to!long(numHoursStr[1..$]);
+ }
+ else
+ isUnknown = true;
+ break;
+ }
+ }
+ catch(ConvException e)
+ isUnknown = true;
+
+ if(isUnknown)
+ return Nullable!Duration(); // Unknown timezone
+
+ auto timeZoneOffset = hours(numHours) + minutes(numMinutes);
+ if(isNegative)
+ timeZoneOffset = -timeZoneOffset;
+
+ // Timezone valid
+ return Nullable!Duration(timeZoneOffset);
+ }
+
+ /// Lex date or datetime (after the initial numeric fragment was lexed)
+ private void lexDate(bool isDateNegative, string yearStr)
+ {
+ assert(ch == '/');
+
+ // Lex months
+ advanceChar(ErrorOnEOF.Yes); // Skip '/'
+ auto monthStr = lexNumericFragment();
+
+ // Lex days
+ if(ch != '/')
+ error("Invalid date format: Missing days.");
+ advanceChar(ErrorOnEOF.Yes); // Skip '/'
+ auto dayStr = lexNumericFragment();
+
+ auto date = makeDate(isDateNegative, yearStr, monthStr, dayStr);
+
+ if(!isEndOfNumber() && ch != '/')
+ error("Dates cannot have suffixes.");
+
+ // Date?
+ if(isEOF)
+ mixin(accept!("Value", "date"));
+
+ auto endOfDate = location;
+
+ while(
+ !isEOF &&
+ ( ch == '\\' || ch == '/' || (isWhite(ch) && !isNewline(ch)) )
+ )
+ {
+ if(ch == '\\' && hasNextCh && isNewline(nextCh))
+ {
+ advanceChar(ErrorOnEOF.Yes);
+ if(isAtNewline())
+ advanceChar(ErrorOnEOF.Yes);
+ advanceChar(ErrorOnEOF.No);
+ }
+
+ eatWhite();
+ }
+
+ // Date?
+ if(isEOF || (!isDigit(ch) && ch != '-'))
+ mixin(accept!("Value", "date", "", "endOfDate.index"));
+
+ auto startOfTime = location;
+
+ // Is time negative?
+ bool isTimeNegative = ch == '-';
+ if(isTimeNegative)
+ advanceChar(ErrorOnEOF.Yes);
+
+ // Lex hours
+ auto hourStr = ch == '.'? "" : lexNumericFragment();
+
+ // Lex minutes
+ if(ch != ':')
+ {
+ // No minutes found. Therefore we had a plain Date followed
+ // by a numeric literal, not a DateTime.
+ lookaheadTokenInfo.exists = true;
+ lookaheadTokenInfo.numericFragment = hourStr;
+ lookaheadTokenInfo.isNegative = isTimeNegative;
+ lookaheadTokenInfo.tokenStart = startOfTime;
+ mixin(accept!("Value", "date", "", "endOfDate.index"));
+ }
+ advanceChar(ErrorOnEOF.Yes); // Skip ':'
+ auto minuteStr = lexNumericFragment();
+
+ // Lex seconds, if exists
+ string secondStr;
+ if(ch == ':')
+ {
+ advanceChar(ErrorOnEOF.Yes); // Skip ':'
+ secondStr = lexNumericFragment();
+ }
+
+ // Lex milliseconds, if exists
+ string millisecondStr;
+ if(ch == '.')
+ {
+ advanceChar(ErrorOnEOF.Yes); // Skip '.'
+ millisecondStr = lexNumericFragment();
+ }
+
+ auto dateTimeFrac = makeDateTimeFrac(isTimeNegative, date, hourStr, minuteStr, secondStr, millisecondStr);
+
+ // Lex zone, if exists
+ if(ch == '-')
+ {
+ advanceChar(ErrorOnEOF.Yes); // Skip '-'
+ auto timezoneStart = location;
+
+ if(!isAlpha(ch))
+ error("Invalid timezone format.");
+
+ while(!isEOF && !isWhite(ch))
+ advanceChar(ErrorOnEOF.No);
+
+ auto timezoneStr = source[timezoneStart.index..location.index];
+ if(timezoneStr.startsWith("GMT"))
+ {
+ auto isoPart = timezoneStr["GMT".length..$];
+ auto offset = getTimeZoneOffset(isoPart);
+
+ if(offset.isNull())
+ {
+ // Unknown time zone
+ mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
+ }
+ else
+ {
+ auto timezone = new immutable SimpleTimeZone(offset.get());
+ mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
+ }
+ }
+
+ try
+ {
+ auto timezone = TimeZone.getTimeZone(timezoneStr);
+ if(timezone)
+ mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
+ }
+ catch(TimeException e)
+ {
+ // Time zone not found. So just move along to "Unknown time zone" below.
+ }
+
+ // Unknown time zone
+ mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
+ }
+
+ if(!isEndOfNumber())
+ error("Date-Times cannot have suffixes.");
+
+ mixin(accept!("Value", "dateTimeFrac"));
+ }
+
+ /// Lex time span (after the initial numeric fragment was lexed)
+ private void lexTimeSpan(bool isNegative, string firstPart)
+ {
+ assert(ch == ':' || ch == 'd');
+
+ string dayStr = "";
+ string hourStr;
+
+ // Lexed days?
+ bool hasDays = ch == 'd';
+ if(hasDays)
+ {
+ dayStr = firstPart;
+ advanceChar(ErrorOnEOF.Yes); // Skip 'd'
+
+ // Lex hours
+ if(ch != ':')
+ error("Invalid time span format: Missing hours.");
+ advanceChar(ErrorOnEOF.Yes); // Skip ':'
+ hourStr = lexNumericFragment();
+ }
+ else
+ hourStr = firstPart;
+
+ // Lex minutes
+ if(ch != ':')
+ error("Invalid time span format: Missing minutes.");
+ advanceChar(ErrorOnEOF.Yes); // Skip ':'
+ auto minuteStr = lexNumericFragment();
+
+ // Lex seconds
+ if(ch != ':')
+ error("Invalid time span format: Missing seconds.");
+ advanceChar(ErrorOnEOF.Yes); // Skip ':'
+ auto secondStr = lexNumericFragment();
+
+ // Lex milliseconds, if exists
+ string millisecondStr = "";
+ if(ch == '.')
+ {
+ advanceChar(ErrorOnEOF.Yes); // Skip '.'
+ millisecondStr = lexNumericFragment();
+ }
+
+ if(!isEndOfNumber())
+ error("Time spans cannot have suffixes.");
+
+ auto duration = makeDuration(isNegative, dayStr, hourStr, minuteStr, secondStr, millisecondStr);
+ mixin(accept!("Value", "duration"));
+ }
+
+ /// Advances past whitespace and comments
+ private void eatWhite(bool allowComments=true)
+ {
+ // -- Comment/Whitepace Lexer -------------
+
+ enum State
+ {
+ normal,
+ lineComment, // Got "#" or "//" or "--", Eating everything until newline
+ blockComment, // Got "/*", Eating everything until "*/"
+ }
+
+ if(isEOF)
+ return;
+
+ Location commentStart;
+ State state = State.normal;
+ bool consumeNewlines = false;
+ bool hasConsumedNewline = false;
+ while(true)
+ {
+ final switch(state)
+ {
+ case State.normal:
+
+ if(ch == '\\')
+ {
+ commentStart = location;
+ consumeNewlines = true;
+ hasConsumedNewline = false;
+ }
+
+ else if(ch == '#')
+ {
+ if(!allowComments)
+ return;
+
+ commentStart = location;
+ state = State.lineComment;
+ continue;
+ }
+
+ else if(ch == '/' || ch == '-')
+ {
+ commentStart = location;
+ if(lookahead(ch))
+ {
+ if(!allowComments)
+ return;
+
+ advanceChar(ErrorOnEOF.No);
+ state = State.lineComment;
+ continue;
+ }
+ else if(ch == '/' && lookahead('*'))
+ {
+ if(!allowComments)
+ return;
+
+ advanceChar(ErrorOnEOF.No);
+ state = State.blockComment;
+ continue;
+ }
+ else
+ return; // Done
+ }
+ else if(isAtNewline())
+ {
+ if(consumeNewlines)
+ hasConsumedNewline = true;
+ else
+ return; // Done
+ }
+ else if(!isWhite(ch))
+ {
+ if(consumeNewlines)
+ {
+ if(hasConsumedNewline)
+ return; // Done
+ else
+ error("Only whitespace can come between a line-continuation backslash and the following newline.");
+ }
+ else
+ return; // Done
+ }
+
+ break;
+
+ case State.lineComment:
+ if(lookahead(&isNewline))
+ state = State.normal;
+ break;
+
+ case State.blockComment:
+ if(ch == '*' && lookahead('/'))
+ {
+ advanceChar(ErrorOnEOF.No);
+ state = State.normal;
+ }
+ break;
+ }
+
+ advanceChar(ErrorOnEOF.No);
+ if(isEOF)
+ {
+ // Reached EOF
+
+ if(consumeNewlines && !hasConsumedNewline)
+ error("Missing newline after line-continuation backslash.");
+
+ else if(state == State.blockComment)
+ error(commentStart, "Unterminated block comment.");
+
+ else
+ return; // Done, reached EOF
+ }
+ }
+ }
+}
+
+version(sdlangUnittest)
+{
+ import std.stdio;
+
+ private auto loc = Location("filename", 0, 0, 0);
+ private auto loc2 = Location("a", 1, 1, 1);
+
+ unittest
+ {
+ assert([Token(symbol!"EOL",loc) ] == [Token(symbol!"EOL",loc) ] );
+ assert([Token(symbol!"EOL",loc,Value(7),"A")] == [Token(symbol!"EOL",loc2,Value(7),"B")] );
+ }
+
+ private int numErrors = 0;
+ private void testLex(string source, Token[] expected, bool test_locations = false, string file=__FILE__, size_t line=__LINE__)
+ {
+ Token[] actual;
+ try
+ actual = lexSource(source, "filename");
+ catch(SDLangParseException e)
+ {
+ numErrors++;
+ stderr.writeln(file, "(", line, "): testLex failed on: ", source);
+ stderr.writeln(" Expected:");
+ stderr.writeln(" ", expected);
+ stderr.writeln(" Actual: SDLangParseException thrown:");
+ stderr.writeln(" ", e.msg);
+ return;
+ }
+
+ bool is_same = actual == expected;
+ if (is_same && test_locations) {
+ is_same = actual.map!(t => t.location).equal(expected.map!(t => t.location));
+ }
+
+ if(!is_same)
+ {
+ numErrors++;
+ stderr.writeln(file, "(", line, "): testLex failed on: ", source);
+ stderr.writeln(" Expected:");
+ stderr.writeln(" ", expected);
+ stderr.writeln(" Actual:");
+ stderr.writeln(" ", actual);
+
+ if(expected.length > 1 || actual.length > 1)
+ {
+ stderr.writeln(" expected.length: ", expected.length);
+ stderr.writeln(" actual.length: ", actual.length);
+
+ if(actual.length == expected.length)
+ foreach(i; 0..actual.length)
+ if(actual[i] != expected[i])
+ {
+ stderr.writeln(" Unequal at index #", i, ":");
+ stderr.writeln(" Expected:");
+ stderr.writeln(" ", expected[i]);
+ stderr.writeln(" Actual:");
+ stderr.writeln(" ", actual[i]);
+ }
+ }
+ }
+ }
+
+ private void testLexThrows(string file=__FILE__, size_t line=__LINE__)(string source)
+ {
+ bool hadException = false;
+ Token[] actual;
+ try
+ actual = lexSource(source, "filename");
+ catch(SDLangParseException e)
+ hadException = true;
+
+ if(!hadException)
+ {
+ numErrors++;
+ stderr.writeln(file, "(", line, "): testLex failed on: ", source);
+ stderr.writeln(" Expected SDLangParseException");
+ stderr.writeln(" Actual:");
+ stderr.writeln(" ", actual);
+ }
+ }
+}
+
+version(sdlangUnittest)
+unittest
+{
+ writeln("Unittesting sdlang lexer...");
+ stdout.flush();
+
+ testLex("", []);
+ testLex(" ", []);
+ testLex("\\\n", []);
+ testLex("/*foo*/", []);
+ testLex("/* multiline \n comment */", []);
+ testLex("/* * */", []);
+ testLexThrows("/* ");
+
+ testLex(":", [ Token(symbol!":", loc) ]);
+ testLex("=", [ Token(symbol!"=", loc) ]);
+ testLex("{", [ Token(symbol!"{", loc) ]);
+ testLex("}", [ Token(symbol!"}", loc) ]);
+ testLex(";", [ Token(symbol!"EOL",loc) ]);
+ testLex("\n", [ Token(symbol!"EOL",loc) ]);
+
+ testLex("foo", [ Token(symbol!"Ident",loc,Value(null),"foo") ]);
+ testLex("_foo", [ Token(symbol!"Ident",loc,Value(null),"_foo") ]);
+ testLex("foo.bar", [ Token(symbol!"Ident",loc,Value(null),"foo.bar") ]);
+ testLex("foo-bar", [ Token(symbol!"Ident",loc,Value(null),"foo-bar") ]);
+ testLex("foo.", [ Token(symbol!"Ident",loc,Value(null),"foo.") ]);
+ testLex("foo-", [ Token(symbol!"Ident",loc,Value(null),"foo-") ]);
+ testLexThrows(".foo");
+
+ testLex("foo bar", [
+ Token(symbol!"Ident",loc,Value(null),"foo"),
+ Token(symbol!"Ident",loc,Value(null),"bar"),
+ ]);
+ testLex("foo \\ \n \n bar", [
+ Token(symbol!"Ident",loc,Value(null),"foo"),
+ Token(symbol!"Ident",loc,Value(null),"bar"),
+ ]);
+ testLex("foo \\ \n \\ \n bar", [
+ Token(symbol!"Ident",loc,Value(null),"foo"),
+ Token(symbol!"Ident",loc,Value(null),"bar"),
+ ]);
+ testLexThrows("foo \\ ");
+ testLexThrows("foo \\ bar");
+ testLexThrows("foo \\ \n \\ ");
+ testLexThrows("foo \\ \n \\ bar");
+
+ testLex("foo : = { } ; \n bar \n", [
+ Token(symbol!"Ident",loc,Value(null),"foo"),
+ Token(symbol!":",loc),
+ Token(symbol!"=",loc),
+ Token(symbol!"{",loc),
+ Token(symbol!"}",loc),
+ Token(symbol!"EOL",loc),
+ Token(symbol!"EOL",loc),
+ Token(symbol!"Ident",loc,Value(null),"bar"),
+ Token(symbol!"EOL",loc),
+ ]);
+
+ testLexThrows("<");
+ testLexThrows("*");
+ testLexThrows(`\`);
+
+ // Integers
+ testLex( "7", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
+ testLex( "-7", [ Token(symbol!"Value",loc,Value(cast( int)-7)) ]);
+ testLex( "7L", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
+ testLex( "7l", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
+ testLex("-7L", [ Token(symbol!"Value",loc,Value(cast(long)-7)) ]);
+ testLex( "0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
+ testLex( "-0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
+
+ testLex("7/**/", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
+ testLex("7#", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
+
+ testLex("7 A", [
+ Token(symbol!"Value",loc,Value(cast(int)7)),
+ Token(symbol!"Ident",loc,Value( null),"A"),
+ ]);
+ testLexThrows("7A");
+ testLexThrows("-A");
+ testLexThrows(`-""`);
+
+ testLex("7;", [
+ Token(symbol!"Value",loc,Value(cast(int)7)),
+ Token(symbol!"EOL",loc),
+ ]);
+
+ // Floats
+ testLex("1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
+ testLex("1.2f" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
+ testLex("1.2" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
+ testLex("1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
+ testLex("1.2d" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
+ testLex("1.2BD", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
+ testLex("1.2bd", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
+ testLex("1.2Bd", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
+ testLex("1.2bD", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
+
+ testLex(".2F" , [ Token(symbol!"Value",loc,Value(cast( float)0.2)) ]);
+ testLex(".2" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
+ testLex(".2D" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
+ testLex(".2BD", [ Token(symbol!"Value",loc,Value(cast( real)0.2)) ]);
+
+ testLex("-1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-1.2)) ]);
+ testLex("-1.2" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
+ testLex("-1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
+ testLex("-1.2BD", [ Token(symbol!"Value",loc,Value(cast( real)-1.2)) ]);
+
+ testLex("-.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-0.2)) ]);
+ testLex("-.2" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
+ testLex("-.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
+ testLex("-.2BD", [ Token(symbol!"Value",loc,Value(cast( real)-0.2)) ]);
+
+ testLex( "0.0" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+ testLex( "0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+ testLex( "0.0BD", [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
+ testLex("-0.0" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+ testLex("-0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+ testLex("-0.0BD", [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
+ testLex( "7F" , [ Token(symbol!"Value",loc,Value(cast( float)7.0)) ]);
+ testLex( "7D" , [ Token(symbol!"Value",loc,Value(cast(double)7.0)) ]);
+ testLex( "7BD" , [ Token(symbol!"Value",loc,Value(cast( real)7.0)) ]);
+ testLex( "0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+ testLex( "0D" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+ testLex( "0BD" , [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
+ testLex("-0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+ testLex("-0D" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+ testLex("-0BD" , [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
+
+ testLex("1.2 F", [
+ Token(symbol!"Value",loc,Value(cast(double)1.2)),
+ Token(symbol!"Ident",loc,Value( null),"F"),
+ ]);
+ testLexThrows("1.2A");
+ testLexThrows("1.2B");
+ testLexThrows("1.2BDF");
+
+ testLex("1.2;", [
+ Token(symbol!"Value",loc,Value(cast(double)1.2)),
+ Token(symbol!"EOL",loc),
+ ]);
+
+ testLex("1.2F;", [
+ Token(symbol!"Value",loc,Value(cast(float)1.2)),
+ Token(symbol!"EOL",loc),
+ ]);
+
+ testLex("1.2BD;", [
+ Token(symbol!"Value",loc,Value(cast(real)1.2)),
+ Token(symbol!"EOL",loc),
+ ]);
+
+ // Booleans and null
+ testLex("true", [ Token(symbol!"Value",loc,Value( true)) ]);
+ testLex("false", [ Token(symbol!"Value",loc,Value(false)) ]);
+ testLex("on", [ Token(symbol!"Value",loc,Value( true)) ]);
+ testLex("off", [ Token(symbol!"Value",loc,Value(false)) ]);
+ testLex("null", [ Token(symbol!"Value",loc,Value( null)) ]);
+
+ testLex("TRUE", [ Token(symbol!"Ident",loc,Value(null),"TRUE") ]);
+ testLex("true ", [ Token(symbol!"Value",loc,Value(true)) ]);
+ testLex("true ", [ Token(symbol!"Value",loc,Value(true)) ]);
+ testLex("tru", [ Token(symbol!"Ident",loc,Value(null),"tru") ]);
+ testLex("truX", [ Token(symbol!"Ident",loc,Value(null),"truX") ]);
+ testLex("trueX", [ Token(symbol!"Ident",loc,Value(null),"trueX") ]);
+
+ // Raw Backtick Strings
+ testLex("`hello world`", [ Token(symbol!"Value",loc,Value(`hello world` )) ]);
+ testLex("` hello world `", [ Token(symbol!"Value",loc,Value(` hello world ` )) ]);
+ testLex("`hello \\t world`", [ Token(symbol!"Value",loc,Value(`hello \t world`)) ]);
+ testLex("`hello \\n world`", [ Token(symbol!"Value",loc,Value(`hello \n world`)) ]);
+ testLex("`hello \n world`", [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
+ testLex("`hello \r\n world`", [ Token(symbol!"Value",loc,Value("hello \r\n world")) ]);
+ testLex("`hello \"world\"`", [ Token(symbol!"Value",loc,Value(`hello "world"` )) ]);
+
+ testLexThrows("`foo");
+ testLexThrows("`");
+
+ // Double-Quote Strings
+ testLex(`"hello world"`, [ Token(symbol!"Value",loc,Value("hello world" )) ]);
+ testLex(`" hello world "`, [ Token(symbol!"Value",loc,Value(" hello world " )) ]);
+ testLex(`"hello \t world"`, [ Token(symbol!"Value",loc,Value("hello \t world")) ]);
+ testLex(`"hello \n world"`, [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
+ testLex("\"hello \\\n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]);
+ testLex("\"hello \\ \n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]);
+ testLex("\"hello \\ \n\n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]);
+ testLex(`"\"hello world\""`, [ Token(symbol!"Value",loc,Value(`"hello world"` )) ]);
+ testLex(`""`, [ Token(symbol!"Value",loc,Value("" )) ]); // issue #34
+
+ testLexThrows("\"hello \n world\"");
+ testLexThrows(`"foo`);
+ testLexThrows(`"`);
+
+ // Characters
+ testLex("'a'", [ Token(symbol!"Value",loc,Value(cast(dchar) 'a')) ]);
+ testLex("'\\n'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\n')) ]);
+ testLex("'\\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
+ testLex("'\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
+ testLex("'\\''", [ Token(symbol!"Value",loc,Value(cast(dchar)'\'')) ]);
+ testLex(`'\\'`, [ Token(symbol!"Value",loc,Value(cast(dchar)'\\')) ]);
+
+ testLexThrows("'a");
+ testLexThrows("'aa'");
+ testLexThrows("''");
+ testLexThrows("'\\\n'");
+ testLexThrows("'\n'");
+ testLexThrows(`'\`);
+ testLexThrows(`'\'`);
+ testLexThrows("'");
+
+ // Unicode
+ testLex("日本語", [ Token(symbol!"Ident",loc,Value(null), "日本語") ]);
+ testLex("`おはよう、日本。`", [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
+ testLex(`"おはよう、日本。"`, [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
+ testLex("'月'", [ Token(symbol!"Value",loc,Value("月"d.dup[0])) ]);
+
+ // Base64 Binary
+ testLex("[aGVsbG8gd29ybGQ=]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
+ testLex("[ aGVsbG8gd29ybGQ= ]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
+ testLex("[\n aGVsbG8g \n \n d29ybGQ= \n]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
+
+ testLexThrows("[aGVsbG8gd29ybGQ]"); // Ie: Not multiple of 4
+ testLexThrows("[ aGVsbG8gd29ybGQ ]");
+
+ // Date
+ testLex( "1999/12/5", [ Token(symbol!"Value",loc,Value(Date( 1999, 12, 5))) ]);
+ testLex( "2013/2/22", [ Token(symbol!"Value",loc,Value(Date( 2013, 2, 22))) ]);
+ testLex("-2013/2/22", [ Token(symbol!"Value",loc,Value(Date(-2013, 2, 22))) ]);
+
+ testLexThrows("7/");
+ testLexThrows("2013/2/22a");
+ testLexThrows("2013/2/22f");
+
+ testLex("1999/12/5\n", [
+ Token(symbol!"Value",loc,Value(Date(1999, 12, 5))),
+ Token(symbol!"EOL",loc),
+ ]);
+
+ // DateTime, no timezone
+ testLex( "2013/2/22 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22 \t 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22/*foo*/07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22 /*foo*/ \\\n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22 /*foo*/ \\\n\n \n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22 /*foo*/ \\\n\\\n \\\n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22/*foo*/\\\n/*bar*/07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
+ testLex("-2013/2/22 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 7, 53, 0)))) ]);
+ testLex( "2013/2/22 -07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53)))) ]);
+ testLex("-2013/2/22 -07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53)))) ]);
+ testLex( "2013/2/22 07:53:34", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34)))) ]);
+ testLex( "2013/2/22 07:53:34.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs))) ]);
+ testLex( "2013/2/22 07:53:34.12", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 120.msecs))) ]);
+ testLex( "2013/2/22 07:53:34.1", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 100.msecs))) ]);
+ testLex( "2013/2/22 07:53.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs))) ]);
+
+ testLex( "2013/2/22 34:65", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0)))) ]);
+ testLex( "2013/2/22 34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds(77), 123.msecs))) ]);
+ testLex( "2013/2/22 34:65.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0), 123.msecs))) ]);
+
+ testLex( "2013/2/22 -34:65", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0)))) ]);
+ testLex( "2013/2/22 -34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds(77), -123.msecs))) ]);
+ testLex( "2013/2/22 -34:65.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), -123.msecs))) ]);
+
+ testLexThrows("2013/2/22 07:53a");
+ testLexThrows("2013/2/22 07:53f");
+ testLexThrows("2013/2/22 07:53:34.123a");
+ testLexThrows("2013/2/22 07:53:34.123f");
+ testLexThrows("2013/2/22a 07:53");
+
+ testLex(`2013/2/22 "foo"`, [
+ Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+ Token(symbol!"Value",loc,Value("foo")),
+ ]);
+
+ testLex("2013/2/22 07", [
+ Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+ Token(symbol!"Value",loc,Value(cast(int)7)),
+ ]);
+
+ testLex("2013/2/22 1.2F", [
+ Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+ Token(symbol!"Value",loc,Value(cast(float)1.2)),
+ ]);
+
+ testLex("2013/2/22 .2F", [
+ Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+ Token(symbol!"Value",loc,Value(cast(float)0.2)),
+ ]);
+
+ testLex("2013/2/22 -1.2F", [
+ Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+ Token(symbol!"Value",loc,Value(cast(float)-1.2)),
+ ]);
+
+ testLex("2013/2/22 -.2F", [
+ Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+ Token(symbol!"Value",loc,Value(cast(float)-0.2)),
+ ]);
+
+ // DateTime, with known timezone
+ testLex( "2013/2/22 07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex("-2013/2/22 07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex( "2013/2/22 -07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex("-2013/2/22 -07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex( "2013/2/22 07:53-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+ testLex( "2013/2/22 07:53-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+ testLex( "2013/2/22 07:53:34-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex( "2013/2/22 07:53:34-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+ testLex( "2013/2/22 07:53:34-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+ testLex( "2013/2/22 07:53:34.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex( "2013/2/22 07:53:34.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+ testLex( "2013/2/22 07:53:34.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+ testLex( "2013/2/22 07:53.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone( hours(0) )))) ]);
+ testLex( "2013/2/22 07:53.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+ testLex( "2013/2/22 07:53.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+
+ testLex( "2013/2/22 -34:65-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+
+ // DateTime, with Java SDL's occasionally weird interpretation of some
+ // "not quite ISO" variations of the "GMT with offset" timezone strings.
+ Token testTokenSimpleTimeZone(Duration d)
+ {
+ auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
+ auto tz = new immutable SimpleTimeZone(d);
+ return Token( symbol!"Value", loc, Value(SysTime(dateTime,tz)) );
+ }
+ Token testTokenUnknownTimeZone(string tzName)
+ {
+ auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
+ auto frac = 0.msecs;
+ return Token( symbol!"Value", loc, Value(DateTimeFracUnknownZone(dateTime,frac,tzName)) );
+ }
+ testLex("2013/2/22 07:53-GMT+", [ testTokenUnknownTimeZone("GMT+") ]);
+ testLex("2013/2/22 07:53-GMT+:", [ testTokenUnknownTimeZone("GMT+:") ]);
+ testLex("2013/2/22 07:53-GMT+:3", [ testTokenUnknownTimeZone("GMT+:3") ]);
+ testLex("2013/2/22 07:53-GMT+:03", [ testTokenSimpleTimeZone(minutes(3)) ]);
+ testLex("2013/2/22 07:53-GMT+:003", [ testTokenUnknownTimeZone("GMT+:003") ]);
+
+ testLex("2013/2/22 07:53-GMT+4", [ testTokenSimpleTimeZone(hours(4)) ]);
+ testLex("2013/2/22 07:53-GMT+4:", [ testTokenUnknownTimeZone("GMT+4:") ]);
+ testLex("2013/2/22 07:53-GMT+4:3", [ testTokenUnknownTimeZone("GMT+4:3") ]);
+ testLex("2013/2/22 07:53-GMT+4:03", [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
+ testLex("2013/2/22 07:53-GMT+4:003", [ testTokenUnknownTimeZone("GMT+4:003") ]);
+
+ testLex("2013/2/22 07:53-GMT+04", [ testTokenSimpleTimeZone(hours(4)) ]);
+ testLex("2013/2/22 07:53-GMT+04:", [ testTokenUnknownTimeZone("GMT+04:") ]);
+ testLex("2013/2/22 07:53-GMT+04:3", [ testTokenUnknownTimeZone("GMT+04:3") ]);
+ testLex("2013/2/22 07:53-GMT+04:03", [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
+ testLex("2013/2/22 07:53-GMT+04:03abc", [ testTokenUnknownTimeZone("GMT+04:03abc") ]);
+ testLex("2013/2/22 07:53-GMT+04:003", [ testTokenUnknownTimeZone("GMT+04:003") ]);
+
+ testLex("2013/2/22 07:53-GMT+004", [ testTokenSimpleTimeZone(minutes(4)) ]);
+ testLex("2013/2/22 07:53-GMT+004:", [ testTokenUnknownTimeZone("GMT+004:") ]);
+ testLex("2013/2/22 07:53-GMT+004:3", [ testTokenUnknownTimeZone("GMT+004:3") ]);
+ testLex("2013/2/22 07:53-GMT+004:03", [ testTokenUnknownTimeZone("GMT+004:03") ]);
+ testLex("2013/2/22 07:53-GMT+004:003", [ testTokenUnknownTimeZone("GMT+004:003") ]);
+
+ testLex("2013/2/22 07:53-GMT+0004", [ testTokenSimpleTimeZone(minutes(4)) ]);
+ testLex("2013/2/22 07:53-GMT+0004:", [ testTokenUnknownTimeZone("GMT+0004:") ]);
+ testLex("2013/2/22 07:53-GMT+0004:3", [ testTokenUnknownTimeZone("GMT+0004:3") ]);
+ testLex("2013/2/22 07:53-GMT+0004:03", [ testTokenUnknownTimeZone("GMT+0004:03") ]);
+ testLex("2013/2/22 07:53-GMT+0004:003", [ testTokenUnknownTimeZone("GMT+0004:003") ]);
+
+ testLex("2013/2/22 07:53-GMT+00004", [ testTokenSimpleTimeZone(minutes(4)) ]);
+ testLex("2013/2/22 07:53-GMT+00004:", [ testTokenUnknownTimeZone("GMT+00004:") ]);
+ testLex("2013/2/22 07:53-GMT+00004:3", [ testTokenUnknownTimeZone("GMT+00004:3") ]);
+ testLex("2013/2/22 07:53-GMT+00004:03", [ testTokenUnknownTimeZone("GMT+00004:03") ]);
+ testLex("2013/2/22 07:53-GMT+00004:003", [ testTokenUnknownTimeZone("GMT+00004:003") ]);
+
+ // DateTime, with unknown timezone
+ testLex( "2013/2/22 07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 0), 0.msecs, "Bogus/Foo")), "2013/2/22 07:53-Bogus/Foo") ]);
+ testLex("-2013/2/22 07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 7, 53, 0), 0.msecs, "Bogus/Foo"))) ]);
+ testLex( "2013/2/22 -07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
+ testLex("-2013/2/22 -07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
+ testLex( "2013/2/22 07:53:34-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 0.msecs, "Bogus/Foo"))) ]);
+ testLex( "2013/2/22 07:53:34.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, "Bogus/Foo"))) ]);
+ testLex( "2013/2/22 07:53.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, "Bogus/Foo"))) ]);
+
+ // Time Span
+ testLex( "12:14:42", [ Token(symbol!"Value",loc,Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs( 0))) ]);
+ testLex("-12:14:42", [ Token(symbol!"Value",loc,Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs( 0))) ]);
+ testLex( "00:09:12", [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs( 0))) ]);
+ testLex( "00:00:01.023", [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23))) ]);
+ testLex( "23d:05:21:23.532", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532))) ]);
+ testLex( "23d:05:21:23.53", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530))) ]);
+ testLex( "23d:05:21:23.5", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500))) ]);
+ testLex("-23d:05:21:23.532", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532))) ]);
+ testLex("-23d:05:21:23.5", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500))) ]);
+ testLex( "23d:05:21:23", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs( 0))) ]);
+
+ testLexThrows("12:14:42a");
+ testLexThrows("23d:05:21:23.532a");
+ testLexThrows("23d:05:21:23.532f");
+
+ // Combination
+ testLex("foo. 7", [
+ Token(symbol!"Ident",loc,Value( null),"foo."),
+ Token(symbol!"Value",loc,Value(cast(int)7))
+ ]);
+
+ testLex(`
+ namespace:person "foo" "bar" 1 23L name.first="ひとみ" name.last="Smith" {
+ namespace:age 37; namespace:favorite_color "blue" // comment
+ somedate 2013/2/22 07:53 -- comment
+
+ inventory /* comment */ {
+ socks
+ }
+ }
+ `,
+ [
+ Token(symbol!"EOL",loc,Value(null),"\n"),
+
+ Token(symbol!"Ident", loc, Value( null ), "namespace"),
+ Token(symbol!":", loc, Value( null ), ":"),
+ Token(symbol!"Ident", loc, Value( null ), "person"),
+ Token(symbol!"Value", loc, Value( "foo" ), `"foo"`),
+ Token(symbol!"Value", loc, Value( "bar" ), `"bar"`),
+ Token(symbol!"Value", loc, Value( cast( int) 1 ), "1"),
+ Token(symbol!"Value", loc, Value( cast(long)23 ), "23L"),
+ Token(symbol!"Ident", loc, Value( null ), "name.first"),
+ Token(symbol!"=", loc, Value( null ), "="),
+ Token(symbol!"Value", loc, Value( "ひとみ" ), `"ひとみ"`),
+ Token(symbol!"Ident", loc, Value( null ), "name.last"),
+ Token(symbol!"=", loc, Value( null ), "="),
+ Token(symbol!"Value", loc, Value( "Smith" ), `"Smith"`),
+ Token(symbol!"{", loc, Value( null ), "{"),
+ Token(symbol!"EOL", loc, Value( null ), "\n"),
+
+ Token(symbol!"Ident", loc, Value( null ), "namespace"),
+ Token(symbol!":", loc, Value( null ), ":"),
+ Token(symbol!"Ident", loc, Value( null ), "age"),
+ Token(symbol!"Value", loc, Value( cast(int)37 ), "37"),
+ Token(symbol!"EOL", loc, Value( null ), ";"),
+ Token(symbol!"Ident", loc, Value( null ), "namespace"),
+ Token(symbol!":", loc, Value( null ), ":"),
+ Token(symbol!"Ident", loc, Value( null ), "favorite_color"),
+ Token(symbol!"Value", loc, Value( "blue" ), `"blue"`),
+ Token(symbol!"EOL", loc, Value( null ), "\n"),
+
+ Token(symbol!"Ident", loc, Value( null ), "somedate"),
+ Token(symbol!"Value", loc, Value( DateTimeFrac(DateTime(2013, 2, 22, 7, 53, 0)) ), "2013/2/22 07:53"),
+ Token(symbol!"EOL", loc, Value( null ), "\n"),
+ Token(symbol!"EOL", loc, Value( null ), "\n"),
+
+ Token(symbol!"Ident", loc, Value(null), "inventory"),
+ Token(symbol!"{", loc, Value(null), "{"),
+ Token(symbol!"EOL", loc, Value(null), "\n"),
+
+ Token(symbol!"Ident", loc, Value(null), "socks"),
+ Token(symbol!"EOL", loc, Value(null), "\n"),
+
+ Token(symbol!"}", loc, Value(null), "}"),
+ Token(symbol!"EOL", loc, Value(null), "\n"),
+
+ Token(symbol!"}", loc, Value(null), "}"),
+ Token(symbol!"EOL", loc, Value(null), "\n"),
+ ]);
+
+ if(numErrors > 0)
+ stderr.writeln(numErrors, " failed test(s)");
+}
+
+version(sdlangUnittest)
+unittest
+{
+ writeln("lexer: Regression test issue #8...");
+ stdout.flush();
+
+ testLex(`"\n \n"`, [ Token(symbol!"Value",loc,Value("\n \n"),`"\n \n"`) ]);
+ testLex(`"\t\t"`, [ Token(symbol!"Value",loc,Value("\t\t"),`"\t\t"`) ]);
+ testLex(`"\n\n"`, [ Token(symbol!"Value",loc,Value("\n\n"),`"\n\n"`) ]);
+}
+
+version(sdlangUnittest)
+unittest
+{
+ writeln("lexer: Regression test issue #11...");
+ stdout.flush();
+
+ void test(string input)
+ {
+ testLex(
+ input,
+ [
+ Token(symbol!"EOL", loc, Value(null), "\n"),
+ Token(symbol!"Ident",loc,Value(null), "a")
+ ]
+ );
+ }
+
+ test("//X\na");
+ test("//\na");
+ test("--\na");
+ test("#\na");
+}
+
+version(sdlangUnittest)
+unittest
+{
+ writeln("lexer: Regression test issue #28...");
+ stdout.flush();
+
+ enum offset = 1; // workaround for an of-by-one error for line numbers
+ testLex("test", [
+ Token(symbol!"Ident", Location("filename", 0, 0, 0), Value(null), "test")
+ ], true);
+ testLex("\ntest", [
+ Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\n"),
+ Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
+ ], true);
+ testLex("\rtest", [
+ Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
+ Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
+ ], true);
+ testLex("\r\ntest", [
+ Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
+ Token(symbol!"Ident", Location("filename", 1, 0, 2), Value(null), "test")
+ ], true);
+ testLex("\r\n\ntest", [
+ Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
+ Token(symbol!"EOL", Location("filename", 1, 0, 2), Value(null), "\n"),
+ Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
+ ], true);
+ testLex("\r\r\ntest", [
+ Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
+ Token(symbol!"EOL", Location("filename", 1, 0, 1), Value(null), "\r\n"),
+ Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
+ ], true);
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-04 13:52:30 +0000