aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/sdlang/lexer.d
diff options
context:
space:
mode:
authorRalph Amissah <ralph@amissah.com>2016-10-01 14:12:13 -0400
committerRalph Amissah <ralph@amissah.com>2019-04-10 15:14:13 -0400
commitba1712e77b31704fd9ba16d14e15518e7a7dd104 (patch)
tree1a0d3233fb611b68dbf43e098a41a0d9378e9ace /src/sdlang/lexer.d
parentupdate sdlang, start looking to using dub remote dependencies (diff)
0.7.0 using dub remote dependencies (local src related to sdlang removed)
Diffstat (limited to 'src/sdlang/lexer.d')
-rw-r--r--src/sdlang/lexer.d2068
1 files changed, 0 insertions, 2068 deletions
diff --git a/src/sdlang/lexer.d b/src/sdlang/lexer.d
deleted file mode 100644
index 3788188..0000000
--- a/src/sdlang/lexer.d
+++ /dev/null
@@ -1,2068 +0,0 @@
-// SDLang-D
-// Written in the D programming language.
-
-module sdlang.lexer;
-
-import std.algorithm;
-import std.array;
-static import std.ascii;
-import std.base64;
-import std.bigint;
-import std.conv;
-import std.datetime;
-import std.file;
-import std.format;
-import std.traits;
-import std.typecons;
-import std.uni;
-import std.utf;
-import std.variant;
-
-import sdlang.exception;
-import sdlang.symbol;
-import sdlang.token;
-import sdlang.util;
-
-alias sdlang.util.startsWith startsWith;
-
-Token[] lexFile(string filename)
-{
- auto source = cast(string)read(filename);
- return lexSource(source, filename);
-}
-
-Token[] lexSource(string source, string filename=null)
-{
- auto lexer = scoped!Lexer(source, filename);
-
- // Can't use 'std.array.array(Range)' because 'lexer' is scoped
- // and therefore cannot have its reference copied.
- Appender!(Token[]) tokens;
- foreach(tok; lexer)
- tokens.put(tok);
-
- return tokens.data;
-}
-
-// Kind of a poor-man's yield, but fast.
-// Only to be used inside Lexer.popFront (and Lexer.this).
-private template accept(string symbolName)
-{
- static assert(symbolName != "Value", "Value symbols must also take a value.");
- enum accept = acceptImpl!(symbolName, "null");
-}
-private template accept(string symbolName, string value)
-{
- static assert(symbolName == "Value", "Only a Value symbol can take a value.");
- enum accept = acceptImpl!(symbolName, value);
-}
-private template accept(string symbolName, string value, string startLocation, string endLocation)
-{
- static assert(symbolName == "Value", "Only a Value symbol can take a value.");
- enum accept = ("
- {
- _front = makeToken!"~symbolName.stringof~";
- _front.value = "~value~";
- _front.location = "~(startLocation==""? "tokenStart" : startLocation)~";
- _front.data = source[
- "~(startLocation==""? "tokenStart.index" : startLocation)~"
- ..
- "~(endLocation==""? "location.index" : endLocation)~"
- ];
- return;
- }
- ").replace("\n", "");
-}
-private template acceptImpl(string symbolName, string value)
-{
- enum acceptImpl = ("
- {
- _front = makeToken!"~symbolName.stringof~";
- _front.value = "~value~";
- return;
- }
- ").replace("\n", "");
-}
-
-class Lexer
-{
- string source;
- string filename;
- Location location; /// Location of current character in source
-
- private dchar ch; // Current character
- private dchar nextCh; // Lookahead character
- private size_t nextPos; // Position of lookahead character (an index into source)
- private bool hasNextCh; // If false, then there's no more lookahead, just EOF
- private size_t posAfterLookahead; // Position after lookahead character (an index into source)
-
- private Location tokenStart; // The starting location of the token being lexed
-
- // Length so far of the token being lexed, not including current char
- private size_t tokenLength; // Length in UTF-8 code units
- private size_t tokenLength32; // Length in UTF-32 code units
-
- // Slight kludge:
- // If a numeric fragment is found after a Date (separated by arbitrary
- // whitespace), it could be the "hours" part of a DateTime, or it could
- // be a separate numeric literal that simply follows a plain Date. If the
- // latter, then the Date must be emitted, but numeric fragment that was
- // found after it needs to be saved for the the lexer's next iteration.
- //
- // It's a slight kludge, and could instead be implemented as a slightly
- // kludgey parser hack, but it's the only situation where SDLang's lexing
- // needs to lookahead more than one character, so this is good enough.
- private struct LookaheadTokenInfo
- {
- bool exists = false;
- string numericFragment = "";
- bool isNegative = false;
- Location tokenStart;
- }
- private LookaheadTokenInfo lookaheadTokenInfo;
-
- this(string source=null, string filename=null)
- {
- this.filename = filename;
- this.source = source;
-
- _front = Token(symbol!"Error", Location());
- lookaheadTokenInfo = LookaheadTokenInfo.init;
-
- if( source.startsWith( ByteOrderMarks[BOM.UTF8] ) )
- {
- source = source[ ByteOrderMarks[BOM.UTF8].length .. $ ];
- this.source = source;
- }
-
- foreach(bom; ByteOrderMarks)
- if( source.startsWith(bom) )
- error(Location(filename,0,0,0), "SDL spec only supports UTF-8, not UTF-16 or UTF-32");
-
- if(source == "")
- mixin(accept!"EOF");
-
- // Prime everything
- hasNextCh = true;
- nextCh = source.decode(posAfterLookahead);
- advanceChar(ErrorOnEOF.Yes);
- location = Location(filename, 0, 0, 0);
- popFront();
- }
-
- @property bool empty()
- {
- return _front.symbol == symbol!"EOF";
- }
-
- Token _front;
- @property Token front()
- {
- return _front;
- }
-
- @property bool isEOF()
- {
- return location.index == source.length && !lookaheadTokenInfo.exists;
- }
-
- private void error(string msg)
- {
- error(location, msg);
- }
-
- //TODO: Take varargs and use output range sink.
- private void error(Location loc, string msg)
- {
- throw new ParseException(loc, "Error: "~msg);
- }
-
- private Token makeToken(string symbolName)()
- {
- auto tok = Token(symbol!symbolName, tokenStart);
- tok.data = tokenData;
- return tok;
- }
-
- private @property string tokenData()
- {
- return source[ tokenStart.index .. location.index ];
- }
-
- /// Check the lookahead character
- private bool lookahead(dchar ch)
- {
- return hasNextCh && nextCh == ch;
- }
-
- private bool lookahead(bool function(dchar) condition)
- {
- return hasNextCh && condition(nextCh);
- }
-
- private static bool isNewline(dchar ch)
- {
- return ch == '\n' || ch == '\r' || ch == lineSep || ch == paraSep;
- }
-
- /// Returns the length of the newline sequence, or zero if the current
- /// character is not a newline
- ///
- /// Note that there are only single character sequences and the two
- /// character sequence `\r\n` as used on Windows.
- private size_t isAtNewline()
- {
- if(ch == '\n' || ch == lineSep || ch == paraSep) return 1;
- else if(ch == '\r') return lookahead('\n') ? 2 : 1;
- else return 0;
- }
-
- /// Is 'ch' a valid base 64 character?
- private bool isBase64(dchar ch)
- {
- if(ch >= 'A' && ch <= 'Z')
- return true;
-
- if(ch >= 'a' && ch <= 'z')
- return true;
-
- if(ch >= '0' && ch <= '9')
- return true;
-
- return ch == '+' || ch == '/' || ch == '=';
- }
-
- /// Is the current character one that's allowed
- /// immediately *after* an int/float literal?
- private bool isEndOfNumber()
- {
- if(isEOF)
- return true;
-
- return !isDigit(ch) && ch != ':' && ch != '_' && !isAlpha(ch);
- }
-
- /// Is current character the last one in an ident?
- private bool isEndOfIdentCached = false;
- private bool _isEndOfIdent;
- private bool isEndOfIdent()
- {
- if(!isEndOfIdentCached)
- {
- if(!hasNextCh)
- _isEndOfIdent = true;
- else
- _isEndOfIdent = !isIdentChar(nextCh);
-
- isEndOfIdentCached = true;
- }
-
- return _isEndOfIdent;
- }
-
- /// Is 'ch' a character that's allowed *somewhere* in an identifier?
- private bool isIdentChar(dchar ch)
- {
- if(isAlpha(ch))
- return true;
-
- else if(isNumber(ch))
- return true;
-
- else
- return
- ch == '-' ||
- ch == '_' ||
- ch == '.' ||
- ch == '$';
- }
-
- private bool isDigit(dchar ch)
- {
- return ch >= '0' && ch <= '9';
- }
-
- private enum KeywordResult
- {
- Accept, // Keyword is matched
- Continue, // Keyword is not matched *yet*
- Failed, // Keyword doesn't match
- }
- private KeywordResult checkKeyword(dstring keyword32)
- {
- // Still within length of keyword
- if(tokenLength32 < keyword32.length)
- {
- if(ch == keyword32[tokenLength32])
- return KeywordResult.Continue;
- else
- return KeywordResult.Failed;
- }
-
- // At position after keyword
- else if(tokenLength32 == keyword32.length)
- {
- if(isEOF || !isIdentChar(ch))
- {
- debug assert(tokenData == to!string(keyword32));
- return KeywordResult.Accept;
- }
- else
- return KeywordResult.Failed;
- }
-
- assert(0, "Fell off end of keyword to check");
- }
-
- enum ErrorOnEOF { No, Yes }
-
- /// Advance one code point.
- private void advanceChar(ErrorOnEOF errorOnEOF)
- {
- if(auto cnt = isAtNewline())
- {
- if (cnt == 1)
- location.line++;
- location.col = 0;
- }
- else
- location.col++;
-
- location.index = nextPos;
-
- nextPos = posAfterLookahead;
- ch = nextCh;
-
- if(!hasNextCh)
- {
- if(errorOnEOF == ErrorOnEOF.Yes)
- error("Unexpected end of file");
-
- return;
- }
-
- tokenLength32++;
- tokenLength = location.index - tokenStart.index;
-
- if(nextPos == source.length)
- {
- nextCh = dchar.init;
- hasNextCh = false;
- return;
- }
-
- nextCh = source.decode(posAfterLookahead);
- isEndOfIdentCached = false;
- }
-
- /// Advances the specified amount of characters
- private void advanceChar(size_t count, ErrorOnEOF errorOnEOF)
- {
- while(count-- > 0)
- advanceChar(errorOnEOF);
- }
-
- void popFront()
- {
- // -- Main Lexer -------------
-
- eatWhite();
-
- if(isEOF)
- mixin(accept!"EOF");
-
- tokenStart = location;
- tokenLength = 0;
- tokenLength32 = 0;
- isEndOfIdentCached = false;
-
- if(lookaheadTokenInfo.exists)
- {
- tokenStart = lookaheadTokenInfo.tokenStart;
-
- auto prevLATokenInfo = lookaheadTokenInfo;
- lookaheadTokenInfo = LookaheadTokenInfo.init;
- lexNumeric(prevLATokenInfo);
- return;
- }
-
- if(ch == '=')
- {
- advanceChar(ErrorOnEOF.No);
- mixin(accept!"=");
- }
-
- else if(ch == '{')
- {
- advanceChar(ErrorOnEOF.No);
- mixin(accept!"{");
- }
-
- else if(ch == '}')
- {
- advanceChar(ErrorOnEOF.No);
- mixin(accept!"}");
- }
-
- else if(ch == ':')
- {
- advanceChar(ErrorOnEOF.No);
- mixin(accept!":");
- }
-
- else if(ch == ';')
- {
- advanceChar(ErrorOnEOF.No);
- mixin(accept!"EOL");
- }
-
- else if(auto cnt = isAtNewline())
- {
- advanceChar(cnt, ErrorOnEOF.No);
- mixin(accept!"EOL");
- }
-
- else if(isAlpha(ch) || ch == '_')
- lexIdentKeyword();
-
- else if(ch == '"')
- lexRegularString();
-
- else if(ch == '`')
- lexRawString();
-
- else if(ch == '\'')
- lexCharacter();
-
- else if(ch == '[')
- lexBinary();
-
- else if(ch == '-' || ch == '.' || isDigit(ch))
- lexNumeric();
-
- else
- {
- if(ch == ',')
- error("Unexpected comma: SDLang is not a comma-separated format.");
- else if(std.ascii.isPrintable(ch))
- error(text("Unexpected: ", ch));
- else
- error("Unexpected character code 0x%02X".format(ch));
-
- advanceChar(ErrorOnEOF.No);
- }
- }
-
- /// Lex Ident or Keyword
- private void lexIdentKeyword()
- {
- assert(isAlpha(ch) || ch == '_');
-
- // Keyword
- struct Key
- {
- dstring name;
- Value value;
- bool failed = false;
- }
- static Key[5] keywords;
- static keywordsInited = false;
- if(!keywordsInited)
- {
- // Value (as a std.variant-based type) can't be statically inited
- keywords[0] = Key("true", Value(true ));
- keywords[1] = Key("false", Value(false));
- keywords[2] = Key("on", Value(true ));
- keywords[3] = Key("off", Value(false));
- keywords[4] = Key("null", Value(null ));
- keywordsInited = true;
- }
-
- foreach(ref key; keywords)
- key.failed = false;
-
- auto numKeys = keywords.length;
-
- do
- {
- foreach(ref key; keywords)
- if(!key.failed)
- {
- final switch(checkKeyword(key.name))
- {
- case KeywordResult.Accept:
- mixin(accept!("Value", "key.value"));
-
- case KeywordResult.Continue:
- break;
-
- case KeywordResult.Failed:
- key.failed = true;
- numKeys--;
- break;
- }
- }
-
- if(numKeys == 0)
- {
- lexIdent();
- return;
- }
-
- advanceChar(ErrorOnEOF.No);
-
- } while(!isEOF);
-
- foreach(ref key; keywords)
- if(!key.failed)
- if(key.name.length == tokenLength32+1)
- mixin(accept!("Value", "key.value"));
-
- mixin(accept!"Ident");
- }
-
- /// Lex Ident
- private void lexIdent()
- {
- if(tokenLength == 0)
- assert(isAlpha(ch) || ch == '_');
-
- while(!isEOF && isIdentChar(ch))
- advanceChar(ErrorOnEOF.No);
-
- mixin(accept!"Ident");
- }
-
- /// Lex regular string
- private void lexRegularString()
- {
- assert(ch == '"');
-
- Appender!string buf;
- size_t spanStart = nextPos;
-
- // Doesn't include current character
- void updateBuf()
- {
- if(location.index == spanStart)
- return;
-
- buf.put( source[spanStart..location.index] );
- }
-
- advanceChar(ErrorOnEOF.Yes);
- while(ch != '"')
- {
- if(ch == '\\')
- {
- updateBuf();
-
- bool wasEscSequence = true;
- if(hasNextCh)
- {
- switch(nextCh)
- {
- case 'n': buf.put('\n'); break;
- case 'r': buf.put('\r'); break;
- case 't': buf.put('\t'); break;
- case '"': buf.put('\"'); break;
- case '\\': buf.put('\\'); break;
- default: wasEscSequence = false; break;
- }
- }
-
- if(wasEscSequence)
- {
- advanceChar(ErrorOnEOF.Yes);
- spanStart = nextPos;
- }
- else
- {
- eatWhite(false);
- spanStart = location.index;
- }
- }
-
- else if(isNewline(ch))
- error("Unescaped newlines are only allowed in raw strings, not regular strings.");
-
- advanceChar(ErrorOnEOF.Yes);
- }
-
- updateBuf();
- advanceChar(ErrorOnEOF.No); // Skip closing double-quote
- mixin(accept!("Value", "buf.data"));
- }
-
- /// Lex raw string
- private void lexRawString()
- {
- assert(ch == '`');
-
- do
- advanceChar(ErrorOnEOF.Yes);
- while(ch != '`');
-
- advanceChar(ErrorOnEOF.No); // Skip closing back-tick
- mixin(accept!("Value", "tokenData[1..$-1]"));
- }
-
- /// Lex character literal
- private void lexCharacter()
- {
- assert(ch == '\'');
- advanceChar(ErrorOnEOF.Yes); // Skip opening single-quote
-
- dchar value;
- if(ch == '\\')
- {
- advanceChar(ErrorOnEOF.Yes); // Skip escape backslash
- switch(ch)
- {
- case 'n': value = '\n'; break;
- case 'r': value = '\r'; break;
- case 't': value = '\t'; break;
- case '\'': value = '\''; break;
- case '\\': value = '\\'; break;
- default: error("Invalid escape sequence.");
- }
- }
- else if(isNewline(ch))
- error("Newline not alowed in character literal.");
- else
- value = ch;
- advanceChar(ErrorOnEOF.Yes); // Skip the character itself
-
- if(ch == '\'')
- advanceChar(ErrorOnEOF.No); // Skip closing single-quote
- else
- error("Expected closing single-quote.");
-
- mixin(accept!("Value", "value"));
- }
-
- /// Lex base64 binary literal
- private void lexBinary()
- {
- assert(ch == '[');
- advanceChar(ErrorOnEOF.Yes);
-
- void eatBase64Whitespace()
- {
- while(!isEOF && isWhite(ch))
- {
- if(isNewline(ch))
- advanceChar(ErrorOnEOF.Yes);
-
- if(!isEOF && isWhite(ch))
- eatWhite();
- }
- }
-
- eatBase64Whitespace();
-
- // Iterates all valid base64 characters, ending at ']'.
- // Skips all whitespace. Throws on invalid chars.
- struct Base64InputRange
- {
- Lexer lexer;
- private bool isInited = false;
- private int numInputCharsMod4 = 0;
-
- @property bool empty()
- {
- if(lexer.ch == ']')
- {
- if(numInputCharsMod4 != 0)
- lexer.error("Length of Base64 encoding must be a multiple of 4. ("~to!string(numInputCharsMod4)~")");
-
- return true;
- }
-
- return false;
- }
-
- @property dchar front()
- {
- return lexer.ch;
- }
-
- void popFront()
- {
- auto lex = lexer;
-
- if(!isInited)
- {
- if(lexer.isBase64(lexer.ch))
- {
- numInputCharsMod4++;
- numInputCharsMod4 %= 4;
- }
-
- isInited = true;
- }
-
- lex.advanceChar(lex.ErrorOnEOF.Yes);
-
- eatBase64Whitespace();
-
- if(lex.isEOF)
- lex.error("Unexpected end of file.");
-
- if(lex.ch != ']')
- {
- if(!lex.isBase64(lex.ch))
- lex.error("Invalid character in base64 binary literal.");
-
- numInputCharsMod4++;
- numInputCharsMod4 %= 4;
- }
- }
- }
-
- // This is a slow ugly hack. It's necessary because Base64.decode
- // currently requires the source to have known length.
- //TODO: Remove this when DMD issue #9543 is fixed.
- dchar[] tmpBuf = array(Base64InputRange(this));
-
- Appender!(ubyte[]) outputBuf;
- // Ugly workaround for DMD issue #9102
- //TODO: Remove this when DMD #9102 is fixed
- struct OutputBuf
- {
- void put(ubyte ch)
- {
- outputBuf.put(ch);
- }
- }
-
- try
- //Base64.decode(Base64InputRange(this), OutputBuf());
- Base64.decode(tmpBuf, OutputBuf());
-
- catch(Base64Exception e)
- error("Invalid character in base64 binary literal.");
-
- advanceChar(ErrorOnEOF.No); // Skip ']'
- mixin(accept!("Value", "outputBuf.data"));
- }
-
- private BigInt toBigInt(bool isNegative, string absValue)
- {
- auto num = BigInt(absValue);
- assert(num >= 0);
-
- if(isNegative)
- num = -num;
-
- return num;
- }
-
- /// Lex [0-9]+, but without emitting a token.
- /// This is used by the other numeric parsing functions.
- private string lexNumericFragment()
- {
- if(!isDigit(ch))
- error("Expected a digit 0-9.");
-
- auto spanStart = location.index;
-
- do
- {
- advanceChar(ErrorOnEOF.No);
- } while(!isEOF && isDigit(ch));
-
- return source[spanStart..location.index];
- }
-
- /// Lex anything that starts with 0-9 or '-'. Ints, floats, dates, etc.
- private void lexNumeric(LookaheadTokenInfo laTokenInfo = LookaheadTokenInfo.init)
- {
- bool isNegative;
- string firstFragment;
- if(laTokenInfo.exists)
- {
- firstFragment = laTokenInfo.numericFragment;
- isNegative = laTokenInfo.isNegative;
- }
- else
- {
- assert(ch == '-' || ch == '.' || isDigit(ch));
-
- // Check for negative
- isNegative = ch == '-';
- if(isNegative)
- advanceChar(ErrorOnEOF.Yes);
-
- // Some floating point with omitted leading zero?
- if(ch == '.')
- {
- lexFloatingPoint("");
- return;
- }
-
- firstFragment = lexNumericFragment();
- }
-
- // Long integer (64-bit signed)?
- if(ch == 'L' || ch == 'l')
- {
- advanceChar(ErrorOnEOF.No);
-
- // BigInt(long.min) is a workaround for DMD issue #9548
- auto num = toBigInt(isNegative, firstFragment);
- if(num < BigInt(long.min) || num > long.max)
- error(tokenStart, "Value doesn't fit in 64-bit signed long integer: "~to!string(num));
-
- mixin(accept!("Value", "num.toLong()"));
- }
-
- // Float (32-bit signed)?
- else if(ch == 'F' || ch == 'f')
- {
- auto value = to!float(tokenData);
- advanceChar(ErrorOnEOF.No);
- mixin(accept!("Value", "value"));
- }
-
- // Double float (64-bit signed) with suffix?
- else if((ch == 'D' || ch == 'd') && !lookahead(':')
- )
- {
- auto value = to!double(tokenData);
- advanceChar(ErrorOnEOF.No);
- mixin(accept!("Value", "value"));
- }
-
- // Decimal (128+ bits signed)?
- else if(
- (ch == 'B' || ch == 'b') &&
- (lookahead('D') || lookahead('d'))
- )
- {
- auto value = to!real(tokenData);
- advanceChar(ErrorOnEOF.No);
- advanceChar(ErrorOnEOF.No);
- mixin(accept!("Value", "value"));
- }
-
- // Some floating point?
- else if(ch == '.')
- lexFloatingPoint(firstFragment);
-
- // Some date?
- else if(ch == '/' && hasNextCh && isDigit(nextCh))
- lexDate(isNegative, firstFragment);
-
- // Some time span?
- else if(ch == ':' || ch == 'd')
- lexTimeSpan(isNegative, firstFragment);
-
- // Integer (32-bit signed)?
- else if(isEndOfNumber())
- {
- auto num = toBigInt(isNegative, firstFragment);
- if(num < int.min || num > int.max)
- error(tokenStart, "Value doesn't fit in 32-bit signed integer: "~to!string(num));
-
- mixin(accept!("Value", "num.toInt()"));
- }
-
- // Invalid suffix
- else
- error("Invalid integer suffix.");
- }
-
- /// Lex any floating-point literal (after the initial numeric fragment was lexed)
- private void lexFloatingPoint(string firstPart)
- {
- assert(ch == '.');
- advanceChar(ErrorOnEOF.No);
-
- auto secondPart = lexNumericFragment();
-
- try
- {
- // Double float (64-bit signed) with suffix?
- if(ch == 'D' || ch == 'd')
- {
- auto value = to!double(tokenData);
- advanceChar(ErrorOnEOF.No);
- mixin(accept!("Value", "value"));
- }
-
- // Float (32-bit signed)?
- else if(ch == 'F' || ch == 'f')
- {
- auto value = to!float(tokenData);
- advanceChar(ErrorOnEOF.No);
- mixin(accept!("Value", "value"));
- }
-
- // Decimal (128+ bits signed)?
- else if(ch == 'B' || ch == 'b')
- {
- auto value = to!real(tokenData);
- advanceChar(ErrorOnEOF.Yes);
-
- if(!isEOF && (ch == 'D' || ch == 'd'))
- {
- advanceChar(ErrorOnEOF.No);
- if(isEndOfNumber())
- mixin(accept!("Value", "value"));
- }
-
- error("Invalid floating point suffix.");
- }
-
- // Double float (64-bit signed) without suffix?
- else if(isEOF || !isIdentChar(ch))
- {
- auto value = to!double(tokenData);
- mixin(accept!("Value", "value"));
- }
-
- // Invalid suffix
- else
- error("Invalid floating point suffix.");
- }
- catch(ConvException e)
- error("Invalid floating point literal.");
- }
-
- private Date makeDate(bool isNegative, string yearStr, string monthStr, string dayStr)
- {
- BigInt biTmp;
-
- biTmp = BigInt(yearStr);
- if(isNegative)
- biTmp = -biTmp;
- if(biTmp < int.min || biTmp > int.max)
- error(tokenStart, "Date's year is out of range. (Must fit within a 32-bit signed int.)");
- auto year = biTmp.toInt();
-
- biTmp = BigInt(monthStr);
- if(biTmp < 1 || biTmp > 12)
- error(tokenStart, "Date's month is out of range.");
- auto month = biTmp.toInt();
-
- biTmp = BigInt(dayStr);
- if(biTmp < 1 || biTmp > 31)
- error(tokenStart, "Date's month is out of range.");
- auto day = biTmp.toInt();
-
- return Date(year, month, day);
- }
-
- private DateTimeFrac makeDateTimeFrac(
- bool isNegative, Date date, string hourStr, string minuteStr,
- string secondStr, string millisecondStr
- )
- {
- BigInt biTmp;
-
- biTmp = BigInt(hourStr);
- if(biTmp < int.min || biTmp > int.max)
- error(tokenStart, "Datetime's hour is out of range.");
- auto numHours = biTmp.toInt();
-
- biTmp = BigInt(minuteStr);
- if(biTmp < 0 || biTmp > int.max)
- error(tokenStart, "Datetime's minute is out of range.");
- auto numMinutes = biTmp.toInt();
-
- int numSeconds = 0;
- if(secondStr != "")
- {
- biTmp = BigInt(secondStr);
- if(biTmp < 0 || biTmp > int.max)
- error(tokenStart, "Datetime's second is out of range.");
- numSeconds = biTmp.toInt();
- }
-
- int millisecond = 0;
- if(millisecondStr != "")
- {
- biTmp = BigInt(millisecondStr);
- if(biTmp < 0 || biTmp > int.max)
- error(tokenStart, "Datetime's millisecond is out of range.");
- millisecond = biTmp.toInt();
-
- if(millisecondStr.length == 1)
- millisecond *= 100;
- else if(millisecondStr.length == 2)
- millisecond *= 10;
- }
-
- Duration fracSecs = millisecond.msecs;
-
- auto offset = hours(numHours) + minutes(numMinutes) + seconds(numSeconds);
-
- if(isNegative)
- {
- offset = -offset;
- fracSecs = -fracSecs;
- }
-
- return DateTimeFrac(DateTime(date) + offset, fracSecs);
- }
-
- private Duration makeDuration(
- bool isNegative, string dayStr,
- string hourStr, string minuteStr, string secondStr,
- string millisecondStr
- )
- {
- BigInt biTmp;
-
- long day = 0;
- if(dayStr != "")
- {
- biTmp = BigInt(dayStr);
- if(biTmp < long.min || biTmp > long.max)
- error(tokenStart, "Time span's day is out of range.");
- day = biTmp.toLong();
- }
-
- biTmp = BigInt(hourStr);
- if(biTmp < long.min || biTmp > long.max)
- error(tokenStart, "Time span's hour is out of range.");
- auto hour = biTmp.toLong();
-
- biTmp = BigInt(minuteStr);
- if(biTmp < long.min || biTmp > long.max)
- error(tokenStart, "Time span's minute is out of range.");
- auto minute = biTmp.toLong();
-
- biTmp = BigInt(secondStr);
- if(biTmp < long.min || biTmp > long.max)
- error(tokenStart, "Time span's second is out of range.");
- auto second = biTmp.toLong();
-
- long millisecond = 0;
- if(millisecondStr != "")
- {
- biTmp = BigInt(millisecondStr);
- if(biTmp < long.min || biTmp > long.max)
- error(tokenStart, "Time span's millisecond is out of range.");
- millisecond = biTmp.toLong();
-
- if(millisecondStr.length == 1)
- millisecond *= 100;
- else if(millisecondStr.length == 2)
- millisecond *= 10;
- }
-
- auto duration =
- dur!"days" (day) +
- dur!"hours" (hour) +
- dur!"minutes"(minute) +
- dur!"seconds"(second) +
- dur!"msecs" (millisecond);
-
- if(isNegative)
- duration = -duration;
-
- return duration;
- }
-
- // This has to reproduce some weird corner case behaviors from the
- // original Java version of SDL. So some of this may seem weird.
- private Nullable!Duration getTimeZoneOffset(string str)
- {
- if(str.length < 2)
- return Nullable!Duration(); // Unknown timezone
-
- if(str[0] != '+' && str[0] != '-')
- return Nullable!Duration(); // Unknown timezone
-
- auto isNegative = str[0] == '-';
-
- string numHoursStr;
- string numMinutesStr;
- if(str[1] == ':')
- {
- numMinutesStr = str[1..$];
- numHoursStr = "";
- }
- else
- {
- numMinutesStr = str.find(':');
- numHoursStr = str[1 .. $-numMinutesStr.length];
- }
-
- long numHours = 0;
- long numMinutes = 0;
- bool isUnknown = false;
- try
- {
- switch(numHoursStr.length)
- {
- case 0:
- if(numMinutesStr.length == 3)
- {
- numHours = 0;
- numMinutes = to!long(numMinutesStr[1..$]);
- }
- else
- isUnknown = true;
- break;
-
- case 1:
- case 2:
- if(numMinutesStr.length == 0)
- {
- numHours = to!long(numHoursStr);
- numMinutes = 0;
- }
- else if(numMinutesStr.length == 3)
- {
- numHours = to!long(numHoursStr);
- numMinutes = to!long(numMinutesStr[1..$]);
- }
- else
- isUnknown = true;
- break;
-
- default:
- if(numMinutesStr.length == 0)
- {
- // Yes, this is correct
- numHours = 0;
- numMinutes = to!long(numHoursStr[1..$]);
- }
- else
- isUnknown = true;
- break;
- }
- }
- catch(ConvException e)
- isUnknown = true;
-
- if(isUnknown)
- return Nullable!Duration(); // Unknown timezone
-
- auto timeZoneOffset = hours(numHours) + minutes(numMinutes);
- if(isNegative)
- timeZoneOffset = -timeZoneOffset;
-
- // Timezone valid
- return Nullable!Duration(timeZoneOffset);
- }
-
- /// Lex date or datetime (after the initial numeric fragment was lexed)
- private void lexDate(bool isDateNegative, string yearStr)
- {
- assert(ch == '/');
-
- // Lex months
- advanceChar(ErrorOnEOF.Yes); // Skip '/'
- auto monthStr = lexNumericFragment();
-
- // Lex days
- if(ch != '/')
- error("Invalid date format: Missing days.");
- advanceChar(ErrorOnEOF.Yes); // Skip '/'
- auto dayStr = lexNumericFragment();
-
- auto date = makeDate(isDateNegative, yearStr, monthStr, dayStr);
-
- if(!isEndOfNumber() && ch != '/')
- error("Dates cannot have suffixes.");
-
- // Date?
- if(isEOF)
- mixin(accept!("Value", "date"));
-
- auto endOfDate = location;
-
- while(
- !isEOF &&
- ( ch == '\\' || ch == '/' || (isWhite(ch) && !isNewline(ch)) )
- )
- {
- if(ch == '\\' && hasNextCh && isNewline(nextCh))
- {
- advanceChar(ErrorOnEOF.Yes);
- if(isAtNewline())
- advanceChar(ErrorOnEOF.Yes);
- advanceChar(ErrorOnEOF.No);
- }
-
- eatWhite();
- }
-
- // Date?
- if(isEOF || (!isDigit(ch) && ch != '-'))
- mixin(accept!("Value", "date", "", "endOfDate.index"));
-
- auto startOfTime = location;
-
- // Is time negative?
- bool isTimeNegative = ch == '-';
- if(isTimeNegative)
- advanceChar(ErrorOnEOF.Yes);
-
- // Lex hours
- auto hourStr = ch == '.'? "" : lexNumericFragment();
-
- // Lex minutes
- if(ch != ':')
- {
- // No minutes found. Therefore we had a plain Date followed
- // by a numeric literal, not a DateTime.
- lookaheadTokenInfo.exists = true;
- lookaheadTokenInfo.numericFragment = hourStr;
- lookaheadTokenInfo.isNegative = isTimeNegative;
- lookaheadTokenInfo.tokenStart = startOfTime;
- mixin(accept!("Value", "date", "", "endOfDate.index"));
- }
- advanceChar(ErrorOnEOF.Yes); // Skip ':'
- auto minuteStr = lexNumericFragment();
-
- // Lex seconds, if exists
- string secondStr;
- if(ch == ':')
- {
- advanceChar(ErrorOnEOF.Yes); // Skip ':'
- secondStr = lexNumericFragment();
- }
-
- // Lex milliseconds, if exists
- string millisecondStr;
- if(ch == '.')
- {
- advanceChar(ErrorOnEOF.Yes); // Skip '.'
- millisecondStr = lexNumericFragment();
- }
-
- auto dateTimeFrac = makeDateTimeFrac(isTimeNegative, date, hourStr, minuteStr, secondStr, millisecondStr);
-
- // Lex zone, if exists
- if(ch == '-')
- {
- advanceChar(ErrorOnEOF.Yes); // Skip '-'
- auto timezoneStart = location;
-
- if(!isAlpha(ch))
- error("Invalid timezone format.");
-
- while(!isEOF && !isWhite(ch))
- advanceChar(ErrorOnEOF.No);
-
- auto timezoneStr = source[timezoneStart.index..location.index];
- if(timezoneStr.startsWith("GMT"))
- {
- auto isoPart = timezoneStr["GMT".length..$];
- auto offset = getTimeZoneOffset(isoPart);
-
- if(offset.isNull())
- {
- // Unknown time zone
- mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
- }
- else
- {
- auto timezone = new immutable SimpleTimeZone(offset.get());
- mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
- }
- }
-
- try
- {
- auto timezone = TimeZone.getTimeZone(timezoneStr);
- if(timezone)
- mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
- }
- catch(TimeException e)
- {
- // Time zone not found. So just move along to "Unknown time zone" below.
- }
-
- // Unknown time zone
- mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
- }
-
- if(!isEndOfNumber())
- error("Date-Times cannot have suffixes.");
-
- mixin(accept!("Value", "dateTimeFrac"));
- }
-
- /// Lex time span (after the initial numeric fragment was lexed)
- private void lexTimeSpan(bool isNegative, string firstPart)
- {
- assert(ch == ':' || ch == 'd');
-
- string dayStr = "";
- string hourStr;
-
- // Lexed days?
- bool hasDays = ch == 'd';
- if(hasDays)
- {
- dayStr = firstPart;
- advanceChar(ErrorOnEOF.Yes); // Skip 'd'
-
- // Lex hours
- if(ch != ':')
- error("Invalid time span format: Missing hours.");
- advanceChar(ErrorOnEOF.Yes); // Skip ':'
- hourStr = lexNumericFragment();
- }
- else
- hourStr = firstPart;
-
- // Lex minutes
- if(ch != ':')
- error("Invalid time span format: Missing minutes.");
- advanceChar(ErrorOnEOF.Yes); // Skip ':'
- auto minuteStr = lexNumericFragment();
-
- // Lex seconds
- if(ch != ':')
- error("Invalid time span format: Missing seconds.");
- advanceChar(ErrorOnEOF.Yes); // Skip ':'
- auto secondStr = lexNumericFragment();
-
- // Lex milliseconds, if exists
- string millisecondStr = "";
- if(ch == '.')
- {
- advanceChar(ErrorOnEOF.Yes); // Skip '.'
- millisecondStr = lexNumericFragment();
- }
-
- if(!isEndOfNumber())
- error("Time spans cannot have suffixes.");
-
- auto duration = makeDuration(isNegative, dayStr, hourStr, minuteStr, secondStr, millisecondStr);
- mixin(accept!("Value", "duration"));
- }
-
- /// Advances past whitespace and comments
- private void eatWhite(bool allowComments=true)
- {
- // -- Comment/Whitepace Lexer -------------
-
- enum State
- {
- normal,
- lineComment, // Got "#" or "//" or "--", Eating everything until newline
- blockComment, // Got "/*", Eating everything until "*/"
- }
-
- if(isEOF)
- return;
-
- Location commentStart;
- State state = State.normal;
- bool consumeNewlines = false;
- bool hasConsumedNewline = false;
- while(true)
- {
- final switch(state)
- {
- case State.normal:
-
- if(ch == '\\')
- {
- commentStart = location;
- consumeNewlines = true;
- hasConsumedNewline = false;
- }
-
- else if(ch == '#')
- {
- if(!allowComments)
- return;
-
- commentStart = location;
- state = State.lineComment;
- continue;
- }
-
- else if(ch == '/' || ch == '-')
- {
- commentStart = location;
- if(lookahead(ch))
- {
- if(!allowComments)
- return;
-
- advanceChar(ErrorOnEOF.No);
- state = State.lineComment;
- continue;
- }
- else if(ch == '/' && lookahead('*'))
- {
- if(!allowComments)
- return;
-
- advanceChar(ErrorOnEOF.No);
- state = State.blockComment;
- continue;
- }
- else
- return; // Done
- }
- else if(isAtNewline())
- {
- if(consumeNewlines)
- hasConsumedNewline = true;
- else
- return; // Done
- }
- else if(!isWhite(ch))
- {
- if(consumeNewlines)
- {
- if(hasConsumedNewline)
- return; // Done
- else
- error("Only whitespace can come between a line-continuation backslash and the following newline.");
- }
- else
- return; // Done
- }
-
- break;
-
- case State.lineComment:
- if(lookahead(&isNewline))
- state = State.normal;
- break;
-
- case State.blockComment:
- if(ch == '*' && lookahead('/'))
- {
- advanceChar(ErrorOnEOF.No);
- state = State.normal;
- }
- break;
- }
-
- advanceChar(ErrorOnEOF.No);
- if(isEOF)
- {
- // Reached EOF
-
- if(consumeNewlines && !hasConsumedNewline)
- error("Missing newline after line-continuation backslash.");
-
- else if(state == State.blockComment)
- error(commentStart, "Unterminated block comment.");
-
- else
- return; // Done, reached EOF
- }
- }
- }
-}
-
-version(unittest)
-{
- import std.stdio;
-
- version(Have_unit_threaded) import unit_threaded;
- else { enum DontTest; }
-
- private auto loc = Location("filename", 0, 0, 0);
- private auto loc2 = Location("a", 1, 1, 1);
-
- @("lexer: EOL")
- unittest
- {
- assert([Token(symbol!"EOL",loc) ] == [Token(symbol!"EOL",loc) ] );
- assert([Token(symbol!"EOL",loc,Value(7),"A")] == [Token(symbol!"EOL",loc2,Value(7),"B")] );
- }
-
- private int numErrors = 0;
- @DontTest
- private void testLex(string source, Token[] expected, bool test_locations = false, string file=__FILE__, size_t line=__LINE__)
- {
- Token[] actual;
- try
- actual = lexSource(source, "filename");
- catch(ParseException e)
- {
- numErrors++;
- stderr.writeln(file, "(", line, "): testLex failed on: ", source);
- stderr.writeln(" Expected:");
- stderr.writeln(" ", expected);
- stderr.writeln(" Actual: ParseException thrown:");
- stderr.writeln(" ", e.msg);
- return;
- }
-
- bool is_same = actual == expected;
- if (is_same && test_locations) {
- is_same = actual.map!(t => t.location).equal(expected.map!(t => t.location));
- }
-
- if(!is_same)
- {
- numErrors++;
- stderr.writeln(file, "(", line, "): testLex failed on: ", source);
- stderr.writeln(" Expected:");
- stderr.writeln(" ", expected);
- stderr.writeln(" Actual:");
- stderr.writeln(" ", actual);
-
- if(expected.length > 1 || actual.length > 1)
- {
- stderr.writeln(" expected.length: ", expected.length);
- stderr.writeln(" actual.length: ", actual.length);
-
- if(actual.length == expected.length)
- foreach(i; 0..actual.length)
- if(actual[i] != expected[i])
- {
- stderr.writeln(" Unequal at index #", i, ":");
- stderr.writeln(" Expected:");
- stderr.writeln(" ", expected[i]);
- stderr.writeln(" Actual:");
- stderr.writeln(" ", actual[i]);
- }
- }
- }
- }
-
- private void testLexThrows(string file=__FILE__, size_t line=__LINE__)(string source)
- {
- bool hadException = false;
- Token[] actual;
- try
- actual = lexSource(source, "filename");
- catch(ParseException e)
- hadException = true;
-
- if(!hadException)
- {
- numErrors++;
- stderr.writeln(file, "(", line, "): testLex failed on: ", source);
- stderr.writeln(" Expected ParseException");
- stderr.writeln(" Actual:");
- stderr.writeln(" ", actual);
- }
- }
-}
-
-@("sdlang lexer")
-unittest
-{
- testLex("", []);
- testLex(" ", []);
- testLex("\\\n", []);
- testLex("/*foo*/", []);
- testLex("/* multiline \n comment */", []);
- testLex("/* * */", []);
- testLexThrows("/* ");
-
- testLex(":", [ Token(symbol!":", loc) ]);
- testLex("=", [ Token(symbol!"=", loc) ]);
- testLex("{", [ Token(symbol!"{", loc) ]);
- testLex("}", [ Token(symbol!"}", loc) ]);
- testLex(";", [ Token(symbol!"EOL",loc) ]);
- testLex("\n", [ Token(symbol!"EOL",loc) ]);
-
- testLex("foo", [ Token(symbol!"Ident",loc,Value(null),"foo") ]);
- testLex("_foo", [ Token(symbol!"Ident",loc,Value(null),"_foo") ]);
- testLex("foo.bar", [ Token(symbol!"Ident",loc,Value(null),"foo.bar") ]);
- testLex("foo-bar", [ Token(symbol!"Ident",loc,Value(null),"foo-bar") ]);
- testLex("foo.", [ Token(symbol!"Ident",loc,Value(null),"foo.") ]);
- testLex("foo-", [ Token(symbol!"Ident",loc,Value(null),"foo-") ]);
- testLexThrows(".foo");
-
- testLex("foo bar", [
- Token(symbol!"Ident",loc,Value(null),"foo"),
- Token(symbol!"Ident",loc,Value(null),"bar"),
- ]);
- testLex("foo \\ \n \n bar", [
- Token(symbol!"Ident",loc,Value(null),"foo"),
- Token(symbol!"Ident",loc,Value(null),"bar"),
- ]);
- testLex("foo \\ \n \\ \n bar", [
- Token(symbol!"Ident",loc,Value(null),"foo"),
- Token(symbol!"Ident",loc,Value(null),"bar"),
- ]);
- testLexThrows("foo \\ ");
- testLexThrows("foo \\ bar");
- testLexThrows("foo \\ \n \\ ");
- testLexThrows("foo \\ \n \\ bar");
-
- testLex("foo : = { } ; \n bar \n", [
- Token(symbol!"Ident",loc,Value(null),"foo"),
- Token(symbol!":",loc),
- Token(symbol!"=",loc),
- Token(symbol!"{",loc),
- Token(symbol!"}",loc),
- Token(symbol!"EOL",loc),
- Token(symbol!"EOL",loc),
- Token(symbol!"Ident",loc,Value(null),"bar"),
- Token(symbol!"EOL",loc),
- ]);
-
- testLexThrows("<");
- testLexThrows("*");
- testLexThrows(`\`);
-
- // Integers
- testLex( "7", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
- testLex( "-7", [ Token(symbol!"Value",loc,Value(cast( int)-7)) ]);
- testLex( "7L", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
- testLex( "7l", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
- testLex("-7L", [ Token(symbol!"Value",loc,Value(cast(long)-7)) ]);
- testLex( "0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
- testLex( "-0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
-
- testLex("7/**/", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
- testLex("7#", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
-
- testLex("7 A", [
- Token(symbol!"Value",loc,Value(cast(int)7)),
- Token(symbol!"Ident",loc,Value( null),"A"),
- ]);
- testLexThrows("7A");
- testLexThrows("-A");
- testLexThrows(`-""`);
-
- testLex("7;", [
- Token(symbol!"Value",loc,Value(cast(int)7)),
- Token(symbol!"EOL",loc),
- ]);
-
- // Floats
- testLex("1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
- testLex("1.2f" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
- testLex("1.2" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
- testLex("1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
- testLex("1.2d" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
- testLex("1.2BD", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
- testLex("1.2bd", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
- testLex("1.2Bd", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
- testLex("1.2bD", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]);
-
- testLex(".2F" , [ Token(symbol!"Value",loc,Value(cast( float)0.2)) ]);
- testLex(".2" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
- testLex(".2D" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
- testLex(".2BD", [ Token(symbol!"Value",loc,Value(cast( real)0.2)) ]);
-
- testLex("-1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-1.2)) ]);
- testLex("-1.2" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
- testLex("-1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
- testLex("-1.2BD", [ Token(symbol!"Value",loc,Value(cast( real)-1.2)) ]);
-
- testLex("-.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-0.2)) ]);
- testLex("-.2" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
- testLex("-.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
- testLex("-.2BD", [ Token(symbol!"Value",loc,Value(cast( real)-0.2)) ]);
-
- testLex( "0.0" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
- testLex( "0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
- testLex( "0.0BD", [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
- testLex("-0.0" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
- testLex("-0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
- testLex("-0.0BD", [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
- testLex( "7F" , [ Token(symbol!"Value",loc,Value(cast( float)7.0)) ]);
- testLex( "7D" , [ Token(symbol!"Value",loc,Value(cast(double)7.0)) ]);
- testLex( "7BD" , [ Token(symbol!"Value",loc,Value(cast( real)7.0)) ]);
- testLex( "0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
- testLex( "0D" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
- testLex( "0BD" , [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
- testLex("-0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
- testLex("-0D" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
- testLex("-0BD" , [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]);
-
- testLex("1.2 F", [
- Token(symbol!"Value",loc,Value(cast(double)1.2)),
- Token(symbol!"Ident",loc,Value( null),"F"),
- ]);
- testLexThrows("1.2A");
- testLexThrows("1.2B");
- testLexThrows("1.2BDF");
-
- testLex("1.2;", [
- Token(symbol!"Value",loc,Value(cast(double)1.2)),
- Token(symbol!"EOL",loc),
- ]);
-
- testLex("1.2F;", [
- Token(symbol!"Value",loc,Value(cast(float)1.2)),
- Token(symbol!"EOL",loc),
- ]);
-
- testLex("1.2BD;", [
- Token(symbol!"Value",loc,Value(cast(real)1.2)),
- Token(symbol!"EOL",loc),
- ]);
-
- // Booleans and null
- testLex("true", [ Token(symbol!"Value",loc,Value( true)) ]);
- testLex("false", [ Token(symbol!"Value",loc,Value(false)) ]);
- testLex("on", [ Token(symbol!"Value",loc,Value( true)) ]);
- testLex("off", [ Token(symbol!"Value",loc,Value(false)) ]);
- testLex("null", [ Token(symbol!"Value",loc,Value( null)) ]);
-
- testLex("TRUE", [ Token(symbol!"Ident",loc,Value(null),"TRUE") ]);
- testLex("true ", [ Token(symbol!"Value",loc,Value(true)) ]);
- testLex("true ", [ Token(symbol!"Value",loc,Value(true)) ]);
- testLex("tru", [ Token(symbol!"Ident",loc,Value(null),"tru") ]);
- testLex("truX", [ Token(symbol!"Ident",loc,Value(null),"truX") ]);
- testLex("trueX", [ Token(symbol!"Ident",loc,Value(null),"trueX") ]);
-
- // Raw Backtick Strings
- testLex("`hello world`", [ Token(symbol!"Value",loc,Value(`hello world` )) ]);
- testLex("` hello world `", [ Token(symbol!"Value",loc,Value(` hello world ` )) ]);
- testLex("`hello \\t world`", [ Token(symbol!"Value",loc,Value(`hello \t world`)) ]);
- testLex("`hello \\n world`", [ Token(symbol!"Value",loc,Value(`hello \n world`)) ]);
- testLex("`hello \n world`", [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
- testLex("`hello \r\n world`", [ Token(symbol!"Value",loc,Value("hello \r\n world")) ]);
- testLex("`hello \"world\"`", [ Token(symbol!"Value",loc,Value(`hello "world"` )) ]);
-
- testLexThrows("`foo");
- testLexThrows("`");
-
- // Double-Quote Strings
- testLex(`"hello world"`, [ Token(symbol!"Value",loc,Value("hello world" )) ]);
- testLex(`" hello world "`, [ Token(symbol!"Value",loc,Value(" hello world " )) ]);
- testLex(`"hello \t world"`, [ Token(symbol!"Value",loc,Value("hello \t world")) ]);
- testLex(`"hello \n world"`, [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
- testLex("\"hello \\\n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]);
- testLex("\"hello \\ \n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]);
- testLex("\"hello \\ \n\n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]);
- testLex(`"\"hello world\""`, [ Token(symbol!"Value",loc,Value(`"hello world"` )) ]);
- testLex(`""`, [ Token(symbol!"Value",loc,Value("" )) ]); // issue #34
-
- testLexThrows("\"hello \n world\"");
- testLexThrows(`"foo`);
- testLexThrows(`"`);
-
- // Characters
- testLex("'a'", [ Token(symbol!"Value",loc,Value(cast(dchar) 'a')) ]);
- testLex("'\\n'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\n')) ]);
- testLex("'\\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
- testLex("'\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
- testLex("'\\''", [ Token(symbol!"Value",loc,Value(cast(dchar)'\'')) ]);
- testLex(`'\\'`, [ Token(symbol!"Value",loc,Value(cast(dchar)'\\')) ]);
-
- testLexThrows("'a");
- testLexThrows("'aa'");
- testLexThrows("''");
- testLexThrows("'\\\n'");
- testLexThrows("'\n'");
- testLexThrows(`'\`);
- testLexThrows(`'\'`);
- testLexThrows("'");
-
- // Unicode
- testLex("日本語", [ Token(symbol!"Ident",loc,Value(null), "日本語") ]);
- testLex("`おはよう、日本。`", [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
- testLex(`"おはよう、日本。"`, [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
- testLex("'月'", [ Token(symbol!"Value",loc,Value("月"d.dup[0])) ]);
-
- // Base64 Binary
- testLex("[aGVsbG8gd29ybGQ=]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
- testLex("[ aGVsbG8gd29ybGQ= ]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
- testLex("[\n aGVsbG8g \n \n d29ybGQ= \n]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
-
- testLexThrows("[aGVsbG8gd29ybGQ]"); // Ie: Not multiple of 4
- testLexThrows("[ aGVsbG8gd29ybGQ ]");
-
- // Date
- testLex( "1999/12/5", [ Token(symbol!"Value",loc,Value(Date( 1999, 12, 5))) ]);
- testLex( "2013/2/22", [ Token(symbol!"Value",loc,Value(Date( 2013, 2, 22))) ]);
- testLex("-2013/2/22", [ Token(symbol!"Value",loc,Value(Date(-2013, 2, 22))) ]);
-
- testLexThrows("7/");
- testLexThrows("2013/2/22a");
- testLexThrows("2013/2/22f");
-
- testLex("1999/12/5\n", [
- Token(symbol!"Value",loc,Value(Date(1999, 12, 5))),
- Token(symbol!"EOL",loc),
- ]);
-
- // DateTime, no timezone
- testLex( "2013/2/22 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22 \t 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22/*foo*/07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22 /*foo*/ \\\n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22 /*foo*/ \\\n\n \n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22 /*foo*/ \\\n\\\n \\\n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22/*foo*/\\\n/*bar*/07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]);
- testLex("-2013/2/22 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 7, 53, 0)))) ]);
- testLex( "2013/2/22 -07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53)))) ]);
- testLex("-2013/2/22 -07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53)))) ]);
- testLex( "2013/2/22 07:53:34", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34)))) ]);
- testLex( "2013/2/22 07:53:34.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs))) ]);
- testLex( "2013/2/22 07:53:34.12", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 120.msecs))) ]);
- testLex( "2013/2/22 07:53:34.1", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 100.msecs))) ]);
- testLex( "2013/2/22 07:53.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs))) ]);
-
- testLex( "2013/2/22 34:65", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0)))) ]);
- testLex( "2013/2/22 34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds(77), 123.msecs))) ]);
- testLex( "2013/2/22 34:65.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0), 123.msecs))) ]);
-
- testLex( "2013/2/22 -34:65", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0)))) ]);
- testLex( "2013/2/22 -34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds(77), -123.msecs))) ]);
- testLex( "2013/2/22 -34:65.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), -123.msecs))) ]);
-
- testLexThrows("2013/2/22 07:53a");
- testLexThrows("2013/2/22 07:53f");
- testLexThrows("2013/2/22 07:53:34.123a");
- testLexThrows("2013/2/22 07:53:34.123f");
- testLexThrows("2013/2/22a 07:53");
-
- testLex(`2013/2/22 "foo"`, [
- Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
- Token(symbol!"Value",loc,Value("foo")),
- ]);
-
- testLex("2013/2/22 07", [
- Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
- Token(symbol!"Value",loc,Value(cast(int)7)),
- ]);
-
- testLex("2013/2/22 1.2F", [
- Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
- Token(symbol!"Value",loc,Value(cast(float)1.2)),
- ]);
-
- testLex("2013/2/22 .2F", [
- Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
- Token(symbol!"Value",loc,Value(cast(float)0.2)),
- ]);
-
- testLex("2013/2/22 -1.2F", [
- Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
- Token(symbol!"Value",loc,Value(cast(float)-1.2)),
- ]);
-
- testLex("2013/2/22 -.2F", [
- Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
- Token(symbol!"Value",loc,Value(cast(float)-0.2)),
- ]);
-
- // DateTime, with known timezone
- testLex( "2013/2/22 07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex("-2013/2/22 07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex( "2013/2/22 -07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex("-2013/2/22 -07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex( "2013/2/22 07:53-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
- testLex( "2013/2/22 07:53-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
- testLex( "2013/2/22 07:53:34-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex( "2013/2/22 07:53:34-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
- testLex( "2013/2/22 07:53:34-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
- testLex( "2013/2/22 07:53:34.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex( "2013/2/22 07:53:34.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
- testLex( "2013/2/22 07:53:34.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
- testLex( "2013/2/22 07:53.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone( hours(0) )))) ]);
- testLex( "2013/2/22 07:53.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
- testLex( "2013/2/22 07:53.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
-
- testLex( "2013/2/22 -34:65-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
-
- // DateTime, with Java SDLang's occasionally weird interpretation of some
- // "not quite ISO" variations of the "GMT with offset" timezone strings.
- Token testTokenSimpleTimeZone(Duration d)
- {
- auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
- auto tz = new immutable SimpleTimeZone(d);
- return Token( symbol!"Value", loc, Value(SysTime(dateTime,tz)) );
- }
- Token testTokenUnknownTimeZone(string tzName)
- {
- auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
- auto frac = 0.msecs;
- return Token( symbol!"Value", loc, Value(DateTimeFracUnknownZone(dateTime,frac,tzName)) );
- }
- testLex("2013/2/22 07:53-GMT+", [ testTokenUnknownTimeZone("GMT+") ]);
- testLex("2013/2/22 07:53-GMT+:", [ testTokenUnknownTimeZone("GMT+:") ]);
- testLex("2013/2/22 07:53-GMT+:3", [ testTokenUnknownTimeZone("GMT+:3") ]);
- testLex("2013/2/22 07:53-GMT+:03", [ testTokenSimpleTimeZone(minutes(3)) ]);
- testLex("2013/2/22 07:53-GMT+:003", [ testTokenUnknownTimeZone("GMT+:003") ]);
-
- testLex("2013/2/22 07:53-GMT+4", [ testTokenSimpleTimeZone(hours(4)) ]);
- testLex("2013/2/22 07:53-GMT+4:", [ testTokenUnknownTimeZone("GMT+4:") ]);
- testLex("2013/2/22 07:53-GMT+4:3", [ testTokenUnknownTimeZone("GMT+4:3") ]);
- testLex("2013/2/22 07:53-GMT+4:03", [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
- testLex("2013/2/22 07:53-GMT+4:003", [ testTokenUnknownTimeZone("GMT+4:003") ]);
-
- testLex("2013/2/22 07:53-GMT+04", [ testTokenSimpleTimeZone(hours(4)) ]);
- testLex("2013/2/22 07:53-GMT+04:", [ testTokenUnknownTimeZone("GMT+04:") ]);
- testLex("2013/2/22 07:53-GMT+04:3", [ testTokenUnknownTimeZone("GMT+04:3") ]);
- testLex("2013/2/22 07:53-GMT+04:03", [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
- testLex("2013/2/22 07:53-GMT+04:03abc", [ testTokenUnknownTimeZone("GMT+04:03abc") ]);
- testLex("2013/2/22 07:53-GMT+04:003", [ testTokenUnknownTimeZone("GMT+04:003") ]);
-
- testLex("2013/2/22 07:53-GMT+004", [ testTokenSimpleTimeZone(minutes(4)) ]);
- testLex("2013/2/22 07:53-GMT+004:", [ testTokenUnknownTimeZone("GMT+004:") ]);
- testLex("2013/2/22 07:53-GMT+004:3", [ testTokenUnknownTimeZone("GMT+004:3") ]);
- testLex("2013/2/22 07:53-GMT+004:03", [ testTokenUnknownTimeZone("GMT+004:03") ]);
- testLex("2013/2/22 07:53-GMT+004:003", [ testTokenUnknownTimeZone("GMT+004:003") ]);
-
- testLex("2013/2/22 07:53-GMT+0004", [ testTokenSimpleTimeZone(minutes(4)) ]);
- testLex("2013/2/22 07:53-GMT+0004:", [ testTokenUnknownTimeZone("GMT+0004:") ]);
- testLex("2013/2/22 07:53-GMT+0004:3", [ testTokenUnknownTimeZone("GMT+0004:3") ]);
- testLex("2013/2/22 07:53-GMT+0004:03", [ testTokenUnknownTimeZone("GMT+0004:03") ]);
- testLex("2013/2/22 07:53-GMT+0004:003", [ testTokenUnknownTimeZone("GMT+0004:003") ]);
-
- testLex("2013/2/22 07:53-GMT+00004", [ testTokenSimpleTimeZone(minutes(4)) ]);
- testLex("2013/2/22 07:53-GMT+00004:", [ testTokenUnknownTimeZone("GMT+00004:") ]);
- testLex("2013/2/22 07:53-GMT+00004:3", [ testTokenUnknownTimeZone("GMT+00004:3") ]);
- testLex("2013/2/22 07:53-GMT+00004:03", [ testTokenUnknownTimeZone("GMT+00004:03") ]);
- testLex("2013/2/22 07:53-GMT+00004:003", [ testTokenUnknownTimeZone("GMT+00004:003") ]);
-
- // DateTime, with unknown timezone
- testLex( "2013/2/22 07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 0), 0.msecs, "Bogus/Foo")), "2013/2/22 07:53-Bogus/Foo") ]);
- testLex("-2013/2/22 07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 7, 53, 0), 0.msecs, "Bogus/Foo"))) ]);
- testLex( "2013/2/22 -07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
- testLex("-2013/2/22 -07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
- testLex( "2013/2/22 07:53:34-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 0.msecs, "Bogus/Foo"))) ]);
- testLex( "2013/2/22 07:53:34.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, "Bogus/Foo"))) ]);
- testLex( "2013/2/22 07:53.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, "Bogus/Foo"))) ]);
-
- // Time Span
- testLex( "12:14:42", [ Token(symbol!"Value",loc,Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs( 0))) ]);
- testLex("-12:14:42", [ Token(symbol!"Value",loc,Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs( 0))) ]);
- testLex( "00:09:12", [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs( 0))) ]);
- testLex( "00:00:01.023", [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23))) ]);
- testLex( "23d:05:21:23.532", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532))) ]);
- testLex( "23d:05:21:23.53", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530))) ]);
- testLex( "23d:05:21:23.5", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500))) ]);
- testLex("-23d:05:21:23.532", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532))) ]);
- testLex("-23d:05:21:23.5", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500))) ]);
- testLex( "23d:05:21:23", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs( 0))) ]);
-
- testLexThrows("12:14:42a");
- testLexThrows("23d:05:21:23.532a");
- testLexThrows("23d:05:21:23.532f");
-
- // Combination
- testLex("foo. 7", [
- Token(symbol!"Ident",loc,Value( null),"foo."),
- Token(symbol!"Value",loc,Value(cast(int)7))
- ]);
-
- testLex(`
- namespace:person "foo" "bar" 1 23L name.first="ひとみ" name.last="Smith" {
- namespace:age 37; namespace:favorite_color "blue" // comment
- somedate 2013/2/22 07:53 -- comment
-
- inventory /* comment */ {
- socks
- }
- }
- `,
- [
- Token(symbol!"EOL",loc,Value(null),"\n"),
-
- Token(symbol!"Ident", loc, Value( null ), "namespace"),
- Token(symbol!":", loc, Value( null ), ":"),
- Token(symbol!"Ident", loc, Value( null ), "person"),
- Token(symbol!"Value", loc, Value( "foo" ), `"foo"`),
- Token(symbol!"Value", loc, Value( "bar" ), `"bar"`),
- Token(symbol!"Value", loc, Value( cast( int) 1 ), "1"),
- Token(symbol!"Value", loc, Value( cast(long)23 ), "23L"),
- Token(symbol!"Ident", loc, Value( null ), "name.first"),
- Token(symbol!"=", loc, Value( null ), "="),
- Token(symbol!"Value", loc, Value( "ひとみ" ), `"ひとみ"`),
- Token(symbol!"Ident", loc, Value( null ), "name.last"),
- Token(symbol!"=", loc, Value( null ), "="),
- Token(symbol!"Value", loc, Value( "Smith" ), `"Smith"`),
- Token(symbol!"{", loc, Value( null ), "{"),
- Token(symbol!"EOL", loc, Value( null ), "\n"),
-
- Token(symbol!"Ident", loc, Value( null ), "namespace"),
- Token(symbol!":", loc, Value( null ), ":"),
- Token(symbol!"Ident", loc, Value( null ), "age"),
- Token(symbol!"Value", loc, Value( cast(int)37 ), "37"),
- Token(symbol!"EOL", loc, Value( null ), ";"),
- Token(symbol!"Ident", loc, Value( null ), "namespace"),
- Token(symbol!":", loc, Value( null ), ":"),
- Token(symbol!"Ident", loc, Value( null ), "favorite_color"),
- Token(symbol!"Value", loc, Value( "blue" ), `"blue"`),
- Token(symbol!"EOL", loc, Value( null ), "\n"),
-
- Token(symbol!"Ident", loc, Value( null ), "somedate"),
- Token(symbol!"Value", loc, Value( DateTimeFrac(DateTime(2013, 2, 22, 7, 53, 0)) ), "2013/2/22 07:53"),
- Token(symbol!"EOL", loc, Value( null ), "\n"),
- Token(symbol!"EOL", loc, Value( null ), "\n"),
-
- Token(symbol!"Ident", loc, Value(null), "inventory"),
- Token(symbol!"{", loc, Value(null), "{"),
- Token(symbol!"EOL", loc, Value(null), "\n"),
-
- Token(symbol!"Ident", loc, Value(null), "socks"),
- Token(symbol!"EOL", loc, Value(null), "\n"),
-
- Token(symbol!"}", loc, Value(null), "}"),
- Token(symbol!"EOL", loc, Value(null), "\n"),
-
- Token(symbol!"}", loc, Value(null), "}"),
- Token(symbol!"EOL", loc, Value(null), "\n"),
- ]);
-
- if(numErrors > 0)
- stderr.writeln(numErrors, " failed test(s)");
-}
-
-@("lexer: Regression test issue #8")
-unittest
-{
- testLex(`"\n \n"`, [ Token(symbol!"Value",loc,Value("\n \n"),`"\n \n"`) ]);
- testLex(`"\t\t"`, [ Token(symbol!"Value",loc,Value("\t\t"),`"\t\t"`) ]);
- testLex(`"\n\n"`, [ Token(symbol!"Value",loc,Value("\n\n"),`"\n\n"`) ]);
-}
-
-@("lexer: Regression test issue #11")
-unittest
-{
- void test(string input)
- {
- testLex(
- input,
- [
- Token(symbol!"EOL", loc, Value(null), "\n"),
- Token(symbol!"Ident",loc,Value(null), "a")
- ]
- );
- }
-
- test("//X\na");
- test("//\na");
- test("--\na");
- test("#\na");
-}
-
-@("ast: Regression test issue #28")
-unittest
-{
- enum offset = 1; // workaround for an of-by-one error for line numbers
- testLex("test", [
- Token(symbol!"Ident", Location("filename", 0, 0, 0), Value(null), "test")
- ], true);
- testLex("\ntest", [
- Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\n"),
- Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
- ], true);
- testLex("\rtest", [
- Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
- Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
- ], true);
- testLex("\r\ntest", [
- Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
- Token(symbol!"Ident", Location("filename", 1, 0, 2), Value(null), "test")
- ], true);
- testLex("\r\n\ntest", [
- Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
- Token(symbol!"EOL", Location("filename", 1, 0, 2), Value(null), "\n"),
- Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
- ], true);
- testLex("\r\r\ntest", [
- Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
- Token(symbol!"EOL", Location("filename", 1, 0, 1), Value(null), "\r\n"),
- Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
- ], true);
-}