step5 sdlang used for config files and doc headers

1 files changed, 2068 insertions, 0 deletions

diff --git a/src/sdlang/lexer.d b/src/sdlang/lexer.d
new file mode 100644
index 0000000..6eeeac2
--- /dev/null
+++ b/src/sdlang/lexer.d
@@ -0,0 +1,2068 @@
+// SDLang-D
+// Written in the D programming language.
+
+module sdlang.lexer;
+
+import std.algorithm;
+import std.array;
+import std.base64;
+import std.bigint;
+import std.conv;
+import std.datetime;
+import std.file;
+import std.stream : ByteOrderMarks, BOM;
+import std.traits;
+import std.typecons;
+import std.uni;
+import std.utf;
+import std.variant;
+
+import sdlang.exception;
+import sdlang.symbol;
+import sdlang.token;
+import sdlang.util;
+
+alias sdlang.util.startsWith startsWith;
+
+Token[] lexFile(string filename)
+{
+	auto source = cast(string)read(filename);
+	return lexSource(source, filename);
+}
+
+Token[] lexSource(string source, string filename=null)
+{
+	auto lexer = scoped!Lexer(source, filename);
+	
+	// Can't use 'std.array.array(Range)' because 'lexer' is scoped
+	// and therefore cannot have its reference copied.
+	Appender!(Token[]) tokens;
+	foreach(tok; lexer)
+		tokens.put(tok);
+
+	return tokens.data;
+}
+
+// Kind of a poor-man's yield, but fast.
+// Only to be used inside Lexer.popFront (and Lexer.this).
+private template accept(string symbolName)
+{
+	static assert(symbolName != "Value", "Value symbols must also take a value.");
+	enum accept = acceptImpl!(symbolName, "null");
+}
+private template accept(string symbolName, string value)
+{
+	static assert(symbolName == "Value", "Only a Value symbol can take a value.");
+	enum accept = acceptImpl!(symbolName, value);
+}
+private template accept(string symbolName, string value, string startLocation, string endLocation)
+{
+	static assert(symbolName == "Value", "Only a Value symbol can take a value.");
+	enum accept = ("
+		{
+			_front = makeToken!"~symbolName.stringof~";
+			_front.value = "~value~";
+			_front.location = "~(startLocation==""? "tokenStart" : startLocation)~";
+			_front.data = source[
+				"~(startLocation==""? "tokenStart.index" : startLocation)~"
+				..
+				"~(endLocation==""? "location.index" : endLocation)~"
+			];
+			return;
+		}
+	").replace("\n", "");
+}
+private template acceptImpl(string symbolName, string value)
+{
+	enum acceptImpl = ("
+		{
+			_front = makeToken!"~symbolName.stringof~";
+			_front.value = "~value~";
+			return;
+		}
+	").replace("\n", "");
+}
+
+class Lexer
+{
+	string source;
+	string filename;
+	Location location; /// Location of current character in source
+
+	private dchar  ch;         // Current character
+	private dchar  nextCh;     // Lookahead character
+	private size_t nextPos;    // Position of lookahead character (an index into source)
+	private bool   hasNextCh;  // If false, then there's no more lookahead, just EOF
+	private size_t posAfterLookahead; // Position after lookahead character (an index into source)
+
+	private Location tokenStart;    // The starting location of the token being lexed
+	
+	// Length so far of the token being lexed, not including current char
+	private size_t tokenLength;   // Length in UTF-8 code units
+	private size_t tokenLength32; // Length in UTF-32 code units
+	
+	// Slight kludge:
+	// If a numeric fragment is found after a Date (separated by arbitrary
+	// whitespace), it could be the "hours" part of a DateTime, or it could
+	// be a separate numeric literal that simply follows a plain Date. If the
+	// latter, then the Date must be emitted, but numeric fragment that was
+	// found after it needs to be saved for the the lexer's next iteration.
+	// 
+	// It's a slight kludge, and could instead be implemented as a slightly
+	// kludgey parser hack, but it's the only situation where SDL's lexing
+	// needs to lookahead more than one character, so this is good enough.
+	private struct LookaheadTokenInfo
+	{
+		bool     exists          = false;
+		string   numericFragment = "";
+		bool     isNegative      = false;
+		Location tokenStart;
+	}
+	private LookaheadTokenInfo lookaheadTokenInfo;
+	
+	this(string source=null, string filename=null)
+	{
+		this.filename = filename;
+		this.source = source;
+		
+		_front = Token(symbol!"Error", Location());
+		lookaheadTokenInfo = LookaheadTokenInfo.init;
+
+		if( source.startsWith( ByteOrderMarks[BOM.UTF8] ) )
+		{
+			source = source[ ByteOrderMarks[BOM.UTF8].length .. $ ];
+			this.source = source;
+		}
+		
+		foreach(bom; ByteOrderMarks)
+		if( source.startsWith(bom) )
+			error(Location(filename,0,0,0), "SDL spec only supports UTF-8, not UTF-16 or UTF-32");
+		
+		if(source == "")
+			mixin(accept!"EOF");
+		
+		// Prime everything
+		hasNextCh = true;
+		nextCh = source.decode(posAfterLookahead);
+		advanceChar(ErrorOnEOF.Yes);
+		location = Location(filename, 0, 0, 0);
+		popFront();
+	}
+	
+	@property bool empty()
+	{
+		return _front.symbol == symbol!"EOF";
+	}
+	
+	Token _front;
+	@property Token front()
+	{
+		return _front;
+	}
+
+	@property bool isEOF()
+	{
+		return location.index == source.length && !lookaheadTokenInfo.exists;
+	}
+
+	private void error(string msg)
+	{
+		error(location, msg);
+	}
+
+	private void error(Location loc, string msg)
+	{
+		throw new SDLangParseException(loc, "Error: "~msg);
+	}
+
+	private Token makeToken(string symbolName)()
+	{
+		auto tok = Token(symbol!symbolName, tokenStart);
+		tok.data = tokenData;
+		return tok;
+	}
+	
+	private @property string tokenData()
+	{
+		return source[ tokenStart.index .. location.index ];
+	}
+	
+	/// Check the lookahead character
+	private bool lookahead(dchar ch)
+	{
+		return hasNextCh && nextCh == ch;
+	}
+
+	private bool lookahead(bool function(dchar) condition)
+	{
+		return hasNextCh && condition(nextCh);
+	}
+
+	private static bool isNewline(dchar ch)
+	{
+		return ch == '\n' || ch == '\r' || ch == lineSep || ch == paraSep;
+	}
+
+	/// Returns the length of the newline sequence, or zero if the current
+	/// character is not a newline
+	///
+	/// Note that there are only single character sequences and the two
+	/// character sequence `\r\n` as used on Windows.
+	private size_t isAtNewline()
+	{
+		if(ch == '\n' || ch == lineSep || ch == paraSep) return 1;
+		else if(ch == '\r') return lookahead('\n') ? 2 : 1;
+		else return 0;
+	}
+
+	/// Is 'ch' a valid base 64 character?
+	private bool isBase64(dchar ch)
+	{
+		if(ch >= 'A' && ch <= 'Z')
+			return true;
+
+		if(ch >= 'a' && ch <= 'z')
+			return true;
+
+		if(ch >= '0' && ch <= '9')
+			return true;
+		
+		return ch == '+' || ch == '/' || ch == '=';
+	}
+	
+	/// Is the current character one that's allowed
+	/// immediately *after* an int/float literal?
+	private bool isEndOfNumber()
+	{
+		if(isEOF)
+			return true;
+		
+		return !isDigit(ch) && ch != ':' && ch != '_' && !isAlpha(ch);
+	}
+	
+	/// Is current character the last one in an ident?
+	private bool isEndOfIdentCached = false;
+	private bool _isEndOfIdent;
+	private bool isEndOfIdent()
+	{
+		if(!isEndOfIdentCached)
+		{
+			if(!hasNextCh)
+				_isEndOfIdent = true;
+			else
+				_isEndOfIdent = !isIdentChar(nextCh);
+			
+			isEndOfIdentCached = true;
+		}
+		
+		return _isEndOfIdent;
+	}
+
+	/// Is 'ch' a character that's allowed *somewhere* in an identifier?
+	private bool isIdentChar(dchar ch)
+	{
+		if(isAlpha(ch))
+			return true;
+		
+		else if(isNumber(ch))
+			return true;
+		
+		else
+			return 
+				ch == '-' ||
+				ch == '_' ||
+				ch == '.' ||
+				ch == '$';
+	}
+
+	private bool isDigit(dchar ch)
+	{
+		return ch >= '0' && ch <= '9';
+	}
+	
+	private enum KeywordResult
+	{
+		Accept,   // Keyword is matched
+		Continue, // Keyword is not matched *yet*
+		Failed,   // Keyword doesn't match
+	}
+	private KeywordResult checkKeyword(dstring keyword32)
+	{
+		// Still within length of keyword
+		if(tokenLength32 < keyword32.length)
+		{
+			if(ch == keyword32[tokenLength32])
+				return KeywordResult.Continue;
+			else
+				return KeywordResult.Failed;
+		}
+
+		// At position after keyword
+		else if(tokenLength32 == keyword32.length)
+		{
+			if(isEOF || !isIdentChar(ch))
+			{
+				debug assert(tokenData == to!string(keyword32));
+				return KeywordResult.Accept;
+			}
+			else
+				return KeywordResult.Failed;
+		}
+
+		assert(0, "Fell off end of keyword to check");
+	}
+
+	enum ErrorOnEOF { No, Yes }
+
+	/// Advance one code point.
+	private void advanceChar(ErrorOnEOF errorOnEOF)
+	{
+		if(auto cnt = isAtNewline())
+		{
+			if (cnt == 1)
+				location.line++;
+			location.col = 0;
+		}
+		else
+			location.col++;
+
+		location.index = nextPos;
+
+		nextPos = posAfterLookahead;
+		ch      = nextCh;
+
+		if(!hasNextCh)
+		{
+			if(errorOnEOF == ErrorOnEOF.Yes)
+				error("Unexpected end of file");
+
+			return;
+		}
+
+		tokenLength32++;
+		tokenLength = location.index - tokenStart.index;
+
+		if(nextPos == source.length)
+		{
+			nextCh = dchar.init;
+			hasNextCh = false;
+			return;
+		}
+		
+		nextCh = source.decode(posAfterLookahead);
+		isEndOfIdentCached = false;
+	}
+
+	/// Advances the specified amount of characters
+	private void advanceChar(size_t count, ErrorOnEOF errorOnEOF)
+	{
+		while(count-- > 0)
+			advanceChar(errorOnEOF);
+	}
+
+	void popFront()
+	{
+		// -- Main Lexer -------------
+
+		eatWhite();
+
+		if(isEOF)
+			mixin(accept!"EOF");
+		
+		tokenStart    = location;
+		tokenLength   = 0;
+		tokenLength32 = 0;
+		isEndOfIdentCached = false;
+		
+		if(lookaheadTokenInfo.exists)
+		{
+			tokenStart = lookaheadTokenInfo.tokenStart;
+
+			auto prevLATokenInfo = lookaheadTokenInfo;
+			lookaheadTokenInfo = LookaheadTokenInfo.init;
+			lexNumeric(prevLATokenInfo);
+			return;
+		}
+		
+		if(ch == '=')
+		{
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!"=");
+		}
+		
+		else if(ch == '{')
+		{
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!"{");
+		}
+		
+		else if(ch == '}')
+		{
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!"}");
+		}
+		
+		else if(ch == ':')
+		{
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!":");
+		}
+		
+		else if(ch == ';')
+		{
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!"EOL");
+		}
+
+		else if(auto cnt = isAtNewline())
+		{
+			advanceChar(cnt, ErrorOnEOF.No);
+			mixin(accept!"EOL");
+		}
+		
+		else if(isAlpha(ch) || ch == '_')
+			lexIdentKeyword();
+
+		else if(ch == '"')
+			lexRegularString();
+
+		else if(ch == '`')
+			lexRawString();
+		
+		else if(ch == '\'')
+			lexCharacter();
+
+		else if(ch == '[')
+			lexBinary();
+
+		else if(ch == '-' || ch == '.' || isDigit(ch))
+			lexNumeric();
+
+		else
+		{
+			advanceChar(ErrorOnEOF.No);
+			error("Syntax error");
+		}
+	}
+
+	/// Lex Ident or Keyword
+	private void lexIdentKeyword()
+	{
+		assert(isAlpha(ch) || ch == '_');
+		
+		// Keyword
+		struct Key
+		{
+			dstring name;
+			Value value;
+			bool failed = false;
+		}
+		static Key[5] keywords;
+		static keywordsInited = false;
+		if(!keywordsInited)
+		{
+			// Value (as a std.variant-based type) can't be statically inited
+			keywords[0] = Key("true",  Value(true ));
+			keywords[1] = Key("false", Value(false));
+			keywords[2] = Key("on",    Value(true ));
+			keywords[3] = Key("off",   Value(false));
+			keywords[4] = Key("null",  Value(null ));
+			keywordsInited = true;
+		}
+		
+		foreach(ref key; keywords)
+			key.failed = false;
+		
+		auto numKeys = keywords.length;
+
+		do
+		{
+			foreach(ref key; keywords)
+			if(!key.failed)
+			{
+				final switch(checkKeyword(key.name))
+				{
+				case KeywordResult.Accept:
+					mixin(accept!("Value", "key.value"));
+				
+				case KeywordResult.Continue:
+					break;
+				
+				case KeywordResult.Failed:
+					key.failed = true;
+					numKeys--;
+					break;
+				}
+			}
+
+			if(numKeys == 0)
+			{
+				lexIdent();
+				return;
+			}
+
+			advanceChar(ErrorOnEOF.No);
+
+		} while(!isEOF);
+
+		foreach(ref key; keywords)
+		if(!key.failed)
+		if(key.name.length == tokenLength32+1)
+			mixin(accept!("Value", "key.value"));
+
+		mixin(accept!"Ident");
+	}
+
+	/// Lex Ident
+	private void lexIdent()
+	{
+		if(tokenLength == 0)
+			assert(isAlpha(ch) || ch == '_');
+		
+		while(!isEOF && isIdentChar(ch))
+			advanceChar(ErrorOnEOF.No);
+
+		mixin(accept!"Ident");
+	}
+	
+	/// Lex regular string
+	private void lexRegularString()
+	{
+		assert(ch == '"');
+
+		Appender!string buf;
+		size_t spanStart = nextPos;
+		
+		// Doesn't include current character
+		void updateBuf()
+		{
+			if(location.index == spanStart)
+				return;
+
+			buf.put( source[spanStart..location.index] );
+		}
+		
+		advanceChar(ErrorOnEOF.Yes);
+		while(ch != '"')
+		{
+			if(ch == '\\')
+			{
+				updateBuf();
+
+				bool wasEscSequence = true;
+				if(hasNextCh)
+				{
+					switch(nextCh)
+					{
+					case 'n':  buf.put('\n'); break;
+					case 'r':  buf.put('\r'); break;
+					case 't':  buf.put('\t'); break;
+					case '"':  buf.put('\"'); break;
+					case '\\': buf.put('\\'); break;
+					default: wasEscSequence = false; break;
+					}
+				}
+				
+				if(wasEscSequence)
+				{
+					advanceChar(ErrorOnEOF.Yes);
+					spanStart = nextPos;
+				}
+				else
+				{
+					eatWhite(false);
+					spanStart = location.index;
+				}
+			}
+
+			else if(isNewline(ch))
+				error("Unescaped newlines are only allowed in raw strings, not regular strings.");
+
+			advanceChar(ErrorOnEOF.Yes);
+		}
+		
+		updateBuf();
+		advanceChar(ErrorOnEOF.No); // Skip closing double-quote
+		mixin(accept!("Value", "buf.data"));
+	}
+
+	/// Lex raw string
+	private void lexRawString()
+	{
+		assert(ch == '`');
+		
+		do
+			advanceChar(ErrorOnEOF.Yes);
+		while(ch != '`');
+		
+		advanceChar(ErrorOnEOF.No); // Skip closing back-tick
+		mixin(accept!("Value", "tokenData[1..$-1]"));
+	}
+	
+	/// Lex character literal
+	private void lexCharacter()
+	{
+		assert(ch == '\'');
+		advanceChar(ErrorOnEOF.Yes); // Skip opening single-quote
+		
+		dchar value;
+		if(ch == '\\')
+		{
+			advanceChar(ErrorOnEOF.Yes); // Skip escape backslash
+			switch(ch)
+			{
+			case 'n':  value = '\n'; break;
+			case 'r':  value = '\r'; break;
+			case 't':  value = '\t'; break;
+			case '\'': value = '\''; break;
+			case '\\': value = '\\'; break;
+			default: error("Invalid escape sequence.");
+			}
+		}
+		else if(isNewline(ch))
+			error("Newline not alowed in character literal.");
+		else
+			value = ch;
+		advanceChar(ErrorOnEOF.Yes); // Skip the character itself
+
+		if(ch == '\'')
+			advanceChar(ErrorOnEOF.No); // Skip closing single-quote
+		else
+			error("Expected closing single-quote.");
+
+		mixin(accept!("Value", "value"));
+	}
+	
+	/// Lex base64 binary literal
+	private void lexBinary()
+	{
+		assert(ch == '[');
+		advanceChar(ErrorOnEOF.Yes);
+		
+		void eatBase64Whitespace()
+		{
+			while(!isEOF && isWhite(ch))
+			{
+				if(isNewline(ch))
+					advanceChar(ErrorOnEOF.Yes);
+				
+				if(!isEOF && isWhite(ch))
+					eatWhite();
+			}
+		}
+		
+		eatBase64Whitespace();
+
+		// Iterates all valid base64 characters, ending at ']'.
+		// Skips all whitespace. Throws on invalid chars.
+		struct Base64InputRange
+		{
+			Lexer lexer;
+			private bool isInited = false;
+			private int numInputCharsMod4 = 0;
+			
+			@property bool empty()
+			{
+				if(lexer.ch == ']')
+				{
+					if(numInputCharsMod4 != 0)
+						lexer.error("Length of Base64 encoding must be a multiple of 4. ("~to!string(numInputCharsMod4)~")");
+					
+					return true;
+				}
+				
+				return false;
+			}
+
+			@property dchar front()
+			{
+				return lexer.ch;
+			}
+			
+			void popFront()
+			{
+				auto lex = lexer;
+
+				if(!isInited)
+				{
+					if(lexer.isBase64(lexer.ch))
+					{
+						numInputCharsMod4++;
+						numInputCharsMod4 %= 4;
+					}
+					
+					isInited = true;
+				}
+				
+				lex.advanceChar(lex.ErrorOnEOF.Yes);
+
+				eatBase64Whitespace();
+				
+				if(lex.isEOF)
+					lex.error("Unexpected end of file.");
+
+				if(lex.ch != ']')
+				{
+					if(!lex.isBase64(lex.ch))
+						lex.error("Invalid character in base64 binary literal.");
+					
+					numInputCharsMod4++;
+					numInputCharsMod4 %= 4;
+				}
+			}
+		}
+		
+		// This is a slow ugly hack. It's necessary because Base64.decode
+		// currently requires the source to have known length.
+		//TODO: Remove this when DMD issue #9543 is fixed.
+		dchar[] tmpBuf = array(Base64InputRange(this));
+
+		Appender!(ubyte[]) outputBuf;
+		// Ugly workaround for DMD issue #9102
+		//TODO: Remove this when DMD #9102 is fixed
+		struct OutputBuf
+		{
+			void put(ubyte ch)
+			{
+				outputBuf.put(ch);
+			}
+		}
+		
+		try
+			//Base64.decode(Base64InputRange(this), OutputBuf());
+			Base64.decode(tmpBuf, OutputBuf());
+
+		//TODO: Starting with dmd 2.062, this should be a Base64Exception
+		catch(Exception e)
+			error("Invalid character in base64 binary literal.");
+		
+		advanceChar(ErrorOnEOF.No); // Skip ']'
+		mixin(accept!("Value", "outputBuf.data"));
+	}
+	
+	private BigInt toBigInt(bool isNegative, string absValue)
+	{
+		auto num = BigInt(absValue);
+		assert(num >= 0);
+
+		if(isNegative)
+			num = -num;
+
+		return num;
+	}
+
+	/// Lex [0-9]+, but without emitting a token.
+	/// This is used by the other numeric parsing functions.
+	private string lexNumericFragment()
+	{
+		if(!isDigit(ch))
+			error("Expected a digit 0-9.");
+		
+		auto spanStart = location.index;
+		
+		do
+		{
+			advanceChar(ErrorOnEOF.No);
+		} while(!isEOF && isDigit(ch));
+		
+		return source[spanStart..location.index];
+	}
+
+	/// Lex anything that starts with 0-9 or '-'. Ints, floats, dates, etc.
+	private void lexNumeric(LookaheadTokenInfo laTokenInfo = LookaheadTokenInfo.init)
+	{
+		bool isNegative;
+		string firstFragment;
+		if(laTokenInfo.exists)
+		{
+			firstFragment = laTokenInfo.numericFragment;
+			isNegative    = laTokenInfo.isNegative;
+		}
+		else
+		{
+			assert(ch == '-' || ch == '.' || isDigit(ch));
+
+			// Check for negative
+			isNegative = ch == '-';
+			if(isNegative)
+				advanceChar(ErrorOnEOF.Yes);
+
+			// Some floating point with omitted leading zero?
+			if(ch == '.')
+			{
+				lexFloatingPoint("");
+				return;
+			}
+
+			firstFragment = lexNumericFragment();
+		}
+
+		// Long integer (64-bit signed)?
+		if(ch == 'L' || ch == 'l')
+		{
+			advanceChar(ErrorOnEOF.No);
+
+			// BigInt(long.min) is a workaround for DMD issue #9548
+			auto num = toBigInt(isNegative, firstFragment);
+			if(num < BigInt(long.min) || num > long.max)
+				error(tokenStart, "Value doesn't fit in 64-bit signed long integer: "~to!string(num));
+
+			mixin(accept!("Value", "num.toLong()"));
+		}
+		
+		// Float (32-bit signed)?
+		else if(ch == 'F' || ch == 'f')
+		{
+			auto value = to!float(tokenData);
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!("Value", "value"));
+		}
+		
+		// Double float (64-bit signed) with suffix?
+		else if((ch == 'D' || ch == 'd') && !lookahead(':')
+		)
+		{
+			auto value = to!double(tokenData);
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!("Value", "value"));
+		}
+		
+		// Decimal (128+ bits signed)?
+		else if(
+			(ch == 'B' || ch == 'b') &&
+			(lookahead('D') || lookahead('d'))
+		)
+		{
+			auto value = to!real(tokenData);
+			advanceChar(ErrorOnEOF.No);
+			advanceChar(ErrorOnEOF.No);
+			mixin(accept!("Value", "value"));
+		}
+		
+		// Some floating point?
+		else if(ch == '.')
+			lexFloatingPoint(firstFragment);
+		
+		// Some date?
+		else if(ch == '/' && hasNextCh && isDigit(nextCh))
+			lexDate(isNegative, firstFragment);
+		
+		// Some time span?
+		else if(ch == ':' || ch == 'd')
+			lexTimeSpan(isNegative, firstFragment);
+
+		// Integer (32-bit signed)?
+		else if(isEndOfNumber())
+		{
+			auto num = toBigInt(isNegative, firstFragment);
+			if(num < int.min || num > int.max)
+				error(tokenStart, "Value doesn't fit in 32-bit signed integer: "~to!string(num));
+
+			mixin(accept!("Value", "num.toInt()"));
+		}
+
+		// Invalid suffix
+		else
+			error("Invalid integer suffix.");
+	}
+	
+	/// Lex any floating-point literal (after the initial numeric fragment was lexed)
+	private void lexFloatingPoint(string firstPart)
+	{
+		assert(ch == '.');
+		advanceChar(ErrorOnEOF.No);
+		
+		auto secondPart = lexNumericFragment();
+		
+		try
+		{
+			// Double float (64-bit signed) with suffix?
+			if(ch == 'D' || ch == 'd')
+			{
+				auto value = to!double(tokenData);
+				advanceChar(ErrorOnEOF.No);
+				mixin(accept!("Value", "value"));
+			}
+
+			// Float (32-bit signed)?
+			else if(ch == 'F' || ch == 'f')
+			{
+				auto value = to!float(tokenData);
+				advanceChar(ErrorOnEOF.No);
+				mixin(accept!("Value", "value"));
+			}
+
+			// Decimal (128+ bits signed)?
+			else if(ch == 'B' || ch == 'b')
+			{
+				auto value = to!real(tokenData);
+				advanceChar(ErrorOnEOF.Yes);
+
+				if(!isEOF && (ch == 'D' || ch == 'd'))
+				{
+					advanceChar(ErrorOnEOF.No);
+					if(isEndOfNumber())
+						mixin(accept!("Value", "value"));
+				}
+
+				error("Invalid floating point suffix.");
+			}
+
+			// Double float (64-bit signed) without suffix?
+			else if(isEOF || !isIdentChar(ch))
+			{
+				auto value = to!double(tokenData);
+				mixin(accept!("Value", "value"));
+			}
+
+			// Invalid suffix
+			else
+				error("Invalid floating point suffix.");
+		}
+		catch(ConvException e)
+			error("Invalid floating point literal.");
+	}
+
+	private Date makeDate(bool isNegative, string yearStr, string monthStr, string dayStr)
+	{
+		BigInt biTmp;
+		
+		biTmp = BigInt(yearStr);
+		if(isNegative)
+			biTmp = -biTmp;
+		if(biTmp < int.min || biTmp > int.max)
+			error(tokenStart, "Date's year is out of range. (Must fit within a 32-bit signed int.)");
+		auto year = biTmp.toInt();
+
+		biTmp = BigInt(monthStr);
+		if(biTmp < 1 || biTmp > 12)
+			error(tokenStart, "Date's month is out of range.");
+		auto month = biTmp.toInt();
+		
+		biTmp = BigInt(dayStr);
+		if(biTmp < 1 || biTmp > 31)
+			error(tokenStart, "Date's month is out of range.");
+		auto day = biTmp.toInt();
+		
+		return Date(year, month, day);
+	}
+	
+	private DateTimeFrac makeDateTimeFrac(
+		bool isNegative, Date date, string hourStr, string minuteStr,
+		string secondStr, string millisecondStr
+	)
+	{
+		BigInt biTmp;
+
+		biTmp = BigInt(hourStr);
+		if(biTmp < int.min || biTmp > int.max)
+			error(tokenStart, "Datetime's hour is out of range.");
+		auto numHours = biTmp.toInt();
+		
+		biTmp = BigInt(minuteStr);
+		if(biTmp < 0 || biTmp > int.max)
+			error(tokenStart, "Datetime's minute is out of range.");
+		auto numMinutes = biTmp.toInt();
+		
+		int numSeconds = 0;
+		if(secondStr != "")
+		{
+			biTmp = BigInt(secondStr);
+			if(biTmp < 0 || biTmp > int.max)
+				error(tokenStart, "Datetime's second is out of range.");
+			numSeconds = biTmp.toInt();
+		}
+		
+		int millisecond = 0;
+		if(millisecondStr != "")
+		{
+			biTmp = BigInt(millisecondStr);
+			if(biTmp < 0 || biTmp > int.max)
+				error(tokenStart, "Datetime's millisecond is out of range.");
+			millisecond = biTmp.toInt();
+
+			if(millisecondStr.length == 1)
+				millisecond *= 100;
+			else if(millisecondStr.length == 2)
+				millisecond *= 10;
+		}
+
+		Duration fracSecs = millisecond.msecs;
+		
+		auto offset = hours(numHours) + minutes(numMinutes) + seconds(numSeconds);
+
+		if(isNegative)
+		{
+			offset   = -offset;
+			fracSecs = -fracSecs;
+		}
+		
+		return DateTimeFrac(DateTime(date) + offset, fracSecs);
+	}
+
+	private Duration makeDuration(
+		bool isNegative, string dayStr,
+		string hourStr, string minuteStr, string secondStr,
+		string millisecondStr
+	)
+	{
+		BigInt biTmp;
+
+		long day = 0;
+		if(dayStr != "")
+		{
+			biTmp = BigInt(dayStr);
+			if(biTmp < long.min || biTmp > long.max)
+				error(tokenStart, "Time span's day is out of range.");
+			day = biTmp.toLong();
+		}
+
+		biTmp = BigInt(hourStr);
+		if(biTmp < long.min || biTmp > long.max)
+			error(tokenStart, "Time span's hour is out of range.");
+		auto hour = biTmp.toLong();
+
+		biTmp = BigInt(minuteStr);
+		if(biTmp < long.min || biTmp > long.max)
+			error(tokenStart, "Time span's minute is out of range.");
+		auto minute = biTmp.toLong();
+
+		biTmp = BigInt(secondStr);
+		if(biTmp < long.min || biTmp > long.max)
+			error(tokenStart, "Time span's second is out of range.");
+		auto second = biTmp.toLong();
+
+		long millisecond = 0;
+		if(millisecondStr != "")
+		{
+			biTmp = BigInt(millisecondStr);
+			if(biTmp < long.min || biTmp > long.max)
+				error(tokenStart, "Time span's millisecond is out of range.");
+			millisecond = biTmp.toLong();
+
+			if(millisecondStr.length == 1)
+				millisecond *= 100;
+			else if(millisecondStr.length == 2)
+				millisecond *= 10;
+		}
+		
+		auto duration =
+			dur!"days"   (day)    +
+			dur!"hours"  (hour)   +
+			dur!"minutes"(minute) +
+			dur!"seconds"(second) +
+			dur!"msecs"  (millisecond);
+
+		if(isNegative)
+			duration = -duration;
+		
+		return duration;
+	}
+
+	// This has to reproduce some weird corner case behaviors from the
+	// original Java version of SDL. So some of this may seem weird.
+	private Nullable!Duration getTimeZoneOffset(string str)
+	{
+		if(str.length < 2)
+			return Nullable!Duration(); // Unknown timezone
+		
+		if(str[0] != '+' && str[0] != '-')
+			return Nullable!Duration(); // Unknown timezone
+
+		auto isNegative = str[0] == '-';
+
+		string numHoursStr;
+		string numMinutesStr;
+		if(str[1] == ':')
+		{
+			numMinutesStr = str[1..$];
+			numHoursStr = "";
+		}
+		else
+		{
+			numMinutesStr = str.find(':');
+			numHoursStr = str[1 .. $-numMinutesStr.length];
+		}
+		
+		long numHours = 0;
+		long numMinutes = 0;
+		bool isUnknown = false;
+		try
+		{
+			switch(numHoursStr.length)
+			{
+			case 0:
+				if(numMinutesStr.length == 3)
+				{
+					numHours   = 0;
+					numMinutes = to!long(numMinutesStr[1..$]);
+				}
+				else
+					isUnknown = true;
+				break;
+
+			case 1:
+			case 2:
+				if(numMinutesStr.length == 0)
+				{
+					numHours   = to!long(numHoursStr);
+					numMinutes = 0;
+				}
+				else if(numMinutesStr.length == 3)
+				{
+					numHours   = to!long(numHoursStr);
+					numMinutes = to!long(numMinutesStr[1..$]);
+				}
+				else
+					isUnknown = true;
+				break;
+
+			default:
+				if(numMinutesStr.length == 0)
+				{
+					// Yes, this is correct
+					numHours   = 0;
+					numMinutes = to!long(numHoursStr[1..$]);
+				}
+				else
+					isUnknown = true;
+				break;
+			}
+		}
+		catch(ConvException e)
+			isUnknown = true;
+		
+		if(isUnknown)
+			return Nullable!Duration(); // Unknown timezone
+
+		auto timeZoneOffset = hours(numHours) + minutes(numMinutes);
+		if(isNegative)
+			timeZoneOffset = -timeZoneOffset;
+
+		// Timezone valid
+		return Nullable!Duration(timeZoneOffset);
+	}
+	
+	/// Lex date or datetime (after the initial numeric fragment was lexed)
+	private void lexDate(bool isDateNegative, string yearStr)
+	{
+		assert(ch == '/');
+		
+		// Lex months
+		advanceChar(ErrorOnEOF.Yes); // Skip '/'
+		auto monthStr = lexNumericFragment();
+
+		// Lex days
+		if(ch != '/')
+			error("Invalid date format: Missing days.");
+		advanceChar(ErrorOnEOF.Yes); // Skip '/'
+		auto dayStr = lexNumericFragment();
+		
+		auto date = makeDate(isDateNegative, yearStr, monthStr, dayStr);
+
+		if(!isEndOfNumber() && ch != '/')
+			error("Dates cannot have suffixes.");
+		
+		// Date?
+		if(isEOF)
+			mixin(accept!("Value", "date"));
+		
+		auto endOfDate = location;
+		
+		while(
+			!isEOF &&
+			( ch == '\\' || ch == '/' || (isWhite(ch) && !isNewline(ch)) )
+		)
+		{
+			if(ch == '\\' && hasNextCh && isNewline(nextCh))
+			{
+				advanceChar(ErrorOnEOF.Yes);
+				if(isAtNewline())
+					advanceChar(ErrorOnEOF.Yes);
+				advanceChar(ErrorOnEOF.No);
+			}
+
+			eatWhite();
+		}
+
+		// Date?
+		if(isEOF || (!isDigit(ch) && ch != '-'))
+			mixin(accept!("Value", "date", "", "endOfDate.index"));
+		
+		auto startOfTime = location;
+
+		// Is time negative?
+		bool isTimeNegative = ch == '-';
+		if(isTimeNegative)
+			advanceChar(ErrorOnEOF.Yes);
+
+		// Lex hours
+		auto hourStr = ch == '.'? "" : lexNumericFragment();
+		
+		// Lex minutes
+		if(ch != ':')
+		{
+			// No minutes found. Therefore we had a plain Date followed
+			// by a numeric literal, not a DateTime.
+			lookaheadTokenInfo.exists          = true;
+			lookaheadTokenInfo.numericFragment = hourStr;
+			lookaheadTokenInfo.isNegative      = isTimeNegative;
+			lookaheadTokenInfo.tokenStart      = startOfTime;
+			mixin(accept!("Value", "date", "", "endOfDate.index"));
+		}
+		advanceChar(ErrorOnEOF.Yes); // Skip ':'
+		auto minuteStr = lexNumericFragment();
+		
+		// Lex seconds, if exists
+		string secondStr;
+		if(ch == ':')
+		{
+			advanceChar(ErrorOnEOF.Yes); // Skip ':'
+			secondStr = lexNumericFragment();
+		}
+		
+		// Lex milliseconds, if exists
+		string millisecondStr;
+		if(ch == '.')
+		{
+			advanceChar(ErrorOnEOF.Yes); // Skip '.'
+			millisecondStr = lexNumericFragment();
+		}
+
+		auto dateTimeFrac = makeDateTimeFrac(isTimeNegative, date, hourStr, minuteStr, secondStr, millisecondStr);
+		
+		// Lex zone, if exists
+		if(ch == '-')
+		{
+			advanceChar(ErrorOnEOF.Yes); // Skip '-'
+			auto timezoneStart = location;
+			
+			if(!isAlpha(ch))
+				error("Invalid timezone format.");
+			
+			while(!isEOF && !isWhite(ch))
+				advanceChar(ErrorOnEOF.No);
+			
+			auto timezoneStr = source[timezoneStart.index..location.index];
+			if(timezoneStr.startsWith("GMT"))
+			{
+				auto isoPart = timezoneStr["GMT".length..$];
+				auto offset = getTimeZoneOffset(isoPart);
+				
+				if(offset.isNull())
+				{
+					// Unknown time zone
+					mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
+				}
+				else
+				{
+					auto timezone = new immutable SimpleTimeZone(offset.get());
+					mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
+				}
+			}
+			
+			try
+			{
+				auto timezone = TimeZone.getTimeZone(timezoneStr);
+				if(timezone)
+					mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
+			}
+			catch(TimeException e)
+			{
+				// Time zone not found. So just move along to "Unknown time zone" below.
+			}
+
+			// Unknown time zone
+			mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
+		}
+
+		if(!isEndOfNumber())
+			error("Date-Times cannot have suffixes.");
+
+		mixin(accept!("Value", "dateTimeFrac"));
+	}
+
+	/// Lex time span (after the initial numeric fragment was lexed)
+	private void lexTimeSpan(bool isNegative, string firstPart)
+	{
+		assert(ch == ':' || ch == 'd');
+		
+		string dayStr = "";
+		string hourStr;
+
+		// Lexed days?
+		bool hasDays = ch == 'd';
+		if(hasDays)
+		{
+			dayStr = firstPart;
+			advanceChar(ErrorOnEOF.Yes); // Skip 'd'
+
+			// Lex hours
+			if(ch != ':')
+				error("Invalid time span format: Missing hours.");
+			advanceChar(ErrorOnEOF.Yes); // Skip ':'
+			hourStr = lexNumericFragment();
+		}
+		else
+			hourStr = firstPart;
+
+		// Lex minutes
+		if(ch != ':')
+			error("Invalid time span format: Missing minutes.");
+		advanceChar(ErrorOnEOF.Yes); // Skip ':'
+		auto minuteStr = lexNumericFragment();
+
+		// Lex seconds
+		if(ch != ':')
+			error("Invalid time span format: Missing seconds.");
+		advanceChar(ErrorOnEOF.Yes); // Skip ':'
+		auto secondStr = lexNumericFragment();
+		
+		// Lex milliseconds, if exists
+		string millisecondStr = "";
+		if(ch == '.')
+		{
+			advanceChar(ErrorOnEOF.Yes); // Skip '.'
+			millisecondStr = lexNumericFragment();
+		}
+
+		if(!isEndOfNumber())
+			error("Time spans cannot have suffixes.");
+		
+		auto duration = makeDuration(isNegative, dayStr, hourStr, minuteStr, secondStr, millisecondStr);
+		mixin(accept!("Value", "duration"));
+	}
+
+	/// Advances past whitespace and comments
+	private void eatWhite(bool allowComments=true)
+	{
+		// -- Comment/Whitepace Lexer -------------
+
+		enum State
+		{
+			normal,
+			lineComment,  // Got "#" or "//" or "--", Eating everything until newline
+			blockComment, // Got "/*", Eating everything until "*/"
+		}
+
+		if(isEOF)
+			return;
+		
+		Location commentStart;
+		State state = State.normal;
+		bool consumeNewlines = false;
+		bool hasConsumedNewline = false;
+		while(true)
+		{
+			final switch(state)
+			{
+			case State.normal:
+
+				if(ch == '\\')
+				{
+					commentStart = location;
+					consumeNewlines = true;
+					hasConsumedNewline = false;
+				}
+
+				else if(ch == '#')
+				{
+					if(!allowComments)
+						return;
+
+					commentStart = location;
+					state = State.lineComment;
+					continue;
+				}
+
+				else if(ch == '/' || ch == '-')
+				{
+					commentStart = location;
+					if(lookahead(ch))
+					{
+						if(!allowComments)
+							return;
+
+						advanceChar(ErrorOnEOF.No);
+						state = State.lineComment;
+						continue;
+					}
+					else if(ch == '/' && lookahead('*'))
+					{
+						if(!allowComments)
+							return;
+
+						advanceChar(ErrorOnEOF.No);
+						state = State.blockComment;
+						continue;
+					}
+					else
+						return; // Done
+				}
+				else if(isAtNewline())
+				{
+					if(consumeNewlines)
+						hasConsumedNewline = true;
+					else
+						return; // Done
+				}
+				else if(!isWhite(ch))
+				{
+					if(consumeNewlines)
+					{
+						if(hasConsumedNewline)
+							return; // Done
+						else
+							error("Only whitespace can come between a line-continuation backslash and the following newline.");
+					}
+					else
+						return; // Done
+				}
+
+				break;
+			
+			case State.lineComment:
+				if(lookahead(&isNewline))
+					state = State.normal;
+				break;
+			
+			case State.blockComment:
+				if(ch == '*' && lookahead('/'))
+				{
+					advanceChar(ErrorOnEOF.No);
+					state = State.normal;
+				}
+				break;
+			}
+			
+			advanceChar(ErrorOnEOF.No);
+			if(isEOF)
+			{
+				// Reached EOF
+
+				if(consumeNewlines && !hasConsumedNewline)
+					error("Missing newline after line-continuation backslash.");
+
+				else if(state == State.blockComment)
+					error(commentStart, "Unterminated block comment.");
+
+				else
+					return; // Done, reached EOF
+			}
+		}
+	}
+}
+
+version(sdlangUnittest)
+{
+	import std.stdio;
+
+	private auto loc  = Location("filename", 0, 0, 0);
+	private auto loc2 = Location("a", 1, 1, 1);
+
+	unittest
+	{
+		assert([Token(symbol!"EOL",loc)             ] == [Token(symbol!"EOL",loc)              ] );
+		assert([Token(symbol!"EOL",loc,Value(7),"A")] == [Token(symbol!"EOL",loc2,Value(7),"B")] );
+	}
+
+	private int numErrors = 0;
+	private void testLex(string source, Token[] expected, bool test_locations = false, string file=__FILE__, size_t line=__LINE__)
+	{
+		Token[] actual;
+		try
+			actual = lexSource(source, "filename");
+		catch(SDLangParseException e)
+		{
+			numErrors++;
+			stderr.writeln(file, "(", line, "): testLex failed on: ", source);
+			stderr.writeln("    Expected:");
+			stderr.writeln("        ", expected);
+			stderr.writeln("    Actual: SDLangParseException thrown:");
+			stderr.writeln("        ", e.msg);
+			return;
+		}
+
+		bool is_same = actual == expected;
+		if (is_same && test_locations) {
+			is_same = actual.map!(t => t.location).equal(expected.map!(t => t.location));
+		}
+		
+		if(!is_same)
+		{
+			numErrors++;
+			stderr.writeln(file, "(", line, "): testLex failed on: ", source);
+			stderr.writeln("    Expected:");
+			stderr.writeln("        ", expected);
+			stderr.writeln("    Actual:");
+			stderr.writeln("        ", actual);
+
+			if(expected.length > 1 || actual.length > 1)
+			{
+				stderr.writeln("    expected.length: ", expected.length);
+				stderr.writeln("    actual.length:   ", actual.length);
+
+				if(actual.length == expected.length)
+				foreach(i; 0..actual.length)
+				if(actual[i] != expected[i])
+				{
+					stderr.writeln("    Unequal at index #", i, ":");
+					stderr.writeln("        Expected:");
+					stderr.writeln("            ", expected[i]);
+					stderr.writeln("        Actual:");
+					stderr.writeln("            ", actual[i]);
+				}
+			}
+		}
+	}
+
+	private void testLexThrows(string file=__FILE__, size_t line=__LINE__)(string source)
+	{
+		bool hadException = false;
+		Token[] actual;
+		try
+			actual = lexSource(source, "filename");
+		catch(SDLangParseException e)
+			hadException = true;
+
+		if(!hadException)
+		{
+			numErrors++;
+			stderr.writeln(file, "(", line, "): testLex failed on: ", source);
+			stderr.writeln("    Expected SDLangParseException");
+			stderr.writeln("    Actual:");
+			stderr.writeln("        ", actual);
+		}
+	}
+}
+
+version(sdlangUnittest)
+unittest
+{
+	writeln("Unittesting sdlang lexer...");
+	stdout.flush();
+	
+	testLex("",        []);
+	testLex(" ",       []);
+	testLex("\\\n",    []);
+	testLex("/*foo*/", []);
+	testLex("/* multiline \n comment */", []);
+	testLex("/* * */", []);
+	testLexThrows("/* ");
+
+	testLex(":",  [ Token(symbol!":",  loc) ]);
+	testLex("=",  [ Token(symbol!"=",  loc) ]);
+	testLex("{",  [ Token(symbol!"{",  loc) ]);
+	testLex("}",  [ Token(symbol!"}",  loc) ]);
+	testLex(";",  [ Token(symbol!"EOL",loc) ]);
+	testLex("\n", [ Token(symbol!"EOL",loc) ]);
+
+	testLex("foo",     [ Token(symbol!"Ident",loc,Value(null),"foo")     ]);
+	testLex("_foo",    [ Token(symbol!"Ident",loc,Value(null),"_foo")    ]);
+	testLex("foo.bar", [ Token(symbol!"Ident",loc,Value(null),"foo.bar") ]);
+	testLex("foo-bar", [ Token(symbol!"Ident",loc,Value(null),"foo-bar") ]);
+	testLex("foo.",    [ Token(symbol!"Ident",loc,Value(null),"foo.")    ]);
+	testLex("foo-",    [ Token(symbol!"Ident",loc,Value(null),"foo-")    ]);
+	testLexThrows(".foo");
+
+	testLex("foo bar", [
+		Token(symbol!"Ident",loc,Value(null),"foo"),
+		Token(symbol!"Ident",loc,Value(null),"bar"),
+	]);
+	testLex("foo \\  \n  \n  bar", [
+		Token(symbol!"Ident",loc,Value(null),"foo"),
+		Token(symbol!"Ident",loc,Value(null),"bar"),
+	]);
+	testLex("foo \\  \n \\ \n  bar", [
+		Token(symbol!"Ident",loc,Value(null),"foo"),
+		Token(symbol!"Ident",loc,Value(null),"bar"),
+	]);
+	testLexThrows("foo \\ ");
+	testLexThrows("foo \\ bar");
+	testLexThrows("foo \\  \n  \\ ");
+	testLexThrows("foo \\  \n  \\ bar");
+
+	testLex("foo : = { } ; \n bar \n", [
+		Token(symbol!"Ident",loc,Value(null),"foo"),
+		Token(symbol!":",loc),
+		Token(symbol!"=",loc),
+		Token(symbol!"{",loc),
+		Token(symbol!"}",loc),
+		Token(symbol!"EOL",loc),
+		Token(symbol!"EOL",loc),
+		Token(symbol!"Ident",loc,Value(null),"bar"),
+		Token(symbol!"EOL",loc),
+	]);
+
+	testLexThrows("<");
+	testLexThrows("*");
+	testLexThrows(`\`);
+	
+	// Integers
+	testLex(  "7", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
+	testLex( "-7", [ Token(symbol!"Value",loc,Value(cast( int)-7)) ]);
+	testLex( "7L", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
+	testLex( "7l", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
+	testLex("-7L", [ Token(symbol!"Value",loc,Value(cast(long)-7)) ]);
+	testLex(  "0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
+	testLex( "-0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
+
+	testLex("7/**/", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
+	testLex("7#",    [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
+
+	testLex("7 A", [
+		Token(symbol!"Value",loc,Value(cast(int)7)),
+		Token(symbol!"Ident",loc,Value(      null),"A"),
+	]);
+	testLexThrows("7A");
+	testLexThrows("-A");
+	testLexThrows(`-""`);
+	
+	testLex("7;", [
+		Token(symbol!"Value",loc,Value(cast(int)7)),
+		Token(symbol!"EOL",loc),
+	]);
+	
+	// Floats
+	testLex("1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
+	testLex("1.2f" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
+	testLex("1.2"  , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
+	testLex("1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
+	testLex("1.2d" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
+	testLex("1.2BD", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
+	testLex("1.2bd", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
+	testLex("1.2Bd", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
+	testLex("1.2bD", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
+
+	testLex(".2F" , [ Token(symbol!"Value",loc,Value(cast( float)0.2)) ]);
+	testLex(".2"  , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
+	testLex(".2D" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
+	testLex(".2BD", [ Token(symbol!"Value",loc,Value(cast(  real)0.2)) ]);
+
+	testLex("-1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-1.2)) ]);
+	testLex("-1.2"  , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
+	testLex("-1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
+	testLex("-1.2BD", [ Token(symbol!"Value",loc,Value(cast(  real)-1.2)) ]);
+
+	testLex("-.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-0.2)) ]);
+	testLex("-.2"  , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
+	testLex("-.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
+	testLex("-.2BD", [ Token(symbol!"Value",loc,Value(cast(  real)-0.2)) ]);
+
+	testLex( "0.0"  , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+	testLex( "0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+	testLex( "0.0BD", [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
+	testLex("-0.0"  , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+	testLex("-0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+	testLex("-0.0BD", [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
+	testLex( "7F"   , [ Token(symbol!"Value",loc,Value(cast( float)7.0)) ]);
+	testLex( "7D"   , [ Token(symbol!"Value",loc,Value(cast(double)7.0)) ]);
+	testLex( "7BD"  , [ Token(symbol!"Value",loc,Value(cast(  real)7.0)) ]);
+	testLex( "0F"   , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+	testLex( "0D"   , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+	testLex( "0BD"  , [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
+	testLex("-0F"   , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
+	testLex("-0D"   , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
+	testLex("-0BD"  , [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
+
+	testLex("1.2 F", [
+		Token(symbol!"Value",loc,Value(cast(double)1.2)),
+		Token(symbol!"Ident",loc,Value(           null),"F"),
+	]);
+	testLexThrows("1.2A");
+	testLexThrows("1.2B");
+	testLexThrows("1.2BDF");
+
+	testLex("1.2;", [
+		Token(symbol!"Value",loc,Value(cast(double)1.2)),
+		Token(symbol!"EOL",loc),
+	]);
+
+	testLex("1.2F;", [
+		Token(symbol!"Value",loc,Value(cast(float)1.2)),
+		Token(symbol!"EOL",loc),
+	]);
+
+	testLex("1.2BD;", [
+		Token(symbol!"Value",loc,Value(cast(real)1.2)),
+		Token(symbol!"EOL",loc),
+	]);
+
+	// Booleans and null
+	testLex("true",   [ Token(symbol!"Value",loc,Value( true)) ]);
+	testLex("false",  [ Token(symbol!"Value",loc,Value(false)) ]);
+	testLex("on",     [ Token(symbol!"Value",loc,Value( true)) ]);
+	testLex("off",    [ Token(symbol!"Value",loc,Value(false)) ]);
+	testLex("null",   [ Token(symbol!"Value",loc,Value( null)) ]);
+
+	testLex("TRUE",   [ Token(symbol!"Ident",loc,Value(null),"TRUE")  ]);
+	testLex("true ",  [ Token(symbol!"Value",loc,Value(true)) ]);
+	testLex("true  ", [ Token(symbol!"Value",loc,Value(true)) ]);
+	testLex("tru",    [ Token(symbol!"Ident",loc,Value(null),"tru")   ]);
+	testLex("truX",   [ Token(symbol!"Ident",loc,Value(null),"truX")  ]);
+	testLex("trueX",  [ Token(symbol!"Ident",loc,Value(null),"trueX") ]);
+
+	// Raw Backtick Strings
+	testLex("`hello world`",      [ Token(symbol!"Value",loc,Value(`hello world`   )) ]);
+	testLex("` hello world `",    [ Token(symbol!"Value",loc,Value(` hello world ` )) ]);
+	testLex("`hello \\t world`",  [ Token(symbol!"Value",loc,Value(`hello \t world`)) ]);
+	testLex("`hello \\n world`",  [ Token(symbol!"Value",loc,Value(`hello \n world`)) ]);
+	testLex("`hello \n world`",   [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
+	testLex("`hello \r\n world`", [ Token(symbol!"Value",loc,Value("hello \r\n world")) ]);
+	testLex("`hello \"world\"`",  [ Token(symbol!"Value",loc,Value(`hello "world"` )) ]);
+
+	testLexThrows("`foo");
+	testLexThrows("`");
+
+	// Double-Quote Strings
+	testLex(`"hello world"`,            [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
+	testLex(`" hello world "`,          [ Token(symbol!"Value",loc,Value(" hello world " )) ]);
+	testLex(`"hello \t world"`,         [ Token(symbol!"Value",loc,Value("hello \t world")) ]);
+	testLex(`"hello \n world"`,         [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
+	testLex("\"hello \\\n world\"",     [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
+	testLex("\"hello \\  \n world\"",   [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
+	testLex("\"hello \\  \n\n world\"", [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
+	testLex(`"\"hello world\""`,        [ Token(symbol!"Value",loc,Value(`"hello world"` )) ]);
+	testLex(`""`,                       [ Token(symbol!"Value",loc,Value(""              )) ]); // issue #34
+
+	testLexThrows("\"hello \n world\"");
+	testLexThrows(`"foo`);
+	testLexThrows(`"`);
+
+	// Characters
+	testLex("'a'",   [ Token(symbol!"Value",loc,Value(cast(dchar) 'a')) ]);
+	testLex("'\\n'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\n')) ]);
+	testLex("'\\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
+	testLex("'\t'",  [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
+	testLex("'\\''", [ Token(symbol!"Value",loc,Value(cast(dchar)'\'')) ]);
+	testLex(`'\\'`,  [ Token(symbol!"Value",loc,Value(cast(dchar)'\\')) ]);
+
+	testLexThrows("'a");
+	testLexThrows("'aa'");
+	testLexThrows("''");
+	testLexThrows("'\\\n'");
+	testLexThrows("'\n'");
+	testLexThrows(`'\`);
+	testLexThrows(`'\'`);
+	testLexThrows("'");
+	
+	// Unicode
+	testLex("日本語",         [ Token(symbol!"Ident",loc,Value(null), "日本語") ]);
+	testLex("`おはよう、日本。`", [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
+	testLex(`"おはよう、日本。"`, [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
+	testLex("'月'",           [ Token(symbol!"Value",loc,Value("月"d.dup[0]))   ]);
+
+	// Base64 Binary
+	testLex("[aGVsbG8gd29ybGQ=]",              [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
+	testLex("[ aGVsbG8gd29ybGQ= ]",            [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
+	testLex("[\n aGVsbG8g \n \n d29ybGQ= \n]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
+
+	testLexThrows("[aGVsbG8gd29ybGQ]"); // Ie: Not multiple of 4
+	testLexThrows("[ aGVsbG8gd29ybGQ ]");
+
+	// Date
+	testLex( "1999/12/5", [ Token(symbol!"Value",loc,Value(Date( 1999, 12, 5))) ]);
+	testLex( "2013/2/22", [ Token(symbol!"Value",loc,Value(Date( 2013, 2, 22))) ]);
+	testLex("-2013/2/22", [ Token(symbol!"Value",loc,Value(Date(-2013, 2, 22))) ]);
+
+	testLexThrows("7/");
+	testLexThrows("2013/2/22a");
+	testLexThrows("2013/2/22f");
+
+	testLex("1999/12/5\n", [
+		Token(symbol!"Value",loc,Value(Date(1999, 12, 5))),
+		Token(symbol!"EOL",loc),
+	]);
+
+	// DateTime, no timezone
+	testLex( "2013/2/22 07:53",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22 \t 07:53",     [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22/*foo*/07:53",  [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22 /*foo*/ \\\n  /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22 /*foo*/ \\\n\n  \n  /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22 /*foo*/ \\\n\\\n  \\\n  /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22/*foo*/\\\n/*bar*/07:53",      [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
+	testLex("-2013/2/22 07:53",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 7, 53,  0)))) ]);
+	testLex( "2013/2/22 -07:53",       [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53)))) ]);
+	testLex("-2013/2/22 -07:53",       [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53)))) ]);
+	testLex( "2013/2/22 07:53:34",     [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34)))) ]);
+	testLex( "2013/2/22 07:53:34.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs))) ]);
+	testLex( "2013/2/22 07:53:34.12",  [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 120.msecs))) ]);
+	testLex( "2013/2/22 07:53:34.1",   [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 100.msecs))) ]);
+	testLex( "2013/2/22 07:53.123",    [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs))) ]);
+
+	testLex( "2013/2/22 34:65",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0)))) ]);
+	testLex( "2013/2/22 34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds(77), 123.msecs))) ]);
+	testLex( "2013/2/22 34:65.123",    [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0), 123.msecs))) ]);
+
+	testLex( "2013/2/22 -34:65",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0)))) ]);
+	testLex( "2013/2/22 -34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds(77), -123.msecs))) ]);
+	testLex( "2013/2/22 -34:65.123",    [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), -123.msecs))) ]);
+
+	testLexThrows("2013/2/22 07:53a");
+	testLexThrows("2013/2/22 07:53f");
+	testLexThrows("2013/2/22 07:53:34.123a");
+	testLexThrows("2013/2/22 07:53:34.123f");
+	testLexThrows("2013/2/22a 07:53");
+
+	testLex(`2013/2/22 "foo"`, [
+		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+		Token(symbol!"Value",loc,Value("foo")),
+	]);
+
+	testLex("2013/2/22 07", [
+		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+		Token(symbol!"Value",loc,Value(cast(int)7)),
+	]);
+
+	testLex("2013/2/22 1.2F", [
+		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+		Token(symbol!"Value",loc,Value(cast(float)1.2)),
+	]);
+
+	testLex("2013/2/22 .2F", [
+		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+		Token(symbol!"Value",loc,Value(cast(float)0.2)),
+	]);
+
+	testLex("2013/2/22 -1.2F", [
+		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+		Token(symbol!"Value",loc,Value(cast(float)-1.2)),
+	]);
+
+	testLex("2013/2/22 -.2F", [
+		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
+		Token(symbol!"Value",loc,Value(cast(float)-0.2)),
+	]);
+
+	// DateTime, with known timezone
+	testLex( "2013/2/22 07:53-GMT+00:00",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex("-2013/2/22 07:53-GMT+00:00",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex( "2013/2/22 -07:53-GMT+00:00",       [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex("-2013/2/22 -07:53-GMT+00:00",       [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex( "2013/2/22 07:53-GMT+02:10",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+	testLex( "2013/2/22 07:53-GMT-05:30",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+	testLex( "2013/2/22 07:53:34-GMT+00:00",     [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex( "2013/2/22 07:53:34-GMT+02:10",     [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+	testLex( "2013/2/22 07:53:34-GMT-05:30",     [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+	testLex( "2013/2/22 07:53:34.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex( "2013/2/22 07:53:34.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+	testLex( "2013/2/22 07:53:34.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+	testLex( "2013/2/22 07:53.123-GMT+00:00",    [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, new immutable SimpleTimeZone( hours(0)            )))) ]);
+	testLex( "2013/2/22 07:53.123-GMT+02:10",    [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
+	testLex( "2013/2/22 07:53.123-GMT-05:30",    [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+
+	testLex( "2013/2/22 -34:65-GMT-05:30",       [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0,  0,  0) - hours(34) - minutes(65) - seconds( 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
+
+	// DateTime, with Java SDL's occasionally weird interpretation of some
+	// "not quite ISO" variations of the "GMT with offset" timezone strings.
+	Token testTokenSimpleTimeZone(Duration d)
+	{
+		auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
+		auto tz = new immutable SimpleTimeZone(d);
+		return Token( symbol!"Value", loc, Value(SysTime(dateTime,tz)) );
+	}
+	Token testTokenUnknownTimeZone(string tzName)
+	{
+		auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
+		auto frac = 0.msecs;
+		return Token( symbol!"Value", loc, Value(DateTimeFracUnknownZone(dateTime,frac,tzName)) );
+	}
+	testLex("2013/2/22 07:53-GMT+",          [ testTokenUnknownTimeZone("GMT+")     ]);
+	testLex("2013/2/22 07:53-GMT+:",         [ testTokenUnknownTimeZone("GMT+:")    ]);
+	testLex("2013/2/22 07:53-GMT+:3",        [ testTokenUnknownTimeZone("GMT+:3")   ]);
+	testLex("2013/2/22 07:53-GMT+:03",       [ testTokenSimpleTimeZone(minutes(3))  ]);
+	testLex("2013/2/22 07:53-GMT+:003",      [ testTokenUnknownTimeZone("GMT+:003") ]);
+
+	testLex("2013/2/22 07:53-GMT+4",         [ testTokenSimpleTimeZone(hours(4))            ]);
+	testLex("2013/2/22 07:53-GMT+4:",        [ testTokenUnknownTimeZone("GMT+4:")           ]);
+	testLex("2013/2/22 07:53-GMT+4:3",       [ testTokenUnknownTimeZone("GMT+4:3")          ]);
+	testLex("2013/2/22 07:53-GMT+4:03",      [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
+	testLex("2013/2/22 07:53-GMT+4:003",     [ testTokenUnknownTimeZone("GMT+4:003")        ]);
+
+	testLex("2013/2/22 07:53-GMT+04",        [ testTokenSimpleTimeZone(hours(4))            ]);
+	testLex("2013/2/22 07:53-GMT+04:",       [ testTokenUnknownTimeZone("GMT+04:")          ]);
+	testLex("2013/2/22 07:53-GMT+04:3",      [ testTokenUnknownTimeZone("GMT+04:3")         ]);
+	testLex("2013/2/22 07:53-GMT+04:03",     [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
+	testLex("2013/2/22 07:53-GMT+04:03abc",  [ testTokenUnknownTimeZone("GMT+04:03abc")     ]);
+	testLex("2013/2/22 07:53-GMT+04:003",    [ testTokenUnknownTimeZone("GMT+04:003")       ]);
+
+	testLex("2013/2/22 07:53-GMT+004",       [ testTokenSimpleTimeZone(minutes(4))     ]);
+	testLex("2013/2/22 07:53-GMT+004:",      [ testTokenUnknownTimeZone("GMT+004:")    ]);
+	testLex("2013/2/22 07:53-GMT+004:3",     [ testTokenUnknownTimeZone("GMT+004:3")   ]);
+	testLex("2013/2/22 07:53-GMT+004:03",    [ testTokenUnknownTimeZone("GMT+004:03")  ]);
+	testLex("2013/2/22 07:53-GMT+004:003",   [ testTokenUnknownTimeZone("GMT+004:003") ]);
+
+	testLex("2013/2/22 07:53-GMT+0004",      [ testTokenSimpleTimeZone(minutes(4))      ]);
+	testLex("2013/2/22 07:53-GMT+0004:",     [ testTokenUnknownTimeZone("GMT+0004:")    ]);
+	testLex("2013/2/22 07:53-GMT+0004:3",    [ testTokenUnknownTimeZone("GMT+0004:3")   ]);
+	testLex("2013/2/22 07:53-GMT+0004:03",   [ testTokenUnknownTimeZone("GMT+0004:03")  ]);
+	testLex("2013/2/22 07:53-GMT+0004:003",  [ testTokenUnknownTimeZone("GMT+0004:003") ]);
+
+	testLex("2013/2/22 07:53-GMT+00004",     [ testTokenSimpleTimeZone(minutes(4))       ]);
+	testLex("2013/2/22 07:53-GMT+00004:",    [ testTokenUnknownTimeZone("GMT+00004:")    ]);
+	testLex("2013/2/22 07:53-GMT+00004:3",   [ testTokenUnknownTimeZone("GMT+00004:3")   ]);
+	testLex("2013/2/22 07:53-GMT+00004:03",  [ testTokenUnknownTimeZone("GMT+00004:03")  ]);
+	testLex("2013/2/22 07:53-GMT+00004:003", [ testTokenUnknownTimeZone("GMT+00004:003") ]);
+
+	// DateTime, with unknown timezone
+	testLex( "2013/2/22 07:53-Bogus/Foo",        [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53,  0),   0.msecs, "Bogus/Foo")), "2013/2/22 07:53-Bogus/Foo") ]);
+	testLex("-2013/2/22 07:53-Bogus/Foo",        [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 7, 53,  0),   0.msecs, "Bogus/Foo"))) ]);
+	testLex( "2013/2/22 -07:53-Bogus/Foo",       [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
+	testLex("-2013/2/22 -07:53-Bogus/Foo",       [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
+	testLex( "2013/2/22 07:53:34-Bogus/Foo",     [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34),   0.msecs, "Bogus/Foo"))) ]);
+	testLex( "2013/2/22 07:53:34.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, "Bogus/Foo"))) ]);
+	testLex( "2013/2/22 07:53.123-Bogus/Foo",    [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, "Bogus/Foo"))) ]);
+
+	// Time Span
+	testLex( "12:14:42",         [ Token(symbol!"Value",loc,Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs(  0))) ]);
+	testLex("-12:14:42",         [ Token(symbol!"Value",loc,Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs(  0))) ]);
+	testLex( "00:09:12",         [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs(  0))) ]);
+	testLex( "00:00:01.023",     [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23))) ]);
+	testLex( "23d:05:21:23.532", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532))) ]);
+	testLex( "23d:05:21:23.53",  [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530))) ]);
+	testLex( "23d:05:21:23.5",   [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500))) ]);
+	testLex("-23d:05:21:23.532", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532))) ]);
+	testLex("-23d:05:21:23.5",   [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500))) ]);
+	testLex( "23d:05:21:23",     [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(  0))) ]);
+
+	testLexThrows("12:14:42a");
+	testLexThrows("23d:05:21:23.532a");
+	testLexThrows("23d:05:21:23.532f");
+
+	// Combination
+	testLex("foo. 7", [
+		Token(symbol!"Ident",loc,Value(      null),"foo."),
+		Token(symbol!"Value",loc,Value(cast(int)7))
+	]);
+	
+	testLex(`
+		namespace:person "foo" "bar" 1 23L name.first="ひとみ" name.last="Smith" {
+			namespace:age 37; namespace:favorite_color "blue" // comment
+			somedate 2013/2/22  07:53 -- comment
+			
+			inventory /* comment */ {
+				socks
+			}
+		}
+	`,
+	[
+		Token(symbol!"EOL",loc,Value(null),"\n"),
+
+		Token(symbol!"Ident", loc, Value(         null ), "namespace"),
+		Token(symbol!":",     loc, Value(         null ), ":"),
+		Token(symbol!"Ident", loc, Value(         null ), "person"),
+		Token(symbol!"Value", loc, Value(        "foo" ), `"foo"`),
+		Token(symbol!"Value", loc, Value(        "bar" ), `"bar"`),
+		Token(symbol!"Value", loc, Value( cast( int) 1 ), "1"),
+		Token(symbol!"Value", loc, Value( cast(long)23 ), "23L"),
+		Token(symbol!"Ident", loc, Value(         null ), "name.first"),
+		Token(symbol!"=",     loc, Value(         null ), "="),
+		Token(symbol!"Value", loc, Value(       "ひとみ" ), `"ひとみ"`),
+		Token(symbol!"Ident", loc, Value(         null ), "name.last"),
+		Token(symbol!"=",     loc, Value(         null ), "="),
+		Token(symbol!"Value", loc, Value(      "Smith" ), `"Smith"`),
+		Token(symbol!"{",     loc, Value(         null ), "{"),
+		Token(symbol!"EOL",   loc, Value(         null ), "\n"),
+
+		Token(symbol!"Ident", loc, Value(        null ), "namespace"),
+		Token(symbol!":",     loc, Value(        null ), ":"),
+		Token(symbol!"Ident", loc, Value(        null ), "age"),
+		Token(symbol!"Value", loc, Value( cast(int)37 ), "37"),
+		Token(symbol!"EOL",   loc, Value(        null ), ";"),
+		Token(symbol!"Ident", loc, Value(        null ), "namespace"),
+		Token(symbol!":",     loc, Value(        null ), ":"),
+		Token(symbol!"Ident", loc, Value(        null ), "favorite_color"),
+		Token(symbol!"Value", loc, Value(      "blue" ), `"blue"`),
+		Token(symbol!"EOL",   loc, Value(        null ), "\n"),
+
+		Token(symbol!"Ident", loc, Value( null ), "somedate"),
+		Token(symbol!"Value", loc, Value( DateTimeFrac(DateTime(2013, 2, 22, 7, 53, 0)) ), "2013/2/22  07:53"),
+		Token(symbol!"EOL",   loc, Value( null ), "\n"),
+		Token(symbol!"EOL",   loc, Value( null ), "\n"),
+
+		Token(symbol!"Ident", loc, Value(null), "inventory"),
+		Token(symbol!"{",     loc, Value(null), "{"),
+		Token(symbol!"EOL",   loc, Value(null), "\n"),
+
+		Token(symbol!"Ident", loc, Value(null), "socks"),
+		Token(symbol!"EOL",   loc, Value(null), "\n"),
+
+		Token(symbol!"}",     loc, Value(null), "}"),
+		Token(symbol!"EOL",   loc, Value(null), "\n"),
+
+		Token(symbol!"}",     loc, Value(null), "}"),
+		Token(symbol!"EOL",   loc, Value(null), "\n"),
+	]);
+	
+	if(numErrors > 0)
+		stderr.writeln(numErrors, " failed test(s)");
+}
+
+version(sdlangUnittest)
+unittest
+{
+	writeln("lexer: Regression test issue #8...");
+	stdout.flush();
+
+	testLex(`"\n \n"`, [ Token(symbol!"Value",loc,Value("\n \n"),`"\n \n"`) ]);
+	testLex(`"\t\t"`, [ Token(symbol!"Value",loc,Value("\t\t"),`"\t\t"`) ]);
+	testLex(`"\n\n"`, [ Token(symbol!"Value",loc,Value("\n\n"),`"\n\n"`) ]);
+}
+
+version(sdlangUnittest)
+unittest
+{
+	writeln("lexer: Regression test issue #11...");
+	stdout.flush();
+	
+	void test(string input)
+	{
+		testLex(
+			input,
+			[
+				Token(symbol!"EOL", loc, Value(null), "\n"),
+				Token(symbol!"Ident",loc,Value(null), "a")
+			]
+		);
+	}
+
+	test("//X\na");
+	test("//\na");
+	test("--\na");
+	test("#\na");
+}
+
+version(sdlangUnittest)
+unittest
+{
+	writeln("lexer: Regression test issue #28...");
+	stdout.flush();
+
+	enum offset = 1; // workaround for an of-by-one error for line numbers
+	testLex("test", [
+		Token(symbol!"Ident", Location("filename", 0, 0, 0), Value(null), "test")
+	], true);
+	testLex("\ntest", [
+		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\n"),
+		Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
+	], true);
+	testLex("\rtest", [
+		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
+		Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
+	], true);
+	testLex("\r\ntest", [
+		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
+		Token(symbol!"Ident", Location("filename", 1, 0, 2), Value(null), "test")
+	], true);
+	testLex("\r\n\ntest", [
+		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
+		Token(symbol!"EOL", Location("filename", 1, 0, 2), Value(null), "\n"),
+		Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
+	], true);
+	testLex("\r\r\ntest", [
+		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
+		Token(symbol!"EOL", Location("filename", 1, 0, 1), Value(null), "\r\n"),
+		Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
+	], true);
+}