step5 sdlang used for config files and doc headers

1 files changed, 505 insertions, 0 deletions

diff --git a/src/sdlang/token.d b/src/sdlang/token.d
new file mode 100644
index 0000000..908d4a3
--- /dev/null
+++ b/src/sdlang/token.d
@@ -0,0 +1,505 @@
+// SDLang-D
+// Written in the D programming language.
+
+module sdlang.token;
+
+import std.array;
+import std.base64;
+import std.conv;
+import std.datetime;
+import std.range;
+import std.string;
+import std.typetuple;
+import std.variant;
+
+import sdlang.symbol;
+import sdlang.util;
+
+/// DateTime doesn't support milliseconds, but SDL's "Date Time" type does.
+/// So this is needed for any SDL "Date Time" that doesn't include a time zone.
+struct DateTimeFrac
+{
+	DateTime dateTime;
+	Duration fracSecs;
+	deprecated("Use fracSecs instead.") {
+		@property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); }
+		@property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; }
+	}
+}
+
+/++
+If a "Date Time" literal in the SDL file has a time zone that's not found in
+your system, you get one of these instead of a SysTime. (Because it's
+impossible to indicate "unknown time zone" with 'std.datetime.TimeZone'.)
+
+The difference between this and 'DateTimeFrac' is that 'DateTimeFrac'
+indicates that no time zone was specified in the SDL at all, whereas
+'DateTimeFracUnknownZone' indicates that a time zone was specified but
+data for it could not be found on your system.
++/
+struct DateTimeFracUnknownZone
+{
+	DateTime dateTime;
+	Duration fracSecs;
+	deprecated("Use fracSecs instead.") {
+		@property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); }
+		@property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; }
+	}
+	string timeZone;
+
+	bool opEquals(const DateTimeFracUnknownZone b) const
+	{
+		return opEquals(b);
+	}
+	bool opEquals(ref const DateTimeFracUnknownZone b) const
+	{
+		return
+			this.dateTime == b.dateTime &&
+			this.fracSecs  == b.fracSecs  &&
+			this.timeZone == b.timeZone;
+	}
+}
+
+/++
+SDL's datatypes map to D's datatypes as described below.
+Most are straightforward, but take special note of the date/time-related types.
+
+Boolean:                       bool
+Null:                          typeof(null)
+Unicode Character:             dchar
+Double-Quote Unicode String:   string
+Raw Backtick Unicode String:   string
+Integer (32 bits signed):      int
+Long Integer (64 bits signed): long
+Float (32 bits signed):        float
+Double Float (64 bits signed): double
+Decimal (128+ bits signed):    real
+Binary (standard Base64):      ubyte[]
+Time Span:                     Duration
+
+Date (with no time at all):           Date
+Date Time (no timezone):              DateTimeFrac
+Date Time (with a known timezone):    SysTime
+Date Time (with an unknown timezone): DateTimeFracUnknownZone
++/
+alias TypeTuple!(
+	bool,
+	string, dchar,
+	int, long,
+	float, double, real,
+	Date, DateTimeFrac, SysTime, DateTimeFracUnknownZone, Duration,
+	ubyte[],
+	typeof(null),
+) ValueTypes;
+
+alias Algebraic!( ValueTypes ) Value; ///ditto
+
+template isSDLSink(T)
+{
+	enum isSink =
+		isOutputRange!T &&
+		is(ElementType!(T)[] == string);
+}
+
+string toSDLString(T)(T value) if(
+	is( T : Value        ) ||
+	is( T : bool         ) ||
+	is( T : string       ) ||
+	is( T : dchar        ) ||
+	is( T : int          ) ||
+	is( T : long         ) ||
+	is( T : float        ) ||
+	is( T : double       ) ||
+	is( T : real         ) ||
+	is( T : Date         ) ||
+	is( T : DateTimeFrac ) ||
+	is( T : SysTime      ) ||
+	is( T : DateTimeFracUnknownZone ) ||
+	is( T : Duration     ) ||
+	is( T : ubyte[]      ) ||
+	is( T : typeof(null) )
+)
+{
+	Appender!string sink;
+	toSDLString(value, sink);
+	return sink.data;
+}
+
+void toSDLString(Sink)(Value value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	foreach(T; ValueTypes)
+	{
+		if(value.type == typeid(T))
+		{
+			toSDLString( value.get!T(), sink );
+			return;
+		}
+	}
+	
+	throw new Exception("Internal SDLang-D error: Unhandled type of Value. Contains: "~value.toString());
+}
+
+void toSDLString(Sink)(typeof(null) value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put("null");
+}
+
+void toSDLString(Sink)(bool value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put(value? "true" : "false");
+}
+
+//TODO: Figure out how to properly handle strings/chars containing lineSep or paraSep
+void toSDLString(Sink)(string value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put('"');
+	
+	// This loop is UTF-safe
+	foreach(char ch; value)
+	{
+		if     (ch == '\n') sink.put(`\n`);
+		else if(ch == '\r') sink.put(`\r`);
+		else if(ch == '\t') sink.put(`\t`);
+		else if(ch == '\"') sink.put(`\"`);
+		else if(ch == '\\') sink.put(`\\`);
+		else
+			sink.put(ch);
+	}
+
+	sink.put('"');
+}
+
+void toSDLString(Sink)(dchar value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put('\'');
+	
+	if     (value == '\n') sink.put(`\n`);
+	else if(value == '\r') sink.put(`\r`);
+	else if(value == '\t') sink.put(`\t`);
+	else if(value == '\'') sink.put(`\'`);
+	else if(value == '\\') sink.put(`\\`);
+	else
+		sink.put(value);
+
+	sink.put('\'');
+}
+
+void toSDLString(Sink)(int value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put( "%s".format(value) );
+}
+
+void toSDLString(Sink)(long value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put( "%sL".format(value) );
+}
+
+void toSDLString(Sink)(float value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put( "%.10sF".format(value) );
+}
+
+void toSDLString(Sink)(double value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put( "%.30sD".format(value) );
+}
+
+void toSDLString(Sink)(real value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put( "%.30sBD".format(value) );
+}
+
+void toSDLString(Sink)(Date value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put(to!string(value.year));
+	sink.put('/');
+	sink.put(to!string(cast(int)value.month));
+	sink.put('/');
+	sink.put(to!string(value.day));
+}
+
+void toSDLString(Sink)(DateTimeFrac value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	toSDLString(value.dateTime.date, sink);
+	sink.put(' ');
+	sink.put("%.2s".format(value.dateTime.hour));
+	sink.put(':');
+	sink.put("%.2s".format(value.dateTime.minute));
+	
+	if(value.dateTime.second != 0)
+	{
+		sink.put(':');
+		sink.put("%.2s".format(value.dateTime.second));
+	}
+
+	if(value.fracSecs != 0.msecs)
+	{
+		sink.put('.');
+		sink.put("%.3s".format(value.fracSecs.total!"msecs"));
+	}
+}
+
+void toSDLString(Sink)(SysTime value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	auto dateTimeFrac = DateTimeFrac(cast(DateTime)value, value.fracSecs);
+	toSDLString(dateTimeFrac, sink);
+	
+	sink.put("-");
+	
+	auto tzString = value.timezone.name;
+	
+	// If name didn't exist, try abbreviation.
+	// Note that according to std.datetime docs, on Windows the
+	// stdName/dstName may not be properly abbreviated.
+	version(Windows) {} else
+	if(tzString == "")
+	{
+		auto tz = value.timezone;
+		auto stdTime = value.stdTime;
+		
+		if(tz.hasDST())
+			tzString = tz.dstInEffect(stdTime)? tz.dstName : tz.stdName;
+		else
+			tzString = tz.stdName;
+	}
+	
+	if(tzString == "")
+	{
+		auto offset = value.timezone.utcOffsetAt(value.stdTime);
+		sink.put("GMT");
+
+		if(offset < seconds(0))
+		{
+			sink.put("-");
+			offset = -offset;
+		}
+		else
+			sink.put("+");
+		
+		sink.put("%.2s".format(offset.split.hours));
+		sink.put(":");
+		sink.put("%.2s".format(offset.split.minutes));
+	}
+	else
+		sink.put(tzString);
+}
+
+void toSDLString(Sink)(DateTimeFracUnknownZone value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	auto dateTimeFrac = DateTimeFrac(value.dateTime, value.fracSecs);
+	toSDLString(dateTimeFrac, sink);
+	
+	sink.put("-");
+	sink.put(value.timeZone);
+}
+
+void toSDLString(Sink)(Duration value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	if(value < seconds(0))
+	{
+		sink.put("-");
+		value = -value;
+	}
+	
+	auto days = value.total!"days"();
+	if(days != 0)
+	{
+		sink.put("%s".format(days));
+		sink.put("d:");
+	}
+
+	sink.put("%.2s".format(value.split.hours));
+	sink.put(':');
+	sink.put("%.2s".format(value.split.minutes));
+	sink.put(':');
+	sink.put("%.2s".format(value.split.seconds));
+
+	if(value.split.msecs != 0)
+	{
+		sink.put('.');
+		sink.put("%.3s".format(value.split.msecs));
+	}
+}
+
+void toSDLString(Sink)(ubyte[] value, ref Sink sink) if(isOutputRange!(Sink,char))
+{
+	sink.put('[');
+	sink.put( Base64.encode(value) );
+	sink.put(']');
+}
+
+/// This only represents terminals. Nonterminals aren't
+/// constructed since the AST is directly built during parsing.
+struct Token
+{
+	Symbol symbol = sdlang.symbol.symbol!"Error"; /// The "type" of this token
+	Location location;
+	Value value; /// Only valid when 'symbol' is symbol!"Value", otherwise null
+	string data; /// Original text from source
+
+	@disable this();
+	this(Symbol symbol, Location location, Value value=Value(null), string data=null)
+	{
+		this.symbol   = symbol;
+		this.location = location;
+		this.value    = value;
+		this.data     = data;
+	}
+	
+	/// Tokens with differing symbols are always unequal.
+	/// Tokens with differing values are always unequal.
+	/// Tokens with differing Value types are always unequal.
+	/// Member 'location' is always ignored for comparison.
+	/// Member 'data' is ignored for comparison *EXCEPT* when the symbol is Ident.
+	bool opEquals(Token b)
+	{
+		return opEquals(b);
+	}
+	bool opEquals(ref Token b) ///ditto
+	{
+		if(
+			this.symbol     != b.symbol     ||
+			this.value.type != b.value.type ||
+			this.value      != b.value
+		)
+			return false;
+		
+		if(this.symbol == .symbol!"Ident")
+			return this.data == b.data;
+		
+		return true;
+	}
+	
+	bool matches(string symbolName)()
+	{
+		return this.symbol == .symbol!symbolName;
+	}
+}
+
+version(sdlangUnittest)
+unittest
+{
+	import std.stdio;
+	writeln("Unittesting sdlang token...");
+	stdout.flush();
+	
+	auto loc  = Location("", 0, 0, 0);
+	auto loc2 = Location("a", 1, 1, 1);
+
+	assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc ));
+	assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc2));
+	assert(Token(symbol!":",  loc) == Token(symbol!":",  loc ));
+	assert(Token(symbol!"EOL",loc) != Token(symbol!":",  loc ));
+	assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),"\n"));
+
+	assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),";" ));
+	assert(Token(symbol!"EOL",loc,Value(null),"A" ) == Token(symbol!"EOL",loc,Value(null),"B" ));
+	assert(Token(symbol!":",  loc,Value(null),"A" ) == Token(symbol!":",  loc,Value(null),"BB"));
+	assert(Token(symbol!"EOL",loc,Value(null),"A" ) != Token(symbol!":",  loc,Value(null),"A" ));
+
+	assert(Token(symbol!"Ident",loc,Value(null),"foo") == Token(symbol!"Ident",loc,Value(null),"foo"));
+	assert(Token(symbol!"Ident",loc,Value(null),"foo") != Token(symbol!"Ident",loc,Value(null),"BAR"));
+
+	assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"foo"));
+	assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc2,Value(null),"foo"));
+	assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"BAR"));
+	assert(Token(symbol!"Value",loc,Value(   7),"foo") == Token(symbol!"Value",loc, Value(   7),"BAR"));
+	assert(Token(symbol!"Value",loc,Value(   7),"foo") != Token(symbol!"Value",loc, Value( "A"),"foo"));
+	assert(Token(symbol!"Value",loc,Value(   7),"foo") != Token(symbol!"Value",loc, Value(   2),"foo"));
+	assert(Token(symbol!"Value",loc,Value(cast(int)7)) != Token(symbol!"Value",loc, Value(cast(long)7)));
+	assert(Token(symbol!"Value",loc,Value(cast(float)1.2)) != Token(symbol!"Value",loc, Value(cast(double)1.2)));
+}
+
+version(sdlangUnittest)
+unittest
+{
+	import std.stdio;
+	writeln("Unittesting sdlang Value.toSDLString()...");
+	stdout.flush();
+	
+	// Bool and null
+	assert(Value(null ).toSDLString() == "null");
+	assert(Value(true ).toSDLString() == "true");
+	assert(Value(false).toSDLString() == "false");
+	
+	// Base64 Binary
+	assert(Value(cast(ubyte[])"hello world".dup).toSDLString() == "[aGVsbG8gd29ybGQ=]");
+
+	// Integer
+	assert(Value(cast( int) 7).toSDLString() ==  "7");
+	assert(Value(cast( int)-7).toSDLString() == "-7");
+	assert(Value(cast( int) 0).toSDLString() ==  "0");
+
+	assert(Value(cast(long) 7).toSDLString() ==  "7L");
+	assert(Value(cast(long)-7).toSDLString() == "-7L");
+	assert(Value(cast(long) 0).toSDLString() ==  "0L");
+
+	// Floating point
+	assert(Value(cast(float) 1.5).toSDLString() ==  "1.5F");
+	assert(Value(cast(float)-1.5).toSDLString() == "-1.5F");
+	assert(Value(cast(float)   0).toSDLString() ==    "0F");
+
+	assert(Value(cast(double) 1.5).toSDLString() ==  "1.5D");
+	assert(Value(cast(double)-1.5).toSDLString() == "-1.5D");
+	assert(Value(cast(double)   0).toSDLString() ==    "0D");
+
+	assert(Value(cast(real) 1.5).toSDLString() ==  "1.5BD");
+	assert(Value(cast(real)-1.5).toSDLString() == "-1.5BD");
+	assert(Value(cast(real)   0).toSDLString() ==    "0BD");
+
+	// String
+	assert(Value("hello"  ).toSDLString() == `"hello"`);
+	assert(Value(" hello ").toSDLString() == `" hello "`);
+	assert(Value(""       ).toSDLString() == `""`);
+	assert(Value("hello \r\n\t\"\\ world").toSDLString() == `"hello \r\n\t\"\\ world"`);
+	assert(Value("日本語").toSDLString() == `"日本語"`);
+
+	// Chars
+	assert(Value(cast(dchar) 'A').toSDLString() ==  `'A'`);
+	assert(Value(cast(dchar)'\r').toSDLString() == `'\r'`);
+	assert(Value(cast(dchar)'\n').toSDLString() == `'\n'`);
+	assert(Value(cast(dchar)'\t').toSDLString() == `'\t'`);
+	assert(Value(cast(dchar)'\'').toSDLString() == `'\''`);
+	assert(Value(cast(dchar)'\\').toSDLString() == `'\\'`);
+	assert(Value(cast(dchar) '月').toSDLString() ==  `'月'`);
+
+	// Date
+	assert(Value(Date( 2004,10,31)).toSDLString() == "2004/10/31");
+	assert(Value(Date(-2004,10,31)).toSDLString() == "-2004/10/31");
+
+	// DateTimeFrac w/o Frac
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,  14,30,15))).toSDLString() == "2004/10/31 14:30:15");
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,   1, 2, 3))).toSDLString() == "2004/10/31 01:02:03");
+	assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15))).toSDLString() == "-2004/10/31 14:30:15");
+
+	// DateTimeFrac w/ Frac
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,  14,30,15), 123.msecs)).toSDLString() == "2004/10/31 14:30:15.123");
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,  14,30,15), 120.msecs)).toSDLString() == "2004/10/31 14:30:15.120");
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,  14,30,15), 100.msecs)).toSDLString() == "2004/10/31 14:30:15.100");
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,  14,30,15),  12.msecs)).toSDLString() == "2004/10/31 14:30:15.012");
+	assert(Value(DateTimeFrac(DateTime(2004,10,31,  14,30,15),   1.msecs)).toSDLString() == "2004/10/31 14:30:15.001");
+	assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "-2004/10/31 14:30:15.123");
+
+	// DateTimeFracUnknownZone
+	assert(Value(DateTimeFracUnknownZone(DateTime(2004,10,31, 14,30,15), 123.msecs, "Foo/Bar")).toSDLString() == "2004/10/31 14:30:15.123-Foo/Bar");
+
+	// SysTime
+	assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(0)             ))).toSDLString() == "2004/10/31 14:30:15-GMT+00:00");
+	assert(Value(SysTime(DateTime(2004,10,31,  1, 2, 3), new immutable SimpleTimeZone( hours(0)             ))).toSDLString() == "2004/10/31 01:02:03-GMT+00:00");
+	assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes(10) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:10");
+	assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone(-hours(5)-minutes(30) ))).toSDLString() == "2004/10/31 14:30:15-GMT-05:30");
+	assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes( 3) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:03");
+	assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), 123.msecs, new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15.123-GMT+00:00");
+
+	// Duration
+	assert( "12:14:42"         == Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs(  0)).toSDLString());
+	assert("-12:14:42"         == Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs(  0)).toSDLString());
+	assert( "00:09:12"         == Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs(  0)).toSDLString());
+	assert( "00:00:01.023"     == Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23)).toSDLString());
+	assert( "23d:05:21:23.532" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532)).toSDLString());
+	assert( "23d:05:21:23.530" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530)).toSDLString());
+	assert( "23d:05:21:23.500" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500)).toSDLString());
+	assert("-23d:05:21:23.532" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532)).toSDLString());
+	assert("-23d:05:21:23.500" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500)).toSDLString());
+	assert( "23d:05:21:23"     == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(  0)).toSDLString());
+}