From 8ab7e935913c102fb039110e20b71f698a68c6ee Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Thu, 16 Jun 2016 01:49:06 -0400 Subject: step5 sdlang used for config files and doc headers --- src/sdlang/token.d | 505 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 505 insertions(+) create mode 100644 src/sdlang/token.d (limited to 'src/sdlang/token.d') diff --git a/src/sdlang/token.d b/src/sdlang/token.d new file mode 100644 index 0000000..908d4a3 --- /dev/null +++ b/src/sdlang/token.d @@ -0,0 +1,505 @@ +// SDLang-D +// Written in the D programming language. + +module sdlang.token; + +import std.array; +import std.base64; +import std.conv; +import std.datetime; +import std.range; +import std.string; +import std.typetuple; +import std.variant; + +import sdlang.symbol; +import sdlang.util; + +/// DateTime doesn't support milliseconds, but SDL's "Date Time" type does. +/// So this is needed for any SDL "Date Time" that doesn't include a time zone. +struct DateTimeFrac +{ + DateTime dateTime; + Duration fracSecs; + deprecated("Use fracSecs instead.") { + @property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); } + @property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; } + } +} + +/++ +If a "Date Time" literal in the SDL file has a time zone that's not found in +your system, you get one of these instead of a SysTime. (Because it's +impossible to indicate "unknown time zone" with 'std.datetime.TimeZone'.) + +The difference between this and 'DateTimeFrac' is that 'DateTimeFrac' +indicates that no time zone was specified in the SDL at all, whereas +'DateTimeFracUnknownZone' indicates that a time zone was specified but +data for it could not be found on your system. ++/ +struct DateTimeFracUnknownZone +{ + DateTime dateTime; + Duration fracSecs; + deprecated("Use fracSecs instead.") { + @property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); } + @property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; } + } + string timeZone; + + bool opEquals(const DateTimeFracUnknownZone b) const + { + return opEquals(b); + } + bool opEquals(ref const DateTimeFracUnknownZone b) const + { + return + this.dateTime == b.dateTime && + this.fracSecs == b.fracSecs && + this.timeZone == b.timeZone; + } +} + +/++ +SDL's datatypes map to D's datatypes as described below. +Most are straightforward, but take special note of the date/time-related types. + +Boolean: bool +Null: typeof(null) +Unicode Character: dchar +Double-Quote Unicode String: string +Raw Backtick Unicode String: string +Integer (32 bits signed): int +Long Integer (64 bits signed): long +Float (32 bits signed): float +Double Float (64 bits signed): double +Decimal (128+ bits signed): real +Binary (standard Base64): ubyte[] +Time Span: Duration + +Date (with no time at all): Date +Date Time (no timezone): DateTimeFrac +Date Time (with a known timezone): SysTime +Date Time (with an unknown timezone): DateTimeFracUnknownZone ++/ +alias TypeTuple!( + bool, + string, dchar, + int, long, + float, double, real, + Date, DateTimeFrac, SysTime, DateTimeFracUnknownZone, Duration, + ubyte[], + typeof(null), +) ValueTypes; + +alias Algebraic!( ValueTypes ) Value; ///ditto + +template isSDLSink(T) +{ + enum isSink = + isOutputRange!T && + is(ElementType!(T)[] == string); +} + +string toSDLString(T)(T value) if( + is( T : Value ) || + is( T : bool ) || + is( T : string ) || + is( T : dchar ) || + is( T : int ) || + is( T : long ) || + is( T : float ) || + is( T : double ) || + is( T : real ) || + is( T : Date ) || + is( T : DateTimeFrac ) || + is( T : SysTime ) || + is( T : DateTimeFracUnknownZone ) || + is( T : Duration ) || + is( T : ubyte[] ) || + is( T : typeof(null) ) +) +{ + Appender!string sink; + toSDLString(value, sink); + return sink.data; +} + +void toSDLString(Sink)(Value value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + foreach(T; ValueTypes) + { + if(value.type == typeid(T)) + { + toSDLString( value.get!T(), sink ); + return; + } + } + + throw new Exception("Internal SDLang-D error: Unhandled type of Value. Contains: "~value.toString()); +} + +void toSDLString(Sink)(typeof(null) value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put("null"); +} + +void toSDLString(Sink)(bool value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put(value? "true" : "false"); +} + +//TODO: Figure out how to properly handle strings/chars containing lineSep or paraSep +void toSDLString(Sink)(string value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put('"'); + + // This loop is UTF-safe + foreach(char ch; value) + { + if (ch == '\n') sink.put(`\n`); + else if(ch == '\r') sink.put(`\r`); + else if(ch == '\t') sink.put(`\t`); + else if(ch == '\"') sink.put(`\"`); + else if(ch == '\\') sink.put(`\\`); + else + sink.put(ch); + } + + sink.put('"'); +} + +void toSDLString(Sink)(dchar value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put('\''); + + if (value == '\n') sink.put(`\n`); + else if(value == '\r') sink.put(`\r`); + else if(value == '\t') sink.put(`\t`); + else if(value == '\'') sink.put(`\'`); + else if(value == '\\') sink.put(`\\`); + else + sink.put(value); + + sink.put('\''); +} + +void toSDLString(Sink)(int value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put( "%s".format(value) ); +} + +void toSDLString(Sink)(long value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put( "%sL".format(value) ); +} + +void toSDLString(Sink)(float value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put( "%.10sF".format(value) ); +} + +void toSDLString(Sink)(double value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put( "%.30sD".format(value) ); +} + +void toSDLString(Sink)(real value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put( "%.30sBD".format(value) ); +} + +void toSDLString(Sink)(Date value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put(to!string(value.year)); + sink.put('/'); + sink.put(to!string(cast(int)value.month)); + sink.put('/'); + sink.put(to!string(value.day)); +} + +void toSDLString(Sink)(DateTimeFrac value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + toSDLString(value.dateTime.date, sink); + sink.put(' '); + sink.put("%.2s".format(value.dateTime.hour)); + sink.put(':'); + sink.put("%.2s".format(value.dateTime.minute)); + + if(value.dateTime.second != 0) + { + sink.put(':'); + sink.put("%.2s".format(value.dateTime.second)); + } + + if(value.fracSecs != 0.msecs) + { + sink.put('.'); + sink.put("%.3s".format(value.fracSecs.total!"msecs")); + } +} + +void toSDLString(Sink)(SysTime value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + auto dateTimeFrac = DateTimeFrac(cast(DateTime)value, value.fracSecs); + toSDLString(dateTimeFrac, sink); + + sink.put("-"); + + auto tzString = value.timezone.name; + + // If name didn't exist, try abbreviation. + // Note that according to std.datetime docs, on Windows the + // stdName/dstName may not be properly abbreviated. + version(Windows) {} else + if(tzString == "") + { + auto tz = value.timezone; + auto stdTime = value.stdTime; + + if(tz.hasDST()) + tzString = tz.dstInEffect(stdTime)? tz.dstName : tz.stdName; + else + tzString = tz.stdName; + } + + if(tzString == "") + { + auto offset = value.timezone.utcOffsetAt(value.stdTime); + sink.put("GMT"); + + if(offset < seconds(0)) + { + sink.put("-"); + offset = -offset; + } + else + sink.put("+"); + + sink.put("%.2s".format(offset.split.hours)); + sink.put(":"); + sink.put("%.2s".format(offset.split.minutes)); + } + else + sink.put(tzString); +} + +void toSDLString(Sink)(DateTimeFracUnknownZone value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + auto dateTimeFrac = DateTimeFrac(value.dateTime, value.fracSecs); + toSDLString(dateTimeFrac, sink); + + sink.put("-"); + sink.put(value.timeZone); +} + +void toSDLString(Sink)(Duration value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + if(value < seconds(0)) + { + sink.put("-"); + value = -value; + } + + auto days = value.total!"days"(); + if(days != 0) + { + sink.put("%s".format(days)); + sink.put("d:"); + } + + sink.put("%.2s".format(value.split.hours)); + sink.put(':'); + sink.put("%.2s".format(value.split.minutes)); + sink.put(':'); + sink.put("%.2s".format(value.split.seconds)); + + if(value.split.msecs != 0) + { + sink.put('.'); + sink.put("%.3s".format(value.split.msecs)); + } +} + +void toSDLString(Sink)(ubyte[] value, ref Sink sink) if(isOutputRange!(Sink,char)) +{ + sink.put('['); + sink.put( Base64.encode(value) ); + sink.put(']'); +} + +/// This only represents terminals. Nonterminals aren't +/// constructed since the AST is directly built during parsing. +struct Token +{ + Symbol symbol = sdlang.symbol.symbol!"Error"; /// The "type" of this token + Location location; + Value value; /// Only valid when 'symbol' is symbol!"Value", otherwise null + string data; /// Original text from source + + @disable this(); + this(Symbol symbol, Location location, Value value=Value(null), string data=null) + { + this.symbol = symbol; + this.location = location; + this.value = value; + this.data = data; + } + + /// Tokens with differing symbols are always unequal. + /// Tokens with differing values are always unequal. + /// Tokens with differing Value types are always unequal. + /// Member 'location' is always ignored for comparison. + /// Member 'data' is ignored for comparison *EXCEPT* when the symbol is Ident. + bool opEquals(Token b) + { + return opEquals(b); + } + bool opEquals(ref Token b) ///ditto + { + if( + this.symbol != b.symbol || + this.value.type != b.value.type || + this.value != b.value + ) + return false; + + if(this.symbol == .symbol!"Ident") + return this.data == b.data; + + return true; + } + + bool matches(string symbolName)() + { + return this.symbol == .symbol!symbolName; + } +} + +version(sdlangUnittest) +unittest +{ + import std.stdio; + writeln("Unittesting sdlang token..."); + stdout.flush(); + + auto loc = Location("", 0, 0, 0); + auto loc2 = Location("a", 1, 1, 1); + + assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc )); + assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc2)); + assert(Token(symbol!":", loc) == Token(symbol!":", loc )); + assert(Token(symbol!"EOL",loc) != Token(symbol!":", loc )); + assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),"\n")); + + assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),";" )); + assert(Token(symbol!"EOL",loc,Value(null),"A" ) == Token(symbol!"EOL",loc,Value(null),"B" )); + assert(Token(symbol!":", loc,Value(null),"A" ) == Token(symbol!":", loc,Value(null),"BB")); + assert(Token(symbol!"EOL",loc,Value(null),"A" ) != Token(symbol!":", loc,Value(null),"A" )); + + assert(Token(symbol!"Ident",loc,Value(null),"foo") == Token(symbol!"Ident",loc,Value(null),"foo")); + assert(Token(symbol!"Ident",loc,Value(null),"foo") != Token(symbol!"Ident",loc,Value(null),"BAR")); + + assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"foo")); + assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc2,Value(null),"foo")); + assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"BAR")); + assert(Token(symbol!"Value",loc,Value( 7),"foo") == Token(symbol!"Value",loc, Value( 7),"BAR")); + assert(Token(symbol!"Value",loc,Value( 7),"foo") != Token(symbol!"Value",loc, Value( "A"),"foo")); + assert(Token(symbol!"Value",loc,Value( 7),"foo") != Token(symbol!"Value",loc, Value( 2),"foo")); + assert(Token(symbol!"Value",loc,Value(cast(int)7)) != Token(symbol!"Value",loc, Value(cast(long)7))); + assert(Token(symbol!"Value",loc,Value(cast(float)1.2)) != Token(symbol!"Value",loc, Value(cast(double)1.2))); +} + +version(sdlangUnittest) +unittest +{ + import std.stdio; + writeln("Unittesting sdlang Value.toSDLString()..."); + stdout.flush(); + + // Bool and null + assert(Value(null ).toSDLString() == "null"); + assert(Value(true ).toSDLString() == "true"); + assert(Value(false).toSDLString() == "false"); + + // Base64 Binary + assert(Value(cast(ubyte[])"hello world".dup).toSDLString() == "[aGVsbG8gd29ybGQ=]"); + + // Integer + assert(Value(cast( int) 7).toSDLString() == "7"); + assert(Value(cast( int)-7).toSDLString() == "-7"); + assert(Value(cast( int) 0).toSDLString() == "0"); + + assert(Value(cast(long) 7).toSDLString() == "7L"); + assert(Value(cast(long)-7).toSDLString() == "-7L"); + assert(Value(cast(long) 0).toSDLString() == "0L"); + + // Floating point + assert(Value(cast(float) 1.5).toSDLString() == "1.5F"); + assert(Value(cast(float)-1.5).toSDLString() == "-1.5F"); + assert(Value(cast(float) 0).toSDLString() == "0F"); + + assert(Value(cast(double) 1.5).toSDLString() == "1.5D"); + assert(Value(cast(double)-1.5).toSDLString() == "-1.5D"); + assert(Value(cast(double) 0).toSDLString() == "0D"); + + assert(Value(cast(real) 1.5).toSDLString() == "1.5BD"); + assert(Value(cast(real)-1.5).toSDLString() == "-1.5BD"); + assert(Value(cast(real) 0).toSDLString() == "0BD"); + + // String + assert(Value("hello" ).toSDLString() == `"hello"`); + assert(Value(" hello ").toSDLString() == `" hello "`); + assert(Value("" ).toSDLString() == `""`); + assert(Value("hello \r\n\t\"\\ world").toSDLString() == `"hello \r\n\t\"\\ world"`); + assert(Value("日本語").toSDLString() == `"日本語"`); + + // Chars + assert(Value(cast(dchar) 'A').toSDLString() == `'A'`); + assert(Value(cast(dchar)'\r').toSDLString() == `'\r'`); + assert(Value(cast(dchar)'\n').toSDLString() == `'\n'`); + assert(Value(cast(dchar)'\t').toSDLString() == `'\t'`); + assert(Value(cast(dchar)'\'').toSDLString() == `'\''`); + assert(Value(cast(dchar)'\\').toSDLString() == `'\\'`); + assert(Value(cast(dchar) '月').toSDLString() == `'月'`); + + // Date + assert(Value(Date( 2004,10,31)).toSDLString() == "2004/10/31"); + assert(Value(Date(-2004,10,31)).toSDLString() == "-2004/10/31"); + + // DateTimeFrac w/o Frac + assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15))).toSDLString() == "2004/10/31 14:30:15"); + assert(Value(DateTimeFrac(DateTime(2004,10,31, 1, 2, 3))).toSDLString() == "2004/10/31 01:02:03"); + assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15))).toSDLString() == "-2004/10/31 14:30:15"); + + // DateTimeFrac w/ Frac + assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "2004/10/31 14:30:15.123"); + assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 120.msecs)).toSDLString() == "2004/10/31 14:30:15.120"); + assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 100.msecs)).toSDLString() == "2004/10/31 14:30:15.100"); + assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 12.msecs)).toSDLString() == "2004/10/31 14:30:15.012"); + assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 1.msecs)).toSDLString() == "2004/10/31 14:30:15.001"); + assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "-2004/10/31 14:30:15.123"); + + // DateTimeFracUnknownZone + assert(Value(DateTimeFracUnknownZone(DateTime(2004,10,31, 14,30,15), 123.msecs, "Foo/Bar")).toSDLString() == "2004/10/31 14:30:15.123-Foo/Bar"); + + // SysTime + assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15-GMT+00:00"); + assert(Value(SysTime(DateTime(2004,10,31, 1, 2, 3), new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 01:02:03-GMT+00:00"); + assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes(10) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:10"); + assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone(-hours(5)-minutes(30) ))).toSDLString() == "2004/10/31 14:30:15-GMT-05:30"); + assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes( 3) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:03"); + assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), 123.msecs, new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15.123-GMT+00:00"); + + // Duration + assert( "12:14:42" == Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs( 0)).toSDLString()); + assert("-12:14:42" == Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs( 0)).toSDLString()); + assert( "00:09:12" == Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs( 0)).toSDLString()); + assert( "00:00:01.023" == Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23)).toSDLString()); + assert( "23d:05:21:23.532" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532)).toSDLString()); + assert( "23d:05:21:23.530" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530)).toSDLString()); + assert( "23d:05:21:23.500" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500)).toSDLString()); + assert("-23d:05:21:23.532" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532)).toSDLString()); + assert("-23d:05:21:23.500" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500)).toSDLString()); + assert( "23d:05:21:23" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs( 0)).toSDLString()); +} -- cgit v1.2.3