diff options
Diffstat (limited to 'src/sdlang')
| -rw-r--r-- | src/sdlang/ast.d | 2945 | ||||
| -rw-r--r-- | src/sdlang/dub.json | 38 | ||||
| -rw-r--r-- | src/sdlang/exception.d | 190 | ||||
| -rw-r--r-- | src/sdlang/lexer.d | 2068 | ||||
| -rw-r--r-- | src/sdlang/libinputvisitor/dub.json | 10 | ||||
| -rw-r--r-- | src/sdlang/libinputvisitor/libInputVisitor.d | 113 | ||||
| -rw-r--r-- | src/sdlang/package.d | 133 | ||||
| -rw-r--r-- | src/sdlang/parser.d | 628 | ||||
| -rw-r--r-- | src/sdlang/symbol.d | 61 | ||||
| -rw-r--r-- | src/sdlang/taggedalgebraic/taggedalgebraic.d | 1085 | ||||
| -rw-r--r-- | src/sdlang/token.d | 550 | ||||
| -rw-r--r-- | src/sdlang/util.d | 200 |
12 files changed, 0 insertions, 8021 deletions
diff --git a/src/sdlang/ast.d b/src/sdlang/ast.d deleted file mode 100644 index 87dd0bd..0000000 --- a/src/sdlang/ast.d +++ /dev/null @@ -1,2945 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.ast; - -import std.algorithm; -import std.array; -import std.conv; -import std.range; -import std.string; - -import sdlang.exception; -import sdlang.token; -import sdlang.util; - -class Attribute -{ - Value value; - Location location; - - private Tag _parent; - /// Get parent tag. To set a parent, attach this Attribute to its intended - /// parent tag by calling `Tag.add(...)`, or by passing it to - /// the parent tag's constructor. - @property Tag parent() - { - return _parent; - } - - private string _namespace; - /++ - This tag's namespace. Empty string if no namespace. - - Note that setting this value is O(n) because internal lookup structures - need to be updated. - - Note also, that setting this may change where this tag is ordered among - its parent's list of tags. - +/ - @property string namespace() - { - return _namespace; - } - ///ditto - @property void namespace(string value) - { - if(_parent && _namespace != value) - { - // Remove - auto saveParent = _parent; - if(_parent) - this.remove(); - - // Change namespace - _namespace = value; - - // Re-add - if(saveParent) - saveParent.add(this); - } - else - _namespace = value; - } - - private string _name; - /++ - This attribute's name, not including namespace. - - Use `getFullName().toString` if you want the namespace included. - - Note that setting this value is O(n) because internal lookup structures - need to be updated. - - Note also, that setting this may change where this attribute is ordered - among its parent's list of tags. - +/ - @property string name() - { - return _name; - } - ///ditto - @property void name(string value) - { - if(_parent && _name != value) - { - _parent.updateId++; - - void removeFromGroupedLookup(string ns) - { - // Remove from _parent._attributes[ns] - auto sameNameAttrs = _parent._attributes[ns][_name]; - auto targetIndex = sameNameAttrs.countUntil(this); - _parent._attributes[ns][_name].removeIndex(targetIndex); - } - - // Remove from _parent._tags - removeFromGroupedLookup(_namespace); - removeFromGroupedLookup("*"); - - // Change name - _name = value; - - // Add to new locations in _parent._attributes - _parent._attributes[_namespace][_name] ~= this; - _parent._attributes["*"][_name] ~= this; - } - else - _name = value; - } - - /// This tag's name, including namespace if one exists. - deprecated("Use 'getFullName().toString()'") - @property string fullName() - { - return getFullName().toString(); - } - - /// This tag's name, including namespace if one exists. - FullName getFullName() - { - return FullName(_namespace, _name); - } - - this(string namespace, string name, Value value, Location location = Location(0, 0, 0)) - { - this._namespace = namespace; - this._name = name; - this.location = location; - this.value = value; - } - - this(string name, Value value, Location location = Location(0, 0, 0)) - { - this._namespace = ""; - this._name = name; - this.location = location; - this.value = value; - } - - /// Copy this Attribute. - /// The clone does $(B $(I not)) have a parent, even if the original does. - Attribute clone() - { - return new Attribute(_namespace, _name, value, location); - } - - /// Removes `this` from its parent, if any. Returns `this` for chaining. - /// Inefficient ATM, but it works. - Attribute remove() - { - if(!_parent) - return this; - - void removeFromGroupedLookup(string ns) - { - // Remove from _parent._attributes[ns] - auto sameNameAttrs = _parent._attributes[ns][_name]; - auto targetIndex = sameNameAttrs.countUntil(this); - _parent._attributes[ns][_name].removeIndex(targetIndex); - } - - // Remove from _parent._attributes - removeFromGroupedLookup(_namespace); - removeFromGroupedLookup("*"); - - // Remove from _parent.allAttributes - auto allAttrsIndex = _parent.allAttributes.countUntil(this); - _parent.allAttributes.removeIndex(allAttrsIndex); - - // Remove from _parent.attributeIndicies - auto sameNamespaceAttrs = _parent.attributeIndicies[_namespace]; - auto attrIndiciesIndex = sameNamespaceAttrs.countUntil(allAttrsIndex); - _parent.attributeIndicies[_namespace].removeIndex(attrIndiciesIndex); - - // Fixup other indicies - foreach(ns, ref nsAttrIndicies; _parent.attributeIndicies) - foreach(k, ref v; nsAttrIndicies) - if(v > allAttrsIndex) - v--; - - _parent.removeNamespaceIfEmpty(_namespace); - _parent.updateId++; - _parent = null; - return this; - } - - override bool opEquals(Object o) - { - auto a = cast(Attribute)o; - if(!a) - return false; - - return - _namespace == a._namespace && - _name == a._name && - value == a.value; - } - - string toSDLString()() - { - Appender!string sink; - this.toSDLString(sink); - return sink.data; - } - - void toSDLString(Sink)(ref Sink sink) if(isOutputRange!(Sink,char)) - { - if(_namespace != "") - { - sink.put(_namespace); - sink.put(':'); - } - - sink.put(_name); - sink.put('='); - value.toSDLString(sink); - } -} - -/// Deep-copy an array of Tag or Attribute. -/// The top-level clones are $(B $(I not)) attached to any parent, even if the originals are. -T[] clone(T)(T[] arr) if(is(T==Tag) || is(T==Attribute)) -{ - T[] newArr; - newArr.length = arr.length; - - foreach(i; 0..arr.length) - newArr[i] = arr[i].clone(); - - return newArr; -} - -class Tag -{ - /// File/Line/Column/Index information for where this tag was located in - /// its original SDLang file. - Location location; - - /// Access all this tag's values, as an array of type `sdlang.token.Value`. - Value[] values; - - private Tag _parent; - /// Get parent tag. To set a parent, attach this Tag to its intended - /// parent tag by calling `Tag.add(...)`, or by passing it to - /// the parent tag's constructor. - @property Tag parent() - { - return _parent; - } - - private string _namespace; - /++ - This tag's namespace. Empty string if no namespace. - - Note that setting this value is O(n) because internal lookup structures - need to be updated. - - Note also, that setting this may change where this tag is ordered among - its parent's list of tags. - +/ - @property string namespace() - { - return _namespace; - } - ///ditto - @property void namespace(string value) - { - //TODO: Can we do this in-place, without removing/adding and thus - // modyfying the internal order? - if(_parent && _namespace != value) - { - // Remove - auto saveParent = _parent; - if(_parent) - this.remove(); - - // Change namespace - _namespace = value; - - // Re-add - if(saveParent) - saveParent.add(this); - } - else - _namespace = value; - } - - private string _name; - /++ - This tag's name, not including namespace. - - Use `getFullName().toString` if you want the namespace included. - - Note that setting this value is O(n) because internal lookup structures - need to be updated. - - Note also, that setting this may change where this tag is ordered among - its parent's list of tags. - +/ - @property string name() - { - return _name; - } - ///ditto - @property void name(string value) - { - //TODO: Seriously? Can't we at least do the "*" modification *in-place*? - - if(_parent && _name != value) - { - _parent.updateId++; - - // Not the most efficient, but it works. - void removeFromGroupedLookup(string ns) - { - // Remove from _parent._tags[ns] - auto sameNameTags = _parent._tags[ns][_name]; - auto targetIndex = sameNameTags.countUntil(this); - _parent._tags[ns][_name].removeIndex(targetIndex); - } - - // Remove from _parent._tags - removeFromGroupedLookup(_namespace); - removeFromGroupedLookup("*"); - - // Change name - _name = value; - - // Add to new locations in _parent._tags - //TODO: Can we re-insert while preserving the original order? - _parent._tags[_namespace][_name] ~= this; - _parent._tags["*"][_name] ~= this; - } - else - _name = value; - } - - /// This tag's name, including namespace if one exists. - deprecated("Use 'getFullName().toString()'") - @property string fullName() - { - return getFullName().toString(); - } - - /// This tag's name, including namespace if one exists. - FullName getFullName() - { - return FullName(_namespace, _name); - } - - // Tracks dirtiness. This is incremented every time a change is made which - // could invalidate existing ranges. This way, the ranges can detect when - // they've been invalidated. - private size_t updateId=0; - - this(Tag parent = null) - { - if(parent) - parent.add(this); - } - - this( - string namespace, string name, - Value[] values=null, Attribute[] attributes=null, Tag[] children=null - ) - { - this(null, namespace, name, values, attributes, children); - } - - this( - Tag parent, string namespace, string name, - Value[] values=null, Attribute[] attributes=null, Tag[] children=null - ) - { - this._namespace = namespace; - this._name = name; - - if(parent) - parent.add(this); - - this.values = values; - this.add(attributes); - this.add(children); - } - - /// Deep-copy this Tag. - /// The clone does $(B $(I not)) have a parent, even if the original does. - Tag clone() - { - auto newTag = new Tag(_namespace, _name, values.dup, allAttributes.clone(), allTags.clone()); - newTag.location = location; - return newTag; - } - - private Attribute[] allAttributes; // In same order as specified in SDL file. - private Tag[] allTags; // In same order as specified in SDL file. - private string[] allNamespaces; // In same order as specified in SDL file. - - private size_t[][string] attributeIndicies; // allAttributes[ attributes[namespace][i] ] - private size_t[][string] tagIndicies; // allTags[ tags[namespace][i] ] - - private Attribute[][string][string] _attributes; // attributes[namespace or "*"][name][i] - private Tag[][string][string] _tags; // tags[namespace or "*"][name][i] - - /// Adds a Value, Attribute, Tag (or array of such) as a member/child of this Tag. - /// Returns `this` for chaining. - /// Throws `ValidationException` if trying to add an Attribute or Tag - /// that already has a parent. - Tag add(Value val) - { - values ~= val; - updateId++; - return this; - } - - ///ditto - Tag add(Value[] vals) - { - foreach(val; vals) - add(val); - - return this; - } - - ///ditto - Tag add(Attribute attr) - { - if(attr._parent) - { - throw new ValidationException( - "Attribute is already attached to a parent tag. "~ - "Use Attribute.remove() before adding it to another tag." - ); - } - - if(!allNamespaces.canFind(attr._namespace)) - allNamespaces ~= attr._namespace; - - attr._parent = this; - - allAttributes ~= attr; - attributeIndicies[attr._namespace] ~= allAttributes.length-1; - _attributes[attr._namespace][attr._name] ~= attr; - _attributes["*"] [attr._name] ~= attr; - - updateId++; - return this; - } - - ///ditto - Tag add(Attribute[] attrs) - { - foreach(attr; attrs) - add(attr); - - return this; - } - - ///ditto - Tag add(Tag tag) - { - if(tag._parent) - { - throw new ValidationException( - "Tag is already attached to a parent tag. "~ - "Use Tag.remove() before adding it to another tag." - ); - } - - if(!allNamespaces.canFind(tag._namespace)) - allNamespaces ~= tag._namespace; - - tag._parent = this; - - allTags ~= tag; - tagIndicies[tag._namespace] ~= allTags.length-1; - _tags[tag._namespace][tag._name] ~= tag; - _tags["*"] [tag._name] ~= tag; - - updateId++; - return this; - } - - ///ditto - Tag add(Tag[] tags) - { - foreach(tag; tags) - add(tag); - - return this; - } - - /// Removes `this` from its parent, if any. Returns `this` for chaining. - /// Inefficient ATM, but it works. - Tag remove() - { - if(!_parent) - return this; - - void removeFromGroupedLookup(string ns) - { - // Remove from _parent._tags[ns] - auto sameNameTags = _parent._tags[ns][_name]; - auto targetIndex = sameNameTags.countUntil(this); - _parent._tags[ns][_name].removeIndex(targetIndex); - } - - // Remove from _parent._tags - removeFromGroupedLookup(_namespace); - removeFromGroupedLookup("*"); - - // Remove from _parent.allTags - auto allTagsIndex = _parent.allTags.countUntil(this); - _parent.allTags.removeIndex(allTagsIndex); - - // Remove from _parent.tagIndicies - auto sameNamespaceTags = _parent.tagIndicies[_namespace]; - auto tagIndiciesIndex = sameNamespaceTags.countUntil(allTagsIndex); - _parent.tagIndicies[_namespace].removeIndex(tagIndiciesIndex); - - // Fixup other indicies - foreach(ns, ref nsTagIndicies; _parent.tagIndicies) - foreach(k, ref v; nsTagIndicies) - if(v > allTagsIndex) - v--; - - _parent.removeNamespaceIfEmpty(_namespace); - _parent.updateId++; - _parent = null; - return this; - } - - private void removeNamespaceIfEmpty(string namespace) - { - // If namespace has no attributes, remove it from attributeIndicies/_attributes - if(namespace in attributeIndicies && attributeIndicies[namespace].length == 0) - { - attributeIndicies.remove(namespace); - _attributes.remove(namespace); - } - - // If namespace has no tags, remove it from tagIndicies/_tags - if(namespace in tagIndicies && tagIndicies[namespace].length == 0) - { - tagIndicies.remove(namespace); - _tags.remove(namespace); - } - - // If namespace is now empty, remove it from allNamespaces - if( - namespace !in tagIndicies && - namespace !in attributeIndicies - ) - { - auto allNamespacesIndex = allNamespaces.length - allNamespaces.find(namespace).length; - allNamespaces = allNamespaces[0..allNamespacesIndex] ~ allNamespaces[allNamespacesIndex+1..$]; - } - } - - struct NamedMemberRange(T, string membersGrouped) - { - private Tag tag; - private string namespace; // "*" indicates "all namespaces" (ok since it's not a valid namespace name) - private string name; - private size_t updateId; // Tag's updateId when this range was created. - - this(Tag tag, string namespace, string name, size_t updateId) - { - this.tag = tag; - this.namespace = namespace; - this.name = name; - this.updateId = updateId; - frontIndex = 0; - - if( - tag !is null && - namespace in mixin("tag."~membersGrouped) && - name in mixin("tag."~membersGrouped~"[namespace]") - ) - endIndex = mixin("tag."~membersGrouped~"[namespace][name].length"); - else - endIndex = 0; - } - - invariant() - { - assert( - this.updateId == tag.updateId, - "This range has been invalidated by a change to the tag." - ); - } - - @property bool empty() - { - return tag is null || frontIndex == endIndex; - } - - private size_t frontIndex; - @property T front() - { - return this[0]; - } - void popFront() - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - frontIndex++; - } - - private size_t endIndex; // One past the last element - @property T back() - { - return this[$-1]; - } - void popBack() - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - endIndex--; - } - - alias length opDollar; - @property size_t length() - { - return endIndex - frontIndex; - } - - @property typeof(this) save() - { - auto r = typeof(this)(this.tag, this.namespace, this.name, this.updateId); - r.frontIndex = this.frontIndex; - r.endIndex = this.endIndex; - return r; - } - - typeof(this) opSlice() - { - return save(); - } - - typeof(this) opSlice(size_t start, size_t end) - { - auto r = save(); - r.frontIndex = this.frontIndex + start; - r.endIndex = this.frontIndex + end; - - if( - r.frontIndex > this.endIndex || - r.endIndex > this.endIndex || - r.frontIndex > r.endIndex - ) - throw new DOMRangeException(tag, "Slice out of range"); - - return r; - } - - T opIndex(size_t index) - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - return mixin("tag."~membersGrouped~"[namespace][name][frontIndex+index]"); - } - } - - struct MemberRange(T, string allMembers, string memberIndicies, string membersGrouped) - { - private Tag tag; - private string namespace; // "*" indicates "all namespaces" (ok since it's not a valid namespace name) - private bool isMaybe; - private size_t updateId; // Tag's updateId when this range was created. - private size_t initialEndIndex; - - this(Tag tag, string namespace, bool isMaybe) - { - this.tag = tag; - this.namespace = namespace; - this.updateId = tag.updateId; - this.isMaybe = isMaybe; - frontIndex = 0; - - if(tag is null) - endIndex = 0; - else - { - - if(namespace == "*") - initialEndIndex = mixin("tag."~allMembers~".length"); - else if(namespace in mixin("tag."~memberIndicies)) - initialEndIndex = mixin("tag."~memberIndicies~"[namespace].length"); - else - initialEndIndex = 0; - - endIndex = initialEndIndex; - } - } - - invariant() - { - assert( - this.updateId == tag.updateId, - "This range has been invalidated by a change to the tag." - ); - } - - @property bool empty() - { - return tag is null || frontIndex == endIndex; - } - - private size_t frontIndex; - @property T front() - { - return this[0]; - } - void popFront() - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - frontIndex++; - } - - private size_t endIndex; // One past the last element - @property T back() - { - return this[$-1]; - } - void popBack() - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - endIndex--; - } - - alias length opDollar; - @property size_t length() - { - return endIndex - frontIndex; - } - - @property typeof(this) save() - { - auto r = typeof(this)(this.tag, this.namespace, this.isMaybe); - r.frontIndex = this.frontIndex; - r.endIndex = this.endIndex; - r.initialEndIndex = this.initialEndIndex; - r.updateId = this.updateId; - return r; - } - - typeof(this) opSlice() - { - return save(); - } - - typeof(this) opSlice(size_t start, size_t end) - { - auto r = save(); - r.frontIndex = this.frontIndex + start; - r.endIndex = this.frontIndex + end; - - if( - r.frontIndex > this.endIndex || - r.endIndex > this.endIndex || - r.frontIndex > r.endIndex - ) - throw new DOMRangeException(tag, "Slice out of range"); - - return r; - } - - T opIndex(size_t index) - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - if(namespace == "*") - return mixin("tag."~allMembers~"[ frontIndex+index ]"); - else - return mixin("tag."~allMembers~"[ tag."~memberIndicies~"[namespace][frontIndex+index] ]"); - } - - alias NamedMemberRange!(T,membersGrouped) ThisNamedMemberRange; - ThisNamedMemberRange opIndex(string name) - { - if(frontIndex != 0 || endIndex != initialEndIndex) - { - throw new DOMRangeException(tag, - "Cannot lookup tags/attributes by name on a subset of a range, "~ - "only across the entire tag. "~ - "Please make sure you haven't called popFront or popBack on this "~ - "range and that you aren't using a slice of the range." - ); - } - - if(!isMaybe && empty) - throw new DOMRangeException(tag, "Range is empty"); - - if(!isMaybe && name !in this) - throw new DOMRangeException(tag, `No such `~T.stringof~` named: "`~name~`"`); - - return ThisNamedMemberRange(tag, namespace, name, updateId); - } - - bool opBinaryRight(string op)(string name) if(op=="in") - { - if(frontIndex != 0 || endIndex != initialEndIndex) - { - throw new DOMRangeException(tag, - "Cannot lookup tags/attributes by name on a subset of a range, "~ - "only across the entire tag. "~ - "Please make sure you haven't called popFront or popBack on this "~ - "range and that you aren't using a slice of the range." - ); - } - - if(tag is null) - return false; - - return - namespace in mixin("tag."~membersGrouped) && - name in mixin("tag."~membersGrouped~"[namespace]") && - mixin("tag."~membersGrouped~"[namespace][name].length") > 0; - } - } - - struct NamespaceRange - { - private Tag tag; - private bool isMaybe; - private size_t updateId; // Tag's updateId when this range was created. - - this(Tag tag, bool isMaybe) - { - this.tag = tag; - this.isMaybe = isMaybe; - this.updateId = tag.updateId; - frontIndex = 0; - endIndex = tag.allNamespaces.length; - } - - invariant() - { - assert( - this.updateId == tag.updateId, - "This range has been invalidated by a change to the tag." - ); - } - - @property bool empty() - { - return frontIndex == endIndex; - } - - private size_t frontIndex; - @property NamespaceAccess front() - { - return this[0]; - } - void popFront() - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - frontIndex++; - } - - private size_t endIndex; // One past the last element - @property NamespaceAccess back() - { - return this[$-1]; - } - void popBack() - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - endIndex--; - } - - alias length opDollar; - @property size_t length() - { - return endIndex - frontIndex; - } - - @property NamespaceRange save() - { - auto r = NamespaceRange(this.tag, this.isMaybe); - r.frontIndex = this.frontIndex; - r.endIndex = this.endIndex; - r.updateId = this.updateId; - return r; - } - - typeof(this) opSlice() - { - return save(); - } - - typeof(this) opSlice(size_t start, size_t end) - { - auto r = save(); - r.frontIndex = this.frontIndex + start; - r.endIndex = this.frontIndex + end; - - if( - r.frontIndex > this.endIndex || - r.endIndex > this.endIndex || - r.frontIndex > r.endIndex - ) - throw new DOMRangeException(tag, "Slice out of range"); - - return r; - } - - NamespaceAccess opIndex(size_t index) - { - if(empty) - throw new DOMRangeException(tag, "Range is empty"); - - auto namespace = tag.allNamespaces[frontIndex+index]; - return NamespaceAccess( - namespace, - AttributeRange(tag, namespace, isMaybe), - TagRange(tag, namespace, isMaybe) - ); - } - - NamespaceAccess opIndex(string namespace) - { - if(!isMaybe && empty) - throw new DOMRangeException(tag, "Range is empty"); - - if(!isMaybe && namespace !in this) - throw new DOMRangeException(tag, `No such namespace: "`~namespace~`"`); - - return NamespaceAccess( - namespace, - AttributeRange(tag, namespace, isMaybe), - TagRange(tag, namespace, isMaybe) - ); - } - - /// Inefficient when range is a slice or has used popFront/popBack, but it works. - bool opBinaryRight(string op)(string namespace) if(op=="in") - { - if(frontIndex == 0 && endIndex == tag.allNamespaces.length) - { - return - namespace in tag.attributeIndicies || - namespace in tag.tagIndicies; - } - else - // Slower fallback method - return tag.allNamespaces[frontIndex..endIndex].canFind(namespace); - } - } - - static struct NamespaceAccess - { - string name; - AttributeRange attributes; - TagRange tags; - } - - alias MemberRange!(Attribute, "allAttributes", "attributeIndicies", "_attributes") AttributeRange; - alias MemberRange!(Tag, "allTags", "tagIndicies", "_tags" ) TagRange; - static assert(isRandomAccessRange!AttributeRange); - static assert(isRandomAccessRange!TagRange); - static assert(isRandomAccessRange!NamespaceRange); - - /++ - Access all attributes that don't have a namespace - - Returns a random access range of `Attribute` objects that supports - numeric-indexing, string-indexing, slicing and length. - - Since SDLang allows multiple attributes with the same name, - string-indexing returns a random access range of all attributes - with the given name. - - The string-indexing does $(B $(I not)) support namespace prefixes. - Use `namespace[string]`.`attributes` or `all`.`attributes` for that. - - See $(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md#tag-and-attribute-api-summary, API Overview) - for a high-level overview (and examples) of how to use this. - +/ - @property AttributeRange attributes() - { - return AttributeRange(this, "", false); - } - - /++ - Access all direct-child tags that don't have a namespace. - - Returns a random access range of `Tag` objects that supports - numeric-indexing, string-indexing, slicing and length. - - Since SDLang allows multiple tags with the same name, string-indexing - returns a random access range of all immediate child tags with the - given name. - - The string-indexing does $(B $(I not)) support namespace prefixes. - Use `namespace[string]`.`attributes` or `all`.`attributes` for that. - - See $(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md#tag-and-attribute-api-summary, API Overview) - for a high-level overview (and examples) of how to use this. - +/ - @property TagRange tags() - { - return TagRange(this, "", false); - } - - /++ - Access all namespaces in this tag, and the attributes/tags within them. - - Returns a random access range of `NamespaceAccess` elements that supports - numeric-indexing, string-indexing, slicing and length. - - See $(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md#tag-and-attribute-api-summary, API Overview) - for a high-level overview (and examples) of how to use this. - +/ - @property NamespaceRange namespaces() - { - return NamespaceRange(this, false); - } - - /// Access all attributes and tags regardless of namespace. - /// - /// See $(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md#tag-and-attribute-api-summary, API Overview) - /// for a better understanding (and examples) of how to use this. - @property NamespaceAccess all() - { - // "*" isn't a valid namespace name, so we can use it to indicate "all namespaces" - return NamespaceAccess( - "*", - AttributeRange(this, "*", false), - TagRange(this, "*", false) - ); - } - - struct MaybeAccess - { - Tag tag; - - /// Access all attributes that don't have a namespace - @property AttributeRange attributes() - { - return AttributeRange(tag, "", true); - } - - /// Access all direct-child tags that don't have a namespace - @property TagRange tags() - { - return TagRange(tag, "", true); - } - - /// Access all namespaces in this tag, and the attributes/tags within them. - @property NamespaceRange namespaces() - { - return NamespaceRange(tag, true); - } - - /// Access all attributes and tags regardless of namespace. - @property NamespaceAccess all() - { - // "*" isn't a valid namespace name, so we can use it to indicate "all namespaces" - return NamespaceAccess( - "*", - AttributeRange(tag, "*", true), - TagRange(tag, "*", true) - ); - } - } - - /// Access `attributes`, `tags`, `namespaces` and `all` like normal, - /// except that looking up a non-existant name/namespace with - /// opIndex(string) results in an empty array instead of - /// a thrown `sdlang.exception.DOMRangeException`. - /// - /// See $(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md#tag-and-attribute-api-summary, API Overview) - /// for a more information (and examples) of how to use this. - @property MaybeAccess maybe() - { - return MaybeAccess(this); - } - - // Internal implementations for the get/expect functions further below: - - private Tag getTagImpl(FullName tagFullName, Tag defaultValue=null, bool useDefaultValue=true) - { - auto tagNS = tagFullName.namespace; - auto tagName = tagFullName.name; - - // Can find namespace? - if(tagNS !in _tags) - { - if(useDefaultValue) - return defaultValue; - else - throw new TagNotFoundException(this, tagFullName, "No tags found in namespace '"~namespace~"'"); - } - - // Can find tag in namespace? - if(tagName !in _tags[tagNS] || _tags[tagNS][tagName].length == 0) - { - if(useDefaultValue) - return defaultValue; - else - throw new TagNotFoundException(this, tagFullName, "Can't find tag '"~tagFullName.toString()~"'"); - } - - // Return last matching tag found - return _tags[tagNS][tagName][$-1]; - } - - private T getValueImpl(T)(T defaultValue, bool useDefaultValue=true) - if(isValueType!T) - { - // Find value - foreach(value; this.values) - { - if(value.type == typeid(T)) - return value.get!T(); - } - - // No value of type T found - if(useDefaultValue) - return defaultValue; - else - { - throw new ValueNotFoundException( - this, - FullName(this.namespace, this.name), - typeid(T), - "No value of type "~T.stringof~" found." - ); - } - } - - private T getAttributeImpl(T)(FullName attrFullName, T defaultValue, bool useDefaultValue=true) - if(isValueType!T) - { - auto attrNS = attrFullName.namespace; - auto attrName = attrFullName.name; - - // Can find namespace and attribute name? - if(attrNS !in this._attributes || attrName !in this._attributes[attrNS]) - { - if(useDefaultValue) - return defaultValue; - else - { - throw new AttributeNotFoundException( - this, this.getFullName(), attrFullName, typeid(T), - "Can't find attribute '"~FullName.combine(attrNS, attrName)~"'" - ); - } - } - - // Find value with chosen type - foreach(attr; this._attributes[attrNS][attrName]) - { - if(attr.value.type == typeid(T)) - return attr.value.get!T(); - } - - // Chosen type not found - if(useDefaultValue) - return defaultValue; - else - { - throw new AttributeNotFoundException( - this, this.getFullName(), attrFullName, typeid(T), - "Can't find attribute '"~FullName.combine(attrNS, attrName)~"' of type "~T.stringof - ); - } - } - - // High-level interfaces for get/expect funtions: - - /++ - Lookup a child tag by name. Returns null if not found. - - Useful if you only expect one, and only one, child tag of a given name. - Only looks for immediate child tags of `this`, doesn't search recursively. - - If you expect multiple tags by the same name and want to get them all, - use `maybe`.`tags[string]` instead. - - The name can optionally include a namespace, as in `"namespace:name"`. - Or, you can search all namespaces using `"*:name"`. Use an empty string - to search for anonymous tags, or `"namespace:"` for anonymous tags inside - a namespace. Wildcard searching is only supported for namespaces, not names. - Use `maybe`.`tags[0]` if you don't care about the name. - - If there are multiple tags by the chosen name, the $(B $(I last tag)) will - always be chosen. That is, this function considers later tags with the - same name to override previous ones. - - If the tag cannot be found, and you provides a default value, the default - value is returned. Otherwise null is returned. If you'd prefer an - exception thrown, use `expectTag` instead. - +/ - Tag getTag(string fullTagName, Tag defaultValue=null) - { - auto parsedName = FullName.parse(fullTagName); - parsedName.ensureNoWildcardName( - "Instead, use 'Tag.maybe.tags[0]', 'Tag.maybe.all.tags[0]' or 'Tag.maybe.namespace[ns].tags[0]'." - ); - return getTagImpl(parsedName, defaultValue); - } - - /// - @("Tag.getTag") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo 1 - foo 2 // getTag considers this to override the first foo - - ns1:foo 3 - ns1:foo 4 // getTag considers this to override the first ns1:foo - ns2:foo 33 - ns2:foo 44 // getTag considers this to override the first ns2:foo - `); - assert( root.getTag("foo" ).values[0].get!int() == 2 ); - assert( root.getTag("ns1:foo").values[0].get!int() == 4 ); - assert( root.getTag("*:foo" ).values[0].get!int() == 44 ); // Search all namespaces - - // Not found - // If you'd prefer an exception, use `expectTag` instead. - assert( root.getTag("doesnt-exist") is null ); - - // Default value - auto foo = root.getTag("foo"); - assert( root.getTag("doesnt-exist", foo) is foo ); - } - - /++ - Lookup a child tag by name. Throws if not found. - - Useful if you only expect one, and only one, child tag of a given name. - Only looks for immediate child tags of `this`, doesn't search recursively. - - If you expect multiple tags by the same name and want to get them all, - use `tags[string]` instead. - - The name can optionally include a namespace, as in `"namespace:name"`. - Or, you can search all namespaces using `"*:name"`. Use an empty string - to search for anonymous tags, or `"namespace:"` for anonymous tags inside - a namespace. Wildcard searching is only supported for namespaces, not names. - Use `tags[0]` if you don't care about the name. - - If there are multiple tags by the chosen name, the $(B $(I last tag)) will - always be chosen. That is, this function considers later tags with the - same name to override previous ones. - - If no such tag is found, an `sdlang.exception.TagNotFoundException` will - be thrown. If you'd rather receive a default value, use `getTag` instead. - +/ - Tag expectTag(string fullTagName) - { - auto parsedName = FullName.parse(fullTagName); - parsedName.ensureNoWildcardName( - "Instead, use 'Tag.tags[0]', 'Tag.all.tags[0]' or 'Tag.namespace[ns].tags[0]'." - ); - return getTagImpl(parsedName, null, false); - } - - /// - @("Tag.expectTag") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo 1 - foo 2 // expectTag considers this to override the first foo - - ns1:foo 3 - ns1:foo 4 // expectTag considers this to override the first ns1:foo - ns2:foo 33 - ns2:foo 44 // expectTag considers this to override the first ns2:foo - `); - assert( root.expectTag("foo" ).values[0].get!int() == 2 ); - assert( root.expectTag("ns1:foo").values[0].get!int() == 4 ); - assert( root.expectTag("*:foo" ).values[0].get!int() == 44 ); // Search all namespaces - - // Not found - // If you'd rather receive a default value than an exception, use `getTag` instead. - assertThrown!TagNotFoundException( root.expectTag("doesnt-exist") ); - } - - /++ - Retrieve a value of type T from `this` tag. Returns a default value if not found. - - Useful if you only expect one value of type T from this tag. Only looks for - values of `this` tag, it does not search child tags. If you wish to search - for a value in a child tag (for example, if this current tag is a root tag), - try `getTagValue`. - - If you want to get more than one value from this tag, use `values` instead. - - If this tag has multiple values, the $(B $(I first)) value matching the - requested type will be returned. Ie, Extra values in the tag are ignored. - - You may provide a default value to be returned in case no value of - the requested type can be found. If you don't provide a default value, - `T.init` will be used. - - If you'd rather an exception be thrown when a value cannot be found, - use `expectValue` instead. - +/ - T getValue(T)(T defaultValue = T.init) if(isValueType!T) - { - return getValueImpl!T(defaultValue, true); - } - - /// - @("Tag.getValue") - unittest - { - import std.exception; - import std.math; - import sdlang.parser; - - auto root = parseSource(` - foo 1 true 2 false - `); - auto foo = root.getTag("foo"); - assert( foo.getValue!int() == 1 ); - assert( foo.getValue!bool() == true ); - - // Value found, default value ignored. - assert( foo.getValue!int(999) == 1 ); - - // No strings found - // If you'd prefer an exception, use `expectValue` instead. - assert( foo.getValue!string("Default") == "Default" ); - assert( foo.getValue!string() is null ); - - // No floats found - assert( foo.getValue!float(99.9).approxEqual(99.9) ); - assert( foo.getValue!float().isNaN() ); - } - - /++ - Retrieve a value of type T from `this` tag. Throws if not found. - - Useful if you only expect one value of type T from this tag. Only looks - for values of `this` tag, it does not search child tags. If you wish to - search for a value in a child tag (for example, if this current tag is a - root tag), try `expectTagValue`. - - If you want to get more than one value from this tag, use `values` instead. - - If this tag has multiple values, the $(B $(I first)) value matching the - requested type will be returned. Ie, Extra values in the tag are ignored. - - An `sdlang.exception.ValueNotFoundException` will be thrown if no value of - the requested type can be found. If you'd rather receive a default value, - use `getValue` instead. - +/ - T expectValue(T)() if(isValueType!T) - { - return getValueImpl!T(T.init, false); - } - - /// - @("Tag.expectValue") - unittest - { - import std.exception; - import std.math; - import sdlang.parser; - - auto root = parseSource(` - foo 1 true 2 false - `); - auto foo = root.getTag("foo"); - assert( foo.expectValue!int() == 1 ); - assert( foo.expectValue!bool() == true ); - - // No strings or floats found - // If you'd rather receive a default value than an exception, use `getValue` instead. - assertThrown!ValueNotFoundException( foo.expectValue!string() ); - assertThrown!ValueNotFoundException( foo.expectValue!float() ); - } - - /++ - Lookup a child tag by name, and retrieve a value of type T from it. - Returns a default value if not found. - - Useful if you only expect one value of type T from a given tag. Only looks - for immediate child tags of `this`, doesn't search recursively. - - This is a shortcut for `getTag().getValue()`, except if the tag isn't found, - then instead of a null reference error, it will return the requested - `defaultValue` (or T.init by default). - +/ - T getTagValue(T)(string fullTagName, T defaultValue = T.init) if(isValueType!T) - { - auto tag = getTag(fullTagName); - if(!tag) - return defaultValue; - - return tag.getValue!T(defaultValue); - } - - /// - @("Tag.getTagValue") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo 1 "a" 2 "b" - foo 3 "c" 4 "d" // getTagValue considers this to override the first foo - - bar "hi" - bar 379 // getTagValue considers this to override the first bar - `); - assert( root.getTagValue!int("foo") == 3 ); - assert( root.getTagValue!string("foo") == "c" ); - - // Value found, default value ignored. - assert( root.getTagValue!int("foo", 999) == 3 ); - - // Tag not found - // If you'd prefer an exception, use `expectTagValue` instead. - assert( root.getTagValue!int("doesnt-exist", 999) == 999 ); - assert( root.getTagValue!int("doesnt-exist") == 0 ); - - // The last "bar" tag doesn't have an int (only the first "bar" tag does) - assert( root.getTagValue!string("bar", "Default") == "Default" ); - assert( root.getTagValue!string("bar") is null ); - - // Using namespaces: - root = parseSource(` - ns1:foo 1 "a" 2 "b" - ns1:foo 3 "c" 4 "d" - ns2:foo 11 "aa" 22 "bb" - ns2:foo 33 "cc" 44 "dd" - - ns1:bar "hi" - ns1:bar 379 // getTagValue considers this to override the first bar - `); - assert( root.getTagValue!int("ns1:foo") == 3 ); - assert( root.getTagValue!int("*:foo" ) == 33 ); // Search all namespaces - - assert( root.getTagValue!string("ns1:foo") == "c" ); - assert( root.getTagValue!string("*:foo" ) == "cc" ); // Search all namespaces - - // The last "bar" tag doesn't have a string (only the first "bar" tag does) - assert( root.getTagValue!string("*:bar", "Default") == "Default" ); - assert( root.getTagValue!string("*:bar") is null ); - } - - /++ - Lookup a child tag by name, and retrieve a value of type T from it. - Throws if not found, - - Useful if you only expect one value of type T from a given tag. Only - looks for immediate child tags of `this`, doesn't search recursively. - - This is a shortcut for `expectTag().expectValue()`. - +/ - T expectTagValue(T)(string fullTagName) if(isValueType!T) - { - return expectTag(fullTagName).expectValue!T(); - } - - /// - @("Tag.expectTagValue") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo 1 "a" 2 "b" - foo 3 "c" 4 "d" // expectTagValue considers this to override the first foo - - bar "hi" - bar 379 // expectTagValue considers this to override the first bar - `); - assert( root.expectTagValue!int("foo") == 3 ); - assert( root.expectTagValue!string("foo") == "c" ); - - // The last "bar" tag doesn't have a string (only the first "bar" tag does) - // If you'd rather receive a default value than an exception, use `getTagValue` instead. - assertThrown!ValueNotFoundException( root.expectTagValue!string("bar") ); - - // Tag not found - assertThrown!TagNotFoundException( root.expectTagValue!int("doesnt-exist") ); - - // Using namespaces: - root = parseSource(` - ns1:foo 1 "a" 2 "b" - ns1:foo 3 "c" 4 "d" - ns2:foo 11 "aa" 22 "bb" - ns2:foo 33 "cc" 44 "dd" - - ns1:bar "hi" - ns1:bar 379 // expectTagValue considers this to override the first bar - `); - assert( root.expectTagValue!int("ns1:foo") == 3 ); - assert( root.expectTagValue!int("*:foo" ) == 33 ); // Search all namespaces - - assert( root.expectTagValue!string("ns1:foo") == "c" ); - assert( root.expectTagValue!string("*:foo" ) == "cc" ); // Search all namespaces - - // The last "bar" tag doesn't have a string (only the first "bar" tag does) - assertThrown!ValueNotFoundException( root.expectTagValue!string("*:bar") ); - - // Namespace not found - assertThrown!TagNotFoundException( root.expectTagValue!int("doesnt-exist:bar") ); - } - - /++ - Lookup an attribute of `this` tag by name, and retrieve a value of type T - from it. Returns a default value if not found. - - Useful if you only expect one attribute of the given name and type. - - Only looks for attributes of `this` tag, it does not search child tags. - If you wish to search for a value in a child tag (for example, if this - current tag is a root tag), try `getTagAttribute`. - - If you expect multiple attributes by the same name and want to get them all, - use `maybe`.`attributes[string]` instead. - - The attribute name can optionally include a namespace, as in - `"namespace:name"`. Or, you can search all namespaces using `"*:name"`. - (Note that unlike tags. attributes can't be anonymous - that's what - values are.) Wildcard searching is only supported for namespaces, not names. - Use `maybe`.`attributes[0]` if you don't care about the name. - - If this tag has multiple attributes, the $(B $(I first)) attribute - matching the requested name and type will be returned. Ie, Extra - attributes in the tag are ignored. - - You may provide a default value to be returned in case no attribute of - the requested name and type can be found. If you don't provide a default - value, `T.init` will be used. - - If you'd rather an exception be thrown when an attribute cannot be found, - use `expectAttribute` instead. - +/ - T getAttribute(T)(string fullAttributeName, T defaultValue = T.init) if(isValueType!T) - { - auto parsedName = FullName.parse(fullAttributeName); - parsedName.ensureNoWildcardName( - "Instead, use 'Attribute.maybe.tags[0]', 'Attribute.maybe.all.tags[0]' or 'Attribute.maybe.namespace[ns].tags[0]'." - ); - return getAttributeImpl!T(parsedName, defaultValue); - } - - /// - @("Tag.getAttribute") - unittest - { - import std.exception; - import std.math; - import sdlang.parser; - - auto root = parseSource(` - foo z=0 X=1 X=true X=2 X=false - `); - auto foo = root.getTag("foo"); - assert( foo.getAttribute!int("X") == 1 ); - assert( foo.getAttribute!bool("X") == true ); - - // Value found, default value ignored. - assert( foo.getAttribute!int("X", 999) == 1 ); - - // Attribute name not found - // If you'd prefer an exception, use `expectValue` instead. - assert( foo.getAttribute!int("doesnt-exist", 999) == 999 ); - assert( foo.getAttribute!int("doesnt-exist") == 0 ); - - // No strings found - assert( foo.getAttribute!string("X", "Default") == "Default" ); - assert( foo.getAttribute!string("X") is null ); - - // No floats found - assert( foo.getAttribute!float("X", 99.9).approxEqual(99.9) ); - assert( foo.getAttribute!float("X").isNaN() ); - - - // Using namespaces: - root = parseSource(` - foo ns1:z=0 ns1:X=1 ns1:X=2 ns2:X=3 ns2:X=4 - `); - foo = root.getTag("foo"); - assert( foo.getAttribute!int("ns2:X") == 3 ); - assert( foo.getAttribute!int("*:X") == 1 ); // Search all namespaces - - // Namespace not found - assert( foo.getAttribute!int("doesnt-exist:X", 999) == 999 ); - - // No attribute X is in the default namespace - assert( foo.getAttribute!int("X", 999) == 999 ); - - // Attribute name not found - assert( foo.getAttribute!int("ns1:doesnt-exist", 999) == 999 ); - } - - /++ - Lookup an attribute of `this` tag by name, and retrieve a value of type T - from it. Throws if not found. - - Useful if you only expect one attribute of the given name and type. - - Only looks for attributes of `this` tag, it does not search child tags. - If you wish to search for a value in a child tag (for example, if this - current tag is a root tag), try `expectTagAttribute`. - - If you expect multiple attributes by the same name and want to get them all, - use `attributes[string]` instead. - - The attribute name can optionally include a namespace, as in - `"namespace:name"`. Or, you can search all namespaces using `"*:name"`. - (Note that unlike tags. attributes can't be anonymous - that's what - values are.) Wildcard searching is only supported for namespaces, not names. - Use `attributes[0]` if you don't care about the name. - - If this tag has multiple attributes, the $(B $(I first)) attribute - matching the requested name and type will be returned. Ie, Extra - attributes in the tag are ignored. - - An `sdlang.exception.AttributeNotFoundException` will be thrown if no - value of the requested type can be found. If you'd rather receive a - default value, use `getAttribute` instead. - +/ - T expectAttribute(T)(string fullAttributeName) if(isValueType!T) - { - auto parsedName = FullName.parse(fullAttributeName); - parsedName.ensureNoWildcardName( - "Instead, use 'Attribute.tags[0]', 'Attribute.all.tags[0]' or 'Attribute.namespace[ns].tags[0]'." - ); - return getAttributeImpl!T(parsedName, T.init, false); - } - - /// - @("Tag.expectAttribute") - unittest - { - import std.exception; - import std.math; - import sdlang.parser; - - auto root = parseSource(` - foo z=0 X=1 X=true X=2 X=false - `); - auto foo = root.getTag("foo"); - assert( foo.expectAttribute!int("X") == 1 ); - assert( foo.expectAttribute!bool("X") == true ); - - // Attribute name not found - // If you'd rather receive a default value than an exception, use `getAttribute` instead. - assertThrown!AttributeNotFoundException( foo.expectAttribute!int("doesnt-exist") ); - - // No strings found - assertThrown!AttributeNotFoundException( foo.expectAttribute!string("X") ); - - // No floats found - assertThrown!AttributeNotFoundException( foo.expectAttribute!float("X") ); - - - // Using namespaces: - root = parseSource(` - foo ns1:z=0 ns1:X=1 ns1:X=2 ns2:X=3 ns2:X=4 - `); - foo = root.getTag("foo"); - assert( foo.expectAttribute!int("ns2:X") == 3 ); - assert( foo.expectAttribute!int("*:X") == 1 ); // Search all namespaces - - // Namespace not found - assertThrown!AttributeNotFoundException( foo.expectAttribute!int("doesnt-exist:X") ); - - // No attribute X is in the default namespace - assertThrown!AttributeNotFoundException( foo.expectAttribute!int("X") ); - - // Attribute name not found - assertThrown!AttributeNotFoundException( foo.expectAttribute!int("ns1:doesnt-exist") ); - } - - /++ - Lookup a child tag and attribute by name, and retrieve a value of type T - from it. Returns a default value if not found. - - Useful if you only expect one attribute of type T from given - the tag and attribute names. Only looks for immediate child tags of - `this`, doesn't search recursively. - - This is a shortcut for `getTag().getAttribute()`, except if the tag isn't - found, then instead of a null reference error, it will return the requested - `defaultValue` (or T.init by default). - +/ - T getTagAttribute(T)(string fullTagName, string fullAttributeName, T defaultValue = T.init) if(isValueType!T) - { - auto tag = getTag(fullTagName); - if(!tag) - return defaultValue; - - return tag.getAttribute!T(fullAttributeName, defaultValue); - } - - /// - @("Tag.getTagAttribute") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo X=1 X="a" X=2 X="b" - foo X=3 X="c" X=4 X="d" // getTagAttribute considers this to override the first foo - - bar X="hi" - bar X=379 // getTagAttribute considers this to override the first bar - `); - assert( root.getTagAttribute!int("foo", "X") == 3 ); - assert( root.getTagAttribute!string("foo", "X") == "c" ); - - // Value found, default value ignored. - assert( root.getTagAttribute!int("foo", "X", 999) == 3 ); - - // Tag not found - // If you'd prefer an exception, use `expectTagAttribute` instead of `getTagAttribute` - assert( root.getTagAttribute!int("doesnt-exist", "X", 999) == 999 ); - assert( root.getTagAttribute!int("doesnt-exist", "X") == 0 ); - assert( root.getTagAttribute!int("foo", "doesnt-exist", 999) == 999 ); - assert( root.getTagAttribute!int("foo", "doesnt-exist") == 0 ); - - // The last "bar" tag doesn't have a string (only the first "bar" tag does) - assert( root.getTagAttribute!string("bar", "X", "Default") == "Default" ); - assert( root.getTagAttribute!string("bar", "X") is null ); - - - // Using namespaces: - root = parseSource(` - ns1:foo X=1 X="a" X=2 X="b" - ns1:foo X=3 X="c" X=4 X="d" - ns2:foo X=11 X="aa" X=22 X="bb" - ns2:foo X=33 X="cc" X=44 X="dd" - - ns1:bar attrNS:X="hi" - ns1:bar attrNS:X=379 // getTagAttribute considers this to override the first bar - `); - assert( root.getTagAttribute!int("ns1:foo", "X") == 3 ); - assert( root.getTagAttribute!int("*:foo", "X") == 33 ); // Search all namespaces - - assert( root.getTagAttribute!string("ns1:foo", "X") == "c" ); - assert( root.getTagAttribute!string("*:foo", "X") == "cc" ); // Search all namespaces - - // bar's attribute X is't in the default namespace - assert( root.getTagAttribute!int("*:bar", "X", 999) == 999 ); - assert( root.getTagAttribute!int("*:bar", "X") == 0 ); - - // The last "bar" tag's "attrNS:X" attribute doesn't have a string (only the first "bar" tag does) - assert( root.getTagAttribute!string("*:bar", "attrNS:X", "Default") == "Default" ); - assert( root.getTagAttribute!string("*:bar", "attrNS:X") is null); - } - - /++ - Lookup a child tag and attribute by name, and retrieve a value of type T - from it. Throws if not found. - - Useful if you only expect one attribute of type T from given - the tag and attribute names. Only looks for immediate child tags of - `this`, doesn't search recursively. - - This is a shortcut for `expectTag().expectAttribute()`. - +/ - T expectTagAttribute(T)(string fullTagName, string fullAttributeName) if(isValueType!T) - { - return expectTag(fullTagName).expectAttribute!T(fullAttributeName); - } - - /// - @("Tag.expectTagAttribute") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo X=1 X="a" X=2 X="b" - foo X=3 X="c" X=4 X="d" // expectTagAttribute considers this to override the first foo - - bar X="hi" - bar X=379 // expectTagAttribute considers this to override the first bar - `); - assert( root.expectTagAttribute!int("foo", "X") == 3 ); - assert( root.expectTagAttribute!string("foo", "X") == "c" ); - - // The last "bar" tag doesn't have an int attribute named "X" (only the first "bar" tag does) - // If you'd rather receive a default value than an exception, use `getAttribute` instead. - assertThrown!AttributeNotFoundException( root.expectTagAttribute!string("bar", "X") ); - - // Tag not found - assertThrown!TagNotFoundException( root.expectTagAttribute!int("doesnt-exist", "X") ); - - // Using namespaces: - root = parseSource(` - ns1:foo X=1 X="a" X=2 X="b" - ns1:foo X=3 X="c" X=4 X="d" - ns2:foo X=11 X="aa" X=22 X="bb" - ns2:foo X=33 X="cc" X=44 X="dd" - - ns1:bar attrNS:X="hi" - ns1:bar attrNS:X=379 // expectTagAttribute considers this to override the first bar - `); - assert( root.expectTagAttribute!int("ns1:foo", "X") == 3 ); - assert( root.expectTagAttribute!int("*:foo", "X") == 33 ); // Search all namespaces - - assert( root.expectTagAttribute!string("ns1:foo", "X") == "c" ); - assert( root.expectTagAttribute!string("*:foo", "X") == "cc" ); // Search all namespaces - - // bar's attribute X is't in the default namespace - assertThrown!AttributeNotFoundException( root.expectTagAttribute!int("*:bar", "X") ); - - // The last "bar" tag's "attrNS:X" attribute doesn't have a string (only the first "bar" tag does) - assertThrown!AttributeNotFoundException( root.expectTagAttribute!string("*:bar", "attrNS:X") ); - - // Tag's namespace not found - assertThrown!TagNotFoundException( root.expectTagAttribute!int("doesnt-exist:bar", "attrNS:X") ); - } - - /++ - Lookup a child tag by name, and retrieve all values from it. - - This just like using `getTag()`.`values`, except if the tag isn't found, - it safely returns null (or an optional array of default values) instead of - a dereferencing null error. - - Note that, unlike `getValue`, this doesn't discriminate by the value's - type. It simply returns all values of a single tag as a `Value[]`. - - If you'd prefer an exception thrown when the tag isn't found, use - `expectTag`.`values` instead. - +/ - Value[] getTagValues(string fullTagName, Value[] defaultValues = null) - { - auto tag = getTag(fullTagName); - if(tag) - return tag.values; - else - return defaultValues; - } - - /// - @("getTagValues") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo 1 "a" 2 "b" - foo 3 "c" 4 "d" // getTagValues considers this to override the first foo - `); - assert( root.getTagValues("foo") == [Value(3), Value("c"), Value(4), Value("d")] ); - - // Tag not found - // If you'd prefer an exception, use `expectTag.values` instead. - assert( root.getTagValues("doesnt-exist") is null ); - assert( root.getTagValues("doesnt-exist", [ Value(999), Value("Not found") ]) == - [ Value(999), Value("Not found") ] ); - } - - /++ - Lookup a child tag by name, and retrieve all attributes in a chosen - (or default) namespace from it. - - This just like using `getTag()`.`attributes` (or - `getTag()`.`namespace[...]`.`attributes`, or `getTag()`.`all`.`attributes`), - except if the tag isn't found, it safely returns an empty range instead - of a dereferencing null error. - - If provided, the `attributeNamespace` parameter can be either the name of - a namespace, or an empty string for the default namespace (the default), - or `"*"` to retreive attributes from all namespaces. - - Note that, unlike `getAttributes`, this doesn't discriminate by the - value's type. It simply returns the usual `attributes` range. - - If you'd prefer an exception thrown when the tag isn't found, use - `expectTag`.`attributes` instead. - +/ - auto getTagAttributes(string fullTagName, string attributeNamespace = null) - { - auto tag = getTag(fullTagName); - if(tag) - { - if(attributeNamespace && attributeNamespace in tag.namespaces) - return tag.namespaces[attributeNamespace].attributes; - else if(attributeNamespace == "*") - return tag.all.attributes; - else - return tag.attributes; - } - - return AttributeRange(null, null, false); - } - - /// - @("getTagAttributes") - unittest - { - import std.exception; - import sdlang.parser; - - auto root = parseSource(` - foo X=1 X=2 - - // getTagAttributes considers this to override the first foo - foo X1=3 X2="c" namespace:bar=7 X3=4 X4="d" - `); - - auto fooAttrs = root.getTagAttributes("foo"); - assert( !fooAttrs.empty ); - assert( fooAttrs.length == 4 ); - assert( fooAttrs[0].name == "X1" && fooAttrs[0].value == Value(3) ); - assert( fooAttrs[1].name == "X2" && fooAttrs[1].value == Value("c") ); - assert( fooAttrs[2].name == "X3" && fooAttrs[2].value == Value(4) ); - assert( fooAttrs[3].name == "X4" && fooAttrs[3].value == Value("d") ); - - fooAttrs = root.getTagAttributes("foo", "namespace"); - assert( !fooAttrs.empty ); - assert( fooAttrs.length == 1 ); - assert( fooAttrs[0].name == "bar" && fooAttrs[0].value == Value(7) ); - - fooAttrs = root.getTagAttributes("foo", "*"); - assert( !fooAttrs.empty ); - assert( fooAttrs.length == 5 ); - assert( fooAttrs[0].name == "X1" && fooAttrs[0].value == Value(3) ); - assert( fooAttrs[1].name == "X2" && fooAttrs[1].value == Value("c") ); - assert( fooAttrs[2].name == "bar" && fooAttrs[2].value == Value(7) ); - assert( fooAttrs[3].name == "X3" && fooAttrs[3].value == Value(4) ); - assert( fooAttrs[4].name == "X4" && fooAttrs[4].value == Value("d") ); - - // Tag not found - // If you'd prefer an exception, use `expectTag.attributes` instead. - assert( root.getTagValues("doesnt-exist").empty ); - } - - @("*: Disallow wildcards for names") - unittest - { - import std.exception; - import std.math; - import sdlang.parser; - - auto root = parseSource(` - foo 1 X=2 - ns:foo 3 ns:X=4 - `); - auto foo = root.getTag("foo"); - auto nsfoo = root.getTag("ns:foo"); - - // Sanity check - assert( foo !is null ); - assert( foo.name == "foo" ); - assert( foo.namespace == "" ); - - assert( nsfoo !is null ); - assert( nsfoo.name == "foo" ); - assert( nsfoo.namespace == "ns" ); - - assert( foo.getValue !int() == 1 ); - assert( foo.expectValue !int() == 1 ); - assert( nsfoo.getValue !int() == 3 ); - assert( nsfoo.expectValue!int() == 3 ); - - assert( root.getTagValue !int("foo") == 1 ); - assert( root.expectTagValue!int("foo") == 1 ); - assert( root.getTagValue !int("ns:foo") == 3 ); - assert( root.expectTagValue!int("ns:foo") == 3 ); - - assert( foo.getAttribute !int("X") == 2 ); - assert( foo.expectAttribute !int("X") == 2 ); - assert( nsfoo.getAttribute !int("ns:X") == 4 ); - assert( nsfoo.expectAttribute!int("ns:X") == 4 ); - - assert( root.getTagAttribute !int("foo", "X") == 2 ); - assert( root.expectTagAttribute!int("foo", "X") == 2 ); - assert( root.getTagAttribute !int("ns:foo", "ns:X") == 4 ); - assert( root.expectTagAttribute!int("ns:foo", "ns:X") == 4 ); - - // No namespace - assertThrown!ArgumentException( root.getTag ("*") ); - assertThrown!ArgumentException( root.expectTag("*") ); - - assertThrown!ArgumentException( root.getTagValue !int("*") ); - assertThrown!ArgumentException( root.expectTagValue!int("*") ); - - assertThrown!ArgumentException( foo.getAttribute !int("*") ); - assertThrown!ArgumentException( foo.expectAttribute !int("*") ); - assertThrown!ArgumentException( root.getTagAttribute !int("*", "X") ); - assertThrown!ArgumentException( root.expectTagAttribute!int("*", "X") ); - assertThrown!ArgumentException( root.getTagAttribute !int("foo", "*") ); - assertThrown!ArgumentException( root.expectTagAttribute!int("foo", "*") ); - - // With namespace - assertThrown!ArgumentException( root.getTag ("ns:*") ); - assertThrown!ArgumentException( root.expectTag("ns:*") ); - - assertThrown!ArgumentException( root.getTagValue !int("ns:*") ); - assertThrown!ArgumentException( root.expectTagValue!int("ns:*") ); - - assertThrown!ArgumentException( nsfoo.getAttribute !int("ns:*") ); - assertThrown!ArgumentException( nsfoo.expectAttribute !int("ns:*") ); - assertThrown!ArgumentException( root.getTagAttribute !int("ns:*", "ns:X") ); - assertThrown!ArgumentException( root.expectTagAttribute!int("ns:*", "ns:X") ); - assertThrown!ArgumentException( root.getTagAttribute !int("ns:foo", "ns:*") ); - assertThrown!ArgumentException( root.expectTagAttribute!int("ns:foo", "ns:*") ); - - // With wildcard namespace - assertThrown!ArgumentException( root.getTag ("*:*") ); - assertThrown!ArgumentException( root.expectTag("*:*") ); - - assertThrown!ArgumentException( root.getTagValue !int("*:*") ); - assertThrown!ArgumentException( root.expectTagValue!int("*:*") ); - - assertThrown!ArgumentException( nsfoo.getAttribute !int("*:*") ); - assertThrown!ArgumentException( nsfoo.expectAttribute !int("*:*") ); - assertThrown!ArgumentException( root.getTagAttribute !int("*:*", "*:X") ); - assertThrown!ArgumentException( root.expectTagAttribute!int("*:*", "*:X") ); - assertThrown!ArgumentException( root.getTagAttribute !int("*:foo", "*:*") ); - assertThrown!ArgumentException( root.expectTagAttribute!int("*:foo", "*:*") ); - } - - override bool opEquals(Object o) - { - auto t = cast(Tag)o; - if(!t) - return false; - - if(_namespace != t._namespace || _name != t._name) - return false; - - if( - values .length != t.values .length || - allAttributes .length != t.allAttributes.length || - allNamespaces .length != t.allNamespaces.length || - allTags .length != t.allTags .length - ) - return false; - - if(values != t.values) - return false; - - if(allNamespaces != t.allNamespaces) - return false; - - if(allAttributes != t.allAttributes) - return false; - - // Ok because cycles are not allowed - //TODO: Actually check for or prevent cycles. - return allTags == t.allTags; - } - - /// Treats `this` as the root tag. Note that root tags cannot have - /// values or attributes, and cannot be part of a namespace. - /// If this isn't a valid root tag, `sdlang.exception.ValidationException` - /// will be thrown. - string toSDLDocument()(string indent="\t", int indentLevel=0) - { - Appender!string sink; - toSDLDocument(sink, indent, indentLevel); - return sink.data; - } - - ///ditto - void toSDLDocument(Sink)(ref Sink sink, string indent="\t", int indentLevel=0) - if(isOutputRange!(Sink,char)) - { - if(values.length > 0) - throw new ValidationException("Root tags cannot have any values, only child tags."); - - if(allAttributes.length > 0) - throw new ValidationException("Root tags cannot have any attributes, only child tags."); - - if(_namespace != "") - throw new ValidationException("Root tags cannot have a namespace."); - - foreach(tag; allTags) - tag.toSDLString(sink, indent, indentLevel); - } - - /// Output this entire tag in SDL format. Does $(B $(I not)) treat `this` as - /// a root tag. If you intend this to be the root of a standard SDL - /// document, use `toSDLDocument` instead. - string toSDLString()(string indent="\t", int indentLevel=0) - { - Appender!string sink; - toSDLString(sink, indent, indentLevel); - return sink.data; - } - - ///ditto - void toSDLString(Sink)(ref Sink sink, string indent="\t", int indentLevel=0) - if(isOutputRange!(Sink,char)) - { - if(_name == "" && values.length == 0) - throw new ValidationException("Anonymous tags must have at least one value."); - - if(_name == "" && _namespace != "") - throw new ValidationException("Anonymous tags cannot have a namespace."); - - // Indent - foreach(i; 0..indentLevel) - sink.put(indent); - - // Name - if(_namespace != "") - { - sink.put(_namespace); - sink.put(':'); - } - sink.put(_name); - - // Values - foreach(i, v; values) - { - // Omit the first space for anonymous tags - if(_name != "" || i > 0) - sink.put(' '); - - v.toSDLString(sink); - } - - // Attributes - foreach(attr; allAttributes) - { - sink.put(' '); - attr.toSDLString(sink); - } - - // Child tags - bool foundChild=false; - foreach(tag; allTags) - { - if(!foundChild) - { - sink.put(" {\n"); - foundChild = true; - } - - tag.toSDLString(sink, indent, indentLevel+1); - } - if(foundChild) - { - foreach(i; 0..indentLevel) - sink.put(indent); - - sink.put("}\n"); - } - else - sink.put("\n"); - } - - /// Outputs full information on the tag. - string toDebugString() - { - import std.algorithm : sort; - - Appender!string buf; - - buf.put("\n"); - buf.put("Tag "); - if(_namespace != "") - { - buf.put("["); - buf.put(_namespace); - buf.put("]"); - } - buf.put("'%s':\n".format(_name)); - - // Values - foreach(val; values) - buf.put(" (%s): %s\n".format(.toString(val.type), val)); - - // Attributes - foreach(attrNamespace; _attributes.keys.sort()) - if(attrNamespace != "*") - foreach(attrName; _attributes[attrNamespace].keys.sort()) - foreach(attr; _attributes[attrNamespace][attrName]) - { - string namespaceStr; - if(attr._namespace != "") - namespaceStr = "["~attr._namespace~"]"; - - buf.put( - " %s%s(%s): %s\n".format( - namespaceStr, attr._name, .toString(attr.value.type), attr.value - ) - ); - } - - // Children - foreach(tagNamespace; _tags.keys.sort()) - if(tagNamespace != "*") - foreach(tagName; _tags[tagNamespace].keys.sort()) - foreach(tag; _tags[tagNamespace][tagName]) - buf.put( tag.toDebugString().replace("\n", "\n ") ); - - return buf.data; - } -} - -version(unittest) -{ - private void testRandomAccessRange(R, E)(R range, E[] expected, bool function(E, E) equals=null) - { - static assert(isRandomAccessRange!R); - static assert(is(ElementType!R == E)); - static assert(hasLength!R); - static assert(!isInfinite!R); - - assert(range.length == expected.length); - if(range.length == 0) - { - assert(range.empty); - return; - } - - static bool defaultEquals(E e1, E e2) - { - return e1 == e2; - } - if(equals is null) - equals = &defaultEquals; - - assert(equals(range.front, expected[0])); - assert(equals(range.front, expected[0])); // Ensure consistent result from '.front' - assert(equals(range.front, expected[0])); // Ensure consistent result from '.front' - - assert(equals(range.back, expected[$-1])); - assert(equals(range.back, expected[$-1])); // Ensure consistent result from '.back' - assert(equals(range.back, expected[$-1])); // Ensure consistent result from '.back' - - // Forward iteration - auto original = range.save; - auto r2 = range.save; - foreach(i; 0..expected.length) - { - //trace("Forward iteration: ", i); - - // Test length/empty - assert(range.length == expected.length - i); - assert(range.length == r2.length); - assert(!range.empty); - assert(!r2.empty); - - // Test front - assert(equals(range.front, expected[i])); - assert(equals(range.front, r2.front)); - - // Test back - assert(equals(range.back, expected[$-1])); - assert(equals(range.back, r2.back)); - - // Test opIndex(0) - assert(equals(range[0], expected[i])); - assert(equals(range[0], r2[0])); - - // Test opIndex($-1) - assert(equals(range[$-1], expected[$-1])); - assert(equals(range[$-1], r2[$-1])); - - // Test popFront - range.popFront(); - assert(range.length == r2.length - 1); - r2.popFront(); - assert(range.length == r2.length); - } - assert(range.empty); - assert(r2.empty); - assert(original.length == expected.length); - - // Backwards iteration - range = original.save; - r2 = original.save; - foreach(i; iota(0, expected.length).retro()) - { - //trace("Backwards iteration: ", i); - - // Test length/empty - assert(range.length == i+1); - assert(range.length == r2.length); - assert(!range.empty); - assert(!r2.empty); - - // Test front - assert(equals(range.front, expected[0])); - assert(equals(range.front, r2.front)); - - // Test back - assert(equals(range.back, expected[i])); - assert(equals(range.back, r2.back)); - - // Test opIndex(0) - assert(equals(range[0], expected[0])); - assert(equals(range[0], r2[0])); - - // Test opIndex($-1) - assert(equals(range[$-1], expected[i])); - assert(equals(range[$-1], r2[$-1])); - - // Test popBack - range.popBack(); - assert(range.length == r2.length - 1); - r2.popBack(); - assert(range.length == r2.length); - } - assert(range.empty); - assert(r2.empty); - assert(original.length == expected.length); - - // Random access - range = original.save; - r2 = original.save; - foreach(i; 0..expected.length) - { - //trace("Random access: ", i); - - // Test length/empty - assert(range.length == expected.length); - assert(range.length == r2.length); - assert(!range.empty); - assert(!r2.empty); - - // Test front - assert(equals(range.front, expected[0])); - assert(equals(range.front, r2.front)); - - // Test back - assert(equals(range.back, expected[$-1])); - assert(equals(range.back, r2.back)); - - // Test opIndex(i) - assert(equals(range[i], expected[i])); - assert(equals(range[i], r2[i])); - } - assert(!range.empty); - assert(!r2.empty); - assert(original.length == expected.length); - } -} - -@("*: Test sdlang ast") -unittest -{ - import std.exception; - import sdlang.parser; - - Tag root; - root = parseSource(""); - testRandomAccessRange(root.attributes, cast( Attribute[])[]); - testRandomAccessRange(root.tags, cast( Tag[])[]); - testRandomAccessRange(root.namespaces, cast(Tag.NamespaceAccess[])[]); - - root = parseSource(` - blue 3 "Lee" isThree=true - blue 5 "Chan" 12345 isThree=false - stuff:orange 1 2 3 2 1 - stuff:square points=4 dimensions=2 points="Still four" - stuff:triangle data:points=3 data:dimensions=2 - nothing - namespaces small:A=1 med:A=2 big:A=3 small:B=10 big:B=30 - - people visitor:a=1 b=2 { - chiyo "Small" "Flies?" nemesis="Car" score=100 - yukari - visitor:sana - tomo - visitor:hayama - } - `); - - auto blue3 = new Tag( - null, "", "blue", - [ Value(3), Value("Lee") ], - [ new Attribute("isThree", Value(true)) ], - null - ); - auto blue5 = new Tag( - null, "", "blue", - [ Value(5), Value("Chan"), Value(12345) ], - [ new Attribute("isThree", Value(false)) ], - null - ); - auto orange = new Tag( - null, "stuff", "orange", - [ Value(1), Value(2), Value(3), Value(2), Value(1) ], - null, - null - ); - auto square = new Tag( - null, "stuff", "square", - null, - [ - new Attribute("points", Value(4)), - new Attribute("dimensions", Value(2)), - new Attribute("points", Value("Still four")), - ], - null - ); - auto triangle = new Tag( - null, "stuff", "triangle", - null, - [ - new Attribute("data", "points", Value(3)), - new Attribute("data", "dimensions", Value(2)), - ], - null - ); - auto nothing = new Tag( - null, "", "nothing", - null, null, null - ); - auto namespaces = new Tag( - null, "", "namespaces", - null, - [ - new Attribute("small", "A", Value(1)), - new Attribute("med", "A", Value(2)), - new Attribute("big", "A", Value(3)), - new Attribute("small", "B", Value(10)), - new Attribute("big", "B", Value(30)), - ], - null - ); - auto chiyo = new Tag( - null, "", "chiyo", - [ Value("Small"), Value("Flies?") ], - [ - new Attribute("nemesis", Value("Car")), - new Attribute("score", Value(100)), - ], - null - ); - auto chiyo_ = new Tag( - null, "", "chiyo_", - [ Value("Small"), Value("Flies?") ], - [ - new Attribute("nemesis", Value("Car")), - new Attribute("score", Value(100)), - ], - null - ); - auto yukari = new Tag( - null, "", "yukari", - null, null, null - ); - auto sana = new Tag( - null, "visitor", "sana", - null, null, null - ); - auto sana_ = new Tag( - null, "visitor", "sana_", - null, null, null - ); - auto sanaVisitor_ = new Tag( - null, "visitor_", "sana_", - null, null, null - ); - auto tomo = new Tag( - null, "", "tomo", - null, null, null - ); - auto hayama = new Tag( - null, "visitor", "hayama", - null, null, null - ); - auto people = new Tag( - null, "", "people", - null, - [ - new Attribute("visitor", "a", Value(1)), - new Attribute("b", Value(2)), - ], - [chiyo, yukari, sana, tomo, hayama] - ); - - assert(blue3 .opEquals( blue3 )); - assert(blue5 .opEquals( blue5 )); - assert(orange .opEquals( orange )); - assert(square .opEquals( square )); - assert(triangle .opEquals( triangle )); - assert(nothing .opEquals( nothing )); - assert(namespaces .opEquals( namespaces )); - assert(people .opEquals( people )); - assert(chiyo .opEquals( chiyo )); - assert(yukari .opEquals( yukari )); - assert(sana .opEquals( sana )); - assert(tomo .opEquals( tomo )); - assert(hayama .opEquals( hayama )); - - assert(!blue3.opEquals(orange)); - assert(!blue3.opEquals(people)); - assert(!blue3.opEquals(sana)); - assert(!blue3.opEquals(blue5)); - assert(!blue5.opEquals(blue3)); - - alias Tag.NamespaceAccess NSA; - static bool namespaceEquals(NSA n1, NSA n2) - { - return n1.name == n2.name; - } - - testRandomAccessRange(root.attributes, cast(Attribute[])[]); - testRandomAccessRange(root.tags, [blue3, blue5, nothing, namespaces, people]); - testRandomAccessRange(root.namespaces, [NSA(""), NSA("stuff")], &namespaceEquals); - testRandomAccessRange(root.namespaces[0].tags, [blue3, blue5, nothing, namespaces, people]); - testRandomAccessRange(root.namespaces[1].tags, [orange, square, triangle]); - assert("" in root.namespaces); - assert("stuff" in root.namespaces); - assert("foobar" !in root.namespaces); - testRandomAccessRange(root.namespaces[ ""].tags, [blue3, blue5, nothing, namespaces, people]); - testRandomAccessRange(root.namespaces["stuff"].tags, [orange, square, triangle]); - testRandomAccessRange(root.all.attributes, cast(Attribute[])[]); - testRandomAccessRange(root.all.tags, [blue3, blue5, orange, square, triangle, nothing, namespaces, people]); - testRandomAccessRange(root.all.tags[], [blue3, blue5, orange, square, triangle, nothing, namespaces, people]); - testRandomAccessRange(root.all.tags[3..6], [square, triangle, nothing]); - assert("blue" in root.tags); - assert("nothing" in root.tags); - assert("people" in root.tags); - assert("orange" !in root.tags); - assert("square" !in root.tags); - assert("foobar" !in root.tags); - assert("blue" in root.all.tags); - assert("nothing" in root.all.tags); - assert("people" in root.all.tags); - assert("orange" in root.all.tags); - assert("square" in root.all.tags); - assert("foobar" !in root.all.tags); - assert("orange" in root.namespaces["stuff"].tags); - assert("square" in root.namespaces["stuff"].tags); - assert("square" in root.namespaces["stuff"].tags); - assert("foobar" !in root.attributes); - assert("foobar" !in root.all.attributes); - assert("foobar" !in root.namespaces["stuff"].attributes); - assert("blue" !in root.attributes); - assert("blue" !in root.all.attributes); - assert("blue" !in root.namespaces["stuff"].attributes); - testRandomAccessRange(root.tags["nothing"], [nothing]); - testRandomAccessRange(root.tags["blue"], [blue3, blue5]); - testRandomAccessRange(root.namespaces["stuff"].tags["orange"], [orange]); - testRandomAccessRange(root.all.tags["nothing"], [nothing]); - testRandomAccessRange(root.all.tags["blue"], [blue3, blue5]); - testRandomAccessRange(root.all.tags["orange"], [orange]); - - assertThrown!DOMRangeException(root.tags["foobar"]); - assertThrown!DOMRangeException(root.all.tags["foobar"]); - assertThrown!DOMRangeException(root.attributes["foobar"]); - assertThrown!DOMRangeException(root.all.attributes["foobar"]); - - // DMD Issue #12585 causes a segfault in these two tests when using 2.064 or 2.065, - // so work around it. - //assertThrown!DOMRangeException(root.namespaces["foobar"].tags["foobar"]); - //assertThrown!DOMRangeException(root.namespaces["foobar"].attributes["foobar"]); - bool didCatch = false; - try - auto x = root.namespaces["foobar"].tags["foobar"]; - catch(DOMRangeException e) - didCatch = true; - assert(didCatch); - - didCatch = false; - try - auto x = root.namespaces["foobar"].attributes["foobar"]; - catch(DOMRangeException e) - didCatch = true; - assert(didCatch); - - testRandomAccessRange(root.maybe.tags["nothing"], [nothing]); - testRandomAccessRange(root.maybe.tags["blue"], [blue3, blue5]); - testRandomAccessRange(root.maybe.namespaces["stuff"].tags["orange"], [orange]); - testRandomAccessRange(root.maybe.all.tags["nothing"], [nothing]); - testRandomAccessRange(root.maybe.all.tags["blue"], [blue3, blue5]); - testRandomAccessRange(root.maybe.all.tags["blue"][], [blue3, blue5]); - testRandomAccessRange(root.maybe.all.tags["blue"][0..1], [blue3]); - testRandomAccessRange(root.maybe.all.tags["blue"][1..2], [blue5]); - testRandomAccessRange(root.maybe.all.tags["orange"], [orange]); - testRandomAccessRange(root.maybe.tags["foobar"], cast(Tag[])[]); - testRandomAccessRange(root.maybe.all.tags["foobar"], cast(Tag[])[]); - testRandomAccessRange(root.maybe.namespaces["foobar"].tags["foobar"], cast(Tag[])[]); - testRandomAccessRange(root.maybe.attributes["foobar"], cast(Attribute[])[]); - testRandomAccessRange(root.maybe.all.attributes["foobar"], cast(Attribute[])[]); - testRandomAccessRange(root.maybe.namespaces["foobar"].attributes["foobar"], cast(Attribute[])[]); - - testRandomAccessRange(blue3.attributes, [ new Attribute("isThree", Value(true)) ]); - testRandomAccessRange(blue3.tags, cast(Tag[])[]); - testRandomAccessRange(blue3.namespaces, [NSA("")], &namespaceEquals); - testRandomAccessRange(blue3.all.attributes, [ new Attribute("isThree", Value(true)) ]); - testRandomAccessRange(blue3.all.tags, cast(Tag[])[]); - - testRandomAccessRange(blue5.attributes, [ new Attribute("isThree", Value(false)) ]); - testRandomAccessRange(blue5.tags, cast(Tag[])[]); - testRandomAccessRange(blue5.namespaces, [NSA("")], &namespaceEquals); - testRandomAccessRange(blue5.all.attributes, [ new Attribute("isThree", Value(false)) ]); - testRandomAccessRange(blue5.all.tags, cast(Tag[])[]); - - testRandomAccessRange(orange.attributes, cast(Attribute[])[]); - testRandomAccessRange(orange.tags, cast(Tag[])[]); - testRandomAccessRange(orange.namespaces, cast(NSA[])[], &namespaceEquals); - testRandomAccessRange(orange.all.attributes, cast(Attribute[])[]); - testRandomAccessRange(orange.all.tags, cast(Tag[])[]); - - testRandomAccessRange(square.attributes, [ - new Attribute("points", Value(4)), - new Attribute("dimensions", Value(2)), - new Attribute("points", Value("Still four")), - ]); - testRandomAccessRange(square.tags, cast(Tag[])[]); - testRandomAccessRange(square.namespaces, [NSA("")], &namespaceEquals); - testRandomAccessRange(square.all.attributes, [ - new Attribute("points", Value(4)), - new Attribute("dimensions", Value(2)), - new Attribute("points", Value("Still four")), - ]); - testRandomAccessRange(square.all.tags, cast(Tag[])[]); - - testRandomAccessRange(triangle.attributes, cast(Attribute[])[]); - testRandomAccessRange(triangle.tags, cast(Tag[])[]); - testRandomAccessRange(triangle.namespaces, [NSA("data")], &namespaceEquals); - testRandomAccessRange(triangle.namespaces[0].attributes, [ - new Attribute("data", "points", Value(3)), - new Attribute("data", "dimensions", Value(2)), - ]); - assert("data" in triangle.namespaces); - assert("foobar" !in triangle.namespaces); - testRandomAccessRange(triangle.namespaces["data"].attributes, [ - new Attribute("data", "points", Value(3)), - new Attribute("data", "dimensions", Value(2)), - ]); - testRandomAccessRange(triangle.all.attributes, [ - new Attribute("data", "points", Value(3)), - new Attribute("data", "dimensions", Value(2)), - ]); - testRandomAccessRange(triangle.all.tags, cast(Tag[])[]); - - testRandomAccessRange(nothing.attributes, cast(Attribute[])[]); - testRandomAccessRange(nothing.tags, cast(Tag[])[]); - testRandomAccessRange(nothing.namespaces, cast(NSA[])[], &namespaceEquals); - testRandomAccessRange(nothing.all.attributes, cast(Attribute[])[]); - testRandomAccessRange(nothing.all.tags, cast(Tag[])[]); - - testRandomAccessRange(namespaces.attributes, cast(Attribute[])[]); - testRandomAccessRange(namespaces.tags, cast(Tag[])[]); - testRandomAccessRange(namespaces.namespaces, [NSA("small"), NSA("med"), NSA("big")], &namespaceEquals); - testRandomAccessRange(namespaces.namespaces[], [NSA("small"), NSA("med"), NSA("big")], &namespaceEquals); - testRandomAccessRange(namespaces.namespaces[1..2], [NSA("med")], &namespaceEquals); - testRandomAccessRange(namespaces.namespaces[0].attributes, [ - new Attribute("small", "A", Value(1)), - new Attribute("small", "B", Value(10)), - ]); - testRandomAccessRange(namespaces.namespaces[1].attributes, [ - new Attribute("med", "A", Value(2)), - ]); - testRandomAccessRange(namespaces.namespaces[2].attributes, [ - new Attribute("big", "A", Value(3)), - new Attribute("big", "B", Value(30)), - ]); - testRandomAccessRange(namespaces.namespaces[1..2][0].attributes, [ - new Attribute("med", "A", Value(2)), - ]); - assert("small" in namespaces.namespaces); - assert("med" in namespaces.namespaces); - assert("big" in namespaces.namespaces); - assert("foobar" !in namespaces.namespaces); - assert("small" !in namespaces.namespaces[1..2]); - assert("med" in namespaces.namespaces[1..2]); - assert("big" !in namespaces.namespaces[1..2]); - assert("foobar" !in namespaces.namespaces[1..2]); - testRandomAccessRange(namespaces.namespaces["small"].attributes, [ - new Attribute("small", "A", Value(1)), - new Attribute("small", "B", Value(10)), - ]); - testRandomAccessRange(namespaces.namespaces["med"].attributes, [ - new Attribute("med", "A", Value(2)), - ]); - testRandomAccessRange(namespaces.namespaces["big"].attributes, [ - new Attribute("big", "A", Value(3)), - new Attribute("big", "B", Value(30)), - ]); - testRandomAccessRange(namespaces.all.attributes, [ - new Attribute("small", "A", Value(1)), - new Attribute("med", "A", Value(2)), - new Attribute("big", "A", Value(3)), - new Attribute("small", "B", Value(10)), - new Attribute("big", "B", Value(30)), - ]); - testRandomAccessRange(namespaces.all.attributes[], [ - new Attribute("small", "A", Value(1)), - new Attribute("med", "A", Value(2)), - new Attribute("big", "A", Value(3)), - new Attribute("small", "B", Value(10)), - new Attribute("big", "B", Value(30)), - ]); - testRandomAccessRange(namespaces.all.attributes[2..4], [ - new Attribute("big", "A", Value(3)), - new Attribute("small", "B", Value(10)), - ]); - testRandomAccessRange(namespaces.all.tags, cast(Tag[])[]); - assert("A" !in namespaces.attributes); - assert("B" !in namespaces.attributes); - assert("foobar" !in namespaces.attributes); - assert("A" in namespaces.all.attributes); - assert("B" in namespaces.all.attributes); - assert("foobar" !in namespaces.all.attributes); - assert("A" in namespaces.namespaces["small"].attributes); - assert("B" in namespaces.namespaces["small"].attributes); - assert("foobar" !in namespaces.namespaces["small"].attributes); - assert("A" in namespaces.namespaces["med"].attributes); - assert("B" !in namespaces.namespaces["med"].attributes); - assert("foobar" !in namespaces.namespaces["med"].attributes); - assert("A" in namespaces.namespaces["big"].attributes); - assert("B" in namespaces.namespaces["big"].attributes); - assert("foobar" !in namespaces.namespaces["big"].attributes); - assert("foobar" !in namespaces.tags); - assert("foobar" !in namespaces.all.tags); - assert("foobar" !in namespaces.namespaces["small"].tags); - assert("A" !in namespaces.tags); - assert("A" !in namespaces.all.tags); - assert("A" !in namespaces.namespaces["small"].tags); - testRandomAccessRange(namespaces.namespaces["small"].attributes["A"], [ - new Attribute("small", "A", Value(1)), - ]); - testRandomAccessRange(namespaces.namespaces["med"].attributes["A"], [ - new Attribute("med", "A", Value(2)), - ]); - testRandomAccessRange(namespaces.namespaces["big"].attributes["A"], [ - new Attribute("big", "A", Value(3)), - ]); - testRandomAccessRange(namespaces.all.attributes["A"], [ - new Attribute("small", "A", Value(1)), - new Attribute("med", "A", Value(2)), - new Attribute("big", "A", Value(3)), - ]); - testRandomAccessRange(namespaces.all.attributes["B"], [ - new Attribute("small", "B", Value(10)), - new Attribute("big", "B", Value(30)), - ]); - - testRandomAccessRange(chiyo.attributes, [ - new Attribute("nemesis", Value("Car")), - new Attribute("score", Value(100)), - ]); - testRandomAccessRange(chiyo.tags, cast(Tag[])[]); - testRandomAccessRange(chiyo.namespaces, [NSA("")], &namespaceEquals); - testRandomAccessRange(chiyo.all.attributes, [ - new Attribute("nemesis", Value("Car")), - new Attribute("score", Value(100)), - ]); - testRandomAccessRange(chiyo.all.tags, cast(Tag[])[]); - - testRandomAccessRange(yukari.attributes, cast(Attribute[])[]); - testRandomAccessRange(yukari.tags, cast(Tag[])[]); - testRandomAccessRange(yukari.namespaces, cast(NSA[])[], &namespaceEquals); - testRandomAccessRange(yukari.all.attributes, cast(Attribute[])[]); - testRandomAccessRange(yukari.all.tags, cast(Tag[])[]); - - testRandomAccessRange(sana.attributes, cast(Attribute[])[]); - testRandomAccessRange(sana.tags, cast(Tag[])[]); - testRandomAccessRange(sana.namespaces, cast(NSA[])[], &namespaceEquals); - testRandomAccessRange(sana.all.attributes, cast(Attribute[])[]); - testRandomAccessRange(sana.all.tags, cast(Tag[])[]); - - testRandomAccessRange(people.attributes, [new Attribute("b", Value(2))]); - testRandomAccessRange(people.tags, [chiyo, yukari, tomo]); - testRandomAccessRange(people.namespaces, [NSA("visitor"), NSA("")], &namespaceEquals); - testRandomAccessRange(people.namespaces[0].attributes, [new Attribute("visitor", "a", Value(1))]); - testRandomAccessRange(people.namespaces[1].attributes, [new Attribute("b", Value(2))]); - testRandomAccessRange(people.namespaces[0].tags, [sana, hayama]); - testRandomAccessRange(people.namespaces[1].tags, [chiyo, yukari, tomo]); - assert("visitor" in people.namespaces); - assert("" in people.namespaces); - assert("foobar" !in people.namespaces); - testRandomAccessRange(people.namespaces["visitor"].attributes, [new Attribute("visitor", "a", Value(1))]); - testRandomAccessRange(people.namespaces[ ""].attributes, [new Attribute("b", Value(2))]); - testRandomAccessRange(people.namespaces["visitor"].tags, [sana, hayama]); - testRandomAccessRange(people.namespaces[ ""].tags, [chiyo, yukari, tomo]); - testRandomAccessRange(people.all.attributes, [ - new Attribute("visitor", "a", Value(1)), - new Attribute("b", Value(2)), - ]); - testRandomAccessRange(people.all.tags, [chiyo, yukari, sana, tomo, hayama]); - - people.attributes["b"][0].name = "b_"; - people.namespaces["visitor"].attributes["a"][0].name = "a_"; - people.tags["chiyo"][0].name = "chiyo_"; - people.namespaces["visitor"].tags["sana"][0].name = "sana_"; - - assert("b_" in people.attributes); - assert("a_" in people.namespaces["visitor"].attributes); - assert("chiyo_" in people.tags); - assert("sana_" in people.namespaces["visitor"].tags); - - assert(people.attributes["b_"][0] == new Attribute("b_", Value(2))); - assert(people.namespaces["visitor"].attributes["a_"][0] == new Attribute("visitor", "a_", Value(1))); - assert(people.tags["chiyo_"][0] == chiyo_); - assert(people.namespaces["visitor"].tags["sana_"][0] == sana_); - - assert("b" !in people.attributes); - assert("a" !in people.namespaces["visitor"].attributes); - assert("chiyo" !in people.tags); - assert("sana" !in people.namespaces["visitor"].tags); - - assert(people.maybe.attributes["b"].length == 0); - assert(people.maybe.namespaces["visitor"].attributes["a"].length == 0); - assert(people.maybe.tags["chiyo"].length == 0); - assert(people.maybe.namespaces["visitor"].tags["sana"].length == 0); - - people.tags["tomo"][0].remove(); - people.namespaces["visitor"].tags["hayama"][0].remove(); - people.tags["chiyo_"][0].remove(); - testRandomAccessRange(people.tags, [yukari]); - testRandomAccessRange(people.namespaces, [NSA("visitor"), NSA("")], &namespaceEquals); - testRandomAccessRange(people.namespaces[0].tags, [sana_]); - testRandomAccessRange(people.namespaces[1].tags, [yukari]); - assert("visitor" in people.namespaces); - assert("" in people.namespaces); - assert("foobar" !in people.namespaces); - testRandomAccessRange(people.namespaces["visitor"].tags, [sana_]); - testRandomAccessRange(people.namespaces[ ""].tags, [yukari]); - testRandomAccessRange(people.all.tags, [yukari, sana_]); - - people.attributes["b_"][0].namespace = "_"; - people.namespaces["visitor"].attributes["a_"][0].namespace = "visitor_"; - assert("_" in people.namespaces); - assert("visitor_" in people.namespaces); - assert("" in people.namespaces); - assert("visitor" in people.namespaces); - people.namespaces["visitor"].tags["sana_"][0].namespace = "visitor_"; - assert("_" in people.namespaces); - assert("visitor_" in people.namespaces); - assert("" in people.namespaces); - assert("visitor" !in people.namespaces); - - assert(people.namespaces["_" ].attributes["b_"][0] == new Attribute("_", "b_", Value(2))); - assert(people.namespaces["visitor_"].attributes["a_"][0] == new Attribute("visitor_", "a_", Value(1))); - assert(people.namespaces["visitor_"].tags["sana_"][0] == sanaVisitor_); - - people.tags["yukari"][0].remove(); - people.namespaces["visitor_"].tags["sana_"][0].remove(); - people.namespaces["visitor_"].attributes["a_"][0].namespace = "visitor"; - people.namespaces["_"].attributes["b_"][0].namespace = ""; - testRandomAccessRange(people.tags, cast(Tag[])[]); - testRandomAccessRange(people.namespaces, [NSA("visitor"), NSA("")], &namespaceEquals); - testRandomAccessRange(people.namespaces[0].tags, cast(Tag[])[]); - testRandomAccessRange(people.namespaces[1].tags, cast(Tag[])[]); - assert("visitor" in people.namespaces); - assert("" in people.namespaces); - assert("foobar" !in people.namespaces); - testRandomAccessRange(people.namespaces["visitor"].tags, cast(Tag[])[]); - testRandomAccessRange(people.namespaces[ ""].tags, cast(Tag[])[]); - testRandomAccessRange(people.all.tags, cast(Tag[])[]); - - people.namespaces["visitor"].attributes["a_"][0].remove(); - testRandomAccessRange(people.attributes, [new Attribute("b_", Value(2))]); - testRandomAccessRange(people.namespaces, [NSA("")], &namespaceEquals); - testRandomAccessRange(people.namespaces[0].attributes, [new Attribute("b_", Value(2))]); - assert("visitor" !in people.namespaces); - assert("" in people.namespaces); - assert("foobar" !in people.namespaces); - testRandomAccessRange(people.namespaces[""].attributes, [new Attribute("b_", Value(2))]); - testRandomAccessRange(people.all.attributes, [ - new Attribute("b_", Value(2)), - ]); - - people.attributes["b_"][0].remove(); - testRandomAccessRange(people.attributes, cast(Attribute[])[]); - testRandomAccessRange(people.namespaces, cast(NSA[])[], &namespaceEquals); - assert("visitor" !in people.namespaces); - assert("" !in people.namespaces); - assert("foobar" !in people.namespaces); - testRandomAccessRange(people.all.attributes, cast(Attribute[])[]); - - // Test clone() - auto rootClone = root.clone(); - assert(rootClone !is root); - assert(rootClone.parent is null); - assert(rootClone.name == root.name); - assert(rootClone.namespace == root.namespace); - assert(rootClone.location == root.location); - assert(rootClone.values == root.values); - assert(rootClone.toSDLDocument() == root.toSDLDocument()); - - auto peopleClone = people.clone(); - assert(peopleClone !is people); - assert(peopleClone.parent is null); - assert(peopleClone.name == people.name); - assert(peopleClone.namespace == people.namespace); - assert(peopleClone.location == people.location); - assert(peopleClone.values == people.values); - assert(peopleClone.toSDLString() == people.toSDLString()); -} - -// Regression test, issue #11: https://github.com/Abscissa/SDLang-D/issues/11 -@("*: Regression test issue #11") -unittest -{ - import sdlang.parser; - - auto root = parseSource( -`// -a`); - - assert("a" in root.tags); - - root = parseSource( -`// -parent { - child -} -`); - - auto child = new Tag( - null, "", "child", - null, null, null - ); - - assert("parent" in root.tags); - assert("child" !in root.tags); - testRandomAccessRange(root.tags["parent"][0].tags, [child]); - assert("child" in root.tags["parent"][0].tags); -} diff --git a/src/sdlang/dub.json b/src/sdlang/dub.json deleted file mode 100644 index d5a0493..0000000 --- a/src/sdlang/dub.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "sdlang-d", - "description": "An SDL (Simple Declarative Language) library for D.", - "homepage": "http://github.com/Abscissa/SDLang-D", - "authors": ["Nick Sabalausky"], - "license": "zlib/libpng", - "copyright": "©2012-2015 Nick Sabalausky", - "sourcePaths": ["."], - "importPaths": ["."], - "buildRequirements": ["allowWarnings"], - "dependencies": { - "libinputvisitor": "~>1.2.0" - }, - "subPackages": [ - "./libinputvisitor" - ], - "configurations": [ - { - "name": "test", - "targetType": "executable", - "versions": ["SDLang_TestApp"], - "targetPath": "../../bin/", - "targetName": "sdlang" - }, - { - "name": "library", - "targetType": "library" - }, - { - "name": "unittest", - "targetType": "executable", - "targetPath": "../../bin/", - "targetName": "sdlang-unittest", - - "versions": ["sdlangUnittest", "sdlangTrace"] - } - ] -} diff --git a/src/sdlang/exception.d b/src/sdlang/exception.d deleted file mode 100644 index 188991e..0000000 --- a/src/sdlang/exception.d +++ /dev/null @@ -1,190 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.exception; - -import std.array; -import std.exception; -import std.range; -import std.stdio; -import std.string; - -import sdlang.ast; -import sdlang.util; - -/// Abstract parent class of all SDLang-D defined exceptions. -abstract class SDLangException : Exception -{ - this(string msg, string file = __FILE__, size_t line = __LINE__) - { - super(msg, file, line); - } -} - -/// Thrown when a syntax error is encounterd while parsing. -class ParseException : SDLangException -{ - Location location; - bool hasLocation; - - this(string msg, string file = __FILE__, size_t line = __LINE__) - { - hasLocation = false; - super(msg, file, line); - } - - this(Location location, string msg, string file = __FILE__, size_t line = __LINE__) - { - hasLocation = true; - super("%s: %s".format(location.toString(), msg), file, line); - } -} - -/// Compatibility alias -deprecated("The new name is ParseException") -alias SDLangParseException = ParseException; - -/++ -Thrown when attempting to do something in the DOM that's unsupported, such as: - -$(UL -$(LI Adding the same instance of a tag or attribute to more than one parent.) -$(LI Writing SDLang where: - $(UL - $(LI The root tag has values, attributes or a namespace. ) - $(LI An anonymous tag has a namespace. ) - $(LI An anonymous tag has no values. ) - $(LI A floating point value is infinity or NaN. ) - ) -)) -+/ -class ValidationException : SDLangException -{ - this(string msg, string file = __FILE__, size_t line = __LINE__) - { - super(msg, file, line); - } -} - -/// Compatibility alias -deprecated("The new name is ValidationException") -alias SDLangValidationException = ValidationException; - -/// Thrown when someting is wrong with the provided arguments to a function. -class ArgumentException : SDLangException -{ - this(string msg, string file = __FILE__, size_t line = __LINE__) - { - super(msg, file, line); - } -} - -/// Thrown by the DOM on empty range and out-of-range conditions. -abstract class DOMException : SDLangException -{ - Tag base; /// The tag searched from - - this(Tag base, string msg, string file = __FILE__, size_t line = __LINE__) - { - this.base = base; - super(msg, file, line); - } - - /// Prefixes a message with file/line information from the tag (if tag exists). - /// Optionally takes output range as a sink. - string customMsg(string msg) - { - if(!base) - return msg; - - Appender!string sink; - this.customMsg(sink, msg); - return sink.data; - } - - ///ditto - void customMsg(Sink)(ref Sink sink, string msg) if(isOutputRange!(Sink,char)) - { - if(base) - { - sink.put(base.location.toString()); - sink.put(": "); - sink.put(msg); - } - else - sink.put(msg); - } - - /// Outputs a message to stderr, prefixed with file/line information - void writeCustomMsg(string msg) - { - stderr.writeln( customMsg(msg) ); - } -} - -/// Thrown by the DOM on empty range and out-of-range conditions. -class DOMRangeException : DOMException -{ - this(Tag base, string msg, string file = __FILE__, size_t line = __LINE__) - { - super(base, msg, file, line); - } -} - -/// Compatibility alias -deprecated("The new name is DOMRangeException") -alias SDLangRangeException = DOMRangeException; - -/// Abstract parent class of `TagNotFoundException`, `ValueNotFoundException` -/// and `AttributeNotFoundException`. -/// -/// Thrown by the DOM's `sdlang.ast.Tag.expectTag`, etc. functions if a matching element isn't found. -abstract class DOMNotFoundException : DOMException -{ - FullName tagName; /// The tag searched for - - this(Tag base, FullName tagName, string msg, string file = __FILE__, size_t line = __LINE__) - { - this.tagName = tagName; - super(base, msg, file, line); - } -} - -/// Thrown by the DOM's `sdlang.ast.Tag.expectTag`, etc. functions if a Tag isn't found. -class TagNotFoundException : DOMNotFoundException -{ - this(Tag base, FullName tagName, string msg, string file = __FILE__, size_t line = __LINE__) - { - super(base, tagName, msg, file, line); - } -} - -/// Thrown by the DOM's `sdlang.ast.Tag.expectValue`, etc. functions if a Value isn't found. -class ValueNotFoundException : DOMNotFoundException -{ - /// Expected type for the not-found value. - TypeInfo valueType; - - this(Tag base, FullName tagName, TypeInfo valueType, string msg, string file = __FILE__, size_t line = __LINE__) - { - this.valueType = valueType; - super(base, tagName, msg, file, line); - } -} - -/// Thrown by the DOM's `sdlang.ast.Tag.expectAttribute`, etc. functions if an Attribute isn't found. -class AttributeNotFoundException : DOMNotFoundException -{ - FullName attributeName; /// The attribute searched for - - /// Expected type for the not-found attribute's value. - TypeInfo valueType; - - this(Tag base, FullName tagName, FullName attributeName, TypeInfo valueType, string msg, - string file = __FILE__, size_t line = __LINE__) - { - this.valueType = valueType; - this.attributeName = attributeName; - super(base, tagName, msg, file, line); - } -} diff --git a/src/sdlang/lexer.d b/src/sdlang/lexer.d deleted file mode 100644 index 3788188..0000000 --- a/src/sdlang/lexer.d +++ /dev/null @@ -1,2068 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.lexer; - -import std.algorithm; -import std.array; -static import std.ascii; -import std.base64; -import std.bigint; -import std.conv; -import std.datetime; -import std.file; -import std.format; -import std.traits; -import std.typecons; -import std.uni; -import std.utf; -import std.variant; - -import sdlang.exception; -import sdlang.symbol; -import sdlang.token; -import sdlang.util; - -alias sdlang.util.startsWith startsWith; - -Token[] lexFile(string filename) -{ - auto source = cast(string)read(filename); - return lexSource(source, filename); -} - -Token[] lexSource(string source, string filename=null) -{ - auto lexer = scoped!Lexer(source, filename); - - // Can't use 'std.array.array(Range)' because 'lexer' is scoped - // and therefore cannot have its reference copied. - Appender!(Token[]) tokens; - foreach(tok; lexer) - tokens.put(tok); - - return tokens.data; -} - -// Kind of a poor-man's yield, but fast. -// Only to be used inside Lexer.popFront (and Lexer.this). -private template accept(string symbolName) -{ - static assert(symbolName != "Value", "Value symbols must also take a value."); - enum accept = acceptImpl!(symbolName, "null"); -} -private template accept(string symbolName, string value) -{ - static assert(symbolName == "Value", "Only a Value symbol can take a value."); - enum accept = acceptImpl!(symbolName, value); -} -private template accept(string symbolName, string value, string startLocation, string endLocation) -{ - static assert(symbolName == "Value", "Only a Value symbol can take a value."); - enum accept = (" - { - _front = makeToken!"~symbolName.stringof~"; - _front.value = "~value~"; - _front.location = "~(startLocation==""? "tokenStart" : startLocation)~"; - _front.data = source[ - "~(startLocation==""? "tokenStart.index" : startLocation)~" - .. - "~(endLocation==""? "location.index" : endLocation)~" - ]; - return; - } - ").replace("\n", ""); -} -private template acceptImpl(string symbolName, string value) -{ - enum acceptImpl = (" - { - _front = makeToken!"~symbolName.stringof~"; - _front.value = "~value~"; - return; - } - ").replace("\n", ""); -} - -class Lexer -{ - string source; - string filename; - Location location; /// Location of current character in source - - private dchar ch; // Current character - private dchar nextCh; // Lookahead character - private size_t nextPos; // Position of lookahead character (an index into source) - private bool hasNextCh; // If false, then there's no more lookahead, just EOF - private size_t posAfterLookahead; // Position after lookahead character (an index into source) - - private Location tokenStart; // The starting location of the token being lexed - - // Length so far of the token being lexed, not including current char - private size_t tokenLength; // Length in UTF-8 code units - private size_t tokenLength32; // Length in UTF-32 code units - - // Slight kludge: - // If a numeric fragment is found after a Date (separated by arbitrary - // whitespace), it could be the "hours" part of a DateTime, or it could - // be a separate numeric literal that simply follows a plain Date. If the - // latter, then the Date must be emitted, but numeric fragment that was - // found after it needs to be saved for the the lexer's next iteration. - // - // It's a slight kludge, and could instead be implemented as a slightly - // kludgey parser hack, but it's the only situation where SDLang's lexing - // needs to lookahead more than one character, so this is good enough. - private struct LookaheadTokenInfo - { - bool exists = false; - string numericFragment = ""; - bool isNegative = false; - Location tokenStart; - } - private LookaheadTokenInfo lookaheadTokenInfo; - - this(string source=null, string filename=null) - { - this.filename = filename; - this.source = source; - - _front = Token(symbol!"Error", Location()); - lookaheadTokenInfo = LookaheadTokenInfo.init; - - if( source.startsWith( ByteOrderMarks[BOM.UTF8] ) ) - { - source = source[ ByteOrderMarks[BOM.UTF8].length .. $ ]; - this.source = source; - } - - foreach(bom; ByteOrderMarks) - if( source.startsWith(bom) ) - error(Location(filename,0,0,0), "SDL spec only supports UTF-8, not UTF-16 or UTF-32"); - - if(source == "") - mixin(accept!"EOF"); - - // Prime everything - hasNextCh = true; - nextCh = source.decode(posAfterLookahead); - advanceChar(ErrorOnEOF.Yes); - location = Location(filename, 0, 0, 0); - popFront(); - } - - @property bool empty() - { - return _front.symbol == symbol!"EOF"; - } - - Token _front; - @property Token front() - { - return _front; - } - - @property bool isEOF() - { - return location.index == source.length && !lookaheadTokenInfo.exists; - } - - private void error(string msg) - { - error(location, msg); - } - - //TODO: Take varargs and use output range sink. - private void error(Location loc, string msg) - { - throw new ParseException(loc, "Error: "~msg); - } - - private Token makeToken(string symbolName)() - { - auto tok = Token(symbol!symbolName, tokenStart); - tok.data = tokenData; - return tok; - } - - private @property string tokenData() - { - return source[ tokenStart.index .. location.index ]; - } - - /// Check the lookahead character - private bool lookahead(dchar ch) - { - return hasNextCh && nextCh == ch; - } - - private bool lookahead(bool function(dchar) condition) - { - return hasNextCh && condition(nextCh); - } - - private static bool isNewline(dchar ch) - { - return ch == '\n' || ch == '\r' || ch == lineSep || ch == paraSep; - } - - /// Returns the length of the newline sequence, or zero if the current - /// character is not a newline - /// - /// Note that there are only single character sequences and the two - /// character sequence `\r\n` as used on Windows. - private size_t isAtNewline() - { - if(ch == '\n' || ch == lineSep || ch == paraSep) return 1; - else if(ch == '\r') return lookahead('\n') ? 2 : 1; - else return 0; - } - - /// Is 'ch' a valid base 64 character? - private bool isBase64(dchar ch) - { - if(ch >= 'A' && ch <= 'Z') - return true; - - if(ch >= 'a' && ch <= 'z') - return true; - - if(ch >= '0' && ch <= '9') - return true; - - return ch == '+' || ch == '/' || ch == '='; - } - - /// Is the current character one that's allowed - /// immediately *after* an int/float literal? - private bool isEndOfNumber() - { - if(isEOF) - return true; - - return !isDigit(ch) && ch != ':' && ch != '_' && !isAlpha(ch); - } - - /// Is current character the last one in an ident? - private bool isEndOfIdentCached = false; - private bool _isEndOfIdent; - private bool isEndOfIdent() - { - if(!isEndOfIdentCached) - { - if(!hasNextCh) - _isEndOfIdent = true; - else - _isEndOfIdent = !isIdentChar(nextCh); - - isEndOfIdentCached = true; - } - - return _isEndOfIdent; - } - - /// Is 'ch' a character that's allowed *somewhere* in an identifier? - private bool isIdentChar(dchar ch) - { - if(isAlpha(ch)) - return true; - - else if(isNumber(ch)) - return true; - - else - return - ch == '-' || - ch == '_' || - ch == '.' || - ch == '$'; - } - - private bool isDigit(dchar ch) - { - return ch >= '0' && ch <= '9'; - } - - private enum KeywordResult - { - Accept, // Keyword is matched - Continue, // Keyword is not matched *yet* - Failed, // Keyword doesn't match - } - private KeywordResult checkKeyword(dstring keyword32) - { - // Still within length of keyword - if(tokenLength32 < keyword32.length) - { - if(ch == keyword32[tokenLength32]) - return KeywordResult.Continue; - else - return KeywordResult.Failed; - } - - // At position after keyword - else if(tokenLength32 == keyword32.length) - { - if(isEOF || !isIdentChar(ch)) - { - debug assert(tokenData == to!string(keyword32)); - return KeywordResult.Accept; - } - else - return KeywordResult.Failed; - } - - assert(0, "Fell off end of keyword to check"); - } - - enum ErrorOnEOF { No, Yes } - - /// Advance one code point. - private void advanceChar(ErrorOnEOF errorOnEOF) - { - if(auto cnt = isAtNewline()) - { - if (cnt == 1) - location.line++; - location.col = 0; - } - else - location.col++; - - location.index = nextPos; - - nextPos = posAfterLookahead; - ch = nextCh; - - if(!hasNextCh) - { - if(errorOnEOF == ErrorOnEOF.Yes) - error("Unexpected end of file"); - - return; - } - - tokenLength32++; - tokenLength = location.index - tokenStart.index; - - if(nextPos == source.length) - { - nextCh = dchar.init; - hasNextCh = false; - return; - } - - nextCh = source.decode(posAfterLookahead); - isEndOfIdentCached = false; - } - - /// Advances the specified amount of characters - private void advanceChar(size_t count, ErrorOnEOF errorOnEOF) - { - while(count-- > 0) - advanceChar(errorOnEOF); - } - - void popFront() - { - // -- Main Lexer ------------- - - eatWhite(); - - if(isEOF) - mixin(accept!"EOF"); - - tokenStart = location; - tokenLength = 0; - tokenLength32 = 0; - isEndOfIdentCached = false; - - if(lookaheadTokenInfo.exists) - { - tokenStart = lookaheadTokenInfo.tokenStart; - - auto prevLATokenInfo = lookaheadTokenInfo; - lookaheadTokenInfo = LookaheadTokenInfo.init; - lexNumeric(prevLATokenInfo); - return; - } - - if(ch == '=') - { - advanceChar(ErrorOnEOF.No); - mixin(accept!"="); - } - - else if(ch == '{') - { - advanceChar(ErrorOnEOF.No); - mixin(accept!"{"); - } - - else if(ch == '}') - { - advanceChar(ErrorOnEOF.No); - mixin(accept!"}"); - } - - else if(ch == ':') - { - advanceChar(ErrorOnEOF.No); - mixin(accept!":"); - } - - else if(ch == ';') - { - advanceChar(ErrorOnEOF.No); - mixin(accept!"EOL"); - } - - else if(auto cnt = isAtNewline()) - { - advanceChar(cnt, ErrorOnEOF.No); - mixin(accept!"EOL"); - } - - else if(isAlpha(ch) || ch == '_') - lexIdentKeyword(); - - else if(ch == '"') - lexRegularString(); - - else if(ch == '`') - lexRawString(); - - else if(ch == '\'') - lexCharacter(); - - else if(ch == '[') - lexBinary(); - - else if(ch == '-' || ch == '.' || isDigit(ch)) - lexNumeric(); - - else - { - if(ch == ',') - error("Unexpected comma: SDLang is not a comma-separated format."); - else if(std.ascii.isPrintable(ch)) - error(text("Unexpected: ", ch)); - else - error("Unexpected character code 0x%02X".format(ch)); - - advanceChar(ErrorOnEOF.No); - } - } - - /// Lex Ident or Keyword - private void lexIdentKeyword() - { - assert(isAlpha(ch) || ch == '_'); - - // Keyword - struct Key - { - dstring name; - Value value; - bool failed = false; - } - static Key[5] keywords; - static keywordsInited = false; - if(!keywordsInited) - { - // Value (as a std.variant-based type) can't be statically inited - keywords[0] = Key("true", Value(true )); - keywords[1] = Key("false", Value(false)); - keywords[2] = Key("on", Value(true )); - keywords[3] = Key("off", Value(false)); - keywords[4] = Key("null", Value(null )); - keywordsInited = true; - } - - foreach(ref key; keywords) - key.failed = false; - - auto numKeys = keywords.length; - - do - { - foreach(ref key; keywords) - if(!key.failed) - { - final switch(checkKeyword(key.name)) - { - case KeywordResult.Accept: - mixin(accept!("Value", "key.value")); - - case KeywordResult.Continue: - break; - - case KeywordResult.Failed: - key.failed = true; - numKeys--; - break; - } - } - - if(numKeys == 0) - { - lexIdent(); - return; - } - - advanceChar(ErrorOnEOF.No); - - } while(!isEOF); - - foreach(ref key; keywords) - if(!key.failed) - if(key.name.length == tokenLength32+1) - mixin(accept!("Value", "key.value")); - - mixin(accept!"Ident"); - } - - /// Lex Ident - private void lexIdent() - { - if(tokenLength == 0) - assert(isAlpha(ch) || ch == '_'); - - while(!isEOF && isIdentChar(ch)) - advanceChar(ErrorOnEOF.No); - - mixin(accept!"Ident"); - } - - /// Lex regular string - private void lexRegularString() - { - assert(ch == '"'); - - Appender!string buf; - size_t spanStart = nextPos; - - // Doesn't include current character - void updateBuf() - { - if(location.index == spanStart) - return; - - buf.put( source[spanStart..location.index] ); - } - - advanceChar(ErrorOnEOF.Yes); - while(ch != '"') - { - if(ch == '\\') - { - updateBuf(); - - bool wasEscSequence = true; - if(hasNextCh) - { - switch(nextCh) - { - case 'n': buf.put('\n'); break; - case 'r': buf.put('\r'); break; - case 't': buf.put('\t'); break; - case '"': buf.put('\"'); break; - case '\\': buf.put('\\'); break; - default: wasEscSequence = false; break; - } - } - - if(wasEscSequence) - { - advanceChar(ErrorOnEOF.Yes); - spanStart = nextPos; - } - else - { - eatWhite(false); - spanStart = location.index; - } - } - - else if(isNewline(ch)) - error("Unescaped newlines are only allowed in raw strings, not regular strings."); - - advanceChar(ErrorOnEOF.Yes); - } - - updateBuf(); - advanceChar(ErrorOnEOF.No); // Skip closing double-quote - mixin(accept!("Value", "buf.data")); - } - - /// Lex raw string - private void lexRawString() - { - assert(ch == '`'); - - do - advanceChar(ErrorOnEOF.Yes); - while(ch != '`'); - - advanceChar(ErrorOnEOF.No); // Skip closing back-tick - mixin(accept!("Value", "tokenData[1..$-1]")); - } - - /// Lex character literal - private void lexCharacter() - { - assert(ch == '\''); - advanceChar(ErrorOnEOF.Yes); // Skip opening single-quote - - dchar value; - if(ch == '\\') - { - advanceChar(ErrorOnEOF.Yes); // Skip escape backslash - switch(ch) - { - case 'n': value = '\n'; break; - case 'r': value = '\r'; break; - case 't': value = '\t'; break; - case '\'': value = '\''; break; - case '\\': value = '\\'; break; - default: error("Invalid escape sequence."); - } - } - else if(isNewline(ch)) - error("Newline not alowed in character literal."); - else - value = ch; - advanceChar(ErrorOnEOF.Yes); // Skip the character itself - - if(ch == '\'') - advanceChar(ErrorOnEOF.No); // Skip closing single-quote - else - error("Expected closing single-quote."); - - mixin(accept!("Value", "value")); - } - - /// Lex base64 binary literal - private void lexBinary() - { - assert(ch == '['); - advanceChar(ErrorOnEOF.Yes); - - void eatBase64Whitespace() - { - while(!isEOF && isWhite(ch)) - { - if(isNewline(ch)) - advanceChar(ErrorOnEOF.Yes); - - if(!isEOF && isWhite(ch)) - eatWhite(); - } - } - - eatBase64Whitespace(); - - // Iterates all valid base64 characters, ending at ']'. - // Skips all whitespace. Throws on invalid chars. - struct Base64InputRange - { - Lexer lexer; - private bool isInited = false; - private int numInputCharsMod4 = 0; - - @property bool empty() - { - if(lexer.ch == ']') - { - if(numInputCharsMod4 != 0) - lexer.error("Length of Base64 encoding must be a multiple of 4. ("~to!string(numInputCharsMod4)~")"); - - return true; - } - - return false; - } - - @property dchar front() - { - return lexer.ch; - } - - void popFront() - { - auto lex = lexer; - - if(!isInited) - { - if(lexer.isBase64(lexer.ch)) - { - numInputCharsMod4++; - numInputCharsMod4 %= 4; - } - - isInited = true; - } - - lex.advanceChar(lex.ErrorOnEOF.Yes); - - eatBase64Whitespace(); - - if(lex.isEOF) - lex.error("Unexpected end of file."); - - if(lex.ch != ']') - { - if(!lex.isBase64(lex.ch)) - lex.error("Invalid character in base64 binary literal."); - - numInputCharsMod4++; - numInputCharsMod4 %= 4; - } - } - } - - // This is a slow ugly hack. It's necessary because Base64.decode - // currently requires the source to have known length. - //TODO: Remove this when DMD issue #9543 is fixed. - dchar[] tmpBuf = array(Base64InputRange(this)); - - Appender!(ubyte[]) outputBuf; - // Ugly workaround for DMD issue #9102 - //TODO: Remove this when DMD #9102 is fixed - struct OutputBuf - { - void put(ubyte ch) - { - outputBuf.put(ch); - } - } - - try - //Base64.decode(Base64InputRange(this), OutputBuf()); - Base64.decode(tmpBuf, OutputBuf()); - - catch(Base64Exception e) - error("Invalid character in base64 binary literal."); - - advanceChar(ErrorOnEOF.No); // Skip ']' - mixin(accept!("Value", "outputBuf.data")); - } - - private BigInt toBigInt(bool isNegative, string absValue) - { - auto num = BigInt(absValue); - assert(num >= 0); - - if(isNegative) - num = -num; - - return num; - } - - /// Lex [0-9]+, but without emitting a token. - /// This is used by the other numeric parsing functions. - private string lexNumericFragment() - { - if(!isDigit(ch)) - error("Expected a digit 0-9."); - - auto spanStart = location.index; - - do - { - advanceChar(ErrorOnEOF.No); - } while(!isEOF && isDigit(ch)); - - return source[spanStart..location.index]; - } - - /// Lex anything that starts with 0-9 or '-'. Ints, floats, dates, etc. - private void lexNumeric(LookaheadTokenInfo laTokenInfo = LookaheadTokenInfo.init) - { - bool isNegative; - string firstFragment; - if(laTokenInfo.exists) - { - firstFragment = laTokenInfo.numericFragment; - isNegative = laTokenInfo.isNegative; - } - else - { - assert(ch == '-' || ch == '.' || isDigit(ch)); - - // Check for negative - isNegative = ch == '-'; - if(isNegative) - advanceChar(ErrorOnEOF.Yes); - - // Some floating point with omitted leading zero? - if(ch == '.') - { - lexFloatingPoint(""); - return; - } - - firstFragment = lexNumericFragment(); - } - - // Long integer (64-bit signed)? - if(ch == 'L' || ch == 'l') - { - advanceChar(ErrorOnEOF.No); - - // BigInt(long.min) is a workaround for DMD issue #9548 - auto num = toBigInt(isNegative, firstFragment); - if(num < BigInt(long.min) || num > long.max) - error(tokenStart, "Value doesn't fit in 64-bit signed long integer: "~to!string(num)); - - mixin(accept!("Value", "num.toLong()")); - } - - // Float (32-bit signed)? - else if(ch == 'F' || ch == 'f') - { - auto value = to!float(tokenData); - advanceChar(ErrorOnEOF.No); - mixin(accept!("Value", "value")); - } - - // Double float (64-bit signed) with suffix? - else if((ch == 'D' || ch == 'd') && !lookahead(':') - ) - { - auto value = to!double(tokenData); - advanceChar(ErrorOnEOF.No); - mixin(accept!("Value", "value")); - } - - // Decimal (128+ bits signed)? - else if( - (ch == 'B' || ch == 'b') && - (lookahead('D') || lookahead('d')) - ) - { - auto value = to!real(tokenData); - advanceChar(ErrorOnEOF.No); - advanceChar(ErrorOnEOF.No); - mixin(accept!("Value", "value")); - } - - // Some floating point? - else if(ch == '.') - lexFloatingPoint(firstFragment); - - // Some date? - else if(ch == '/' && hasNextCh && isDigit(nextCh)) - lexDate(isNegative, firstFragment); - - // Some time span? - else if(ch == ':' || ch == 'd') - lexTimeSpan(isNegative, firstFragment); - - // Integer (32-bit signed)? - else if(isEndOfNumber()) - { - auto num = toBigInt(isNegative, firstFragment); - if(num < int.min || num > int.max) - error(tokenStart, "Value doesn't fit in 32-bit signed integer: "~to!string(num)); - - mixin(accept!("Value", "num.toInt()")); - } - - // Invalid suffix - else - error("Invalid integer suffix."); - } - - /// Lex any floating-point literal (after the initial numeric fragment was lexed) - private void lexFloatingPoint(string firstPart) - { - assert(ch == '.'); - advanceChar(ErrorOnEOF.No); - - auto secondPart = lexNumericFragment(); - - try - { - // Double float (64-bit signed) with suffix? - if(ch == 'D' || ch == 'd') - { - auto value = to!double(tokenData); - advanceChar(ErrorOnEOF.No); - mixin(accept!("Value", "value")); - } - - // Float (32-bit signed)? - else if(ch == 'F' || ch == 'f') - { - auto value = to!float(tokenData); - advanceChar(ErrorOnEOF.No); - mixin(accept!("Value", "value")); - } - - // Decimal (128+ bits signed)? - else if(ch == 'B' || ch == 'b') - { - auto value = to!real(tokenData); - advanceChar(ErrorOnEOF.Yes); - - if(!isEOF && (ch == 'D' || ch == 'd')) - { - advanceChar(ErrorOnEOF.No); - if(isEndOfNumber()) - mixin(accept!("Value", "value")); - } - - error("Invalid floating point suffix."); - } - - // Double float (64-bit signed) without suffix? - else if(isEOF || !isIdentChar(ch)) - { - auto value = to!double(tokenData); - mixin(accept!("Value", "value")); - } - - // Invalid suffix - else - error("Invalid floating point suffix."); - } - catch(ConvException e) - error("Invalid floating point literal."); - } - - private Date makeDate(bool isNegative, string yearStr, string monthStr, string dayStr) - { - BigInt biTmp; - - biTmp = BigInt(yearStr); - if(isNegative) - biTmp = -biTmp; - if(biTmp < int.min || biTmp > int.max) - error(tokenStart, "Date's year is out of range. (Must fit within a 32-bit signed int.)"); - auto year = biTmp.toInt(); - - biTmp = BigInt(monthStr); - if(biTmp < 1 || biTmp > 12) - error(tokenStart, "Date's month is out of range."); - auto month = biTmp.toInt(); - - biTmp = BigInt(dayStr); - if(biTmp < 1 || biTmp > 31) - error(tokenStart, "Date's month is out of range."); - auto day = biTmp.toInt(); - - return Date(year, month, day); - } - - private DateTimeFrac makeDateTimeFrac( - bool isNegative, Date date, string hourStr, string minuteStr, - string secondStr, string millisecondStr - ) - { - BigInt biTmp; - - biTmp = BigInt(hourStr); - if(biTmp < int.min || biTmp > int.max) - error(tokenStart, "Datetime's hour is out of range."); - auto numHours = biTmp.toInt(); - - biTmp = BigInt(minuteStr); - if(biTmp < 0 || biTmp > int.max) - error(tokenStart, "Datetime's minute is out of range."); - auto numMinutes = biTmp.toInt(); - - int numSeconds = 0; - if(secondStr != "") - { - biTmp = BigInt(secondStr); - if(biTmp < 0 || biTmp > int.max) - error(tokenStart, "Datetime's second is out of range."); - numSeconds = biTmp.toInt(); - } - - int millisecond = 0; - if(millisecondStr != "") - { - biTmp = BigInt(millisecondStr); - if(biTmp < 0 || biTmp > int.max) - error(tokenStart, "Datetime's millisecond is out of range."); - millisecond = biTmp.toInt(); - - if(millisecondStr.length == 1) - millisecond *= 100; - else if(millisecondStr.length == 2) - millisecond *= 10; - } - - Duration fracSecs = millisecond.msecs; - - auto offset = hours(numHours) + minutes(numMinutes) + seconds(numSeconds); - - if(isNegative) - { - offset = -offset; - fracSecs = -fracSecs; - } - - return DateTimeFrac(DateTime(date) + offset, fracSecs); - } - - private Duration makeDuration( - bool isNegative, string dayStr, - string hourStr, string minuteStr, string secondStr, - string millisecondStr - ) - { - BigInt biTmp; - - long day = 0; - if(dayStr != "") - { - biTmp = BigInt(dayStr); - if(biTmp < long.min || biTmp > long.max) - error(tokenStart, "Time span's day is out of range."); - day = biTmp.toLong(); - } - - biTmp = BigInt(hourStr); - if(biTmp < long.min || biTmp > long.max) - error(tokenStart, "Time span's hour is out of range."); - auto hour = biTmp.toLong(); - - biTmp = BigInt(minuteStr); - if(biTmp < long.min || biTmp > long.max) - error(tokenStart, "Time span's minute is out of range."); - auto minute = biTmp.toLong(); - - biTmp = BigInt(secondStr); - if(biTmp < long.min || biTmp > long.max) - error(tokenStart, "Time span's second is out of range."); - auto second = biTmp.toLong(); - - long millisecond = 0; - if(millisecondStr != "") - { - biTmp = BigInt(millisecondStr); - if(biTmp < long.min || biTmp > long.max) - error(tokenStart, "Time span's millisecond is out of range."); - millisecond = biTmp.toLong(); - - if(millisecondStr.length == 1) - millisecond *= 100; - else if(millisecondStr.length == 2) - millisecond *= 10; - } - - auto duration = - dur!"days" (day) + - dur!"hours" (hour) + - dur!"minutes"(minute) + - dur!"seconds"(second) + - dur!"msecs" (millisecond); - - if(isNegative) - duration = -duration; - - return duration; - } - - // This has to reproduce some weird corner case behaviors from the - // original Java version of SDL. So some of this may seem weird. - private Nullable!Duration getTimeZoneOffset(string str) - { - if(str.length < 2) - return Nullable!Duration(); // Unknown timezone - - if(str[0] != '+' && str[0] != '-') - return Nullable!Duration(); // Unknown timezone - - auto isNegative = str[0] == '-'; - - string numHoursStr; - string numMinutesStr; - if(str[1] == ':') - { - numMinutesStr = str[1..$]; - numHoursStr = ""; - } - else - { - numMinutesStr = str.find(':'); - numHoursStr = str[1 .. $-numMinutesStr.length]; - } - - long numHours = 0; - long numMinutes = 0; - bool isUnknown = false; - try - { - switch(numHoursStr.length) - { - case 0: - if(numMinutesStr.length == 3) - { - numHours = 0; - numMinutes = to!long(numMinutesStr[1..$]); - } - else - isUnknown = true; - break; - - case 1: - case 2: - if(numMinutesStr.length == 0) - { - numHours = to!long(numHoursStr); - numMinutes = 0; - } - else if(numMinutesStr.length == 3) - { - numHours = to!long(numHoursStr); - numMinutes = to!long(numMinutesStr[1..$]); - } - else - isUnknown = true; - break; - - default: - if(numMinutesStr.length == 0) - { - // Yes, this is correct - numHours = 0; - numMinutes = to!long(numHoursStr[1..$]); - } - else - isUnknown = true; - break; - } - } - catch(ConvException e) - isUnknown = true; - - if(isUnknown) - return Nullable!Duration(); // Unknown timezone - - auto timeZoneOffset = hours(numHours) + minutes(numMinutes); - if(isNegative) - timeZoneOffset = -timeZoneOffset; - - // Timezone valid - return Nullable!Duration(timeZoneOffset); - } - - /// Lex date or datetime (after the initial numeric fragment was lexed) - private void lexDate(bool isDateNegative, string yearStr) - { - assert(ch == '/'); - - // Lex months - advanceChar(ErrorOnEOF.Yes); // Skip '/' - auto monthStr = lexNumericFragment(); - - // Lex days - if(ch != '/') - error("Invalid date format: Missing days."); - advanceChar(ErrorOnEOF.Yes); // Skip '/' - auto dayStr = lexNumericFragment(); - - auto date = makeDate(isDateNegative, yearStr, monthStr, dayStr); - - if(!isEndOfNumber() && ch != '/') - error("Dates cannot have suffixes."); - - // Date? - if(isEOF) - mixin(accept!("Value", "date")); - - auto endOfDate = location; - - while( - !isEOF && - ( ch == '\\' || ch == '/' || (isWhite(ch) && !isNewline(ch)) ) - ) - { - if(ch == '\\' && hasNextCh && isNewline(nextCh)) - { - advanceChar(ErrorOnEOF.Yes); - if(isAtNewline()) - advanceChar(ErrorOnEOF.Yes); - advanceChar(ErrorOnEOF.No); - } - - eatWhite(); - } - - // Date? - if(isEOF || (!isDigit(ch) && ch != '-')) - mixin(accept!("Value", "date", "", "endOfDate.index")); - - auto startOfTime = location; - - // Is time negative? - bool isTimeNegative = ch == '-'; - if(isTimeNegative) - advanceChar(ErrorOnEOF.Yes); - - // Lex hours - auto hourStr = ch == '.'? "" : lexNumericFragment(); - - // Lex minutes - if(ch != ':') - { - // No minutes found. Therefore we had a plain Date followed - // by a numeric literal, not a DateTime. - lookaheadTokenInfo.exists = true; - lookaheadTokenInfo.numericFragment = hourStr; - lookaheadTokenInfo.isNegative = isTimeNegative; - lookaheadTokenInfo.tokenStart = startOfTime; - mixin(accept!("Value", "date", "", "endOfDate.index")); - } - advanceChar(ErrorOnEOF.Yes); // Skip ':' - auto minuteStr = lexNumericFragment(); - - // Lex seconds, if exists - string secondStr; - if(ch == ':') - { - advanceChar(ErrorOnEOF.Yes); // Skip ':' - secondStr = lexNumericFragment(); - } - - // Lex milliseconds, if exists - string millisecondStr; - if(ch == '.') - { - advanceChar(ErrorOnEOF.Yes); // Skip '.' - millisecondStr = lexNumericFragment(); - } - - auto dateTimeFrac = makeDateTimeFrac(isTimeNegative, date, hourStr, minuteStr, secondStr, millisecondStr); - - // Lex zone, if exists - if(ch == '-') - { - advanceChar(ErrorOnEOF.Yes); // Skip '-' - auto timezoneStart = location; - - if(!isAlpha(ch)) - error("Invalid timezone format."); - - while(!isEOF && !isWhite(ch)) - advanceChar(ErrorOnEOF.No); - - auto timezoneStr = source[timezoneStart.index..location.index]; - if(timezoneStr.startsWith("GMT")) - { - auto isoPart = timezoneStr["GMT".length..$]; - auto offset = getTimeZoneOffset(isoPart); - - if(offset.isNull()) - { - // Unknown time zone - mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)")); - } - else - { - auto timezone = new immutable SimpleTimeZone(offset.get()); - mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)")); - } - } - - try - { - auto timezone = TimeZone.getTimeZone(timezoneStr); - if(timezone) - mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)")); - } - catch(TimeException e) - { - // Time zone not found. So just move along to "Unknown time zone" below. - } - - // Unknown time zone - mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)")); - } - - if(!isEndOfNumber()) - error("Date-Times cannot have suffixes."); - - mixin(accept!("Value", "dateTimeFrac")); - } - - /// Lex time span (after the initial numeric fragment was lexed) - private void lexTimeSpan(bool isNegative, string firstPart) - { - assert(ch == ':' || ch == 'd'); - - string dayStr = ""; - string hourStr; - - // Lexed days? - bool hasDays = ch == 'd'; - if(hasDays) - { - dayStr = firstPart; - advanceChar(ErrorOnEOF.Yes); // Skip 'd' - - // Lex hours - if(ch != ':') - error("Invalid time span format: Missing hours."); - advanceChar(ErrorOnEOF.Yes); // Skip ':' - hourStr = lexNumericFragment(); - } - else - hourStr = firstPart; - - // Lex minutes - if(ch != ':') - error("Invalid time span format: Missing minutes."); - advanceChar(ErrorOnEOF.Yes); // Skip ':' - auto minuteStr = lexNumericFragment(); - - // Lex seconds - if(ch != ':') - error("Invalid time span format: Missing seconds."); - advanceChar(ErrorOnEOF.Yes); // Skip ':' - auto secondStr = lexNumericFragment(); - - // Lex milliseconds, if exists - string millisecondStr = ""; - if(ch == '.') - { - advanceChar(ErrorOnEOF.Yes); // Skip '.' - millisecondStr = lexNumericFragment(); - } - - if(!isEndOfNumber()) - error("Time spans cannot have suffixes."); - - auto duration = makeDuration(isNegative, dayStr, hourStr, minuteStr, secondStr, millisecondStr); - mixin(accept!("Value", "duration")); - } - - /// Advances past whitespace and comments - private void eatWhite(bool allowComments=true) - { - // -- Comment/Whitepace Lexer ------------- - - enum State - { - normal, - lineComment, // Got "#" or "//" or "--", Eating everything until newline - blockComment, // Got "/*", Eating everything until "*/" - } - - if(isEOF) - return; - - Location commentStart; - State state = State.normal; - bool consumeNewlines = false; - bool hasConsumedNewline = false; - while(true) - { - final switch(state) - { - case State.normal: - - if(ch == '\\') - { - commentStart = location; - consumeNewlines = true; - hasConsumedNewline = false; - } - - else if(ch == '#') - { - if(!allowComments) - return; - - commentStart = location; - state = State.lineComment; - continue; - } - - else if(ch == '/' || ch == '-') - { - commentStart = location; - if(lookahead(ch)) - { - if(!allowComments) - return; - - advanceChar(ErrorOnEOF.No); - state = State.lineComment; - continue; - } - else if(ch == '/' && lookahead('*')) - { - if(!allowComments) - return; - - advanceChar(ErrorOnEOF.No); - state = State.blockComment; - continue; - } - else - return; // Done - } - else if(isAtNewline()) - { - if(consumeNewlines) - hasConsumedNewline = true; - else - return; // Done - } - else if(!isWhite(ch)) - { - if(consumeNewlines) - { - if(hasConsumedNewline) - return; // Done - else - error("Only whitespace can come between a line-continuation backslash and the following newline."); - } - else - return; // Done - } - - break; - - case State.lineComment: - if(lookahead(&isNewline)) - state = State.normal; - break; - - case State.blockComment: - if(ch == '*' && lookahead('/')) - { - advanceChar(ErrorOnEOF.No); - state = State.normal; - } - break; - } - - advanceChar(ErrorOnEOF.No); - if(isEOF) - { - // Reached EOF - - if(consumeNewlines && !hasConsumedNewline) - error("Missing newline after line-continuation backslash."); - - else if(state == State.blockComment) - error(commentStart, "Unterminated block comment."); - - else - return; // Done, reached EOF - } - } - } -} - -version(unittest) -{ - import std.stdio; - - version(Have_unit_threaded) import unit_threaded; - else { enum DontTest; } - - private auto loc = Location("filename", 0, 0, 0); - private auto loc2 = Location("a", 1, 1, 1); - - @("lexer: EOL") - unittest - { - assert([Token(symbol!"EOL",loc) ] == [Token(symbol!"EOL",loc) ] ); - assert([Token(symbol!"EOL",loc,Value(7),"A")] == [Token(symbol!"EOL",loc2,Value(7),"B")] ); - } - - private int numErrors = 0; - @DontTest - private void testLex(string source, Token[] expected, bool test_locations = false, string file=__FILE__, size_t line=__LINE__) - { - Token[] actual; - try - actual = lexSource(source, "filename"); - catch(ParseException e) - { - numErrors++; - stderr.writeln(file, "(", line, "): testLex failed on: ", source); - stderr.writeln(" Expected:"); - stderr.writeln(" ", expected); - stderr.writeln(" Actual: ParseException thrown:"); - stderr.writeln(" ", e.msg); - return; - } - - bool is_same = actual == expected; - if (is_same && test_locations) { - is_same = actual.map!(t => t.location).equal(expected.map!(t => t.location)); - } - - if(!is_same) - { - numErrors++; - stderr.writeln(file, "(", line, "): testLex failed on: ", source); - stderr.writeln(" Expected:"); - stderr.writeln(" ", expected); - stderr.writeln(" Actual:"); - stderr.writeln(" ", actual); - - if(expected.length > 1 || actual.length > 1) - { - stderr.writeln(" expected.length: ", expected.length); - stderr.writeln(" actual.length: ", actual.length); - - if(actual.length == expected.length) - foreach(i; 0..actual.length) - if(actual[i] != expected[i]) - { - stderr.writeln(" Unequal at index #", i, ":"); - stderr.writeln(" Expected:"); - stderr.writeln(" ", expected[i]); - stderr.writeln(" Actual:"); - stderr.writeln(" ", actual[i]); - } - } - } - } - - private void testLexThrows(string file=__FILE__, size_t line=__LINE__)(string source) - { - bool hadException = false; - Token[] actual; - try - actual = lexSource(source, "filename"); - catch(ParseException e) - hadException = true; - - if(!hadException) - { - numErrors++; - stderr.writeln(file, "(", line, "): testLex failed on: ", source); - stderr.writeln(" Expected ParseException"); - stderr.writeln(" Actual:"); - stderr.writeln(" ", actual); - } - } -} - -@("sdlang lexer") -unittest -{ - testLex("", []); - testLex(" ", []); - testLex("\\\n", []); - testLex("/*foo*/", []); - testLex("/* multiline \n comment */", []); - testLex("/* * */", []); - testLexThrows("/* "); - - testLex(":", [ Token(symbol!":", loc) ]); - testLex("=", [ Token(symbol!"=", loc) ]); - testLex("{", [ Token(symbol!"{", loc) ]); - testLex("}", [ Token(symbol!"}", loc) ]); - testLex(";", [ Token(symbol!"EOL",loc) ]); - testLex("\n", [ Token(symbol!"EOL",loc) ]); - - testLex("foo", [ Token(symbol!"Ident",loc,Value(null),"foo") ]); - testLex("_foo", [ Token(symbol!"Ident",loc,Value(null),"_foo") ]); - testLex("foo.bar", [ Token(symbol!"Ident",loc,Value(null),"foo.bar") ]); - testLex("foo-bar", [ Token(symbol!"Ident",loc,Value(null),"foo-bar") ]); - testLex("foo.", [ Token(symbol!"Ident",loc,Value(null),"foo.") ]); - testLex("foo-", [ Token(symbol!"Ident",loc,Value(null),"foo-") ]); - testLexThrows(".foo"); - - testLex("foo bar", [ - Token(symbol!"Ident",loc,Value(null),"foo"), - Token(symbol!"Ident",loc,Value(null),"bar"), - ]); - testLex("foo \\ \n \n bar", [ - Token(symbol!"Ident",loc,Value(null),"foo"), - Token(symbol!"Ident",loc,Value(null),"bar"), - ]); - testLex("foo \\ \n \\ \n bar", [ - Token(symbol!"Ident",loc,Value(null),"foo"), - Token(symbol!"Ident",loc,Value(null),"bar"), - ]); - testLexThrows("foo \\ "); - testLexThrows("foo \\ bar"); - testLexThrows("foo \\ \n \\ "); - testLexThrows("foo \\ \n \\ bar"); - - testLex("foo : = { } ; \n bar \n", [ - Token(symbol!"Ident",loc,Value(null),"foo"), - Token(symbol!":",loc), - Token(symbol!"=",loc), - Token(symbol!"{",loc), - Token(symbol!"}",loc), - Token(symbol!"EOL",loc), - Token(symbol!"EOL",loc), - Token(symbol!"Ident",loc,Value(null),"bar"), - Token(symbol!"EOL",loc), - ]); - - testLexThrows("<"); - testLexThrows("*"); - testLexThrows(`\`); - - // Integers - testLex( "7", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]); - testLex( "-7", [ Token(symbol!"Value",loc,Value(cast( int)-7)) ]); - testLex( "7L", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]); - testLex( "7l", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]); - testLex("-7L", [ Token(symbol!"Value",loc,Value(cast(long)-7)) ]); - testLex( "0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]); - testLex( "-0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]); - - testLex("7/**/", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]); - testLex("7#", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]); - - testLex("7 A", [ - Token(symbol!"Value",loc,Value(cast(int)7)), - Token(symbol!"Ident",loc,Value( null),"A"), - ]); - testLexThrows("7A"); - testLexThrows("-A"); - testLexThrows(`-""`); - - testLex("7;", [ - Token(symbol!"Value",loc,Value(cast(int)7)), - Token(symbol!"EOL",loc), - ]); - - // Floats - testLex("1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]); - testLex("1.2f" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]); - testLex("1.2" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]); - testLex("1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]); - testLex("1.2d" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]); - testLex("1.2BD", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]); - testLex("1.2bd", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]); - testLex("1.2Bd", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]); - testLex("1.2bD", [ Token(symbol!"Value",loc,Value(cast( real)1.2)) ]); - - testLex(".2F" , [ Token(symbol!"Value",loc,Value(cast( float)0.2)) ]); - testLex(".2" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]); - testLex(".2D" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]); - testLex(".2BD", [ Token(symbol!"Value",loc,Value(cast( real)0.2)) ]); - - testLex("-1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-1.2)) ]); - testLex("-1.2" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]); - testLex("-1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]); - testLex("-1.2BD", [ Token(symbol!"Value",loc,Value(cast( real)-1.2)) ]); - - testLex("-.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-0.2)) ]); - testLex("-.2" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]); - testLex("-.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]); - testLex("-.2BD", [ Token(symbol!"Value",loc,Value(cast( real)-0.2)) ]); - - testLex( "0.0" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]); - testLex( "0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]); - testLex( "0.0BD", [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]); - testLex("-0.0" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]); - testLex("-0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]); - testLex("-0.0BD", [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]); - testLex( "7F" , [ Token(symbol!"Value",loc,Value(cast( float)7.0)) ]); - testLex( "7D" , [ Token(symbol!"Value",loc,Value(cast(double)7.0)) ]); - testLex( "7BD" , [ Token(symbol!"Value",loc,Value(cast( real)7.0)) ]); - testLex( "0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]); - testLex( "0D" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]); - testLex( "0BD" , [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]); - testLex("-0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]); - testLex("-0D" , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]); - testLex("-0BD" , [ Token(symbol!"Value",loc,Value(cast( real)0.0)) ]); - - testLex("1.2 F", [ - Token(symbol!"Value",loc,Value(cast(double)1.2)), - Token(symbol!"Ident",loc,Value( null),"F"), - ]); - testLexThrows("1.2A"); - testLexThrows("1.2B"); - testLexThrows("1.2BDF"); - - testLex("1.2;", [ - Token(symbol!"Value",loc,Value(cast(double)1.2)), - Token(symbol!"EOL",loc), - ]); - - testLex("1.2F;", [ - Token(symbol!"Value",loc,Value(cast(float)1.2)), - Token(symbol!"EOL",loc), - ]); - - testLex("1.2BD;", [ - Token(symbol!"Value",loc,Value(cast(real)1.2)), - Token(symbol!"EOL",loc), - ]); - - // Booleans and null - testLex("true", [ Token(symbol!"Value",loc,Value( true)) ]); - testLex("false", [ Token(symbol!"Value",loc,Value(false)) ]); - testLex("on", [ Token(symbol!"Value",loc,Value( true)) ]); - testLex("off", [ Token(symbol!"Value",loc,Value(false)) ]); - testLex("null", [ Token(symbol!"Value",loc,Value( null)) ]); - - testLex("TRUE", [ Token(symbol!"Ident",loc,Value(null),"TRUE") ]); - testLex("true ", [ Token(symbol!"Value",loc,Value(true)) ]); - testLex("true ", [ Token(symbol!"Value",loc,Value(true)) ]); - testLex("tru", [ Token(symbol!"Ident",loc,Value(null),"tru") ]); - testLex("truX", [ Token(symbol!"Ident",loc,Value(null),"truX") ]); - testLex("trueX", [ Token(symbol!"Ident",loc,Value(null),"trueX") ]); - - // Raw Backtick Strings - testLex("`hello world`", [ Token(symbol!"Value",loc,Value(`hello world` )) ]); - testLex("` hello world `", [ Token(symbol!"Value",loc,Value(` hello world ` )) ]); - testLex("`hello \\t world`", [ Token(symbol!"Value",loc,Value(`hello \t world`)) ]); - testLex("`hello \\n world`", [ Token(symbol!"Value",loc,Value(`hello \n world`)) ]); - testLex("`hello \n world`", [ Token(symbol!"Value",loc,Value("hello \n world")) ]); - testLex("`hello \r\n world`", [ Token(symbol!"Value",loc,Value("hello \r\n world")) ]); - testLex("`hello \"world\"`", [ Token(symbol!"Value",loc,Value(`hello "world"` )) ]); - - testLexThrows("`foo"); - testLexThrows("`"); - - // Double-Quote Strings - testLex(`"hello world"`, [ Token(symbol!"Value",loc,Value("hello world" )) ]); - testLex(`" hello world "`, [ Token(symbol!"Value",loc,Value(" hello world " )) ]); - testLex(`"hello \t world"`, [ Token(symbol!"Value",loc,Value("hello \t world")) ]); - testLex(`"hello \n world"`, [ Token(symbol!"Value",loc,Value("hello \n world")) ]); - testLex("\"hello \\\n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]); - testLex("\"hello \\ \n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]); - testLex("\"hello \\ \n\n world\"", [ Token(symbol!"Value",loc,Value("hello world" )) ]); - testLex(`"\"hello world\""`, [ Token(symbol!"Value",loc,Value(`"hello world"` )) ]); - testLex(`""`, [ Token(symbol!"Value",loc,Value("" )) ]); // issue #34 - - testLexThrows("\"hello \n world\""); - testLexThrows(`"foo`); - testLexThrows(`"`); - - // Characters - testLex("'a'", [ Token(symbol!"Value",loc,Value(cast(dchar) 'a')) ]); - testLex("'\\n'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\n')) ]); - testLex("'\\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]); - testLex("'\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]); - testLex("'\\''", [ Token(symbol!"Value",loc,Value(cast(dchar)'\'')) ]); - testLex(`'\\'`, [ Token(symbol!"Value",loc,Value(cast(dchar)'\\')) ]); - - testLexThrows("'a"); - testLexThrows("'aa'"); - testLexThrows("''"); - testLexThrows("'\\\n'"); - testLexThrows("'\n'"); - testLexThrows(`'\`); - testLexThrows(`'\'`); - testLexThrows("'"); - - // Unicode - testLex("日本語", [ Token(symbol!"Ident",loc,Value(null), "日本語") ]); - testLex("`おはよう、日本。`", [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]); - testLex(`"おはよう、日本。"`, [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]); - testLex("'月'", [ Token(symbol!"Value",loc,Value("月"d.dup[0])) ]); - - // Base64 Binary - testLex("[aGVsbG8gd29ybGQ=]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]); - testLex("[ aGVsbG8gd29ybGQ= ]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]); - testLex("[\n aGVsbG8g \n \n d29ybGQ= \n]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]); - - testLexThrows("[aGVsbG8gd29ybGQ]"); // Ie: Not multiple of 4 - testLexThrows("[ aGVsbG8gd29ybGQ ]"); - - // Date - testLex( "1999/12/5", [ Token(symbol!"Value",loc,Value(Date( 1999, 12, 5))) ]); - testLex( "2013/2/22", [ Token(symbol!"Value",loc,Value(Date( 2013, 2, 22))) ]); - testLex("-2013/2/22", [ Token(symbol!"Value",loc,Value(Date(-2013, 2, 22))) ]); - - testLexThrows("7/"); - testLexThrows("2013/2/22a"); - testLexThrows("2013/2/22f"); - - testLex("1999/12/5\n", [ - Token(symbol!"Value",loc,Value(Date(1999, 12, 5))), - Token(symbol!"EOL",loc), - ]); - - // DateTime, no timezone - testLex( "2013/2/22 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22 \t 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22/*foo*/07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22 /*foo*/ \\\n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22 /*foo*/ \\\n\n \n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22 /*foo*/ \\\n\\\n \\\n /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22/*foo*/\\\n/*bar*/07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0)))) ]); - testLex("-2013/2/22 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 7, 53, 0)))) ]); - testLex( "2013/2/22 -07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53)))) ]); - testLex("-2013/2/22 -07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53)))) ]); - testLex( "2013/2/22 07:53:34", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34)))) ]); - testLex( "2013/2/22 07:53:34.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs))) ]); - testLex( "2013/2/22 07:53:34.12", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 120.msecs))) ]); - testLex( "2013/2/22 07:53:34.1", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 100.msecs))) ]); - testLex( "2013/2/22 07:53.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs))) ]); - - testLex( "2013/2/22 34:65", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0)))) ]); - testLex( "2013/2/22 34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds(77), 123.msecs))) ]); - testLex( "2013/2/22 34:65.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0), 123.msecs))) ]); - - testLex( "2013/2/22 -34:65", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0)))) ]); - testLex( "2013/2/22 -34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds(77), -123.msecs))) ]); - testLex( "2013/2/22 -34:65.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), -123.msecs))) ]); - - testLexThrows("2013/2/22 07:53a"); - testLexThrows("2013/2/22 07:53f"); - testLexThrows("2013/2/22 07:53:34.123a"); - testLexThrows("2013/2/22 07:53:34.123f"); - testLexThrows("2013/2/22a 07:53"); - - testLex(`2013/2/22 "foo"`, [ - Token(symbol!"Value",loc,Value(Date(2013, 2, 22))), - Token(symbol!"Value",loc,Value("foo")), - ]); - - testLex("2013/2/22 07", [ - Token(symbol!"Value",loc,Value(Date(2013, 2, 22))), - Token(symbol!"Value",loc,Value(cast(int)7)), - ]); - - testLex("2013/2/22 1.2F", [ - Token(symbol!"Value",loc,Value(Date(2013, 2, 22))), - Token(symbol!"Value",loc,Value(cast(float)1.2)), - ]); - - testLex("2013/2/22 .2F", [ - Token(symbol!"Value",loc,Value(Date(2013, 2, 22))), - Token(symbol!"Value",loc,Value(cast(float)0.2)), - ]); - - testLex("2013/2/22 -1.2F", [ - Token(symbol!"Value",loc,Value(Date(2013, 2, 22))), - Token(symbol!"Value",loc,Value(cast(float)-1.2)), - ]); - - testLex("2013/2/22 -.2F", [ - Token(symbol!"Value",loc,Value(Date(2013, 2, 22))), - Token(symbol!"Value",loc,Value(cast(float)-0.2)), - ]); - - // DateTime, with known timezone - testLex( "2013/2/22 07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(0) )))) ]); - testLex("-2013/2/22 07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(0) )))) ]); - testLex( "2013/2/22 -07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0) )))) ]); - testLex("-2013/2/22 -07:53-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0) )))) ]); - testLex( "2013/2/22 07:53-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]); - testLex( "2013/2/22 07:53-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]); - testLex( "2013/2/22 07:53:34-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(0) )))) ]); - testLex( "2013/2/22 07:53:34-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]); - testLex( "2013/2/22 07:53:34-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]); - testLex( "2013/2/22 07:53:34.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(0) )))) ]); - testLex( "2013/2/22 07:53:34.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]); - testLex( "2013/2/22 07:53:34.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]); - testLex( "2013/2/22 07:53.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone( hours(0) )))) ]); - testLex( "2013/2/22 07:53.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]); - testLex( "2013/2/22 07:53.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]); - - testLex( "2013/2/22 -34:65-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]); - - // DateTime, with Java SDLang's occasionally weird interpretation of some - // "not quite ISO" variations of the "GMT with offset" timezone strings. - Token testTokenSimpleTimeZone(Duration d) - { - auto dateTime = DateTime(2013, 2, 22, 7, 53, 0); - auto tz = new immutable SimpleTimeZone(d); - return Token( symbol!"Value", loc, Value(SysTime(dateTime,tz)) ); - } - Token testTokenUnknownTimeZone(string tzName) - { - auto dateTime = DateTime(2013, 2, 22, 7, 53, 0); - auto frac = 0.msecs; - return Token( symbol!"Value", loc, Value(DateTimeFracUnknownZone(dateTime,frac,tzName)) ); - } - testLex("2013/2/22 07:53-GMT+", [ testTokenUnknownTimeZone("GMT+") ]); - testLex("2013/2/22 07:53-GMT+:", [ testTokenUnknownTimeZone("GMT+:") ]); - testLex("2013/2/22 07:53-GMT+:3", [ testTokenUnknownTimeZone("GMT+:3") ]); - testLex("2013/2/22 07:53-GMT+:03", [ testTokenSimpleTimeZone(minutes(3)) ]); - testLex("2013/2/22 07:53-GMT+:003", [ testTokenUnknownTimeZone("GMT+:003") ]); - - testLex("2013/2/22 07:53-GMT+4", [ testTokenSimpleTimeZone(hours(4)) ]); - testLex("2013/2/22 07:53-GMT+4:", [ testTokenUnknownTimeZone("GMT+4:") ]); - testLex("2013/2/22 07:53-GMT+4:3", [ testTokenUnknownTimeZone("GMT+4:3") ]); - testLex("2013/2/22 07:53-GMT+4:03", [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]); - testLex("2013/2/22 07:53-GMT+4:003", [ testTokenUnknownTimeZone("GMT+4:003") ]); - - testLex("2013/2/22 07:53-GMT+04", [ testTokenSimpleTimeZone(hours(4)) ]); - testLex("2013/2/22 07:53-GMT+04:", [ testTokenUnknownTimeZone("GMT+04:") ]); - testLex("2013/2/22 07:53-GMT+04:3", [ testTokenUnknownTimeZone("GMT+04:3") ]); - testLex("2013/2/22 07:53-GMT+04:03", [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]); - testLex("2013/2/22 07:53-GMT+04:03abc", [ testTokenUnknownTimeZone("GMT+04:03abc") ]); - testLex("2013/2/22 07:53-GMT+04:003", [ testTokenUnknownTimeZone("GMT+04:003") ]); - - testLex("2013/2/22 07:53-GMT+004", [ testTokenSimpleTimeZone(minutes(4)) ]); - testLex("2013/2/22 07:53-GMT+004:", [ testTokenUnknownTimeZone("GMT+004:") ]); - testLex("2013/2/22 07:53-GMT+004:3", [ testTokenUnknownTimeZone("GMT+004:3") ]); - testLex("2013/2/22 07:53-GMT+004:03", [ testTokenUnknownTimeZone("GMT+004:03") ]); - testLex("2013/2/22 07:53-GMT+004:003", [ testTokenUnknownTimeZone("GMT+004:003") ]); - - testLex("2013/2/22 07:53-GMT+0004", [ testTokenSimpleTimeZone(minutes(4)) ]); - testLex("2013/2/22 07:53-GMT+0004:", [ testTokenUnknownTimeZone("GMT+0004:") ]); - testLex("2013/2/22 07:53-GMT+0004:3", [ testTokenUnknownTimeZone("GMT+0004:3") ]); - testLex("2013/2/22 07:53-GMT+0004:03", [ testTokenUnknownTimeZone("GMT+0004:03") ]); - testLex("2013/2/22 07:53-GMT+0004:003", [ testTokenUnknownTimeZone("GMT+0004:003") ]); - - testLex("2013/2/22 07:53-GMT+00004", [ testTokenSimpleTimeZone(minutes(4)) ]); - testLex("2013/2/22 07:53-GMT+00004:", [ testTokenUnknownTimeZone("GMT+00004:") ]); - testLex("2013/2/22 07:53-GMT+00004:3", [ testTokenUnknownTimeZone("GMT+00004:3") ]); - testLex("2013/2/22 07:53-GMT+00004:03", [ testTokenUnknownTimeZone("GMT+00004:03") ]); - testLex("2013/2/22 07:53-GMT+00004:003", [ testTokenUnknownTimeZone("GMT+00004:003") ]); - - // DateTime, with unknown timezone - testLex( "2013/2/22 07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 0), 0.msecs, "Bogus/Foo")), "2013/2/22 07:53-Bogus/Foo") ]); - testLex("-2013/2/22 07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 7, 53, 0), 0.msecs, "Bogus/Foo"))) ]); - testLex( "2013/2/22 -07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]); - testLex("-2013/2/22 -07:53-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 0, 0, 0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]); - testLex( "2013/2/22 07:53:34-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 0.msecs, "Bogus/Foo"))) ]); - testLex( "2013/2/22 07:53:34.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, "Bogus/Foo"))) ]); - testLex( "2013/2/22 07:53.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 0), 123.msecs, "Bogus/Foo"))) ]); - - // Time Span - testLex( "12:14:42", [ Token(symbol!"Value",loc,Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs( 0))) ]); - testLex("-12:14:42", [ Token(symbol!"Value",loc,Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs( 0))) ]); - testLex( "00:09:12", [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs( 0))) ]); - testLex( "00:00:01.023", [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23))) ]); - testLex( "23d:05:21:23.532", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532))) ]); - testLex( "23d:05:21:23.53", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530))) ]); - testLex( "23d:05:21:23.5", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500))) ]); - testLex("-23d:05:21:23.532", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532))) ]); - testLex("-23d:05:21:23.5", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500))) ]); - testLex( "23d:05:21:23", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs( 0))) ]); - - testLexThrows("12:14:42a"); - testLexThrows("23d:05:21:23.532a"); - testLexThrows("23d:05:21:23.532f"); - - // Combination - testLex("foo. 7", [ - Token(symbol!"Ident",loc,Value( null),"foo."), - Token(symbol!"Value",loc,Value(cast(int)7)) - ]); - - testLex(` - namespace:person "foo" "bar" 1 23L name.first="ひとみ" name.last="Smith" { - namespace:age 37; namespace:favorite_color "blue" // comment - somedate 2013/2/22 07:53 -- comment - - inventory /* comment */ { - socks - } - } - `, - [ - Token(symbol!"EOL",loc,Value(null),"\n"), - - Token(symbol!"Ident", loc, Value( null ), "namespace"), - Token(symbol!":", loc, Value( null ), ":"), - Token(symbol!"Ident", loc, Value( null ), "person"), - Token(symbol!"Value", loc, Value( "foo" ), `"foo"`), - Token(symbol!"Value", loc, Value( "bar" ), `"bar"`), - Token(symbol!"Value", loc, Value( cast( int) 1 ), "1"), - Token(symbol!"Value", loc, Value( cast(long)23 ), "23L"), - Token(symbol!"Ident", loc, Value( null ), "name.first"), - Token(symbol!"=", loc, Value( null ), "="), - Token(symbol!"Value", loc, Value( "ひとみ" ), `"ひとみ"`), - Token(symbol!"Ident", loc, Value( null ), "name.last"), - Token(symbol!"=", loc, Value( null ), "="), - Token(symbol!"Value", loc, Value( "Smith" ), `"Smith"`), - Token(symbol!"{", loc, Value( null ), "{"), - Token(symbol!"EOL", loc, Value( null ), "\n"), - - Token(symbol!"Ident", loc, Value( null ), "namespace"), - Token(symbol!":", loc, Value( null ), ":"), - Token(symbol!"Ident", loc, Value( null ), "age"), - Token(symbol!"Value", loc, Value( cast(int)37 ), "37"), - Token(symbol!"EOL", loc, Value( null ), ";"), - Token(symbol!"Ident", loc, Value( null ), "namespace"), - Token(symbol!":", loc, Value( null ), ":"), - Token(symbol!"Ident", loc, Value( null ), "favorite_color"), - Token(symbol!"Value", loc, Value( "blue" ), `"blue"`), - Token(symbol!"EOL", loc, Value( null ), "\n"), - - Token(symbol!"Ident", loc, Value( null ), "somedate"), - Token(symbol!"Value", loc, Value( DateTimeFrac(DateTime(2013, 2, 22, 7, 53, 0)) ), "2013/2/22 07:53"), - Token(symbol!"EOL", loc, Value( null ), "\n"), - Token(symbol!"EOL", loc, Value( null ), "\n"), - - Token(symbol!"Ident", loc, Value(null), "inventory"), - Token(symbol!"{", loc, Value(null), "{"), - Token(symbol!"EOL", loc, Value(null), "\n"), - - Token(symbol!"Ident", loc, Value(null), "socks"), - Token(symbol!"EOL", loc, Value(null), "\n"), - - Token(symbol!"}", loc, Value(null), "}"), - Token(symbol!"EOL", loc, Value(null), "\n"), - - Token(symbol!"}", loc, Value(null), "}"), - Token(symbol!"EOL", loc, Value(null), "\n"), - ]); - - if(numErrors > 0) - stderr.writeln(numErrors, " failed test(s)"); -} - -@("lexer: Regression test issue #8") -unittest -{ - testLex(`"\n \n"`, [ Token(symbol!"Value",loc,Value("\n \n"),`"\n \n"`) ]); - testLex(`"\t\t"`, [ Token(symbol!"Value",loc,Value("\t\t"),`"\t\t"`) ]); - testLex(`"\n\n"`, [ Token(symbol!"Value",loc,Value("\n\n"),`"\n\n"`) ]); -} - -@("lexer: Regression test issue #11") -unittest -{ - void test(string input) - { - testLex( - input, - [ - Token(symbol!"EOL", loc, Value(null), "\n"), - Token(symbol!"Ident",loc,Value(null), "a") - ] - ); - } - - test("//X\na"); - test("//\na"); - test("--\na"); - test("#\na"); -} - -@("ast: Regression test issue #28") -unittest -{ - enum offset = 1; // workaround for an of-by-one error for line numbers - testLex("test", [ - Token(symbol!"Ident", Location("filename", 0, 0, 0), Value(null), "test") - ], true); - testLex("\ntest", [ - Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\n"), - Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test") - ], true); - testLex("\rtest", [ - Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"), - Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test") - ], true); - testLex("\r\ntest", [ - Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"), - Token(symbol!"Ident", Location("filename", 1, 0, 2), Value(null), "test") - ], true); - testLex("\r\n\ntest", [ - Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"), - Token(symbol!"EOL", Location("filename", 1, 0, 2), Value(null), "\n"), - Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test") - ], true); - testLex("\r\r\ntest", [ - Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"), - Token(symbol!"EOL", Location("filename", 1, 0, 1), Value(null), "\r\n"), - Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test") - ], true); -} diff --git a/src/sdlang/libinputvisitor/dub.json b/src/sdlang/libinputvisitor/dub.json deleted file mode 100644 index 6e273c8..0000000 --- a/src/sdlang/libinputvisitor/dub.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "libinputvisitor", - "description": "Write D input range generators in a straightforward coroutine style", - "authors": ["Nick Sabalausky"], - "homepage": "https://github.com/abscissa/libInputVisitor", - "license": "WTFPL", - "sourcePaths": ["."], - "importPaths": ["."], - "excludedSourceFiles": ["libInputVisitorExample.d"] -} diff --git a/src/sdlang/libinputvisitor/libInputVisitor.d b/src/sdlang/libinputvisitor/libInputVisitor.d deleted file mode 100644 index f29dc4f..0000000 --- a/src/sdlang/libinputvisitor/libInputVisitor.d +++ /dev/null @@ -1,113 +0,0 @@ -/++ -Copyright (C) 2012 Nick Sabalausky <http://semitwist.com/contact> - -This program is free software. It comes without any warranty, to -the extent permitted by applicable law. You can redistribute it -and/or modify it under the terms of the Do What The Fuck You Want -To Public License, Version 2, as published by Sam Hocevar. See -http://www.wtfpl.net/ for more details. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - -Copyright (C) 2004 Sam Hocevar <sam@hocevar.net> - -Everyone is permitted to copy and distribute verbatim or modified -copies of this license document, and changing it is allowed as long -as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. You just DO WHAT THE FUCK YOU WANT TO. -+/ - -/++ -Should work with DMD 2.059 and up - -For more info on this, see: -http://semitwist.com/articles/article/view/combine-coroutines-and-input-ranges-for-dead-simple-d-iteration -+/ - -import core.thread; - -class InputVisitor(Obj, Elem) : Fiber -{ - bool started = false; - Obj obj; - this(Obj obj) - { - this.obj = obj; - - version(Windows) // Issue #1 - { - import core.sys.windows.windows : SYSTEM_INFO, GetSystemInfo; - SYSTEM_INFO info; - GetSystemInfo(&info); - auto PAGESIZE = info.dwPageSize; - - super(&run, PAGESIZE * 16); - } - else - super(&run); - } - - this(Obj obj, size_t stackSize) - { - this.obj = obj; - super(&run, stackSize); - } - - private void run() - { - obj.visit(this); - } - - private void ensureStarted() - { - if(!started) - { - call(); - started = true; - } - } - - // Member 'front' must be a function due to DMD Issue #5403 - private Elem _front = Elem.init; // Default initing here avoids "Error: field _front must be initialized in constructor" - @property Elem front() - { - ensureStarted(); - return _front; - } - - void popFront() - { - ensureStarted(); - call(); - } - - @property bool empty() - { - ensureStarted(); - return state == Fiber.State.TERM; - } - - void yield(Elem elem) - { - _front = elem; - Fiber.yield(); - } -} - -template inputVisitor(Elem) -{ - @property InputVisitor!(Obj, Elem) inputVisitor(Obj)(Obj obj) - { - return new InputVisitor!(Obj, Elem)(obj); - } - - @property InputVisitor!(Obj, Elem) inputVisitor(Obj)(Obj obj, size_t stackSize) - { - return new InputVisitor!(Obj, Elem)(obj, stackSize); - } -} diff --git a/src/sdlang/package.d b/src/sdlang/package.d deleted file mode 100644 index dd8df1a..0000000 --- a/src/sdlang/package.d +++ /dev/null @@ -1,133 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -/++ -$(H2 SDLang-D v0.10.0) - -Library for parsing and generating SDL (Simple Declarative Language). - -Import this module to use SDLang-D as a library. - -For the list of officially supported compiler versions, see the -$(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/.travis.yml, .travis.yml) -file included with your version of SDLang-D. - -Links: -$(UL - $(LI $(LINK2 http://sdlang.org/, SDLang Language Homepage) ) - $(LI $(LINK2 https://github.com/Abscissa/SDLang-D, SDLang-D Homepage) ) - $(LI $(LINK2 http://semitwist.com/sdlang-d, SDLang-D API Reference (latest version) ) ) - $(LI $(LINK2 http://semitwist.com/sdlang-d-docs, SDLang-D API Reference (earlier versions) ) ) - $(LI $(LINK2 http://sdl.ikayzo.org/display/SDL/Language+Guide, Old Official SDL Site) [$(LINK2 http://semitwist.com/sdl-mirror/Language+Guide.html, mirror)] ) -) - -Authors: Nick Sabalausky ("Abscissa") http://semitwist.com/contact -Copyright: -Copyright (C) 2012-2016 Nick Sabalausky. - -License: $(LINK2 https://github.com/Abscissa/SDLang-D/blob/master/LICENSE.txt, zlib/libpng) -+/ - -module sdlang; - -import std.array; -import std.datetime; -import std.file; -import std.stdio; - -import sdlang.ast; -import sdlang.exception; -import sdlang.lexer; -import sdlang.parser; -import sdlang.symbol; -import sdlang.token; -import sdlang.util; - -// Expose main public API -public import sdlang.ast : Attribute, Tag; -public import sdlang.exception; -public import sdlang.parser : parseFile, parseSource; -public import sdlang.token : Value, Token, DateTimeFrac, DateTimeFracUnknownZone; -public import sdlang.util : sdlangVersion, Location; - -version(sdlangUsingBuiltinTestRunner) - void main() {} - -version(sdlangCliApp) -{ - int main(string[] args) - { - if( - args.length != 3 || - (args[1] != "lex" && args[1] != "parse" && args[1] != "to-sdl") - ) - { - stderr.writeln("SDLang-D v", sdlangVersion); - stderr.writeln("Usage: sdlang [lex|parse|to-sdl] filename.sdl"); - return 1; - } - - auto filename = args[2]; - - try - { - if(args[1] == "lex") - doLex(filename); - else if(args[1] == "parse") - doParse(filename); - else - doToSDL(filename); - } - catch(ParseException e) - { - stderr.writeln(e.msg); - return 1; - } - - return 0; - } - - void doLex(string filename) - { - auto source = cast(string)read(filename); - auto lexer = new Lexer(source, filename); - - foreach(tok; lexer) - { - // Value - string value; - if(tok.symbol == symbol!"Value") - value = tok.value.hasValue? toString(tok.value.type) : "{null}"; - - value = value==""? "\t" : "("~value~":"~tok.value.toString()~") "; - - // Data - auto data = tok.data.replace("\n", "").replace("\r", ""); - if(data != "") - data = "\t|"~tok.data~"|"; - - // Display - writeln( - tok.location.toString, ":\t", - tok.symbol.name, value, - data - ); - - if(tok.symbol.name == "Error") - break; - } - } - - void doParse(string filename) - { - auto root = parseFile(filename); - stdout.rawWrite(root.toDebugString()); - writeln(); - } - - void doToSDL(string filename) - { - auto root = parseFile(filename); - stdout.rawWrite(root.toSDLDocument()); - } -} diff --git a/src/sdlang/parser.d b/src/sdlang/parser.d deleted file mode 100644 index c9b8d4f..0000000 --- a/src/sdlang/parser.d +++ /dev/null @@ -1,628 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.parser; - -import std.file; - -import libInputVisitor; -import taggedalgebraic; - -import sdlang.ast; -import sdlang.exception; -import sdlang.lexer; -import sdlang.symbol; -import sdlang.token; -import sdlang.util; - -/// Returns root tag. -Tag parseFile(string filename) -{ - auto source = cast(string)read(filename); - return parseSource(source, filename); -} - -/// Returns root tag. The optional `filename` parameter can be included -/// so that the SDLang document's filename (if any) can be displayed with -/// any syntax error messages. -Tag parseSource(string source, string filename=null) -{ - auto lexer = new Lexer(source, filename); - auto parser = DOMParser(lexer); - return parser.parseRoot(); -} - -/++ -Parses an SDL document using StAX/Pull-style. Returns an InputRange with -element type ParserEvent. - -The pullParseFile version reads a file and parses it, while pullParseSource -parses a string passed in. The optional `filename` parameter in pullParseSource -can be included so that the SDLang document's filename (if any) can be displayed -with any syntax error messages. - -Note: The old FileStartEvent and FileEndEvent events -$(LINK2 https://github.com/Abscissa/SDLang-D/issues/17, were deemed unnessecary) -and removed as of SDLang-D v0.10.0. - -Note: Previously, in SDLang-D v0.9.x, ParserEvent was a -$(LINK2 http://dlang.org/phobos/std_variant.html#.Algebraic, std.variant.Algebraic). -As of SDLang-D v0.10.0, it is now a -$(LINK2 https://github.com/s-ludwig/taggedalgebraic, TaggedAlgebraic), -so usage has changed somewhat. - -Example: ------------------- -parent 12 attr="q" { - childA 34 - childB 56 -} -lastTag ------------------- - -The ParserEvent sequence emitted for that SDL document would be as -follows (indented for readability): ------------------- -TagStartEvent (parent) - ValueEvent (12) - AttributeEvent (attr, "q") - TagStartEvent (childA) - ValueEvent (34) - TagEndEvent - TagStartEvent (childB) - ValueEvent (56) - TagEndEvent -TagEndEvent -TagStartEvent (lastTag) -TagEndEvent ------------------- -+/ -auto pullParseFile(string filename) -{ - auto source = cast(string)read(filename); - return parseSource(source, filename); -} - -///ditto -auto pullParseSource(string source, string filename=null) -{ - auto lexer = new Lexer(source, filename); - auto parser = PullParser(lexer); - return inputVisitor!ParserEvent( parser ); -} - -/// -@("pullParseFile/pullParseSource example") -unittest -{ - // stuff.sdl - immutable stuffSdl = ` - name "sdlang-d" - description "An SDL (Simple Declarative Language) library for D." - homepage "http://github.com/Abscissa/SDLang-D" - - configuration "library" { - targetType "library" - } - `; - - import std.stdio; - - foreach(event; pullParseSource(stuffSdl)) - final switch(event.kind) - { - case ParserEvent.Kind.tagStart: - auto e = cast(TagStartEvent) event; - writeln("TagStartEvent: ", e.namespace, ":", e.name, " @ ", e.location); - break; - - case ParserEvent.Kind.tagEnd: - auto e = cast(TagEndEvent) event; - writeln("TagEndEvent"); - break; - - case ParserEvent.Kind.value: - auto e = cast(ValueEvent) event; - writeln("ValueEvent: ", e.value); - break; - - case ParserEvent.Kind.attribute: - auto e = cast(AttributeEvent) event; - writeln("AttributeEvent: ", e.namespace, ":", e.name, "=", e.value); - break; - } -} - -private union ParserEventUnion -{ - TagStartEvent tagStart; - TagEndEvent tagEnd; - ValueEvent value; - AttributeEvent attribute; -} - -/++ -The element of the InputRange returned by pullParseFile and pullParseSource. - -This is a tagged union, built from the following: -------- -alias ParserEvent = TaggedAlgebraic!ParserEventUnion; -private union ParserEventUnion -{ - TagStartEvent tagStart; - TagEndEvent tagEnd; - ValueEvent value; - AttributeEvent attribute; -} -------- - -Note: The old FileStartEvent and FileEndEvent events -$(LINK2 https://github.com/Abscissa/SDLang-D/issues/17, were deemed unnessecary) -and removed as of SDLang-D v0.10.0. - -Note: Previously, in SDLang-D v0.9.x, ParserEvent was a -$(LINK2 http://dlang.org/phobos/std_variant.html#.Algebraic, std.variant.Algebraic). -As of SDLang-D v0.10.0, it is now a -$(LINK2 https://github.com/s-ludwig/taggedalgebraic, TaggedAlgebraic), -so usage has changed somewhat. -+/ -alias ParserEvent = TaggedAlgebraic!ParserEventUnion; - -/// -@("ParserEvent example") -unittest -{ - // Create - ParserEvent event1 = TagStartEvent(); - ParserEvent event2 = TagEndEvent(); - ParserEvent event3 = ValueEvent(); - ParserEvent event4 = AttributeEvent(); - - // Check type - assert(event1.kind == ParserEvent.Kind.tagStart); - assert(event2.kind == ParserEvent.Kind.tagEnd); - assert(event3.kind == ParserEvent.Kind.value); - assert(event4.kind == ParserEvent.Kind.attribute); - - // Cast to base type - auto e1 = cast(TagStartEvent) event1; - auto e2 = cast(TagEndEvent) event2; - auto e3 = cast(ValueEvent) event3; - auto e4 = cast(AttributeEvent) event4; - //auto noGood = cast(AttributeEvent) event1; // AssertError: event1 is a TagStartEvent, not AttributeEvent. - - // Use as base type. - // In many cases, no casting is even needed. - event1.name = "foo"; - //auto noGood = event3.name; // AssertError: ValueEvent doesn't have a member 'name'. - - // Final switch is supported: - final switch(event1.kind) - { - case ParserEvent.Kind.tagStart: break; - case ParserEvent.Kind.tagEnd: break; - case ParserEvent.Kind.value: break; - case ParserEvent.Kind.attribute: break; - } -} - -/// Event: Start of tag -struct TagStartEvent -{ - Location location; - string namespace; - string name; -} - -/// Event: End of tag -struct TagEndEvent -{ - //Location location; -} - -/// Event: Found a Value in the current tag -struct ValueEvent -{ - Location location; - Value value; -} - -/// Event: Found an Attribute in the current tag -struct AttributeEvent -{ - Location location; - string namespace; - string name; - Value value; -} - -// The actual pull parser -private struct PullParser -{ - private Lexer lexer; - - private struct IDFull - { - string namespace; - string name; - } - - private void error(string msg) - { - error(lexer.front.location, msg); - } - - private void error(Location loc, string msg) - { - throw new ParseException(loc, "Error: "~msg); - } - - private InputVisitor!(PullParser, ParserEvent) v; - - void visit(InputVisitor!(PullParser, ParserEvent) v) - { - this.v = v; - parseRoot(); - } - - private void emit(Event)(Event event) - { - v.yield( ParserEvent(event) ); - } - - /// <Root> ::= <Tags> EOF (Lookaheads: Anything) - private void parseRoot() - { - //trace("Starting parse of file: ", lexer.filename); - //trace(__FUNCTION__, ": <Root> ::= <Tags> EOF (Lookaheads: Anything)"); - - auto startLocation = Location(lexer.filename, 0, 0, 0); - - parseTags(); - - auto token = lexer.front; - if(token.matches!":"()) - { - lexer.popFront(); - token = lexer.front; - if(token.matches!"Ident"()) - { - error("Missing namespace. If you don't wish to use a namespace, then say '"~token.data~"', not ':"~token.data~"'"); - assert(0); - } - else - { - error("Missing namespace. If you don't wish to use a namespace, then omit the ':'"); - assert(0); - } - } - else if(!token.matches!"EOF"()) - error("Expected a tag or end-of-file, not " ~ token.symbol.name); - } - - /// <Tags> ::= <Tag> <Tags> (Lookaheads: Ident Value) - /// | EOL <Tags> (Lookaheads: EOL) - /// | {empty} (Lookaheads: Anything else, except '{') - void parseTags() - { - //trace("Enter ", __FUNCTION__); - while(true) - { - auto token = lexer.front; - if(token.matches!"Ident"() || token.matches!"Value"()) - { - //trace(__FUNCTION__, ": <Tags> ::= <Tag> <Tags> (Lookaheads: Ident Value)"); - parseTag(); - continue; - } - else if(token.matches!"EOL"()) - { - //trace(__FUNCTION__, ": <Tags> ::= EOL <Tags> (Lookaheads: EOL)"); - lexer.popFront(); - continue; - } - else if(token.matches!"{"()) - { - error("Found start of child block, but no tag name. If you intended an anonymous "~ - "tag, you must have at least one value before any attributes or child tags."); - } - else - { - //trace(__FUNCTION__, ": <Tags> ::= {empty} (Lookaheads: Anything else, except '{')"); - break; - } - } - } - - /// <Tag> - /// ::= <IDFull> <Values> <Attributes> <OptChild> <TagTerminator> (Lookaheads: Ident) - /// | <Value> <Values> <Attributes> <OptChild> <TagTerminator> (Lookaheads: Value) - void parseTag() - { - auto token = lexer.front; - if(token.matches!"Ident"()) - { - //trace(__FUNCTION__, ": <Tag> ::= <IDFull> <Values> <Attributes> <OptChild> <TagTerminator> (Lookaheads: Ident)"); - //trace("Found tag named: ", tag.fullName); - auto id = parseIDFull(); - emit( TagStartEvent(token.location, id.namespace, id.name) ); - } - else if(token.matches!"Value"()) - { - //trace(__FUNCTION__, ": <Tag> ::= <Value> <Values> <Attributes> <OptChild> <TagTerminator> (Lookaheads: Value)"); - //trace("Found anonymous tag."); - emit( TagStartEvent(token.location, null, null) ); - } - else - error("Expected tag name or value, not " ~ token.symbol.name); - - if(lexer.front.matches!"="()) - error("Found attribute, but no tag name. If you intended an anonymous "~ - "tag, you must have at least one value before any attributes."); - - parseValues(); - parseAttributes(); - parseOptChild(); - parseTagTerminator(); - - emit( TagEndEvent() ); - } - - /// <IDFull> ::= Ident <IDSuffix> (Lookaheads: Ident) - IDFull parseIDFull() - { - auto token = lexer.front; - if(token.matches!"Ident"()) - { - //trace(__FUNCTION__, ": <IDFull> ::= Ident <IDSuffix> (Lookaheads: Ident)"); - lexer.popFront(); - return parseIDSuffix(token.data); - } - else - { - error("Expected namespace or identifier, not " ~ token.symbol.name); - assert(0); - } - } - - /// <IDSuffix> - /// ::= ':' Ident (Lookaheads: ':') - /// ::= {empty} (Lookaheads: Anything else) - IDFull parseIDSuffix(string firstIdent) - { - auto token = lexer.front; - if(token.matches!":"()) - { - //trace(__FUNCTION__, ": <IDSuffix> ::= ':' Ident (Lookaheads: ':')"); - lexer.popFront(); - token = lexer.front; - if(token.matches!"Ident"()) - { - lexer.popFront(); - return IDFull(firstIdent, token.data); - } - else - { - error("Expected name, not " ~ token.symbol.name); - assert(0); - } - } - else - { - //trace(__FUNCTION__, ": <IDSuffix> ::= {empty} (Lookaheads: Anything else)"); - return IDFull("", firstIdent); - } - } - - /// <Values> - /// ::= Value <Values> (Lookaheads: Value) - /// | {empty} (Lookaheads: Anything else) - void parseValues() - { - while(true) - { - auto token = lexer.front; - if(token.matches!"Value"()) - { - //trace(__FUNCTION__, ": <Values> ::= Value <Values> (Lookaheads: Value)"); - parseValue(); - continue; - } - else - { - //trace(__FUNCTION__, ": <Values> ::= {empty} (Lookaheads: Anything else)"); - break; - } - } - } - - /// Handle Value terminals that aren't part of an attribute - void parseValue() - { - auto token = lexer.front; - if(token.matches!"Value"()) - { - //trace(__FUNCTION__, ": (Handle Value terminals that aren't part of an attribute)"); - auto value = token.value; - //trace("In tag '", parent.fullName, "', found value: ", value); - emit( ValueEvent(token.location, value) ); - - lexer.popFront(); - } - else - error("Expected value, not "~token.symbol.name); - } - - /// <Attributes> - /// ::= <Attribute> <Attributes> (Lookaheads: Ident) - /// | {empty} (Lookaheads: Anything else) - void parseAttributes() - { - while(true) - { - auto token = lexer.front; - if(token.matches!"Ident"()) - { - //trace(__FUNCTION__, ": <Attributes> ::= <Attribute> <Attributes> (Lookaheads: Ident)"); - parseAttribute(); - continue; - } - else - { - //trace(__FUNCTION__, ": <Attributes> ::= {empty} (Lookaheads: Anything else)"); - break; - } - } - } - - /// <Attribute> ::= <IDFull> '=' Value (Lookaheads: Ident) - void parseAttribute() - { - //trace(__FUNCTION__, ": <Attribute> ::= <IDFull> '=' Value (Lookaheads: Ident)"); - auto token = lexer.front; - if(!token.matches!"Ident"()) - error("Expected attribute name, not "~token.symbol.name); - - auto id = parseIDFull(); - - token = lexer.front; - if(!token.matches!"="()) - error("Expected '=' after attribute name, not "~token.symbol.name); - - lexer.popFront(); - token = lexer.front; - if(!token.matches!"Value"()) - error("Expected attribute value, not "~token.symbol.name); - - //trace("In tag '", parent.fullName, "', found attribute '", attr.fullName, "'"); - emit( AttributeEvent(token.location, id.namespace, id.name, token.value) ); - - lexer.popFront(); - } - - /// <OptChild> - /// ::= '{' EOL <Tags> '}' (Lookaheads: '{') - /// | {empty} (Lookaheads: Anything else) - void parseOptChild() - { - auto token = lexer.front; - if(token.matches!"{") - { - //trace(__FUNCTION__, ": <OptChild> ::= '{' EOL <Tags> '}' (Lookaheads: '{')"); - lexer.popFront(); - token = lexer.front; - if(!token.matches!"EOL"()) - error("Expected newline or semicolon after '{', not "~token.symbol.name); - - lexer.popFront(); - parseTags(); - - token = lexer.front; - if(!token.matches!"}"()) - error("Expected '}' after child tags, not "~token.symbol.name); - lexer.popFront(); - } - else - { - //trace(__FUNCTION__, ": <OptChild> ::= {empty} (Lookaheads: Anything else)"); - // Do nothing, no error. - } - } - - /// <TagTerminator> - /// ::= EOL (Lookahead: EOL) - /// | {empty} (Lookahead: EOF) - void parseTagTerminator() - { - auto token = lexer.front; - if(token.matches!"EOL") - { - //trace(__FUNCTION__, ": <TagTerminator> ::= EOL (Lookahead: EOL)"); - lexer.popFront(); - } - else if(token.matches!"EOF") - { - //trace(__FUNCTION__, ": <TagTerminator> ::= {empty} (Lookahead: EOF)"); - // Do nothing - } - else - error("Expected end of tag (newline, semicolon or end-of-file), not " ~ token.symbol.name); - } -} - -private struct DOMParser -{ - Lexer lexer; - - Tag parseRoot() - { - auto currTag = new Tag(null, null, "root"); - currTag.location = Location(lexer.filename, 0, 0, 0); - - auto parser = PullParser(lexer); - auto eventRange = inputVisitor!ParserEvent( parser ); - - foreach(event; eventRange) - final switch(event.kind) - { - case ParserEvent.Kind.tagStart: - auto newTag = new Tag(currTag, event.namespace, event.name); - newTag.location = event.location; - - currTag = newTag; - break; - - case ParserEvent.Kind.tagEnd: - currTag = currTag.parent; - - if(!currTag) - parser.error("Internal Error: Received an extra TagEndEvent"); - break; - - case ParserEvent.Kind.value: - currTag.add((cast(ValueEvent)event).value); - break; - - case ParserEvent.Kind.attribute: - auto e = cast(AttributeEvent) event; - auto attr = new Attribute(e.namespace, e.name, e.value, e.location); - currTag.add(attr); - break; - } - - return currTag; - } -} - -// Other parser tests are part of the AST's tests over in the ast module. - -// Regression test, issue #13: https://github.com/Abscissa/SDLang-D/issues/13 -// "Incorrectly accepts ":tagname" (blank namespace, tagname prefixed with colon)" -@("parser: Regression test issue #13") -unittest -{ - import std.exception; - assertThrown!ParseException(parseSource(`:test`)); - assertThrown!ParseException(parseSource(`:4`)); -} - -// Regression test, issue #16: https://github.com/Abscissa/SDLang-D/issues/16 -@("parser: Regression test issue #16") -unittest -{ - // Shouldn't crash - foreach(event; pullParseSource(`tag "data"`)) - { - if(event.kind == ParserEvent.Kind.tagStart) - auto e = cast(TagStartEvent) event; - } -} - -// Regression test, issue #31: https://github.com/Abscissa/SDLang-D/issues/31 -// "Escape sequence results in range violation error" -@("parser: Regression test issue #31") -unittest -{ - // Shouldn't get a Range violation - parseSource(`test "\"foo\""`); -} diff --git a/src/sdlang/symbol.d b/src/sdlang/symbol.d deleted file mode 100644 index ebb2b93..0000000 --- a/src/sdlang/symbol.d +++ /dev/null @@ -1,61 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.symbol; - -import std.algorithm; - -static immutable validSymbolNames = [ - "Error", - "EOF", - "EOL", - - ":", - "=", - "{", - "}", - - "Ident", - "Value", -]; - -/// Use this to create a Symbol. Ex: symbol!"Value" or symbol!"=" -/// Invalid names (such as symbol!"FooBar") are rejected at compile-time. -template symbol(string name) -{ - static assert(validSymbolNames.find(name), "Invalid Symbol: '"~name~"'"); - immutable symbol = _symbol(name); -} - -private Symbol _symbol(string name) -{ - return Symbol(name); -} - -/// Symbol is essentially the "type" of a Token. -/// Token is like an instance of a Symbol. -/// -/// This only represents terminals. Nonterminal tokens aren't -/// constructed since the AST is built directly during parsing. -/// -/// You can't create a Symbol directly. Instead, use the `symbol` -/// template. -struct Symbol -{ - private string _name; - @property string name() - { - return _name; - } - - @disable this(); - private this(string name) - { - this._name = name; - } - - string toString() - { - return _name; - } -} diff --git a/src/sdlang/taggedalgebraic/taggedalgebraic.d b/src/sdlang/taggedalgebraic/taggedalgebraic.d deleted file mode 100644 index ffaac49..0000000 --- a/src/sdlang/taggedalgebraic/taggedalgebraic.d +++ /dev/null @@ -1,1085 +0,0 @@ -/** - * Algebraic data type implementation based on a tagged union. - * - * Copyright: Copyright 2015, Sönke Ludwig. - * License: $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0). - * Authors: Sönke Ludwig -*/ -module taggedalgebraic; - -import std.typetuple; - -// TODO: -// - distinguish between @property and non@-property methods. -// - verify that static methods are handled properly - -/** Implements a generic algebraic type using an enum to identify the stored type. - - This struct takes a `union` or `struct` declaration as an input and builds - an algebraic data type from its fields, using an automatically generated - `Kind` enumeration to identify which field of the union is currently used. - Multiple fields with the same value are supported. - - All operators and methods are transparently forwarded to the contained - value. The caller has to make sure that the contained value supports the - requested operation. Failure to do so will result in an assertion failure. - - The return value of forwarded operations is determined as follows: - $(UL - $(LI If the type can be uniquely determined, it is used as the return - value) - $(LI If there are multiple possible return values and all of them match - the unique types defined in the `TaggedAlgebraic`, a - `TaggedAlgebraic` is returned.) - $(LI If there are multiple return values and none of them is a - `Variant`, an `Algebraic` of the set of possible return types is - returned.) - $(LI If any of the possible operations returns a `Variant`, this is used - as the return value.) - ) -*/ -struct TaggedAlgebraic(U) if (is(U == union) || is(U == struct)) -{ - import std.algorithm : among; - import std.string : format; - import std.traits : FieldTypeTuple, FieldNameTuple, Largest, hasElaborateCopyConstructor, hasElaborateDestructor; - - private alias Union = U; - private alias FieldTypes = FieldTypeTuple!U; - private alias fieldNames = FieldNameTuple!U; - - static assert(FieldTypes.length > 0, "The TaggedAlgebraic's union type must have at least one field."); - static assert(FieldTypes.length == fieldNames.length); - - - private { - void[Largest!FieldTypes.sizeof] m_data = void; - Kind m_kind; - } - - /// A type enum that identifies the type of value currently stored. - alias Kind = TypeEnum!U; - - /// Compatibility alias - deprecated("Use 'Kind' instead.") alias Type = Kind; - - /// The type ID of the currently stored value. - @property Kind kind() const { return m_kind; } - - // Compatibility alias - deprecated("Use 'kind' instead.") - alias typeID = kind; - - // constructors - //pragma(msg, generateConstructors!U()); - mixin(generateConstructors!U); - - this(TaggedAlgebraic other) - { - import std.algorithm : swap; - swap(this, other); - } - - void opAssign(TaggedAlgebraic other) - { - import std.algorithm : swap; - swap(this, other); - } - - // postblit constructor - static if (anySatisfy!(hasElaborateCopyConstructor, FieldTypes)) - { - this(this) - { - switch (m_kind) { - default: break; - foreach (i, tname; fieldNames) { - alias T = typeof(__traits(getMember, U, tname)); - static if (hasElaborateCopyConstructor!T) - { - case __traits(getMember, Kind, tname): - typeid(T).postblit(cast(void*)&trustedGet!tname()); - return; - } - } - } - } - } - - // destructor - static if (anySatisfy!(hasElaborateDestructor, FieldTypes)) - { - ~this() - { - final switch (m_kind) { - foreach (i, tname; fieldNames) { - alias T = typeof(__traits(getMember, U, tname)); - case __traits(getMember, Kind, tname): - static if (hasElaborateDestructor!T) { - .destroy(trustedGet!tname); - } - return; - } - } - } - } - - /// Enables conversion or extraction of the stored value. - T opCast(T)() - { - import std.conv : to; - - final switch (m_kind) { - foreach (i, FT; FieldTypes) { - case __traits(getMember, Kind, fieldNames[i]): - static if (is(typeof(to!T(trustedGet!(fieldNames[i]))))) { - return to!T(trustedGet!(fieldNames[i])); - } else { - assert(false, "Cannot cast a "~(cast(Kind)m_kind).to!string~" value ("~FT.stringof~") to "~T.stringof); - } - } - } - assert(false); // never reached - } - /// ditto - T opCast(T)() const - { - // this method needs to be duplicated because inout doesn't work with to!() - import std.conv : to; - - final switch (m_kind) { - foreach (i, FT; FieldTypes) { - case __traits(getMember, Kind, fieldNames[i]): - static if (is(typeof(to!T(trustedGet!(fieldNames[i]))))) { - return to!T(trustedGet!(fieldNames[i])); - } else { - assert(false, "Cannot cast a "~(cast(Kind)m_kind).to!string~" value ("~FT.stringof~") to "~T.stringof); - } - } - } - assert(false); // never reached - } - - /// Uses `cast(string)`/`to!string` to return a string representation of the enclosed value. - string toString() const { return cast(string)this; } - - // NOTE: "this TA" is used here as the functional equivalent of inout, - // just that it generates one template instantiation per modifier - // combination, so that we can actually decide what to do for each - // case. - - /// Enables the invocation of methods of the stored value. - auto opDispatch(string name, this TA, ARGS...)(auto ref ARGS args) if (hasOp!(TA, OpKind.method, name, ARGS)) { return implementOp!(OpKind.method, name)(this, args); } - /// Enables accessing properties/fields of the stored value. - @property auto opDispatch(string name, this TA, ARGS...)(auto ref ARGS args) if (hasOp!(TA, OpKind.field, name, ARGS) && !hasOp!(TA, OpKind.method, name, ARGS)) { return implementOp!(OpKind.field, name)(this, args); } - /// Enables equality comparison with the stored value. - auto opEquals(T, this TA)(auto ref T other) if (hasOp!(TA, OpKind.binary, "==", T)) { return implementOp!(OpKind.binary, "==")(this, other); } - /// Enables relational comparisons with the stored value. - auto opCmp(T, this TA)(auto ref T other) if (hasOp!(TA, OpKind.binary, "<", T)) { assert(false, "TODO!"); } - /// Enables the use of unary operators with the stored value. - auto opUnary(string op, this TA)() if (hasOp!(TA, OpKind.unary, op)) { return implementOp!(OpKind.unary, op)(this); } - /// Enables the use of binary operators with the stored value. - auto opBinary(string op, T, this TA)(auto ref T other) if (hasOp!(TA, OpKind.binary, op, T)) { return implementOp!(OpKind.binary, op)(this, other); } - /// Enables the use of binary operators with the stored value. - auto opBinaryRight(string op, T, this TA)(auto ref T other) if (hasOp!(TA, OpKind.binaryRight, op, T)) { return implementOp!(OpKind.binaryRight, op)(this, other); } - /// Enables operator assignments on the stored value. - auto opOpAssign(string op, T, this TA)(auto ref T other) if (hasOp!(TA, OpKind.binary, op~"=", T)) { return implementOp!(OpKind.binary, op~"=")(this, other); } - /// Enables indexing operations on the stored value. - auto opIndex(this TA, ARGS...)(auto ref ARGS args) if (hasOp!(TA, OpKind.index, null, ARGS)) { return implementOp!(OpKind.index, null)(this, args); } - /// Enables index assignments on the stored value. - auto opIndexAssign(this TA, ARGS...)(auto ref ARGS args) if (hasOp!(TA, OpKind.indexAssign, null, ARGS)) { return implementOp!(OpKind.indexAssign, null)(this, args); } - /// Enables call syntax operations on the stored value. - auto opCall(this TA, ARGS...)(auto ref ARGS args) if (hasOp!(TA, OpKind.call, null, ARGS)) { return implementOp!(OpKind.call, null)(this, args); } - - private @trusted @property ref inout(typeof(__traits(getMember, U, f))) trustedGet(string f)() inout { return trustedGet!(inout(typeof(__traits(getMember, U, f)))); } - private @trusted @property ref inout(T) trustedGet(T)() inout { return *cast(inout(T)*)m_data.ptr; } -} - -/// -unittest -{ - import taggedalgebraic; - - struct Foo { - string name; - void bar() {} - } - - union Base { - int i; - string str; - Foo foo; - } - - alias Tagged = TaggedAlgebraic!Base; - - // Instantiate - Tagged taggedInt = 5; - Tagged taggedString = "Hello"; - Tagged taggedFoo = Foo(); - Tagged taggedAny = taggedInt; - taggedAny = taggedString; - taggedAny = taggedFoo; - - // Check type: Tagged.Kind is an enum - assert(taggedInt.kind == Tagged.Kind.i); - assert(taggedString.kind == Tagged.Kind.str); - assert(taggedFoo.kind == Tagged.Kind.foo); - assert(taggedAny.kind == Tagged.Kind.foo); - - // In most cases, can simply use as-is - auto num = 4 + taggedInt; - auto msg = taggedString ~ " World!"; - taggedFoo.bar(); - if (taggedAny.kind == Tagged.Kind.foo) // Make sure to check type first! - taggedAny.bar(); - //taggedString.bar(); // AssertError: Not a Foo! - - // Convert back by casting - auto i = cast(int) taggedInt; - auto str = cast(string) taggedString; - auto foo = cast(Foo) taggedFoo; - if (taggedAny.kind == Tagged.Kind.foo) // Make sure to check type first! - auto foo2 = cast(Foo) taggedAny; - //cast(Foo) taggedString; // AssertError! - - // Kind is an enum, so final switch is supported: - final switch (taggedAny.kind) { - case Tagged.Kind.i: - // It's "int i" - break; - - case Tagged.Kind.str: - // It's "string str" - break; - - case Tagged.Kind.foo: - // It's "Foo foo" - break; - } -} - -/** Operators and methods of the contained type can be used transparently. -*/ -@safe unittest { - static struct S { - int v; - int test() { return v / 2; } - } - - static union Test { - typeof(null) null_; - int integer; - string text; - string[string] dictionary; - S custom; - } - - alias TA = TaggedAlgebraic!Test; - - TA ta; - assert(ta.kind == TA.Kind.null_); - - ta = 12; - assert(ta.kind == TA.Kind.integer); - assert(ta == 12); - assert(cast(int)ta == 12); - assert(cast(long)ta == 12); - assert(cast(short)ta == 12); - - ta += 12; - assert(ta == 24); - assert(ta - 10 == 14); - - ta = ["foo" : "bar"]; - assert(ta.kind == TA.Kind.dictionary); - assert(ta["foo"] == "bar"); - - ta["foo"] = "baz"; - assert(ta["foo"] == "baz"); - - ta = S(8); - assert(ta.test() == 4); -} - -unittest { // std.conv integration - import std.conv : to; - - static struct S { - int v; - int test() { return v / 2; } - } - - static union Test { - typeof(null) null_; - int number; - string text; - } - - alias TA = TaggedAlgebraic!Test; - - TA ta; - assert(ta.kind == TA.Kind.null_); - ta = "34"; - assert(ta == "34"); - assert(to!int(ta) == 34, to!string(to!int(ta))); - assert(to!string(ta) == "34", to!string(ta)); -} - -/** Multiple fields are allowed to have the same type, in which case the type - ID enum is used to disambiguate. -*/ -@safe unittest { - static union Test { - typeof(null) null_; - int count; - int difference; - } - - alias TA = TaggedAlgebraic!Test; - - TA ta; - ta = TA(12, TA.Kind.count); - assert(ta.kind == TA.Kind.count); - assert(ta == 12); - - ta = null; - assert(ta.kind == TA.Kind.null_); -} - -unittest { - // test proper type modifier support - static struct S { - void test() {} - void testI() immutable {} - void testC() const {} - void testS() shared {} - void testSC() shared const {} - } - static union U { - S s; - } - - auto u = TaggedAlgebraic!U(S.init); - const uc = u; - immutable ui = cast(immutable)u; - //const shared usc = cast(shared)u; - //shared us = cast(shared)u; - - static assert( is(typeof(u.test()))); - static assert(!is(typeof(u.testI()))); - static assert( is(typeof(u.testC()))); - static assert(!is(typeof(u.testS()))); - static assert(!is(typeof(u.testSC()))); - - static assert(!is(typeof(uc.test()))); - static assert(!is(typeof(uc.testI()))); - static assert( is(typeof(uc.testC()))); - static assert(!is(typeof(uc.testS()))); - static assert(!is(typeof(uc.testSC()))); - - static assert(!is(typeof(ui.test()))); - static assert( is(typeof(ui.testI()))); - static assert( is(typeof(ui.testC()))); - static assert(!is(typeof(ui.testS()))); - static assert( is(typeof(ui.testSC()))); - - /*static assert(!is(typeof(us.test()))); - static assert(!is(typeof(us.testI()))); - static assert(!is(typeof(us.testC()))); - static assert( is(typeof(us.testS()))); - static assert( is(typeof(us.testSC()))); - - static assert(!is(typeof(usc.test()))); - static assert(!is(typeof(usc.testI()))); - static assert(!is(typeof(usc.testC()))); - static assert(!is(typeof(usc.testS()))); - static assert( is(typeof(usc.testSC())));*/ -} - -unittest { - // test attributes on contained values - import std.typecons : Rebindable, rebindable; - - class C { - void test() {} - void testC() const {} - void testI() immutable {} - } - union U { - Rebindable!(immutable(C)) c; - } - - auto ta = TaggedAlgebraic!U(rebindable(new immutable C)); - static assert(!is(typeof(ta.test()))); - static assert( is(typeof(ta.testC()))); - static assert( is(typeof(ta.testI()))); -} - -version (unittest) { - // test recursive definition using a wrapper dummy struct - // (needed to avoid "no size yet for forward reference" errors) - template ID(What) { alias ID = What; } - private struct _test_Wrapper { - TaggedAlgebraic!_test_U u; - alias u this; - this(ARGS...)(ARGS args) { u = TaggedAlgebraic!_test_U(args); } - } - private union _test_U { - _test_Wrapper[] children; - int value; - } - unittest { - alias TA = _test_Wrapper; - auto ta = TA(null); - ta ~= TA(0); - ta ~= TA(1); - ta ~= TA([TA(2)]); - assert(ta[0] == 0); - assert(ta[1] == 1); - assert(ta[2][0] == 2); - } -} - -unittest { // postblit/destructor test - static struct S { - static int i = 0; - bool initialized = false; - this(bool) { initialized = true; i++; } - this(this) { if (initialized) i++; } - ~this() { if (initialized) i--; } - } - - static struct U { - S s; - int t; - } - alias TA = TaggedAlgebraic!U; - { - assert(S.i == 0); - auto ta = TA(S(true)); - assert(S.i == 1); - { - auto tb = ta; - assert(S.i == 2); - ta = tb; - assert(S.i == 2); - ta = 1; - assert(S.i == 1); - ta = S(true); - assert(S.i == 2); - } - assert(S.i == 1); - } - assert(S.i == 0); - - static struct U2 { - S a; - S b; - } - alias TA2 = TaggedAlgebraic!U2; - { - auto ta2 = TA2(S(true), TA2.Kind.a); - assert(S.i == 1); - } - assert(S.i == 0); -} - -unittest { - static struct S { - union U { - int i; - string s; - U[] a; - } - alias TA = TaggedAlgebraic!U; - TA p; - alias p this; - } - S s = S(S.TA("hello")); - assert(cast(string)s == "hello"); -} - -unittest { // multiple operator choices - union U { - int i; - double d; - } - alias TA = TaggedAlgebraic!U; - TA ta = 12; - static assert(is(typeof(ta + 10) == TA)); // ambiguous, could be int or double - assert((ta + 10).kind == TA.Kind.i); - assert(ta + 10 == 22); - static assert(is(typeof(ta + 10.5) == double)); - assert(ta + 10.5 == 22.5); -} - -unittest { // Binary op between two TaggedAlgebraic values - union U { int i; } - alias TA = TaggedAlgebraic!U; - - TA a = 1, b = 2; - static assert(is(typeof(a + b) == int)); - assert(a + b == 3); -} - -unittest { // Ambiguous binary op between two TaggedAlgebraic values - union U { int i; double d; } - alias TA = TaggedAlgebraic!U; - - TA a = 1, b = 2; - static assert(is(typeof(a + b) == TA)); - assert((a + b).kind == TA.Kind.i); - assert(a + b == 3); -} - -unittest { - struct S { - union U { - @disableIndex string str; - S[] array; - S[string] object; - } - alias TA = TaggedAlgebraic!U; - TA payload; - alias payload this; - } - - S a = S(S.TA("hello")); - S b = S(S.TA(["foo": a])); - S c = S(S.TA([a])); - assert(b["foo"] == a); - assert(b["foo"] == "hello"); - assert(c[0] == a); - assert(c[0] == "hello"); -} - - -/** Tests if the algebraic type stores a value of a certain data type. -*/ -bool hasType(T, U)(in ref TaggedAlgebraic!U ta) -{ - alias Fields = Filter!(fieldMatchesType!(U, T), ta.fieldNames); - static assert(Fields.length > 0, "Type "~T.stringof~" cannot be stored in a "~(TaggedAlgebraic!U).stringof~"."); - - switch (ta.kind) { - default: return false; - foreach (i, fname; Fields) - case __traits(getMember, ta.Kind, fname): - return true; - } - assert(false); // never reached -} - -/// -unittest { - union Fields { - int number; - string text; - } - - TaggedAlgebraic!Fields ta = "test"; - - assert(ta.hasType!string); - assert(!ta.hasType!int); - - ta = 42; - assert(ta.hasType!int); - assert(!ta.hasType!string); -} - -unittest { // issue #1 - union U { - int a; - int b; - } - alias TA = TaggedAlgebraic!U; - - TA ta = TA(0, TA.Kind.b); - static assert(!is(typeof(ta.hasType!double))); - assert(ta.hasType!int); -} - -/** Gets the value stored in an algebraic type based on its data type. -*/ -ref inout(T) get(T, U)(ref inout(TaggedAlgebraic!U) ta) -{ - assert(hasType!(T, U)(ta)); - return ta.trustedGet!T; -} - -/// Convenience type that can be used for union fields that have no value (`void` is not allowed). -struct Void {} - -/// User-defined attibute to disable `opIndex` forwarding for a particular tagged union member. -@property auto disableIndex() { assert(__ctfe, "disableIndex must only be used as an attribute."); return DisableOpAttribute(OpKind.index, null); } - -private struct DisableOpAttribute { - OpKind kind; - string name; -} - - -private template hasOp(TA, OpKind kind, string name, ARGS...) -{ - import std.traits : CopyTypeQualifiers; - alias UQ = CopyTypeQualifiers!(TA, TA.Union); - enum hasOp = TypeTuple!(OpInfo!(UQ, kind, name, ARGS).fields).length > 0; -} - -unittest { - static struct S { - void m(int i) {} - bool opEquals(int i) { return true; } - bool opEquals(S s) { return true; } - } - - static union U { int i; string s; S st; } - alias TA = TaggedAlgebraic!U; - - static assert(hasOp!(TA, OpKind.binary, "+", int)); - static assert(hasOp!(TA, OpKind.binary, "~", string)); - static assert(hasOp!(TA, OpKind.binary, "==", int)); - static assert(hasOp!(TA, OpKind.binary, "==", string)); - static assert(hasOp!(TA, OpKind.binary, "==", int)); - static assert(hasOp!(TA, OpKind.binary, "==", S)); - static assert(hasOp!(TA, OpKind.method, "m", int)); - static assert(hasOp!(TA, OpKind.binary, "+=", int)); - static assert(!hasOp!(TA, OpKind.binary, "~", int)); - static assert(!hasOp!(TA, OpKind.binary, "~", int)); - static assert(!hasOp!(TA, OpKind.method, "m", string)); - static assert(!hasOp!(TA, OpKind.method, "m")); - static assert(!hasOp!(const(TA), OpKind.binary, "+=", int)); - static assert(!hasOp!(const(TA), OpKind.method, "m", int)); -} - -unittest { - struct S { - union U { - string s; - S[] arr; - S[string] obj; - } - alias TA = TaggedAlgebraic!(S.U); - TA payload; - alias payload this; - } - static assert(hasOp!(S.TA, OpKind.index, null, size_t)); - static assert(hasOp!(S.TA, OpKind.index, null, int)); - static assert(hasOp!(S.TA, OpKind.index, null, string)); - static assert(hasOp!(S.TA, OpKind.field, "length")); -} - -unittest { // "in" operator - union U { - string[string] dict; - } - alias TA = TaggedAlgebraic!U; - auto ta = TA(["foo": "bar"]); - assert("foo" in ta); - assert(*("foo" in ta) == "bar"); -} - -private static auto implementOp(OpKind kind, string name, T, ARGS...)(ref T self, auto ref ARGS args) -{ - import std.array : join; - import std.traits : CopyTypeQualifiers; - import std.variant : Algebraic, Variant; - alias UQ = CopyTypeQualifiers!(T, T.Union); - - alias info = OpInfo!(UQ, kind, name, ARGS); - - static assert(hasOp!(T, kind, name, ARGS)); - - static assert(info.fields.length > 0, "Implementing operator that has no valid implementation for any supported type."); - - //pragma(msg, "Fields for "~kind.stringof~" "~name~", "~T.stringof~": "~info.fields.stringof); - //pragma(msg, "Return types for "~kind.stringof~" "~name~", "~T.stringof~": "~info.ReturnTypes.stringof); - //pragma(msg, typeof(T.Union.tupleof)); - //import std.meta : staticMap; pragma(msg, staticMap!(isMatchingUniqueType!(T.Union), info.ReturnTypes)); - - switch (self.m_kind) { - default: assert(false, "Operator "~name~" ("~kind.stringof~") can only be used on values of the following types: "~[info.fields].join(", ")); - foreach (i, f; info.fields) { - alias FT = typeof(__traits(getMember, T.Union, f)); - case __traits(getMember, T.Kind, f): - static if (NoDuplicates!(info.ReturnTypes).length == 1) - return info.perform(self.trustedGet!FT, args); - else static if (allSatisfy!(isMatchingUniqueType!(T.Union), info.ReturnTypes)) - return TaggedAlgebraic!(T.Union)(info.perform(self.trustedGet!FT, args)); - else static if (allSatisfy!(isNoVariant, info.ReturnTypes)) { - alias Alg = Algebraic!(NoDuplicates!(info.ReturnTypes)); - info.ReturnTypes[i] ret = info.perform(self.trustedGet!FT, args); - import std.traits : isInstanceOf; - static if (isInstanceOf!(TaggedAlgebraic, typeof(ret))) return Alg(ret.payload); - else return Alg(ret); - } - else static if (is(FT == Variant)) - return info.perform(self.trustedGet!FT, args); - else - return Variant(info.perform(self.trustedGet!FT, args)); - } - } - - assert(false); // never reached -} - -unittest { // opIndex on recursive TA with closed return value set - static struct S { - union U { - char ch; - string str; - S[] arr; - } - alias TA = TaggedAlgebraic!U; - TA payload; - alias payload this; - - this(T)(T t) { this.payload = t; } - } - S a = S("foo"); - S s = S([a]); - - assert(implementOp!(OpKind.field, "length")(s.payload) == 1); - static assert(is(typeof(implementOp!(OpKind.index, null)(s.payload, 0)) == S.TA)); - assert(implementOp!(OpKind.index, null)(s.payload, 0) == "foo"); -} - -unittest { // opIndex on recursive TA with closed return value set using @disableIndex - static struct S { - union U { - @disableIndex string str; - S[] arr; - } - alias TA = TaggedAlgebraic!U; - TA payload; - alias payload this; - - this(T)(T t) { this.payload = t; } - } - S a = S("foo"); - S s = S([a]); - - assert(implementOp!(OpKind.field, "length")(s.payload) == 1); - static assert(is(typeof(implementOp!(OpKind.index, null)(s.payload, 0)) == S)); - assert(implementOp!(OpKind.index, null)(s.payload, 0) == "foo"); -} - - -private auto performOpRaw(U, OpKind kind, string name, T, ARGS...)(ref T value, /*auto ref*/ ARGS args) -{ - static if (kind == OpKind.binary) return mixin("value "~name~" args[0]"); - else static if (kind == OpKind.binaryRight) return mixin("args[0] "~name~" value"); - else static if (kind == OpKind.unary) return mixin("name "~value); - else static if (kind == OpKind.method) return __traits(getMember, value, name)(args); - else static if (kind == OpKind.field) return __traits(getMember, value, name); - else static if (kind == OpKind.index) return value[args]; - else static if (kind == OpKind.indexAssign) return value[args[1 .. $]] = args[0]; - else static if (kind == OpKind.call) return value(args); - else static assert(false, "Unsupported kind of operator: "~kind.stringof); -} - -unittest { - union U { int i; string s; } - - { int v = 1; assert(performOpRaw!(U, OpKind.binary, "+")(v, 3) == 4); } - { string v = "foo"; assert(performOpRaw!(U, OpKind.binary, "~")(v, "bar") == "foobar"); } -} - - -private auto performOp(U, OpKind kind, string name, T, ARGS...)(ref T value, /*auto ref*/ ARGS args) -{ - import std.traits : isInstanceOf; - static if (ARGS.length > 0 && isInstanceOf!(TaggedAlgebraic, ARGS[0])) { - static if (is(typeof(performOpRaw!(U, kind, name, T, ARGS)(value, args)))) { - return performOpRaw!(U, kind, name, T, ARGS)(value, args); - } else { - alias TA = ARGS[0]; - template MTypesImpl(size_t i) { - static if (i < TA.FieldTypes.length) { - alias FT = TA.FieldTypes[i]; - static if (is(typeof(&performOpRaw!(U, kind, name, T, FT, ARGS[1 .. $])))) - alias MTypesImpl = TypeTuple!(FT, MTypesImpl!(i+1)); - else alias MTypesImpl = TypeTuple!(MTypesImpl!(i+1)); - } else alias MTypesImpl = TypeTuple!(); - } - alias MTypes = NoDuplicates!(MTypesImpl!0); - static assert(MTypes.length > 0, "No type of the TaggedAlgebraic parameter matches any function declaration."); - static if (MTypes.length == 1) { - if (args[0].hasType!(MTypes[0])) - return performOpRaw!(U, kind, name)(value, args[0].get!(MTypes[0]), args[1 .. $]); - } else { - // TODO: allow all return types (fall back to Algebraic or Variant) - foreach (FT; MTypes) { - if (args[0].hasType!FT) - return ARGS[0](performOpRaw!(U, kind, name)(value, args[0].get!FT, args[1 .. $])); - } - } - throw new /*InvalidAgument*/Exception("Algebraic parameter type mismatch"); - } - } else return performOpRaw!(U, kind, name, T, ARGS)(value, args); -} - -unittest { - union U { int i; double d; string s; } - - { int v = 1; assert(performOp!(U, OpKind.binary, "+")(v, 3) == 4); } - { string v = "foo"; assert(performOp!(U, OpKind.binary, "~")(v, "bar") == "foobar"); } - { string v = "foo"; assert(performOp!(U, OpKind.binary, "~")(v, TaggedAlgebraic!U("bar")) == "foobar"); } - { int v = 1; assert(performOp!(U, OpKind.binary, "+")(v, TaggedAlgebraic!U(3)) == 4); } -} - - -private template OpInfo(U, OpKind kind, string name, ARGS...) -{ - import std.traits : CopyTypeQualifiers, FieldTypeTuple, FieldNameTuple, ReturnType; - - private alias FieldTypes = FieldTypeTuple!U; - private alias fieldNames = FieldNameTuple!U; - - private template isOpEnabled(string field) - { - alias attribs = TypeTuple!(__traits(getAttributes, __traits(getMember, U, field))); - template impl(size_t i) { - static if (i < attribs.length) { - static if (is(typeof(attribs[i]) == DisableOpAttribute)) { - static if (kind == attribs[i].kind && name == attribs[i].name) - enum impl = false; - else enum impl = impl!(i+1); - } else enum impl = impl!(i+1); - } else enum impl = true; - } - enum isOpEnabled = impl!0; - } - - template fieldsImpl(size_t i) - { - static if (i < FieldTypes.length) { - static if (isOpEnabled!(fieldNames[i]) && is(typeof(&performOp!(U, kind, name, FieldTypes[i], ARGS)))) { - alias fieldsImpl = TypeTuple!(fieldNames[i], fieldsImpl!(i+1)); - } else alias fieldsImpl = fieldsImpl!(i+1); - } else alias fieldsImpl = TypeTuple!(); - } - alias fields = fieldsImpl!0; - - template ReturnTypesImpl(size_t i) { - static if (i < fields.length) { - alias FT = CopyTypeQualifiers!(U, typeof(__traits(getMember, U, fields[i]))); - alias ReturnTypesImpl = TypeTuple!(ReturnType!(performOp!(U, kind, name, FT, ARGS)), ReturnTypesImpl!(i+1)); - } else alias ReturnTypesImpl = TypeTuple!(); - } - alias ReturnTypes = ReturnTypesImpl!0; - - static auto perform(T)(ref T value, auto ref ARGS args) { return performOp!(U, kind, name)(value, args); } -} - -private template ImplicitUnqual(T) { - import std.traits : Unqual, hasAliasing; - static if (is(T == void)) alias ImplicitUnqual = void; - else { - private static struct S { T t; } - static if (hasAliasing!S) alias ImplicitUnqual = T; - else alias ImplicitUnqual = Unqual!T; - } -} - -private enum OpKind { - binary, - binaryRight, - unary, - method, - field, - index, - indexAssign, - call -} - -private template TypeEnum(U) -{ - import std.array : join; - import std.traits : FieldNameTuple; - mixin("enum TypeEnum { " ~ [FieldNameTuple!U].join(", ") ~ " }"); -} - -private string generateConstructors(U)() -{ - import std.algorithm : map; - import std.array : join; - import std.string : format; - import std.traits : FieldTypeTuple; - - string ret; - - // disable default construction if first type is not a null/Void type - static if (!is(FieldTypeTuple!U[0] == typeof(null)) && !is(FieldTypeTuple!U[0] == Void)) - { - ret ~= q{ - @disable this(); - }; - } - - // normal type constructors - foreach (tname; UniqueTypeFields!U) - ret ~= q{ - this(typeof(U.%s) value) - { - m_data.rawEmplace(value); - m_kind = Kind.%s; - } - - void opAssign(typeof(U.%s) value) - { - if (m_kind != Kind.%s) { - // NOTE: destroy(this) doesn't work for some opDispatch-related reason - static if (is(typeof(&this.__xdtor))) - this.__xdtor(); - m_data.rawEmplace(value); - } else { - trustedGet!"%s" = value; - } - m_kind = Kind.%s; - } - }.format(tname, tname, tname, tname, tname, tname); - - // type constructors with explicit type tag - foreach (tname; AmbiguousTypeFields!U) - ret ~= q{ - this(typeof(U.%s) value, Kind type) - { - assert(type.among!(%s), format("Invalid type ID for type %%s: %%s", typeof(U.%s).stringof, type)); - m_data.rawEmplace(value); - m_kind = type; - } - }.format(tname, [SameTypeFields!(U, tname)].map!(f => "Kind."~f).join(", "), tname); - - return ret; -} - -private template UniqueTypeFields(U) { - import std.traits : FieldTypeTuple, FieldNameTuple; - - alias Types = FieldTypeTuple!U; - - template impl(size_t i) { - static if (i < Types.length) { - enum name = FieldNameTuple!U[i]; - alias T = Types[i]; - static if (staticIndexOf!(T, Types) == i && staticIndexOf!(T, Types[i+1 .. $]) < 0) - alias impl = TypeTuple!(name, impl!(i+1)); - else alias impl = TypeTuple!(impl!(i+1)); - } else alias impl = TypeTuple!(); - } - alias UniqueTypeFields = impl!0; -} - -private template AmbiguousTypeFields(U) { - import std.traits : FieldTypeTuple, FieldNameTuple; - - alias Types = FieldTypeTuple!U; - - template impl(size_t i) { - static if (i < Types.length) { - enum name = FieldNameTuple!U[i]; - alias T = Types[i]; - static if (staticIndexOf!(T, Types) == i && staticIndexOf!(T, Types[i+1 .. $]) >= 0) - alias impl = TypeTuple!(name, impl!(i+1)); - else alias impl = impl!(i+1); - } else alias impl = TypeTuple!(); - } - alias AmbiguousTypeFields = impl!0; -} - -unittest { - union U { - int a; - string b; - int c; - double d; - } - static assert([UniqueTypeFields!U] == ["b", "d"]); - static assert([AmbiguousTypeFields!U] == ["a"]); -} - -private template SameTypeFields(U, string field) { - import std.traits : FieldTypeTuple, FieldNameTuple; - - alias Types = FieldTypeTuple!U; - - alias T = typeof(__traits(getMember, U, field)); - template impl(size_t i) { - static if (i < Types.length) { - enum name = FieldNameTuple!U[i]; - static if (is(Types[i] == T)) - alias impl = TypeTuple!(name, impl!(i+1)); - else alias impl = TypeTuple!(impl!(i+1)); - } else alias impl = TypeTuple!(); - } - alias SameTypeFields = impl!0; -} - -private template MemberType(U) { - template MemberType(string name) { - alias MemberType = typeof(__traits(getMember, U, name)); - } -} - -private template isMatchingType(U) { - import std.traits : FieldTypeTuple; - enum isMatchingType(T) = staticIndexOf!(T, FieldTypeTuple!U) >= 0; -} - -private template isMatchingUniqueType(U) { - import std.traits : staticMap; - alias UniqueTypes = staticMap!(FieldTypeOf!U, UniqueTypeFields!U); - template isMatchingUniqueType(T) { - static if (is(T : TaggedAlgebraic!U)) enum isMatchingUniqueType = true; - else enum isMatchingUniqueType = staticIndexOfImplicit!(T, UniqueTypes) >= 0; - } -} - -private template fieldMatchesType(U, T) -{ - enum fieldMatchesType(string field) = is(typeof(__traits(getMember, U, field)) == T); -} - -private template FieldTypeOf(U) { - template FieldTypeOf(string name) { - alias FieldTypeOf = typeof(__traits(getMember, U, name)); - } -} - -private template staticIndexOfImplicit(T, Types...) { - template impl(size_t i) { - static if (i < Types.length) { - static if (is(T : Types[i])) enum impl = i; - else enum impl = impl!(i+1); - } else enum impl = -1; - } - enum staticIndexOfImplicit = impl!0; -} - -unittest { - static assert(staticIndexOfImplicit!(immutable(char), char) == 0); - static assert(staticIndexOfImplicit!(int, long) == 0); - static assert(staticIndexOfImplicit!(long, int) < 0); - static assert(staticIndexOfImplicit!(int, int, double) == 0); - static assert(staticIndexOfImplicit!(double, int, double) == 1); -} - - -private template isNoVariant(T) { - import std.variant : Variant; - enum isNoVariant = !is(T == Variant); -} - -private void rawEmplace(T)(void[] dst, ref T src) -{ - T* tdst = () @trusted { return cast(T*)dst.ptr; } (); - static if (is(T == class)) { - *tdst = src; - } else { - import std.conv : emplace; - emplace(tdst); - *tdst = src; - } -} diff --git a/src/sdlang/token.d b/src/sdlang/token.d deleted file mode 100644 index 0a5b2fd..0000000 --- a/src/sdlang/token.d +++ /dev/null @@ -1,550 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.token; - -import std.array; -import std.base64; -import std.conv; -import std.datetime; -import std.meta; -import std.range; -import std.string; -import std.traits; -import std.typetuple; -import std.variant; - -import sdlang.exception; -import sdlang.symbol; -import sdlang.util; - -/// DateTime doesn't support milliseconds, but SDLang's "Date Time" type does. -/// So this is needed for any SDL "Date Time" that doesn't include a time zone. -struct DateTimeFrac -{ - DateTime dateTime; - Duration fracSecs; - deprecated("Use fracSecs instead.") { - @property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); } - @property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; } - } -} - -/++ -If a "Date Time" literal in the SDL file has a time zone that's not found in -your system, you get one of these instead of a SysTime. (Because it's -impossible to indicate "unknown time zone" with `std.datetime.TimeZone`.) - -The difference between this and `DateTimeFrac` is that `DateTimeFrac` -indicates that no time zone was specified in the SDL at all, whereas -`DateTimeFracUnknownZone` indicates that a time zone was specified but -data for it could not be found on your system. -+/ -struct DateTimeFracUnknownZone -{ - DateTime dateTime; - Duration fracSecs; - deprecated("Use fracSecs instead.") { - @property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); } - @property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; } - } - string timeZone; - - bool opEquals(const DateTimeFracUnknownZone b) const - { - return opEquals(b); - } - bool opEquals(ref const DateTimeFracUnknownZone b) const - { - return - this.dateTime == b.dateTime && - this.fracSecs == b.fracSecs && - this.timeZone == b.timeZone; - } -} - -/++ -SDLang's datatypes map to D's datatypes as described below. -Most are straightforward, but take special note of the date/time-related types. - ---------------------------------------------------------------- -Boolean: bool -Null: typeof(null) -Unicode Character: dchar -Double-Quote Unicode String: string -Raw Backtick Unicode String: string -Integer (32 bits signed): int -Long Integer (64 bits signed): long -Float (32 bits signed): float -Double Float (64 bits signed): double -Decimal (128+ bits signed): real -Binary (standard Base64): ubyte[] -Time Span: Duration - -Date (with no time at all): Date -Date Time (no timezone): DateTimeFrac -Date Time (with a known timezone): SysTime -Date Time (with an unknown timezone): DateTimeFracUnknownZone ---------------------------------------------------------------- -+/ -alias ValueTypes = TypeTuple!( - bool, - string, dchar, - int, long, - float, double, real, - Date, DateTimeFrac, SysTime, DateTimeFracUnknownZone, Duration, - ubyte[], - typeof(null), -); - -alias Value = Algebraic!( ValueTypes ); ///ditto -enum isValueType(T) = staticIndexOf!(T, ValueTypes) != -1; - -enum isSink(T) = - isOutputRange!T && - is(ElementType!(T)[] == string); - -string toSDLString(T)(T value) if(is(T==Value) || isValueType!T) -{ - Appender!string sink; - toSDLString(value, sink); - return sink.data; -} - -/// Throws SDLangException if value is infinity, -infinity or NaN, because -/// those are not currently supported by the SDLang spec. -void toSDLString(Sink)(Value value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - foreach(T; ValueTypes) - { - if(value.type == typeid(T)) - { - toSDLString( value.get!T(), sink ); - return; - } - } - - throw new Exception("Internal SDLang-D error: Unhandled type of Value. Contains: "~value.toString()); -} - -@("toSDLString on infinity and NaN") -unittest -{ - import std.exception; - - auto floatInf = float.infinity; - auto floatNegInf = -float.infinity; - auto floatNaN = float.nan; - - auto doubleInf = double.infinity; - auto doubleNegInf = -double.infinity; - auto doubleNaN = double.nan; - - auto realInf = real.infinity; - auto realNegInf = -real.infinity; - auto realNaN = real.nan; - - assertNotThrown( toSDLString(0.0F) ); - assertNotThrown( toSDLString(0.0) ); - assertNotThrown( toSDLString(0.0L) ); - - assertThrown!ValidationException( toSDLString(floatInf) ); - assertThrown!ValidationException( toSDLString(floatNegInf) ); - assertThrown!ValidationException( toSDLString(floatNaN) ); - - assertThrown!ValidationException( toSDLString(doubleInf) ); - assertThrown!ValidationException( toSDLString(doubleNegInf) ); - assertThrown!ValidationException( toSDLString(doubleNaN) ); - - assertThrown!ValidationException( toSDLString(realInf) ); - assertThrown!ValidationException( toSDLString(realNegInf) ); - assertThrown!ValidationException( toSDLString(realNaN) ); - - assertThrown!ValidationException( toSDLString(Value(floatInf)) ); - assertThrown!ValidationException( toSDLString(Value(floatNegInf)) ); - assertThrown!ValidationException( toSDLString(Value(floatNaN)) ); - - assertThrown!ValidationException( toSDLString(Value(doubleInf)) ); - assertThrown!ValidationException( toSDLString(Value(doubleNegInf)) ); - assertThrown!ValidationException( toSDLString(Value(doubleNaN)) ); - - assertThrown!ValidationException( toSDLString(Value(realInf)) ); - assertThrown!ValidationException( toSDLString(Value(realNegInf)) ); - assertThrown!ValidationException( toSDLString(Value(realNaN)) ); -} - -void toSDLString(Sink)(typeof(null) value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put("null"); -} - -void toSDLString(Sink)(bool value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put(value? "true" : "false"); -} - -//TODO: Figure out how to properly handle strings/chars containing lineSep or paraSep -void toSDLString(Sink)(string value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put('"'); - - // This loop is UTF-safe - foreach(char ch; value) - { - if (ch == '\n') sink.put(`\n`); - else if(ch == '\r') sink.put(`\r`); - else if(ch == '\t') sink.put(`\t`); - else if(ch == '\"') sink.put(`\"`); - else if(ch == '\\') sink.put(`\\`); - else - sink.put(ch); - } - - sink.put('"'); -} - -void toSDLString(Sink)(dchar value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put('\''); - - if (value == '\n') sink.put(`\n`); - else if(value == '\r') sink.put(`\r`); - else if(value == '\t') sink.put(`\t`); - else if(value == '\'') sink.put(`\'`); - else if(value == '\\') sink.put(`\\`); - else - sink.put(value); - - sink.put('\''); -} - -void toSDLString(Sink)(int value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put( "%s".format(value) ); -} - -void toSDLString(Sink)(long value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put( "%sL".format(value) ); -} - -private void checkUnsupportedFloatingPoint(T)(T value) if(isFloatingPoint!T) -{ - import std.exception; - import std.math; - - enforce!ValidationException( - !isInfinity(value), - "SDLang does not currently support infinity for floating-point types" - ); - - enforce!ValidationException( - !isNaN(value), - "SDLang does not currently support NaN for floating-point types" - ); -} - -void toSDLString(Sink)(float value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - checkUnsupportedFloatingPoint(value); - sink.put( "%.10sF".format(value) ); -} - -void toSDLString(Sink)(double value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - checkUnsupportedFloatingPoint(value); - sink.put( "%.30sD".format(value) ); -} - -void toSDLString(Sink)(real value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - checkUnsupportedFloatingPoint(value); - sink.put( "%.30sBD".format(value) ); -} - -void toSDLString(Sink)(Date value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put(to!string(value.year)); - sink.put('/'); - sink.put(to!string(cast(int)value.month)); - sink.put('/'); - sink.put(to!string(value.day)); -} - -void toSDLString(Sink)(DateTimeFrac value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - toSDLString(value.dateTime.date, sink); - sink.put(' '); - sink.put("%.2s".format(value.dateTime.hour)); - sink.put(':'); - sink.put("%.2s".format(value.dateTime.minute)); - - if(value.dateTime.second != 0) - { - sink.put(':'); - sink.put("%.2s".format(value.dateTime.second)); - } - - if(value.fracSecs != 0.msecs) - { - sink.put('.'); - sink.put("%.3s".format(value.fracSecs.total!"msecs")); - } -} - -void toSDLString(Sink)(SysTime value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - auto dateTimeFrac = DateTimeFrac(cast(DateTime)value, value.fracSecs); - toSDLString(dateTimeFrac, sink); - - sink.put("-"); - - auto tzString = value.timezone.name; - - // If name didn't exist, try abbreviation. - // Note that according to std.datetime docs, on Windows the - // stdName/dstName may not be properly abbreviated. - version(Windows) {} else - if(tzString == "") - { - auto tz = value.timezone; - auto stdTime = value.stdTime; - - if(tz.hasDST()) - tzString = tz.dstInEffect(stdTime)? tz.dstName : tz.stdName; - else - tzString = tz.stdName; - } - - if(tzString == "") - { - auto offset = value.timezone.utcOffsetAt(value.stdTime); - sink.put("GMT"); - - if(offset < seconds(0)) - { - sink.put("-"); - offset = -offset; - } - else - sink.put("+"); - - sink.put("%.2s".format(offset.split.hours)); - sink.put(":"); - sink.put("%.2s".format(offset.split.minutes)); - } - else - sink.put(tzString); -} - -void toSDLString(Sink)(DateTimeFracUnknownZone value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - auto dateTimeFrac = DateTimeFrac(value.dateTime, value.fracSecs); - toSDLString(dateTimeFrac, sink); - - sink.put("-"); - sink.put(value.timeZone); -} - -void toSDLString(Sink)(Duration value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - if(value < seconds(0)) - { - sink.put("-"); - value = -value; - } - - auto days = value.total!"days"(); - if(days != 0) - { - sink.put("%s".format(days)); - sink.put("d:"); - } - - sink.put("%.2s".format(value.split.hours)); - sink.put(':'); - sink.put("%.2s".format(value.split.minutes)); - sink.put(':'); - sink.put("%.2s".format(value.split.seconds)); - - if(value.split.msecs != 0) - { - sink.put('.'); - sink.put("%.3s".format(value.split.msecs)); - } -} - -void toSDLString(Sink)(ubyte[] value, ref Sink sink) if(isOutputRange!(Sink,char)) -{ - sink.put('['); - sink.put( Base64.encode(value) ); - sink.put(']'); -} - -/// This only represents terminals. Nonterminals aren't -/// constructed since the AST is directly built during parsing. -struct Token -{ - Symbol symbol = sdlang.symbol.symbol!"Error"; /// The "type" of this token - Location location; - Value value; /// Only valid when `symbol` is `symbol!"Value"`, otherwise null - string data; /// Original text from source - - @disable this(); - this(Symbol symbol, Location location, Value value=Value(null), string data=null) - { - this.symbol = symbol; - this.location = location; - this.value = value; - this.data = data; - } - - /// Tokens with differing symbols are always unequal. - /// Tokens with differing values are always unequal. - /// Tokens with differing Value types are always unequal. - /// Member `location` is always ignored for comparison. - /// Member `data` is ignored for comparison *EXCEPT* when the symbol is Ident. - bool opEquals(Token b) - { - return opEquals(b); - } - bool opEquals(ref Token b) ///ditto - { - if( - this.symbol != b.symbol || - this.value.type != b.value.type || - this.value != b.value - ) - return false; - - if(this.symbol == .symbol!"Ident") - return this.data == b.data; - - return true; - } - - bool matches(string symbolName)() - { - return this.symbol == .symbol!symbolName; - } -} - -@("sdlang token") -unittest -{ - auto loc = Location("", 0, 0, 0); - auto loc2 = Location("a", 1, 1, 1); - - assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc )); - assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc2)); - assert(Token(symbol!":", loc) == Token(symbol!":", loc )); - assert(Token(symbol!"EOL",loc) != Token(symbol!":", loc )); - assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),"\n")); - - assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),";" )); - assert(Token(symbol!"EOL",loc,Value(null),"A" ) == Token(symbol!"EOL",loc,Value(null),"B" )); - assert(Token(symbol!":", loc,Value(null),"A" ) == Token(symbol!":", loc,Value(null),"BB")); - assert(Token(symbol!"EOL",loc,Value(null),"A" ) != Token(symbol!":", loc,Value(null),"A" )); - - assert(Token(symbol!"Ident",loc,Value(null),"foo") == Token(symbol!"Ident",loc,Value(null),"foo")); - assert(Token(symbol!"Ident",loc,Value(null),"foo") != Token(symbol!"Ident",loc,Value(null),"BAR")); - - assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"foo")); - assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc2,Value(null),"foo")); - assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"BAR")); - assert(Token(symbol!"Value",loc,Value( 7),"foo") == Token(symbol!"Value",loc, Value( 7),"BAR")); - assert(Token(symbol!"Value",loc,Value( 7),"foo") != Token(symbol!"Value",loc, Value( "A"),"foo")); - assert(Token(symbol!"Value",loc,Value( 7),"foo") != Token(symbol!"Value",loc, Value( 2),"foo")); - assert(Token(symbol!"Value",loc,Value(cast(int)7)) != Token(symbol!"Value",loc, Value(cast(long)7))); - assert(Token(symbol!"Value",loc,Value(cast(float)1.2)) != Token(symbol!"Value",loc, Value(cast(double)1.2))); -} - -@("sdlang Value.toSDLString()") -unittest -{ - // Bool and null - assert(Value(null ).toSDLString() == "null"); - assert(Value(true ).toSDLString() == "true"); - assert(Value(false).toSDLString() == "false"); - - // Base64 Binary - assert(Value(cast(ubyte[])"hello world".dup).toSDLString() == "[aGVsbG8gd29ybGQ=]"); - - // Integer - assert(Value(cast( int) 7).toSDLString() == "7"); - assert(Value(cast( int)-7).toSDLString() == "-7"); - assert(Value(cast( int) 0).toSDLString() == "0"); - - assert(Value(cast(long) 7).toSDLString() == "7L"); - assert(Value(cast(long)-7).toSDLString() == "-7L"); - assert(Value(cast(long) 0).toSDLString() == "0L"); - - // Floating point - assert(Value(cast(float) 1.5).toSDLString() == "1.5F"); - assert(Value(cast(float)-1.5).toSDLString() == "-1.5F"); - assert(Value(cast(float) 0).toSDLString() == "0F"); - - assert(Value(cast(double) 1.5).toSDLString() == "1.5D"); - assert(Value(cast(double)-1.5).toSDLString() == "-1.5D"); - assert(Value(cast(double) 0).toSDLString() == "0D"); - - assert(Value(cast(real) 1.5).toSDLString() == "1.5BD"); - assert(Value(cast(real)-1.5).toSDLString() == "-1.5BD"); - assert(Value(cast(real) 0).toSDLString() == "0BD"); - - // String - assert(Value("hello" ).toSDLString() == `"hello"`); - assert(Value(" hello ").toSDLString() == `" hello "`); - assert(Value("" ).toSDLString() == `""`); - assert(Value("hello \r\n\t\"\\ world").toSDLString() == `"hello \r\n\t\"\\ world"`); - assert(Value("日本語").toSDLString() == `"日本語"`); - - // Chars - assert(Value(cast(dchar) 'A').toSDLString() == `'A'`); - assert(Value(cast(dchar)'\r').toSDLString() == `'\r'`); - assert(Value(cast(dchar)'\n').toSDLString() == `'\n'`); - assert(Value(cast(dchar)'\t').toSDLString() == `'\t'`); - assert(Value(cast(dchar)'\'').toSDLString() == `'\''`); - assert(Value(cast(dchar)'\\').toSDLString() == `'\\'`); - assert(Value(cast(dchar) '月').toSDLString() == `'月'`); - - // Date - assert(Value(Date( 2004,10,31)).toSDLString() == "2004/10/31"); - assert(Value(Date(-2004,10,31)).toSDLString() == "-2004/10/31"); - - // DateTimeFrac w/o Frac - assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15))).toSDLString() == "2004/10/31 14:30:15"); - assert(Value(DateTimeFrac(DateTime(2004,10,31, 1, 2, 3))).toSDLString() == "2004/10/31 01:02:03"); - assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15))).toSDLString() == "-2004/10/31 14:30:15"); - - // DateTimeFrac w/ Frac - assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "2004/10/31 14:30:15.123"); - assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 120.msecs)).toSDLString() == "2004/10/31 14:30:15.120"); - assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 100.msecs)).toSDLString() == "2004/10/31 14:30:15.100"); - assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 12.msecs)).toSDLString() == "2004/10/31 14:30:15.012"); - assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 1.msecs)).toSDLString() == "2004/10/31 14:30:15.001"); - assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "-2004/10/31 14:30:15.123"); - - // DateTimeFracUnknownZone - assert(Value(DateTimeFracUnknownZone(DateTime(2004,10,31, 14,30,15), 123.msecs, "Foo/Bar")).toSDLString() == "2004/10/31 14:30:15.123-Foo/Bar"); - - // SysTime - assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15-GMT+00:00"); - assert(Value(SysTime(DateTime(2004,10,31, 1, 2, 3), new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 01:02:03-GMT+00:00"); - assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes(10) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:10"); - assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone(-hours(5)-minutes(30) ))).toSDLString() == "2004/10/31 14:30:15-GMT-05:30"); - assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes( 3) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:03"); - assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), 123.msecs, new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15.123-GMT+00:00"); - - // Duration - assert( "12:14:42" == Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs( 0)).toSDLString()); - assert("-12:14:42" == Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs( 0)).toSDLString()); - assert( "00:09:12" == Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs( 0)).toSDLString()); - assert( "00:00:01.023" == Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23)).toSDLString()); - assert( "23d:05:21:23.532" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532)).toSDLString()); - assert( "23d:05:21:23.530" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530)).toSDLString()); - assert( "23d:05:21:23.500" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500)).toSDLString()); - assert("-23d:05:21:23.532" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532)).toSDLString()); - assert("-23d:05:21:23.500" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500)).toSDLString()); - assert( "23d:05:21:23" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs( 0)).toSDLString()); -} diff --git a/src/sdlang/util.d b/src/sdlang/util.d deleted file mode 100644 index d192ea2..0000000 --- a/src/sdlang/util.d +++ /dev/null @@ -1,200 +0,0 @@ -// SDLang-D -// Written in the D programming language. - -module sdlang.util; - -import std.algorithm; -import std.array; -import std.conv; -import std.datetime; -import std.range; -import std.stdio; -import std.string; - -import sdlang.exception; -import sdlang.token; - -enum sdlangVersion = "0.9.1"; - -alias immutable(ubyte)[] ByteString; - -auto startsWith(T)(string haystack, T needle) - if( is(T:ByteString) || is(T:string) ) -{ - return std.algorithm.startsWith( cast(ByteString)haystack, cast(ByteString)needle ); -} - -struct Location -{ - string file; /// Filename (including path) - int line; /// Zero-indexed - int col; /// Zero-indexed, Tab counts as 1 - size_t index; /// Index into the source - - this(int line, int col, int index) - { - this.line = line; - this.col = col; - this.index = index; - } - - this(string file, int line, int col, int index) - { - this.file = file; - this.line = line; - this.col = col; - this.index = index; - } - - /// Convert to string. Optionally takes output range as a sink. - string toString() - { - Appender!string sink; - this.toString(sink); - return sink.data; - } - - ///ditto - void toString(Sink)(ref Sink sink) if(isOutputRange!(Sink,char)) - { - sink.put(file); - sink.put("("); - sink.put(to!string(line+1)); - sink.put(":"); - sink.put(to!string(col+1)); - sink.put(")"); - } -} - -struct FullName -{ - string namespace; - string name; - - /// Convert to string. Optionally takes output range as a sink. - string toString() - { - if(namespace == "") - return name; - - Appender!string sink; - this.toString(sink); - return sink.data; - } - - ///ditto - void toString(Sink)(ref Sink sink) if(isOutputRange!(Sink,char)) - { - if(namespace != "") - { - sink.put(namespace); - sink.put(":"); - } - - sink.put(name); - } - - /// - static string combine(string namespace, string name) - { - return FullName(namespace, name).toString(); - } - /// - @("FullName.combine example") - unittest - { - assert(FullName.combine("", "name") == "name"); - assert(FullName.combine("*", "name") == "*:name"); - assert(FullName.combine("namespace", "name") == "namespace:name"); - } - - /// - static FullName parse(string fullName) - { - FullName result; - - auto parts = fullName.findSplit(":"); - if(parts[1] == "") // No colon - { - result.namespace = ""; - result.name = parts[0]; - } - else - { - result.namespace = parts[0]; - result.name = parts[2]; - } - - return result; - } - /// - @("FullName.parse example") - unittest - { - assert(FullName.parse("name") == FullName("", "name")); - assert(FullName.parse("*:name") == FullName("*", "name")); - assert(FullName.parse("namespace:name") == FullName("namespace", "name")); - } - - /// Throws with appropriate message if this.name is "*". - /// Wildcards are only supported for namespaces, not names. - void ensureNoWildcardName(string extaMsg = null) - { - if(name == "*") - throw new ArgumentException(`Wildcards ("*") only allowed for namespaces, not names. `~extaMsg); - } -} -struct Foo { string foo; } - -void removeIndex(E)(ref E[] arr, ptrdiff_t index) -{ - arr = arr[0..index] ~ arr[index+1..$]; -} - -void trace(string file=__FILE__, size_t line=__LINE__, TArgs...)(TArgs args) -{ - version(sdlangTrace) - { - writeln(file, "(", line, "): ", args); - stdout.flush(); - } -} - -string toString(TypeInfo ti) -{ - if (ti == typeid( bool )) return "bool"; - else if(ti == typeid( string )) return "string"; - else if(ti == typeid( dchar )) return "dchar"; - else if(ti == typeid( int )) return "int"; - else if(ti == typeid( long )) return "long"; - else if(ti == typeid( float )) return "float"; - else if(ti == typeid( double )) return "double"; - else if(ti == typeid( real )) return "real"; - else if(ti == typeid( Date )) return "Date"; - else if(ti == typeid( DateTimeFrac )) return "DateTimeFrac"; - else if(ti == typeid( DateTimeFracUnknownZone )) return "DateTimeFracUnknownZone"; - else if(ti == typeid( SysTime )) return "SysTime"; - else if(ti == typeid( Duration )) return "Duration"; - else if(ti == typeid( ubyte[] )) return "ubyte[]"; - else if(ti == typeid( typeof(null) )) return "null"; - - return "{unknown}"; -} - -enum BOM { - UTF8, /// UTF-8 - UTF16LE, /// UTF-16 (little-endian) - UTF16BE, /// UTF-16 (big-endian) - UTF32LE, /// UTF-32 (little-endian) - UTF32BE, /// UTF-32 (big-endian) -} - -enum NBOM = __traits(allMembers, BOM).length; -immutable ubyte[][NBOM] ByteOrderMarks = -[ - [0xEF, 0xBB, 0xBF], //UTF8 - [0xFF, 0xFE], //UTF16LE - [0xFE, 0xFF], //UTF16BE - [0xFF, 0xFE, 0x00, 0x00], //UTF32LE - [0x00, 0x00, 0xFE, 0xFF] //UTF32BE -]; |