diff options
Diffstat (limited to 'src/ext_depends/D-YAML/source')
27 files changed, 1432 insertions, 1800 deletions
diff --git a/src/ext_depends/D-YAML/source/dyaml/composer.d b/src/ext_depends/D-YAML/source/dyaml/composer.d index e7b083a..5467af8 100644 --- a/src/ext_depends/D-YAML/source/dyaml/composer.d +++ b/src/ext_depends/D-YAML/source/dyaml/composer.d @@ -29,15 +29,6 @@ import dyaml.resolver; package: -/** - * Exception thrown at composer errors. - * - * See_Also: MarkedYAMLException - */ -class ComposerException : MarkedYAMLException -{ - mixin MarkedExceptionCtors; -} ///Composes YAML documents from events provided by a Parser. struct Composer @@ -70,7 +61,7 @@ struct Composer * Params: parser = Parser to provide YAML events. * resolver = Resolver to resolve tags (data types). */ - this(Parser parser, Resolver resolver) @safe + this(Parser parser, Resolver resolver) @safe nothrow { parser_ = parser; resolver_ = resolver; @@ -101,6 +92,22 @@ struct Composer return composeDocument(); } + /// Set file name. + ref inout(string) name() inout @safe return pure nothrow @nogc + { + return parser_.name; + } + /// Get a mark from the current reader position + Mark mark() const @safe pure nothrow @nogc + { + return parser_.mark; + } + + /// Get resolver + ref Resolver resolver() @safe return pure nothrow @nogc { + return resolver_; + } + private: void skipExpected(const EventID id) @safe @@ -158,8 +165,8 @@ struct Composer //it's not finished, i.e. we're currently composing it //and trying to use it recursively here. enforce(anchors_[anchor] != Node(), - new ComposerException("Found recursive alias: " ~ anchor, - event.startMark)); + new ComposerException(text("Found recursive alias: ", anchor), + event.startMark, "defined here", anchors_[anchor].startMark)); return anchors_[anchor]; } @@ -168,8 +175,8 @@ struct Composer const anchor = event.anchor; if((anchor !is null) && (anchor in anchors_) !is null) { - throw new ComposerException("Found duplicate anchor: " ~ anchor, - event.startMark); + throw new ComposerException(text("Found duplicate anchor: ", anchor), + event.startMark, "defined here", anchors_[anchor].startMark); } Node result; @@ -177,7 +184,9 @@ struct Composer //used to detect duplicate and recursive anchors. if(anchor !is null) { - anchors_[anchor] = Node(); + Node tempNode; + tempNode.startMark_ = event.startMark; + anchors_[anchor] = tempNode; } switch (parser_.front.id) @@ -265,12 +274,10 @@ struct Composer { //this is Composer, but the code is related to Constructor. throw new ConstructorException("While constructing a mapping, " ~ - "expected a mapping or a list of " ~ - "mappings for merging, but found: " ~ - text(node.type) ~ - " NOTE: line/column shows topmost parent " ~ - "to which the content is being merged", - startMark, endMark); + "expected a mapping or a list of " ~ + "mappings for merging, but found: " ~ + text(node.type), + endMark, "mapping started here", startMark); } ensureAppendersExist(pairAppenderLevel, nodeAppenderLevel); @@ -360,14 +367,14 @@ struct Composer } auto sorted = pairAppender.data.dup.sort!((x,y) => x.key > y.key); - if (sorted.length) { + if (sorted.length) + { foreach (index, const ref value; sorted[0 .. $ - 1].enumerate) - if (value.key == sorted[index + 1].key) { - const message = () @trusted { - return format("Key '%s' appears multiple times in mapping (first: %s)", - value.key.get!string, value.key.startMark); - }(); - throw new ComposerException(message, sorted[index + 1].key.startMark); + if (value.key == sorted[index + 1].key) + { + throw new ComposerException( + text("Key '", value.key.get!string, "' appears multiple times in mapping"), + sorted[index + 1].key.startMark, "defined here", value.key.startMark); } } @@ -380,22 +387,3 @@ struct Composer return node; } } - -// Provide good error message on multiple keys (which JSON supports) -@safe unittest -{ - import dyaml.loader : Loader; - - const str = `{ - "comment": "This is a common technique", - "name": "foobar", - "comment": "To write down comments pre-JSON5" -}`; - - try - auto node = Loader.fromString(str).load(); - catch (ComposerException exc) - assert(exc.message() == - "Key 'comment' appears multiple times in mapping " ~ - "(first: file <unknown>,line 2,column 5)\nfile <unknown>,line 4,column 5"); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/constructor.d b/src/ext_depends/D-YAML/source/dyaml/constructor.d index 4cd1546..72c32f4 100644 --- a/src/ext_depends/D-YAML/source/dyaml/constructor.d +++ b/src/ext_depends/D-YAML/source/dyaml/constructor.d @@ -30,22 +30,6 @@ import dyaml.style; package: -// Exception thrown at constructor errors. -class ConstructorException : YAMLException -{ - /// Construct a ConstructorException. - /// - /// Params: msg = Error message. - /// start = Start position of the error context. - /// end = End position of the error context. - this(string msg, Mark start, Mark end, string file = __FILE__, size_t line = __LINE__) - @safe pure nothrow - { - super(msg ~ "\nstart: " ~ start.toString() ~ "\nend: " ~ end.toString(), - file, line); - } -} - /** Constructs YAML values. * * Each YAML scalar, sequence or mapping has a tag specifying its data type. @@ -76,109 +60,106 @@ Node constructNode(T)(const Mark start, const Mark end, const string tag, if((is(T : string) || is(T == Node[]) || is(T == Node.Pair[]))) { Node newNode; - try + noreturn error(string a, string b)() { - switch(tag) - { - case "tag:yaml.org,2002:null": - newNode = Node(YAMLNull(), tag); + enum msg = "Error constructing " ~ T.stringof ~ ": Only " ~ a ~ " can be " ~ b; + throw new ConstructorException(msg, start, "end", end); + } + switch(tag) + { + case "tag:yaml.org,2002:null": + newNode = Node(YAMLNull(), tag); + break; + case "tag:yaml.org,2002:bool": + static if(is(T == string)) + { + newNode = Node(constructBool(value, start, end), tag); break; - case "tag:yaml.org,2002:bool": - static if(is(T == string)) - { - newNode = Node(constructBool(value), tag); - break; - } - else throw new Exception("Only scalars can be bools"); - case "tag:yaml.org,2002:int": - static if(is(T == string)) - { - newNode = Node(constructLong(value), tag); - break; - } - else throw new Exception("Only scalars can be ints"); - case "tag:yaml.org,2002:float": - static if(is(T == string)) - { - newNode = Node(constructReal(value), tag); - break; - } - else throw new Exception("Only scalars can be floats"); - case "tag:yaml.org,2002:binary": - static if(is(T == string)) - { - newNode = Node(constructBinary(value), tag); - break; - } - else throw new Exception("Only scalars can be binary data"); - case "tag:yaml.org,2002:timestamp": - static if(is(T == string)) - { - newNode = Node(constructTimestamp(value), tag); - break; - } - else throw new Exception("Only scalars can be timestamps"); - case "tag:yaml.org,2002:str": - static if(is(T == string)) - { - newNode = Node(constructString(value), tag); - break; - } - else throw new Exception("Only scalars can be strings"); - case "tag:yaml.org,2002:value": - static if(is(T == string)) - { - newNode = Node(constructString(value), tag); - break; - } - else throw new Exception("Only scalars can be values"); - case "tag:yaml.org,2002:omap": - static if(is(T == Node[])) - { - newNode = Node(constructOrderedMap(value), tag); - break; - } - else throw new Exception("Only sequences can be ordered maps"); - case "tag:yaml.org,2002:pairs": - static if(is(T == Node[])) - { - newNode = Node(constructPairs(value), tag); - break; - } - else throw new Exception("Only sequences can be pairs"); - case "tag:yaml.org,2002:set": - static if(is(T == Node.Pair[])) - { - newNode = Node(constructSet(value), tag); - break; - } - else throw new Exception("Only mappings can be sets"); - case "tag:yaml.org,2002:seq": - static if(is(T == Node[])) - { - newNode = Node(constructSequence(value), tag); - break; - } - else throw new Exception("Only sequences can be sequences"); - case "tag:yaml.org,2002:map": - static if(is(T == Node.Pair[])) - { - newNode = Node(constructMap(value), tag); - break; - } - else throw new Exception("Only mappings can be maps"); - case "tag:yaml.org,2002:merge": - newNode = Node(YAMLMerge(), tag); + } + else error!("scalars", "bools"); + case "tag:yaml.org,2002:int": + static if(is(T == string)) + { + newNode = Node(constructLong(value, start, end), tag); break; - default: - newNode = Node(value, tag); + } + else error!("scalars", "ints"); + case "tag:yaml.org,2002:float": + static if(is(T == string)) + { + newNode = Node(constructReal(value, start, end), tag); break; - } - } - catch(Exception e) - { - throw new ConstructorException("Error constructing " ~ typeid(T).toString() - ~ ":\n" ~ e.msg, start, end); + } + else error!("scalars", "floats"); + case "tag:yaml.org,2002:binary": + static if(is(T == string)) + { + newNode = Node(constructBinary(value, start, end), tag); + break; + } + else error!("scalars", "binary data"); + case "tag:yaml.org,2002:timestamp": + static if(is(T == string)) + { + newNode = Node(constructTimestamp(value, start, end), tag); + break; + } + else error!("scalars", "timestamps"); + case "tag:yaml.org,2002:str": + static if(is(T == string)) + { + newNode = Node(constructString(value, start, end), tag); + break; + } + else error!("scalars", "strings"); + case "tag:yaml.org,2002:value": + static if(is(T == string)) + { + newNode = Node(constructString(value, start, end), tag); + break; + } + else error!("scalars", "values"); + case "tag:yaml.org,2002:omap": + static if(is(T == Node[])) + { + newNode = Node(constructOrderedMap(value, start, end), tag); + break; + } + else error!("sequences", "ordered maps"); + case "tag:yaml.org,2002:pairs": + static if(is(T == Node[])) + { + newNode = Node(constructPairs(value, start, end), tag); + break; + } + else error!("sequences", "pairs"); + case "tag:yaml.org,2002:set": + static if(is(T == Node.Pair[])) + { + newNode = Node(constructSet(value, start, end), tag); + break; + } + else error!("mappings", "sets"); + case "tag:yaml.org,2002:seq": + static if(is(T == Node[])) + { + newNode = Node(constructSequence(value, start, end), tag); + break; + } + else error!("sequences", "sequences"); + case "tag:yaml.org,2002:map": + static if(is(T == Node.Pair[])) + { + newNode = Node(constructMap(value, start, end), tag); + break; + } + else error!("mappings", "maps"); + case "tag:yaml.org,2002:merge": + newNode = Node(YAMLMerge(), tag); + break; + default: + newNode = Node(value, tag); + break; } newNode.startMark_ = start; @@ -188,16 +169,21 @@ Node constructNode(T)(const Mark start, const Mark end, const string tag, private: // Construct a boolean _node. -bool constructBool(const string str) @safe +bool constructBool(const string str, const Mark start, const Mark end) @safe { string value = str.toLower(); if(value.among!("yes", "true", "on")){return true;} if(value.among!("no", "false", "off")){return false;} - throw new Exception("Unable to parse boolean value: " ~ value); + throw new ConstructorException("Invalid boolean value: " ~ str, start, "ending at", end); +} + +@safe unittest +{ + assert(collectException!ConstructorException(constructBool("foo", Mark("unittest", 1, 0), Mark("unittest", 1, 3))).msg == "Invalid boolean value: foo"); } // Construct an integer (long) _node. -long constructLong(const string str) @safe +long constructLong(const string str, const Mark start, const Mark end) @safe { string value = str.replace("_", ""); const char c = value[0]; @@ -207,7 +193,7 @@ long constructLong(const string str) @safe value = value[1 .. $]; } - enforce(value != "", new Exception("Unable to parse float value: " ~ value)); + enforce(value != "", new ConstructorException("Unable to parse integer value: " ~ str, start, "ending at", end)); long result; try @@ -237,7 +223,7 @@ long constructLong(const string str) @safe } catch(ConvException e) { - throw new Exception("Unable to parse integer value: " ~ value); + throw new ConstructorException("Unable to parse integer value: " ~ str, start, "ending at", end); } return result; @@ -251,16 +237,18 @@ long constructLong(const string str) @safe string binary = "0b1010_0111_0100_1010_1110"; string sexagesimal = "190:20:30"; - assert(685230 == constructLong(canonical)); - assert(685230 == constructLong(decimal)); - assert(685230 == constructLong(octal)); - assert(685230 == constructLong(hexadecimal)); - assert(685230 == constructLong(binary)); - assert(685230 == constructLong(sexagesimal)); + assert(685230 == constructLong(canonical, Mark.init, Mark.init)); + assert(685230 == constructLong(decimal, Mark.init, Mark.init)); + assert(685230 == constructLong(octal, Mark.init, Mark.init)); + assert(685230 == constructLong(hexadecimal, Mark.init, Mark.init)); + assert(685230 == constructLong(binary, Mark.init, Mark.init)); + assert(685230 == constructLong(sexagesimal, Mark.init, Mark.init)); + assert(collectException!ConstructorException(constructLong("+", Mark.init, Mark.init)).msg == "Unable to parse integer value: +"); + assert(collectException!ConstructorException(constructLong("0xINVALID", Mark.init, Mark.init)).msg == "Unable to parse integer value: 0xINVALID"); } // Construct a floating point (real) _node. -real constructReal(const string str) @safe +real constructReal(const string str, const Mark start, const Mark end) @safe { string value = str.replace("_", "").toLower(); const char c = value[0]; @@ -271,7 +259,7 @@ real constructReal(const string str) @safe } enforce(value != "" && value != "nan" && value != "inf" && value != "-inf", - new Exception("Unable to parse float value: " ~ value)); + new ConstructorException("Unable to parse float value: \"" ~ str ~ "\"", start, "ending at", end)); real result; try @@ -297,7 +285,7 @@ real constructReal(const string str) @safe } catch(ConvException e) { - throw new Exception("Unable to parse float value: \"" ~ value ~ "\""); + throw new ConstructorException("Unable to parse float value: \"" ~ str ~ "\"", start, "ending at", end); } return result; @@ -316,16 +304,18 @@ real constructReal(const string str) @safe string negativeInf = "-.inf"; string NaN = ".NaN"; - assert(eq(685230.15, constructReal(canonical))); - assert(eq(685230.15, constructReal(exponential))); - assert(eq(685230.15, constructReal(fixed))); - assert(eq(685230.15, constructReal(sexagesimal))); - assert(eq(-real.infinity, constructReal(negativeInf))); - assert(to!string(constructReal(NaN)) == "nan"); + assert(eq(685230.15, constructReal(canonical, Mark.init, Mark.init))); + assert(eq(685230.15, constructReal(exponential, Mark.init, Mark.init))); + assert(eq(685230.15, constructReal(fixed, Mark.init, Mark.init))); + assert(eq(685230.15, constructReal(sexagesimal, Mark.init, Mark.init))); + assert(eq(-real.infinity, constructReal(negativeInf, Mark.init, Mark.init))); + assert(to!string(constructReal(NaN, Mark.init, Mark.init)) == "nan"); + assert(collectException!ConstructorException(constructReal("+", Mark.init, Mark.init)).msg == "Unable to parse float value: \"+\""); + assert(collectException!ConstructorException(constructReal("74.invalid", Mark.init, Mark.init)).msg == "Unable to parse float value: \"74.invalid\""); } // Construct a binary (base64) _node. -ubyte[] constructBinary(const string value) @safe +ubyte[] constructBinary(const string value, const Mark start, const Mark end) @safe { import std.ascii : newline; import std.array : array; @@ -337,7 +327,7 @@ ubyte[] constructBinary(const string value) @safe } catch(Base64Exception e) { - throw new Exception("Unable to decode base64 value: " ~ e.msg); + throw new ConstructorException("Unable to decode base64 value: " ~ e.msg, start, "ending at", end); } } @@ -347,13 +337,13 @@ ubyte[] constructBinary(const string value) @safe char[] buffer; buffer.length = 256; string input = Base64.encode(test, buffer).idup; - const value = constructBinary(input); + const value = constructBinary(input, Mark.init, Mark.init); assert(value == test); assert(value == [84, 104, 101, 32, 65, 110, 115, 119, 101, 114, 58, 32, 52, 50]); } // Construct a timestamp (SysTime) _node. -SysTime constructTimestamp(const string str) @safe +SysTime constructTimestamp(const string str, const Mark start, const Mark end) @safe { string value = str; @@ -429,7 +419,7 @@ SysTime constructTimestamp(const string str) @safe { string timestamp(string value) { - return constructTimestamp(value).toISOString(); + return constructTimestamp(value, Mark.init, Mark.init).toISOString(); } string canonical = "2001-12-15T02:59:43.1Z"; @@ -452,21 +442,21 @@ SysTime constructTimestamp(const string str) @safe } // Construct a string _node. -string constructString(const string str) @safe +string constructString(const string str, const Mark start, const Mark end) @safe { return str; } // Convert a sequence of single-element mappings into a sequence of pairs. -Node.Pair[] getPairs(string type, const Node[] nodes) @safe +Node.Pair[] getPairs(string type)(const Node[] nodes) @safe { + enum msg = "While constructing " ~ type ~ ", expected a mapping with single element"; Node.Pair[] pairs; pairs.reserve(nodes.length); foreach(node; nodes) { enforce(node.nodeID == NodeID.mapping && node.length == 1, - new Exception("While constructing " ~ type ~ - ", expected a mapping with single element")); + new ConstructorException(msg, node.startMark)); pairs ~= node.as!(Node.Pair[]); } @@ -475,30 +465,33 @@ Node.Pair[] getPairs(string type, const Node[] nodes) @safe } // Construct an ordered map (ordered sequence of key:value pairs without duplicates) _node. -Node.Pair[] constructOrderedMap(const Node[] nodes) @safe +Node.Pair[] constructOrderedMap(const Node[] nodes, const Mark start, const Mark end) @safe { - auto pairs = getPairs("ordered map", nodes); + auto pairs = getPairs!"an ordered map"(nodes); //Detect duplicates. //TODO this should be replaced by something with deterministic memory allocation. auto keys = new RedBlackTree!Node(); foreach(ref pair; pairs) { - enforce(!(pair.key in keys), - new Exception("Duplicate entry in an ordered map: " - ~ pair.key.debugString())); + auto foundMatch = keys.equalRange(pair.key); + enforce(foundMatch.empty, new ConstructorException( + "Duplicate entry in an ordered map", pair.key.startMark, + "first occurrence here", foundMatch.front.startMark)); keys.insert(pair.key); } return pairs; } @safe unittest { + uint lines; Node[] alternateTypes(uint length) @safe { Node[] pairs; foreach(long i; 0 .. length) { auto pair = (i % 2) ? Node.Pair(i.to!string, i) : Node.Pair(i, i.to!string); + pair.key.startMark_ = Mark("unittest", lines++, 0); pairs ~= Node([pair]); } return pairs; @@ -510,27 +503,29 @@ Node.Pair[] constructOrderedMap(const Node[] nodes) @safe foreach(long i; 0 .. length) { auto pair = Node.Pair(i.to!string, i); + pair.key.startMark_ = Mark("unittest", lines++, 0); pairs ~= Node([pair]); } return pairs; } - assertThrown(constructOrderedMap(alternateTypes(8) ~ alternateTypes(2))); - assertNotThrown(constructOrderedMap(alternateTypes(8))); - assertThrown(constructOrderedMap(sameType(64) ~ sameType(16))); - assertThrown(constructOrderedMap(alternateTypes(64) ~ alternateTypes(16))); - assertNotThrown(constructOrderedMap(sameType(64))); - assertNotThrown(constructOrderedMap(alternateTypes(64))); + assert(collectException!ConstructorException(constructOrderedMap(alternateTypes(8) ~ alternateTypes(2), Mark.init, Mark.init)).message == "Duplicate entry in an ordered map\nunittest:9,1\nfirst occurrence here: unittest:1,1"); + assertNotThrown(constructOrderedMap(alternateTypes(8), Mark.init, Mark.init)); + assert(collectException!ConstructorException(constructOrderedMap(sameType(64) ~ sameType(16), Mark.init, Mark.init)).message == "Duplicate entry in an ordered map\nunittest:83,1\nfirst occurrence here: unittest:19,1"); + assert(collectException!ConstructorException(constructOrderedMap(alternateTypes(64) ~ alternateTypes(16), Mark.init, Mark.init)).message == "Duplicate entry in an ordered map\nunittest:163,1\nfirst occurrence here: unittest:99,1"); + assertNotThrown(constructOrderedMap(sameType(64), Mark.init, Mark.init)); + assertNotThrown(constructOrderedMap(alternateTypes(64), Mark.init, Mark.init)); + assert(collectException!ConstructorException(constructOrderedMap([Node([Node(1), Node(2)])], Mark.init, Mark.init)).message == "While constructing an ordered map, expected a mapping with single element\n<unknown>:1,1"); } // Construct a pairs (ordered sequence of key: value pairs allowing duplicates) _node. -Node.Pair[] constructPairs(const Node[] nodes) @safe +Node.Pair[] constructPairs(const Node[] nodes, const Mark start, const Mark end) @safe { - return getPairs("pairs", nodes); + return getPairs!"pairs"(nodes); } // Construct a set _node. -Node[] constructSet(const Node.Pair[] pairs) @safe +Node[] constructSet(const Node.Pair[] pairs, const Mark start, const Mark end) @safe { // In future, the map here should be replaced with something with deterministic // memory allocation if possible. @@ -583,20 +578,20 @@ Node[] constructSet(const Node.Pair[] pairs) @safe auto nodeDuplicatesLong = DuplicatesLong.dup; auto nodeNoDuplicatesLong = noDuplicatesLong.dup; - assertThrown(constructSet(nodeDuplicatesShort)); - assertNotThrown(constructSet(nodeNoDuplicatesShort)); - assertThrown(constructSet(nodeDuplicatesLong)); - assertNotThrown(constructSet(nodeNoDuplicatesLong)); + assertThrown(constructSet(nodeDuplicatesShort, Mark.init, Mark.init)); + assertNotThrown(constructSet(nodeNoDuplicatesShort, Mark.init, Mark.init)); + assertThrown(constructSet(nodeDuplicatesLong, Mark.init, Mark.init)); + assertNotThrown(constructSet(nodeNoDuplicatesLong, Mark.init, Mark.init)); } // Construct a sequence (array) _node. -Node[] constructSequence(Node[] nodes) @safe +Node[] constructSequence(Node[] nodes, const Mark start, const Mark end) @safe { return nodes; } // Construct an unordered map (unordered set of key:value _pairs without duplicates) _node. -Node.Pair[] constructMap(Node.Pair[] pairs) @safe +Node.Pair[] constructMap(Node.Pair[] pairs, const Mark start, const Mark end) @safe { //Detect duplicates. //TODO this should be replaced by something with deterministic memory allocation. diff --git a/src/ext_depends/D-YAML/source/dyaml/dumper.d b/src/ext_depends/D-YAML/source/dyaml/dumper.d index 03d3620..e4aa1d3 100644 --- a/src/ext_depends/D-YAML/source/dyaml/dumper.d +++ b/src/ext_depends/D-YAML/source/dyaml/dumper.d @@ -73,9 +73,6 @@ struct Dumper // Default style for collection nodes. If style is $(D CollectionStyle.invalid), the _style is chosen automatically. CollectionStyle defaultCollectionStyle = CollectionStyle.invalid; - @disable bool opEquals(ref Dumper); - @disable int opCmp(ref Dumper); - ///Set indentation width. 2 by default. Must not be zero. @property void indent(uint indent) pure @safe nothrow in diff --git a/src/ext_depends/D-YAML/source/dyaml/emitter.d b/src/ext_depends/D-YAML/source/dyaml/emitter.d index 5aafc0e..b61fea1 100644 --- a/src/ext_depends/D-YAML/source/dyaml/emitter.d +++ b/src/ext_depends/D-YAML/source/dyaml/emitter.d @@ -153,9 +153,6 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) ScalarStyle style_ = ScalarStyle.invalid; public: - @disable int opCmp(ref Emitter); - @disable bool opEquals(ref Emitter); - /** * Construct an emitter. * @@ -775,7 +772,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) { if(style_ == ScalarStyle.invalid){style_ = chooseScalarStyle();} if((!canonical_ || (tag is null)) && - ((tag == "tag:yaml.org,2002:str") || (style_ == ScalarStyle.plain ? event_.implicit : !event_.implicit && (tag is null)))) + (((tag == "tag:yaml.org,2002:str") && event_.implicit) || (style_ == ScalarStyle.plain ? event_.implicit : !event_.implicit && (tag is null)))) { preparedTag_ = null; return; @@ -1283,9 +1280,6 @@ struct ScalarWriter(Range, CharType) } private: - @disable int opCmp(ref Emitter!(Range, CharType)); - @disable bool opEquals(ref Emitter!(Range, CharType)); - ///Used as "null" UTF-32 character. static immutable dcharNone = dchar.max; diff --git a/src/ext_depends/D-YAML/source/dyaml/event.d b/src/ext_depends/D-YAML/source/dyaml/event.d index f4a747f..36638e3 100644 --- a/src/ext_depends/D-YAML/source/dyaml/event.d +++ b/src/ext_depends/D-YAML/source/dyaml/event.d @@ -10,6 +10,7 @@ */ module dyaml.event; +import std.algorithm; import std.array; import std.conv; @@ -43,42 +44,30 @@ enum EventID : ubyte */ struct Event { - @disable int opCmp(ref Event); - ///Value of the event, if any. string value; ///Start position of the event in file/stream. Mark startMark; ///End position of the event in file/stream. Mark endMark; - union - { - struct - { - ///Anchor of the event, if any. - string _anchor; - ///Tag of the event, if any. - string _tag; - } - ///Tag directives, if this is a DocumentStart. - //TagDirectives tagDirectives; - TagDirective[] _tagDirectives; - } + ///Anchor of the event, if any. + string _anchor; + ///Tag of the event, if any. + string _tag; + ///Tag directives, if this is a DocumentStart. + TagDirective[] _tagDirectives; ///Event type. EventID id = EventID.invalid; ///Style of scalar event, if this is a scalar event. ScalarStyle scalarStyle = ScalarStyle.invalid; - union - { - ///Should the tag be implicitly resolved? - bool implicit; - /** - * Is this document event explicit? - * - * Used if this is a DocumentStart or DocumentEnd. - */ - bool explicitDocument; - } + ///Should the tag be implicitly resolved? + bool implicit; + /** + * Is this document event explicit? + * + * Used if this is a DocumentStart or DocumentEnd. + */ + alias explicitDocument = implicit; ///Collection style, if this is a SequenceStart or MappingStart. CollectionStyle collectionStyle = CollectionStyle.invalid; @@ -102,6 +91,117 @@ struct Event assert(id == EventID.documentStart, "Only DocumentStart events have tag directives."); return _tagDirectives; } + void toString(W)(ref W writer) const + { + import std.algorithm.iteration : substitute; + import std.format : formattedWrite; + import std.range : put; + final switch (id) + { + case EventID.scalar: + put(writer, "=VAL "); + if (anchor != "") + { + writer.formattedWrite!"&%s " (anchor); + } + if (tag != "") + { + writer.formattedWrite!"<%s> " (tag); + } + final switch(scalarStyle) + { + case ScalarStyle.singleQuoted: + put(writer, "'"); + break; + case ScalarStyle.doubleQuoted: + put(writer, "\""); + break; + case ScalarStyle.literal: + put(writer, "|"); + break; + case ScalarStyle.folded: + put(writer, ">"); + break; + case ScalarStyle.invalid: //default to plain + case ScalarStyle.plain: + put(writer, ":"); + break; + } + if (value != "") + { + writer.formattedWrite!"%s"(value.substitute("\n", "\\n", `\`, `\\`, "\r", "\\r", "\t", "\\t", "\b", "\\b")); + } + break; + case EventID.streamStart: + put(writer, "+STR"); + break; + case EventID.documentStart: + put(writer, "+DOC"); + if (explicitDocument) + { + put(writer, " ---"); + } + break; + case EventID.mappingStart: + put(writer, "+MAP"); + if (collectionStyle == CollectionStyle.flow) + { + put(writer, " {}"); + } + if (anchor != "") + { + put(writer, " &"); + put(writer, anchor); + } + if (tag != "") + { + put(writer, " <"); + put(writer, tag); + put(writer, ">"); + } + break; + case EventID.sequenceStart: + put(writer, "+SEQ"); + if (collectionStyle == CollectionStyle.flow) + { + put(writer, " []"); + } + if (anchor != "") + { + put(writer, " &"); + put(writer, anchor); + } + if (tag != "") + { + put(writer, " <"); + put(writer, tag); + put(writer, ">"); + } + break; + case EventID.streamEnd: + put(writer, "-STR"); + break; + case EventID.documentEnd: + put(writer, "-DOC"); + if (explicitDocument) + { + put(writer, " ..."); + } + break; + case EventID.mappingEnd: + put(writer, "-MAP"); + break; + case EventID.sequenceEnd: + put(writer, "-SEQ"); + break; + case EventID.alias_: + put(writer, "=ALI *"); + put(writer, anchor); + break; + case EventID.invalid: + assert(0, "Invalid EventID produced"); + } + } } /** diff --git a/src/ext_depends/D-YAML/source/dyaml/exception.d b/src/ext_depends/D-YAML/source/dyaml/exception.d index 145e9c3..8a2fe0d 100644 --- a/src/ext_depends/D-YAML/source/dyaml/exception.d +++ b/src/ext_depends/D-YAML/source/dyaml/exception.d @@ -7,165 +7,201 @@ ///Exceptions thrown by D:YAML and _exception related code. module dyaml.exception; - import std.algorithm; import std.array; -import std.string; import std.conv; +import std.exception; +import std.format; +import std.range; +import std.string; +import std.typecons; /// Base class for all exceptions thrown by D:YAML. class YAMLException : Exception { - /// Construct a YAMLException with specified message and position where it was thrown. - public this(string msg, string file = __FILE__, size_t line = __LINE__) - @safe pure nothrow @nogc - { - super(msg, file, line); - } + mixin basicExceptionCtors; } /// Position in a YAML stream, used for error messages. struct Mark { - package: - /// File name. - string name_; - /// Line number. - ushort line_; - /// Column number. - ushort column_; + /// File name. + string name = "<unknown>"; + /// Line number. + ushort line; + /// Column number. + ushort column; public: /// Construct a Mark with specified line and column in the file. this(string name, const uint line, const uint column) @safe pure nothrow @nogc { - name_ = name; - line_ = cast(ushort)min(ushort.max, line); + this.name = name; + this.line = cast(ushort)min(ushort.max, line); // This *will* overflow on extremely wide files but saves CPU time // (mark ctor takes ~5% of time) - column_ = cast(ushort)column; - } - - /// Get a file name. - @property string name() @safe pure nothrow @nogc const - { - return name_; - } - - /// Get a line number. - @property ushort line() @safe pure nothrow @nogc const - { - return line_; - } - - /// Get a column number. - @property ushort column() @safe pure nothrow @nogc const - { - return column_; - } - - /// Duplicate a mark - Mark dup () const scope @safe pure nothrow - { - return Mark(this.name_.idup, this.line_, this.column_); + this.column = cast(ushort)column; } /// Get a string representation of the mark. - string toString() const scope @safe pure nothrow + void toString(W)(ref W writer) const scope { // Line/column numbers start at zero internally, make them start at 1. - static string clamped(ushort v) @safe pure nothrow + void writeClamped(ushort v) { - return text(v + 1, v == ushort.max ? " or higher" : ""); + writer.formattedWrite!"%s"(v + 1); + if (v == ushort.max) + { + put(writer, "or higher"); + } } - return "file " ~ name_ ~ ",line " ~ clamped(line_) ~ ",column " ~ clamped(column_); + put(writer, name); + put(writer, ":"); + writeClamped(line); + put(writer, ","); + writeClamped(column); } } -// Base class of YAML exceptions with marked positions of the problem. +/// Base class of YAML exceptions with marked positions of the problem. abstract class MarkedYAMLException : YAMLException { /// Position of the error. Mark mark; + /// Additional position information, usually the start of a token or scalar + Nullable!Mark mark2; + /// A label for the extra information + string mark2Label; - // Construct a MarkedYAMLException with specified context and problem. - this(string context, scope const Mark contextMark, - string problem, scope const Mark problemMark, + // Construct a MarkedYAMLException with two marks + this(string context, const Mark mark, string mark2Label, const Nullable!Mark mark2, string file = __FILE__, size_t line = __LINE__) @safe pure nothrow { - const msg = context ~ '\n' ~ - (contextMark != problemMark ? contextMark.toString() ~ '\n' : "") ~ - problem ~ '\n' ~ problemMark.toString() ~ '\n'; - super(msg, file, line); - mark = problemMark.dup; + super(context, file, line); + this.mark = mark; + this.mark2 = mark2; + this.mark2Label = mark2Label; } // Construct a MarkedYAMLException with specified problem. - this(string problem, scope const Mark problemMark, + this(string msg, const Mark mark, string file = __FILE__, size_t line = __LINE__) @safe pure nothrow { - super(problem ~ '\n' ~ problemMark.toString(), file, line); - mark = problemMark.dup; + super(msg, file, line); + this.mark = mark; } - /// Construct a MarkedYAMLException from a struct storing constructor parameters. - this(ref const(MarkedYAMLExceptionData) data) @safe pure nothrow + /// Custom toString to add context without requiring allocation up-front + void toString(W)(ref W sink) const + { + sink.formattedWrite!"%s@%s(%s): "(typeid(this).name, file, line); + put(sink, msg); + put(sink, "\n"); + mark.toString(sink); + if (!mark2.isNull) + { + put(sink, "\n"); + put(sink, mark2Label); + put(sink, ":"); + mark2.get.toString(sink); + } + put(sink, "\n"); + put(sink, info.toString()); + } + /// Ditto + override void toString(scope void delegate(in char[]) sink) const { - with(data) this(context, contextMark, problem, problemMark); + toString!(typeof(sink))(sink); + } + /// An override of message + override const(char)[] message() const @safe nothrow + { + if (mark2.isNull) + { + return assertNotThrown(text(msg, "\n", mark)); + } + else + { + return assertNotThrown(text(msg, "\n", mark, "\n", mark2Label, ": ", mark2.get)); + } } } -package: -// A struct storing parameters to the MarkedYAMLException constructor. -struct MarkedYAMLExceptionData +/// Exception thrown on composer errors. +class ComposerException : MarkedYAMLException { - // Context of the error. - string context; - // Position of the context in a YAML buffer. - Mark contextMark; - // The error itself. - string problem; - // Position if the error. - Mark problemMark; + mixin MarkedExceptionCtors; } -// Constructors of YAML exceptions are mostly the same, so we use a mixin. -// -// See_Also: YAMLException -template ExceptionCtors() +/// Exception thrown on constructor errors. +class ConstructorException : MarkedYAMLException { - public this(string msg, string file = __FILE__, size_t line = __LINE__) - @safe pure nothrow - { - super(msg, file, line); - } + mixin MarkedExceptionCtors; +} + +/// Exception thrown on loader errors. +class LoaderException : MarkedYAMLException +{ + mixin MarkedExceptionCtors; } -// Constructors of marked YAML exceptions are mostly the same, so we use a mixin. -// -// See_Also: MarkedYAMLException +/// Exception thrown on node related errors. +class NodeException : MarkedYAMLException +{ + mixin MarkedExceptionCtors; +} + +/// Exception thrown on parser errors. +class ParserException : MarkedYAMLException +{ + mixin MarkedExceptionCtors; +} + +/// Exception thrown on Reader errors. +class ReaderException : MarkedYAMLException +{ + mixin MarkedExceptionCtors; +} + +/// Exception thrown on Representer errors. +class RepresenterException : YAMLException +{ + mixin basicExceptionCtors; +} + +/// Exception thrown on scanner errors. +class ScannerException : MarkedYAMLException +{ + mixin MarkedExceptionCtors; +} + +private: + +/// Constructors of marked YAML exceptions are identical, so we use a mixin. +/// +/// See_Also: MarkedYAMLException template MarkedExceptionCtors() { public: - this(string context, const Mark contextMark, string problem, - const Mark problemMark, string file = __FILE__, size_t line = __LINE__) + this(string msg, const Mark mark1, string mark2Label, + const Mark mark2, string file = __FILE__, size_t line = __LINE__) @safe pure nothrow { - super(context, contextMark, problem, problemMark, - file, line); + super(msg, mark1, mark2Label, Nullable!Mark(mark2), file, line); } - this(string problem, const Mark problemMark, + this(string msg, const Mark mark, string file = __FILE__, size_t line = __LINE__) @safe pure nothrow { - super(problem, problemMark, file, line); + super(msg, mark, file, line); } - - this(ref const(MarkedYAMLExceptionData) data) @safe pure nothrow + this(string msg, const Mark mark1, string mark2Label, + const Nullable!Mark mark2, string file = __FILE__, size_t line = __LINE__) + @safe pure nothrow { - super(data); + super(msg, mark1, mark2Label, mark2, file, line); } } diff --git a/src/ext_depends/D-YAML/source/dyaml/loader.d b/src/ext_depends/D-YAML/source/dyaml/loader.d index 6638dfc..358855a 100644 --- a/src/ext_depends/D-YAML/source/dyaml/loader.d +++ b/src/ext_depends/D-YAML/source/dyaml/loader.d @@ -33,14 +33,8 @@ import dyaml.token; struct Loader { private: - // Processes character data to YAML tokens. - Scanner scanner_; - // Processes tokens to YAML events. - Parser parser_; - // Resolves tags (data types). - Resolver resolver_; - // Name of the input file or stream, used in error messages. - string name_ = "<unknown>"; + // Assembles YAML documents + Composer composer_; // Are we done loading? bool done_; // Last node read from stream @@ -49,10 +43,6 @@ struct Loader bool rangeInitialized; public: - @disable this(); - @disable int opCmp(ref Loader); - @disable bool opEquals(ref Loader); - /** Construct a Loader to load YAML from a file. * * Params: filename = Name of the file to load from. @@ -147,33 +137,30 @@ struct Loader /// Ditto private this(ubyte[] yamlData, string name = "<unknown>") @safe { - resolver_ = Resolver.withDefaultResolvers; - name_ = name; try { - auto reader_ = new Reader(yamlData, name); - scanner_ = Scanner(reader_); - parser_ = new Parser(scanner_); + auto reader = Reader(yamlData, name); + auto parser = new Parser(Scanner(reader)); + composer_ = Composer(parser, Resolver.withDefaultResolvers); } - catch(YAMLException e) + catch(MarkedYAMLException e) { - throw new YAMLException("Unable to open %s for YAML loading: %s" - .format(name_, e.msg), e.file, e.line); + throw new LoaderException("Unable to open %s for YAML loading: %s" + .format(name, e.msg), e.mark, e.file, e.line); } } /// Set stream _name. Used in debugging messages. - void name(string name) pure @safe nothrow @nogc + ref inout(string) name() inout @safe return pure nothrow @nogc { - name_ = name; - scanner_.name = name; + return composer_.name; } /// Specify custom Resolver to use. auto ref resolver() pure @safe nothrow @nogc { - return resolver_; + return composer_.resolver; } /** Load single YAML document. @@ -189,10 +176,12 @@ struct Loader */ Node load() @safe { - enforce!YAMLException(!empty, "Zero documents in stream"); + enforce(!empty, + new LoaderException("Zero documents in stream", composer_.mark)); auto output = front; popFront(); - enforce!YAMLException(empty, "More than one document in stream"); + enforce(empty, + new LoaderException("More than one document in stream", composer_.mark)); return output; } @@ -217,22 +206,23 @@ struct Loader */ void popFront() @safe { - // Composer initialization is done here in case the constructor is - // modified, which is a pretty common case. - static Composer composer; - if (!rangeInitialized) - { - composer = Composer(parser_, resolver_); - rangeInitialized = true; - } + scope(success) rangeInitialized = true; assert(!done_, "Loader.popFront called on empty range"); - if (composer.checkNode()) + try { - currentNode = composer.getNode(); + if (composer_.checkNode()) + { + currentNode = composer_.getNode(); + } + else + { + done_ = true; + } } - else + catch(MarkedYAMLException e) { - done_ = true; + throw new LoaderException("Unable to load %s: %s" + .format(name, e.msg), e.mark, e.mark2Label, e.mark2, e.file, e.line); } } /** Implements the front range primitive. @@ -248,30 +238,6 @@ struct Loader } return currentNode; } - - // Scan all tokens, throwing them away. Used for benchmarking. - void scanBench() @safe - { - try - { - while(!scanner_.empty) - { - scanner_.popFront(); - } - } - catch(YAMLException e) - { - throw new YAMLException("Unable to scan YAML from stream " ~ - name_ ~ " : " ~ e.msg, e.file, e.line); - } - } - - - // Parse and return all events. Used for debugging. - auto parse() @safe - { - return parser_; - } } /// Load single YAML document from a file: @safe unittest @@ -408,6 +374,89 @@ EOS"; loader.name = filename; Node unused; - auto e = loader.load().collectException!ScannerException(unused); + auto e = loader.load().collectException!LoaderException(unused); assert(e.mark.name == filename); } +/// https://github.com/dlang-community/D-YAML/issues/325 +@safe unittest +{ + assert(Loader.fromString("--- {x: a}").load()["x"] == "a"); +} + +// Ensure exceptions are thrown as appropriate +@safe unittest +{ + LoaderException e; + // No documents + e = collectException!LoaderException(Loader.fromString("", "filename.yaml").load()); + assert(e); + with(e) + { + assert(mark.name == "filename.yaml"); + assert(mark.line == 0); + assert(mark.column == 0); + } + // Too many documents + e = collectException!LoaderException(Loader.fromString("--- 4\n--- 6\n--- 5", "filename.yaml").load()); + assert(e, "No exception thrown"); + with(e) + { + assert(mark.name == "filename.yaml"); + // FIXME: should be position of second document, not end of file + //assert(mark.line == 1); + //assert(mark.column == 0); + } + // Invalid document + e = collectException!LoaderException(Loader.fromString("[", "filename.yaml").load()); + assert(e, "No exception thrown"); + with(e) + { + assert(mark.name == "filename.yaml"); + // FIXME: should be position of second document, not end of file + assert(mark.line == 0); + assert(mark.column == 1); + } +} + +@safe unittest +{ + assertThrown(Loader.fromString("Invalid character: \xFF").load()); +} + +@safe unittest +{ + assertThrown(Loader.fromFile("test/data/odd-utf16.stream-error").load()); +} + +// UTF-16 and 32 test +@safe unittest +{ + import std.conv : to; + import std.range : only; + enum string yaml = `ABCDØ`; + enum bom = '\uFEFF'; + foreach (doc; only( + cast(ubyte[])(bom~yaml.to!(wchar[])), + cast(ubyte[])(bom~yaml.to!(dchar[])), + )) + { + assert(Loader.fromBuffer(doc).load().as!string == yaml); + } +} +// Invalid unicode test +@safe unittest +{ + import std.conv : to; + import std.range : only; + enum string yaml = `ABCDØ`; + enum badBOM = '\uFFFE'; + foreach (doc; only( + cast(ubyte[])yaml.to!(wchar[]), + cast(ubyte[])yaml.to!(dchar[]), + cast(ubyte[])(badBOM~yaml.to!(wchar[])), + cast(ubyte[])(badBOM~yaml.to!(dchar[])), + )) + { + assertThrown(Loader.fromBuffer(doc).load()); + } +} diff --git a/src/ext_depends/D-YAML/source/dyaml/node.d b/src/ext_depends/D-YAML/source/dyaml/node.d index 4c3c5eb..fd47f7e 100644 --- a/src/ext_depends/D-YAML/source/dyaml/node.d +++ b/src/ext_depends/D-YAML/source/dyaml/node.d @@ -30,22 +30,6 @@ import dyaml.event; import dyaml.exception; import dyaml.style; -/// Exception thrown at node related errors. -class NodeException : MarkedYAMLException -{ - package: - // Construct a NodeException. - // - // Params: msg = Error message. - // start = Start position of the node. - this(string msg, const scope Mark start, - string file = __FILE__, size_t line = __LINE__) - @safe pure nothrow - { - super(msg, start, file, line); - } -} - // Node kinds. enum NodeID : ubyte { diff --git a/src/ext_depends/D-YAML/source/dyaml/parser.d b/src/ext_depends/D-YAML/source/dyaml/parser.d index befdfa4..cc2ea47 100644 --- a/src/ext_depends/D-YAML/source/dyaml/parser.d +++ b/src/ext_depends/D-YAML/source/dyaml/parser.d @@ -88,16 +88,6 @@ import dyaml.tagdirective; */ -/** - * Marked exception thrown at parser errors. - * - * See_Also: MarkedYAMLException - */ -class ParserException : MarkedYAMLException -{ - mixin MarkedExceptionCtors; -} - package: /// Generates events from tokens provided by a Scanner. /// @@ -173,6 +163,17 @@ final class Parser ensureState(); } + /// Set file name. + ref inout(string) name() inout @safe return pure nothrow @nogc + { + return scanner_.name; + } + /// Get a mark from the current reader position + Mark mark() const @safe pure nothrow @nogc + { + return scanner_.mark; + } + private: /// If current event is invalid, load the next valid one if possible. void ensureState() @safe @@ -508,9 +509,9 @@ final class Parser } const token = scanner_.front; - throw new ParserException("While parsing a " ~ (block ? "block" : "flow") ~ " node", - startMark, "expected node content, but found: " - ~ token.idString, token.startMark); + throw new ParserException("While parsing a " ~ (block ? "block" : "flow") + ~ " node, expected node content, but found: " ~ token.idString, + token.startMark, "node started here", startMark); } /// Handle escape sequences in a double quoted scalar. @@ -618,8 +619,8 @@ final class Parser } //handle must be in tagDirectives_ enforce(replacement !is null, - new ParserException("While parsing a node", startMark, - "found undefined tag handle: " ~ handle, tagMark)); + new ParserException("While parsing a node, found undefined tag handle: " + ~ handle, tagMark, "node started here", startMark)); return replacement ~ suffix; } return suffix; @@ -658,9 +659,8 @@ final class Parser if(scanner_.front.id != TokenID.blockEnd) { const token = scanner_.front; - throw new ParserException("While parsing a block collection", marks_.data.back, - "expected block end, but found " ~ token.idString, - token.startMark); + throw new ParserException("While parsing a block sequence, expected block end, but found: " + ~ token.idString, token.startMark, "sequence started here", marks_.data.back); } state_ = popState(); @@ -730,9 +730,8 @@ final class Parser if(scanner_.front.id != TokenID.blockEnd) { const token = scanner_.front; - throw new ParserException("While parsing a block mapping", marks_.data.back, - "expected block end, but found: " ~ token.idString, - token.startMark); + throw new ParserException("While parsing a block mapping, expected block end, but found: " + ~ token.idString, token.startMark, "mapping started here", marks_.data.back); } state_ = popState(); @@ -797,9 +796,8 @@ final class Parser else { const token = scanner_.front; - throw new ParserException("While parsing a flow sequence", marks_.data.back, - "expected ',' or ']', but got: " ~ - token.idString, token.startMark); + throw new ParserException("While parsing a flow sequence, expected ',' or ']', but got: " ~ + token.idString, token.startMark, "sequence started here", marks_.data.back); } } @@ -912,9 +910,8 @@ final class Parser else { const token = scanner_.front; - throw new ParserException("While parsing a flow mapping", marks_.data.back, - "expected ',' or '}', but got: " ~ - token.idString, token.startMark); + throw new ParserException("While parsing a flow mapping, expected ',' or '}', but got: " + ~ token.idString, token.startMark, "mapping started here", marks_.data.back); } } diff --git a/src/ext_depends/D-YAML/source/dyaml/queue.d b/src/ext_depends/D-YAML/source/dyaml/queue.d index 57b0d34..35b45c8 100644 --- a/src/ext_depends/D-YAML/source/dyaml/queue.d +++ b/src/ext_depends/D-YAML/source/dyaml/queue.d @@ -88,11 +88,6 @@ private: } public: - - @disable void opAssign(ref Queue); - @disable bool opEquals(ref Queue); - @disable int opCmp(ref Queue); - this(this) @safe nothrow @nogc { auto node = first_; diff --git a/src/ext_depends/D-YAML/source/dyaml/reader.d b/src/ext_depends/D-YAML/source/dyaml/reader.d index ae44c80..824c1d1 100644 --- a/src/ext_depends/D-YAML/source/dyaml/reader.d +++ b/src/ext_depends/D-YAML/source/dyaml/reader.d @@ -31,19 +31,8 @@ alias isBreak = among!('\n', '\u0085', '\u2028', '\u2029'); package: -///Exception thrown at Reader errors. -class ReaderException : YAMLException -{ - this(string msg, string file = __FILE__, size_t line = __LINE__) - @safe pure nothrow - { - super("Reader error: " ~ msg, file, line); - } -} - -/// Provides an API to read characters from a UTF-8 buffer and build slices into that -/// buffer to avoid allocations (see SliceBuilder). -final class Reader +/// Provides an API to read characters from a UTF-8 buffer. +struct Reader { private: // Buffer of currently loaded characters. @@ -102,8 +91,9 @@ final class Reader auto endianResult = fixUTFByteOrder(buffer); if(endianResult.bytesStripped > 0) { + // TODO: add line and column throw new ReaderException("Size of UTF-16 or UTF-32 input not aligned " ~ - "to 2 or 4 bytes, respectively"); + "to 2 or 4 bytes, respectively", Mark(name, 0, 0)); } version(unittest) { endian_ = endianResult.endian; } @@ -113,17 +103,18 @@ final class Reader const msg = utf8Result.errorMessage; if(msg !is null) { - throw new ReaderException("Error when converting to UTF-8: " ~ msg); + // TODO: add line and column + throw new ReaderException("Error when converting to UTF-8: " ~ msg, Mark(name, 0, 0)); } buffer_ = utf8Result.utf8; characterCount_ = utf8Result.characterCount; // Check that all characters in buffer are printable. + // TODO: add line and column enforce(isPrintableValidUTF8(buffer_), - new ReaderException("Special unicode characters are not allowed")); + new ReaderException("Special unicode characters are not allowed", Mark(name, 0, 0))); - this.sliceBuilder = SliceBuilder(this); checkASCII(); } @@ -212,8 +203,7 @@ final class Reader /// Get specified number of characters starting at current position. /// /// Note: This gets only a "view" into the internal buffer, which will be - /// invalidated after other Reader calls. Use SliceBuilder to build slices - /// for permanent use. + /// invalidated after other Reader calls. /// /// Params: length = Number of characters (code points, not bytes) to get. May /// reach past the end of the buffer; in that case the returned @@ -228,8 +218,7 @@ final class Reader /// Get specified number of bytes, not code points, starting at current position. /// /// Note: This gets only a "view" into the internal buffer, which will be - /// invalidated after other Reader calls. Use SliceBuilder to build slices - /// for permanent use. + /// invalidated after other Reader calls. /// /// Params: length = Number bytes (not code points) to get. May NOT reach past /// the end of the buffer; should be used with peek() to avoid @@ -396,17 +385,34 @@ final class Reader checkASCII(); } - /// Used to build slices of read data in Reader; to avoid allocations. - SliceBuilder sliceBuilder; - - /// Get a string describing current buffer position, used for error messages. + /// Get filename, line and column of current position. Mark mark() const pure nothrow @nogc @safe { return Mark(name_, line_, column_); } - /// Get file name. - string name() const @safe pure nothrow @nogc { return name_; } + /// Get filename, line and column of current position + some number of chars + Mark mark(size_t advance) const pure @safe + { + auto lineTemp = cast()line_; + auto columnTemp = cast()column_; + auto bufferOffsetTemp = cast()bufferOffset_; + for (size_t pos = 0; pos < advance; pos++) + { + if (bufferOffsetTemp >= buffer_.length) + { + break; + } + const c = decode(buffer_, bufferOffsetTemp); + if (c.isBreak || (c == '\r' && buffer_[bufferOffsetTemp] == '\n')) + { + lineTemp++; + columnTemp = 0; + } + columnTemp++; + } + return Mark(name_, lineTemp, columnTemp); + } - /// Set file name. - void name(string name) pure @safe nothrow @nogc { name_ = name; } + /// Get file name. + ref inout(string) name() inout @safe return pure nothrow @nogc { return name_; } /// Get current line number. uint line() const @safe pure nothrow @nogc { return line_; } @@ -448,267 +454,6 @@ private: } } -/// Used to build slices of already read data in Reader buffer, avoiding allocations. -/// -/// Usually these slices point to unchanged Reader data, but sometimes the data is -/// changed due to how YAML interprets certain characters/strings. -/// -/// See begin() documentation. -struct SliceBuilder -{ -private: - // No copying by the user. - @disable this(this); - @disable void opAssign(ref SliceBuilder); - - // Reader this builder works in. - Reader reader_; - - // Start of the slice om reader_.buffer_ (size_t.max while no slice being build) - size_t start_ = size_t.max; - // End of the slice om reader_.buffer_ (size_t.max while no slice being build) - size_t end_ = size_t.max; - - // Stack of slice ends to revert to (see Transaction) - // - // Very few levels as we don't want arbitrarily nested transactions. - size_t[4] endStack_; - // The number of elements currently in endStack_. - size_t endStackUsed_; - - @safe const pure nothrow @nogc invariant() - { - if(!inProgress) { return; } - assert(end_ <= reader_.bufferOffset_, "Slice ends after buffer position"); - assert(start_ <= end_, "Slice start after slice end"); - } - - // Is a slice currently being built? - bool inProgress() @safe const pure nothrow @nogc - in(start_ == size_t.max ? end_ == size_t.max : end_ != size_t.max, "start_/end_ are not consistent") - { - return start_ != size_t.max; - } - -public: - /// Begin building a slice. - /// - /// Only one slice can be built at any given time; before beginning a new slice, - /// finish the previous one (if any). - /// - /// The slice starts at the current position in the Reader buffer. It can only be - /// extended up to the current position in the buffer; Reader methods get() and - /// forward() move the position. E.g. it is valid to extend a slice by write()-ing - /// a string just returned by get() - but not one returned by prefix() unless the - /// position has changed since the prefix() call. - void begin() @safe pure nothrow @nogc - in(!inProgress, "Beginning a slice while another slice is being built") - in(endStackUsed_ == 0, "Slice stack not empty at slice begin") - { - - start_ = reader_.bufferOffset_; - end_ = reader_.bufferOffset_; - } - - /// Finish building a slice and return it. - /// - /// Any Transactions on the slice must be committed or destroyed before the slice - /// is finished. - /// - /// Returns a string; once a slice is finished it is definitive that its contents - /// will not be changed. - char[] finish() @safe pure nothrow @nogc - in(inProgress, "finish called without begin") - in(endStackUsed_ == 0, "Finishing a slice with running transactions.") - { - - auto result = reader_.buffer_[start_ .. end_]; - start_ = end_ = size_t.max; - return result; - } - - /// Write a string to the slice being built. - /// - /// Data can only be written up to the current position in the Reader buffer. - /// - /// If str is a string returned by a Reader method, and str starts right after the - /// end of the slice being built, the slice is extended (trivial operation). - /// - /// See_Also: begin - void write(scope char[] str) @safe pure nothrow @nogc - { - assert(inProgress, "write called without begin"); - assert(end_ <= reader_.bufferOffset_, - "AT START: Slice ends after buffer position"); - - // Nothing? Already done. - if (str.length == 0) { return; } - // If str starts at the end of the slice (is a string returned by a Reader - // method), just extend the slice to contain str. - if(&str[0] == &reader_.buffer_[end_]) - { - end_ += str.length; - } - // Even if str does not start at the end of the slice, it still may be returned - // by a Reader method and point to buffer. So we need to memmove. - else - { - copy(str, reader_.buffer_[end_..end_ + str.length * char.sizeof]); - end_ += str.length; - } - } - - /// Write a character to the slice being built. - /// - /// Data can only be written up to the current position in the Reader buffer. - /// - /// See_Also: begin - void write(dchar c) @safe pure - in(inProgress, "write called without begin") - { - if(c < 0x80) - { - reader_.buffer_[end_++] = cast(char)c; - return; - } - - // We need to encode a non-ASCII dchar into UTF-8 - char[4] encodeBuf; - const bytes = encode(encodeBuf, c); - reader_.buffer_[end_ .. end_ + bytes] = encodeBuf[0 .. bytes]; - end_ += bytes; - } - - /// Insert a character to a specified position in the slice. - /// - /// Enlarges the slice by 1 char. Note that the slice can only extend up to the - /// current position in the Reader buffer. - /// - /// Params: - /// - /// c = The character to insert. - /// position = Position to insert the character at in code units, not code points. - /// Must be less than slice length(); a previously returned length() - /// can be used. - void insert(const dchar c, const size_t position) @safe pure - in(inProgress, "insert called without begin") - in(start_ + position <= end_, "Trying to insert after the end of the slice") - { - - const point = start_ + position; - const movedLength = end_ - point; - - // Encode c into UTF-8 - char[4] encodeBuf; - if(c < 0x80) { encodeBuf[0] = cast(char)c; } - const size_t bytes = c < 0x80 ? 1 : encode(encodeBuf, c); - - if(movedLength > 0) - { - copy(reader_.buffer_[point..point + movedLength * char.sizeof], - reader_.buffer_[point + bytes..point + bytes + movedLength * char.sizeof]); - } - reader_.buffer_[point .. point + bytes] = encodeBuf[0 .. bytes]; - end_ += bytes; - } - - /// Get the current length of the slice. - size_t length() @safe const pure nothrow @nogc - { - return end_ - start_; - } - - /// A slice building transaction. - /// - /// Can be used to save and revert back to slice state. - struct Transaction - { - private: - // The slice builder affected by the transaction. - SliceBuilder* builder_; - // Index of the return point of the transaction in StringBuilder.endStack_. - size_t stackLevel_; - // True after commit() has been called. - bool committed_; - - public: - /// Begins a transaction on a SliceBuilder object. - /// - /// The transaction must end $(B after) any transactions created within the - /// transaction but $(B before) the slice is finish()-ed. A transaction can be - /// ended either by commit()-ing or reverting through the destructor. - /// - /// Saves the current state of a slice. - this(SliceBuilder* builder) @safe pure nothrow @nogc - { - builder_ = builder; - stackLevel_ = builder_.endStackUsed_; - builder_.push(); - } - - /// Commit changes to the slice. - /// - /// Ends the transaction - can only be called once, and removes the possibility - /// to revert slice state. - /// - /// Does nothing for a default-initialized transaction (the transaction has not - /// been started yet). - void commit() @safe pure nothrow @nogc - in(!committed_, "Can't commit a transaction more than once") - { - - if(builder_ is null) { return; } - assert(builder_.endStackUsed_ == stackLevel_ + 1, - "Parent transactions don't fully contain child transactions"); - builder_.apply(); - committed_ = true; - } - - /// Destroy the transaction and revert it if it hasn't been committed yet. - void end() @safe pure nothrow @nogc - in(builder_ && builder_.endStackUsed_ == stackLevel_ + 1, "Parent transactions don't fully contain child transactions") - { - builder_.pop(); - builder_ = null; - } - - } - -private: - // Push the current end of the slice so we can revert to it if needed. - // - // Used by Transaction. - void push() @safe pure nothrow @nogc - in(inProgress, "push called without begin") - in(endStackUsed_ < endStack_.length, "Slice stack overflow") - { - endStack_[endStackUsed_++] = end_; - } - - // Pop the current end of endStack_ and set the end of the slice to the popped - // value, reverting changes since the old end was pushed. - // - // Used by Transaction. - void pop() @safe pure nothrow @nogc - in(inProgress, "pop called without begin") - in(endStackUsed_ > 0, "Trying to pop an empty slice stack") - { - end_ = endStack_[--endStackUsed_]; - } - - // Pop the current end of endStack_, but keep the current end of the slice, applying - // changes made since pushing the old end. - // - // Used by Transaction. - void apply() @safe pure nothrow @nogc - in(inProgress, "apply called without begin") - in(endStackUsed_ > 0, "Trying to apply an empty slice stack") - { - --endStackUsed_; - } -} - - private: // Convert a UTF-8/16/32 buffer to UTF-8, in-place if possible. @@ -728,7 +473,7 @@ private: // this first. // $(D char[] utf8) input converted to UTF-8. May be a slice of input. // $(D size_t characterCount) Number of characters (code points) in input. -auto toUTF8(ubyte[] input, const UTFEncoding encoding) @safe pure nothrow +public auto toUTF8(ubyte[] input, const UTFEncoding encoding) @safe pure nothrow { // Documented in function ddoc. struct Result diff --git a/src/ext_depends/D-YAML/source/dyaml/representer.d b/src/ext_depends/D-YAML/source/dyaml/representer.d index f903b60..4d36ec6 100644 --- a/src/ext_depends/D-YAML/source/dyaml/representer.d +++ b/src/ext_depends/D-YAML/source/dyaml/representer.d @@ -31,11 +31,6 @@ import dyaml.serializer; import dyaml.style; package: -///Exception thrown on Representer errors. -class RepresenterException : YAMLException -{ - mixin ExceptionCtors; -} /** * Represents YAML nodes as scalar, sequence and mapping nodes ready for output. diff --git a/src/ext_depends/D-YAML/source/dyaml/resolver.d b/src/ext_depends/D-YAML/source/dyaml/resolver.d index 16d8419..4e82931 100644 --- a/src/ext_depends/D-YAML/source/dyaml/resolver.d +++ b/src/ext_depends/D-YAML/source/dyaml/resolver.d @@ -100,9 +100,6 @@ struct Resolver } public: - @disable bool opEquals(ref Resolver); - @disable int opCmp(ref Resolver); - /** * Add an implicit scalar resolver. * @@ -209,45 +206,6 @@ struct Resolver assert(false, "Cannot resolve an invalid node"); } } - @safe unittest - { - auto resolver = Resolver.withDefaultResolvers; - - bool tagMatch(string tag, string[] values) @safe - { - const string expected = tag; - foreach(value; values) - { - const string resolved = resolver.resolve(NodeID.scalar, null, value, true); - if(expected != resolved) - { - return false; - } - } - return true; - } - - assert(tagMatch("tag:yaml.org,2002:bool", - ["yes", "NO", "True", "on"])); - assert(tagMatch("tag:yaml.org,2002:float", - ["6.8523015e+5", "685.230_15e+03", "685_230.15", - "190:20:30.15", "-.inf", ".NaN"])); - assert(tagMatch("tag:yaml.org,2002:int", - ["685230", "+685_230", "02472256", "0x_0A_74_AE", - "0b1010_0111_0100_1010_1110", "190:20:30"])); - assert(tagMatch("tag:yaml.org,2002:merge", ["<<"])); - assert(tagMatch("tag:yaml.org,2002:null", ["~", "null", ""])); - assert(tagMatch("tag:yaml.org,2002:str", - ["abcd", "9a8b", "9.1adsf"])); - assert(tagMatch("tag:yaml.org,2002:timestamp", - ["2001-12-15T02:59:43.1Z", - "2001-12-14t21:59:43.10-05:00", - "2001-12-14 21:59:43.10 -5", - "2001-12-15 2:59:43.10", - "2002-12-14"])); - assert(tagMatch("tag:yaml.org,2002:value", ["="])); - assert(tagMatch("tag:yaml.org,2002:yaml", ["!", "&", "*"])); - } ///Returns: Default scalar tag. @property string defaultScalarTag() const pure @safe nothrow {return defaultScalarTag_;} diff --git a/src/ext_depends/D-YAML/source/dyaml/scanner.d b/src/ext_depends/D-YAML/source/dyaml/scanner.d index 77c3e38..f30ce79 100644 --- a/src/ext_depends/D-YAML/source/dyaml/scanner.d +++ b/src/ext_depends/D-YAML/source/dyaml/scanner.d @@ -74,14 +74,6 @@ alias isFlowScalarBreakSpace = among!(' ', '\t', '\0', '\n', '\r', '\u0085', '\u alias isNSAnchorName = c => !c.isWhiteSpace && !c.among!('[', ']', '{', '}', ',', '\uFEFF'); -/// Marked exception thrown at scanner errors. -/// -/// See_Also: MarkedYAMLException -class ScannerException : MarkedYAMLException -{ - mixin MarkedExceptionCtors; -} - /// Generates tokens from data provided by a Reader. struct Scanner { @@ -95,18 +87,12 @@ struct Scanner /// We emit the KEY token before all keys, so when we find a potential simple /// key, we try to locate the corresponding ':' indicator. Simple keys should be /// limited to a single line and 1024 characters. - /// - /// 16 bytes on 64-bit. static struct SimpleKey { - /// Character index in reader where the key starts. - uint charIndex = uint.max; + /// Position of the key + Mark mark; /// Index of the key token from start (first token scanned being 0). uint tokenIndex; - /// Line the key starts at. - uint line; - /// Column the key starts at. - ushort column; /// Is this required to be a simple key? bool required; /// Is this struct "null" (invalid)?. @@ -188,9 +174,14 @@ struct Scanner } /// Set file name. - void name(string name) @safe pure nothrow @nogc + ref inout(string) name() inout @safe return pure nothrow @nogc { - reader_.name = name; + return reader_.name; + } + /// Get a mark from the current reader position + Mark mark() const @safe pure nothrow @nogc + { + return reader_.mark; } private: @@ -198,7 +189,7 @@ struct Scanner /// function. string expected(T)(string expected, T found) { - return text("expected ", expected, ", but found ", found); + return text(expected, ", but found ", found); } /// Determine whether or not we need to fetch more tokens before peeking/getting a token. @@ -284,12 +275,11 @@ struct Scanner foreach(level, ref key; possibleSimpleKeys_) { if(key.isNull) { continue; } - if(key.line != reader_.line || reader_.charIndex - key.charIndex > 1024) + if(key.mark.line != reader_.mark.line || reader_.mark.column - key.mark.column > 1024) { enforce(!key.required, - new ScannerException("While scanning a simple key", - Mark(reader_.name, key.line, key.column), - "could not find expected ':'", reader_.mark)); + new ScannerException("While scanning a simple key, could not find expected ':'", + reader_.mark, "key started here", key.mark)); key.isNull = true; } } @@ -311,17 +301,19 @@ struct Scanner removePossibleSimpleKey(); const tokenCount = tokensTaken_ + cast(uint)tokens_.length; - const line = reader_.line; + const line = reader_.line; const column = reader_.column; - const key = SimpleKey(cast(uint)reader_.charIndex, tokenCount, line, - cast(ushort)min(column, ushort.max), required); + const key = SimpleKey(reader_.mark, tokenCount, required); if(possibleSimpleKeys_.length <= flowLevel_) { const oldLength = possibleSimpleKeys_.length; possibleSimpleKeys_.length = flowLevel_ + 1; - //No need to initialize the last element, it's already done in the next line. - possibleSimpleKeys_[oldLength .. flowLevel_] = SimpleKey.init; + // Make sure all the empty keys are null + foreach (ref emptyKey; possibleSimpleKeys_[oldLength .. flowLevel_]) + { + emptyKey.isNull = true; + } } possibleSimpleKeys_[flowLevel_] = key; } @@ -335,9 +327,8 @@ struct Scanner { const key = possibleSimpleKeys_[flowLevel_]; enforce(!key.required, - new ScannerException("While scanning a simple key", - Mark(reader_.name, key.line, key.column), - "could not find expected ':'", reader_.mark)); + new ScannerException("While scanning a simple key, could not find expected ':'", + reader_.mark, "key started here", key.mark)); possibleSimpleKeys_[flowLevel_].isNull = true; } } @@ -547,18 +538,18 @@ struct Scanner !possibleSimpleKeys_[flowLevel_].isNull) { const key = possibleSimpleKeys_[flowLevel_]; + assert(key.tokenIndex >= tokensTaken_); + possibleSimpleKeys_[flowLevel_].isNull = true; - Mark keyMark = Mark(reader_.name, key.line, key.column); + Mark keyMark = key.mark; const idx = key.tokenIndex - tokensTaken_; - assert(idx >= 0); - // Add KEY. // Manually inserting since tokens are immutable (need linked list). tokens_.insert(keyToken(keyMark, keyMark), idx); // If this key starts a new block mapping, we need to add BLOCK-MAPPING-START. - if(flowLevel_ == 0 && addIndent(key.column)) + if(flowLevel_ == 0 && addIndent(key.mark.column)) { tokens_.insert(blockMappingStartToken(keyMark, keyMark), idx); } @@ -753,23 +744,23 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanAlphaNumericToSlice(string name)(const Mark startMark) + void scanAlphaNumericToSlice(string name)(ref char[] slice, const Mark startMark) { size_t length; dchar c = reader_.peek(); while(c.isAlphaNum || c.among!('-', '_')) { c = reader_.peek(++length); } - enforce(length > 0, new ScannerException("While scanning " ~ name, - startMark, expected("alphanumeric, '-' or '_'", c), reader_.mark)); + enforce(length > 0, new ScannerException(expected("While scanning a " ~ name ~ ", expected alphanumeric, '-' or '_'", c), + reader_.mark, name~" started here", startMark)); - reader_.sliceBuilder.write(reader_.get(length)); + slice ~= reader_.get(length); } /// Scan a string. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanAnchorAliasToSlice(const Mark startMark) @safe + char[] readAnchorAlias(const Mark startMark) @safe { size_t length; dchar c = reader_.peek(); @@ -778,10 +769,11 @@ struct Scanner c = reader_.peek(++length); } - enforce(length > 0, new ScannerException("While scanning an anchor or alias", - startMark, expected("a printable character besides '[', ']', '{', '}' and ','", c), reader_.mark)); + enforce(length > 0, new ScannerException( + expected("While scanning an anchor or alias, expected a printable character besides '[', ']', '{', '}' and ','", c), + reader_.mark, "started here", startMark)); - reader_.sliceBuilder.write(reader_.get(length)); + return reader_.get(length); } /// Scan and throw away all characters until next line break. @@ -794,14 +786,14 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanToNextBreakToSlice() @safe + void scanToNextBreakToSlice(ref char[] slice) @safe { uint length; while(!reader_.peek(length).isBreak) { ++length; } - reader_.sliceBuilder.write(reader_.get(length)); + slice ~= reader_.get(length); } @@ -859,17 +851,15 @@ struct Scanner reader_.forward(); // Scan directive name - reader_.sliceBuilder.begin(); - scanDirectiveNameToSlice(startMark); - const name = reader_.sliceBuilder.finish(); + char[] name; + scanDirectiveNameToSlice(name, startMark); - reader_.sliceBuilder.begin(); + char[] value; // Index where tag handle ends and suffix starts in a tag directive value. uint tagHandleEnd = uint.max; - if(name == "YAML") { scanYAMLDirectiveValueToSlice(startMark); } - else if(name == "TAG") { tagHandleEnd = scanTagDirectiveValueToSlice(startMark); } - char[] value = reader_.sliceBuilder.finish(); + if(name == "YAML") { scanYAMLDirectiveValueToSlice(value, startMark); } + else if(name == "TAG") { tagHandleEnd = scanTagDirectiveValueToSlice(value, startMark); } Mark endMark = reader_.mark; @@ -891,55 +881,55 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanDirectiveNameToSlice(const Mark startMark) @safe + void scanDirectiveNameToSlice(ref char[] slice, const Mark startMark) @safe { // Scan directive name. - scanAlphaNumericToSlice!"a directive"(startMark); + scanAlphaNumericToSlice!"directive"(slice, startMark); enforce(reader_.peek().among!(' ', '\0', '\n', '\r', '\u0085', '\u2028', '\u2029'), - new ScannerException("While scanning a directive", startMark, - expected("alphanumeric, '-' or '_'", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive, expected alphanumeric, '-' or '_'", reader_.peek()), + reader_.mark, "directive started here", startMark)); } /// Scan value of a YAML directive token. Returns major, minor version separated by '.'. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanYAMLDirectiveValueToSlice(const Mark startMark) @safe + void scanYAMLDirectiveValueToSlice(ref char[] slice, const Mark startMark) @safe { findNextNonSpace(); - scanYAMLDirectiveNumberToSlice(startMark); + scanYAMLDirectiveNumberToSlice(slice, startMark); enforce(reader_.peekByte() == '.', - new ScannerException("While scanning a directive", startMark, - expected("digit or '.'", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive, expected digit or '.'", reader_.peek()), + reader_.mark, "directive started here", startMark)); // Skip the '.'. reader_.forward(); - reader_.sliceBuilder.write('.'); - scanYAMLDirectiveNumberToSlice(startMark); + slice ~= '.'; + scanYAMLDirectiveNumberToSlice(slice, startMark); enforce(reader_.peek().among!(' ', '\0', '\n', '\r', '\u0085', '\u2028', '\u2029'), - new ScannerException("While scanning a directive", startMark, - expected("digit or '.'", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive, expected digit or '.'", reader_.peek()), + reader_.mark, "directive started here", startMark)); } /// Scan a number from a YAML directive. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanYAMLDirectiveNumberToSlice(const Mark startMark) @safe + void scanYAMLDirectiveNumberToSlice(ref char[] slice, const Mark startMark) @safe { enforce(isDigit(reader_.peek()), - new ScannerException("While scanning a directive", startMark, - expected("digit", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive, expected a digit", reader_.peek()), + reader_.mark, "directive started here", startMark)); // Already found the first digit in the enforce(), so set length to 1. uint length = 1; while(reader_.peek(length).isDigit) { ++length; } - reader_.sliceBuilder.write(reader_.get(length)); + slice ~= reader_.get(length); } /// Scan value of a tag directive. @@ -948,14 +938,14 @@ struct Scanner /// characters into that slice. /// /// Returns: Length of tag handle (which is before tag prefix) in scanned data - uint scanTagDirectiveValueToSlice(const Mark startMark) @safe + uint scanTagDirectiveValueToSlice(ref char[] slice, const Mark startMark) @safe { findNextNonSpace(); - const startLength = reader_.sliceBuilder.length; - scanTagDirectiveHandleToSlice(startMark); - const handleLength = cast(uint)(reader_.sliceBuilder.length - startLength); + const startLength = slice.length; + scanTagDirectiveHandleToSlice(slice, startMark); + const handleLength = cast(uint)(slice.length - startLength); findNextNonSpace(); - scanTagDirectivePrefixToSlice(startMark); + scanTagDirectivePrefixToSlice(slice, startMark); return handleLength; } @@ -964,24 +954,24 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanTagDirectiveHandleToSlice(const Mark startMark) @safe + void scanTagDirectiveHandleToSlice(ref char[] slice, const Mark startMark) @safe { - scanTagHandleToSlice!"directive"(startMark); + scanTagHandleToSlice!"directive"(slice, startMark); enforce(reader_.peekByte() == ' ', - new ScannerException("While scanning a directive handle", startMark, - expected("' '", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive handle, expected ' '", reader_.peek()), + reader_.mark, "directive started here", startMark)); } /// Scan prefix of a tag directive. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanTagDirectivePrefixToSlice(const Mark startMark) @safe + void scanTagDirectivePrefixToSlice(ref char[] slice, const Mark startMark) @safe { - scanTagURIToSlice!"directive"(startMark); + scanTagURIToSlice!"directive"(slice, startMark); enforce(reader_.peek().among!(' ', '\0', '\n', '\r', '\u0085', '\u2028', '\u2029'), - new ScannerException("While scanning a directive prefix", startMark, - expected("' '", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive prefix, expected ' '", reader_.peek()), + reader_.mark, "directive started here", startMark)); } /// Scan (and ignore) ignored line after a directive. @@ -990,8 +980,8 @@ struct Scanner findNextNonSpace(); if(reader_.peekByte() == '#') { scanToNextBreak(); } enforce(reader_.peek().isBreak, - new ScannerException("While scanning a directive", startMark, - expected("comment or a line break", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a directive, expected a comment or a line break", reader_.peek()), + reader_.mark, "directive started here", startMark)); scanLineBreak(); } @@ -1011,10 +1001,7 @@ struct Scanner const startMark = reader_.mark; reader_.forward(); // The */& character was only peeked, so we drop it now - reader_.sliceBuilder.begin(); - scanAnchorAliasToSlice(startMark); - // On error, value is discarded as we return immediately - char[] value = reader_.sliceBuilder.finish(); + char[] value = readAnchorAlias(startMark); assert(!reader_.peek().isNSAnchorName, "Anchor/alias name not fully scanned"); @@ -1035,8 +1022,7 @@ struct Scanner const startMark = reader_.mark; dchar c = reader_.peek(1); - reader_.sliceBuilder.begin(); - scope(failure) { reader_.sliceBuilder.finish(); } + char[] slice; // Index where tag handle ends and tag suffix starts in the tag value // (slice) we will produce. uint handleEnd; @@ -1046,17 +1032,17 @@ struct Scanner reader_.forward(2); handleEnd = 0; - scanTagURIToSlice!"tag"(startMark); + scanTagURIToSlice!"tag"(slice, startMark); enforce(reader_.peekByte() == '>', - new ScannerException("While scanning a tag", startMark, - expected("'>'", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a tag, expected a '>'", reader_.peek()), + reader_.mark, "tag started here", startMark)); reader_.forward(); } else if(c.isWhiteSpace) { reader_.forward(); handleEnd = 0; - reader_.sliceBuilder.write('!'); + slice ~= '!'; } else { @@ -1076,24 +1062,23 @@ struct Scanner if(useHandle) { - scanTagHandleToSlice!"tag"(startMark); - handleEnd = cast(uint)reader_.sliceBuilder.length; + scanTagHandleToSlice!"tag"(slice, startMark); + handleEnd = cast(uint)slice.length; } else { reader_.forward(); - reader_.sliceBuilder.write('!'); - handleEnd = cast(uint)reader_.sliceBuilder.length; + slice ~= '!'; + handleEnd = cast(uint)slice.length; } - scanTagURIToSlice!"tag"(startMark); + scanTagURIToSlice!"tag"(slice, startMark); } enforce(reader_.peek().isBreakOrSpace, - new ScannerException("While scanning a tag", startMark, expected("' '", reader_.peek()), - reader_.mark)); + new ScannerException(expected("While scanning a tag, expected a ' '", reader_.peek()), + reader_.mark, "tag started here", startMark)); - char[] slice = reader_.sliceBuilder.finish(); return tagToken(startMark, reader_.mark, slice, handleEnd); } @@ -1115,23 +1100,22 @@ struct Scanner Mark endMark; uint indent = max(1, indent_ + 1); - reader_.sliceBuilder.begin(); - alias Transaction = SliceBuilder.Transaction; + char[] slice; // Used to strip the last line breaks written to the slice at the end of the // scalar, which may be needed based on chomping. - Transaction breaksTransaction = Transaction(&reader_.sliceBuilder); + char[] newBreakSlice; // Read the first indentation/line breaks before the scalar. - size_t startLen = reader_.sliceBuilder.length; + size_t startLen = newBreakSlice.length; if(increment == int.min) { - auto indentation = scanBlockScalarIndentationToSlice(); + auto indentation = scanBlockScalarIndentationToSlice(newBreakSlice); endMark = indentation[1]; indent = max(indent, indentation[0]); } else { indent += increment - 1; - endMark = scanBlockScalarBreaksToSlice(indent); + endMark = scanBlockScalarBreaksToSlice(newBreakSlice, indent); } // int.max means there's no line break (int.max is outside UTF-32). @@ -1140,24 +1124,22 @@ struct Scanner // Scan the inner part of the block scalar. while(reader_.column == indent && reader_.peekByte() != '\0') { - breaksTransaction.commit(); + slice ~= newBreakSlice; const bool leadingNonSpace = !reader_.peekByte().among!(' ', '\t'); // This is where the 'interesting' non-whitespace data gets read. - scanToNextBreakToSlice(); + scanToNextBreakToSlice(slice); lineBreak = scanLineBreak(); // This transaction serves to rollback data read in the // scanBlockScalarBreaksToSlice() call. - breaksTransaction = Transaction(&reader_.sliceBuilder); - startLen = reader_.sliceBuilder.length; + newBreakSlice = []; + startLen = slice.length; // The line breaks should actually be written _after_ the if() block // below. We work around that by inserting - endMark = scanBlockScalarBreaksToSlice(indent); + endMark = scanBlockScalarBreaksToSlice(newBreakSlice, indent); - // This will not run during the last iteration (see the if() vs the - // while()), hence breaksTransaction rollback (which happens after this - // loop) will never roll back data written in this if() block. + // This will not run during the last iteration if(reader_.column == indent && reader_.peekByte() != '\0') { // Unfortunately, folding rules are ambiguous. @@ -1168,16 +1150,16 @@ struct Scanner { // No breaks were scanned; no need to insert the space in the // middle of slice. - if(startLen == reader_.sliceBuilder.length) + if(startLen == slice.length + newBreakSlice.length) { - reader_.sliceBuilder.write(' '); + newBreakSlice ~= ' '; } } else { // We need to insert in the middle of the slice in case any line // breaks were scanned. - reader_.sliceBuilder.insert(lineBreak, startLen); + newBreakSlice.insert(lineBreak, 0); } ////this is Clark Evans's interpretation (also in the spec @@ -1208,11 +1190,10 @@ struct Scanner } } - // If chompint is Keep, we keep (commit) the last scanned line breaks + // If chomping is Keep, we keep (commit) the last scanned line breaks // (which are at the end of the scalar). Otherwise re remove them (end the // transaction). - if(chomping == Chomping.keep) { breaksTransaction.commit(); } - else { breaksTransaction.end(); } + if(chomping == Chomping.keep) { slice ~= newBreakSlice; } if(chomping != Chomping.strip && lineBreak != int.max) { // If chomping is Keep, we keep the line break but the first line break @@ -1220,21 +1201,18 @@ struct Scanner // be inserted _before_ the other line breaks. if(chomping == Chomping.keep) { - reader_.sliceBuilder.insert(lineBreak, startLen); + slice.insert(lineBreak, startLen); } - // If chomping is not Keep, breaksTransaction was cancelled so we can - // directly write the first line break (as it isn't stripped - chomping - // is not Strip) + // If chomping is not Keep, discard the line break else { if (lineBreak != '\0') { - reader_.sliceBuilder.write(lineBreak); + slice ~= lineBreak; } } } - char[] slice = reader_.sliceBuilder.finish(); return scalarToken(startMark, endMark, slice, style); } @@ -1257,8 +1235,8 @@ struct Scanner } enforce(c.among!(' ', '\0', '\n', '\r', '\u0085', '\u2028', '\u2029'), - new ScannerException("While scanning a block scalar", startMark, - expected("chomping or indentation indicator", c), reader_.mark)); + new ScannerException(expected("While scanning a block scalar, expected a chomping or indentation indicator", c), + reader_.mark, "scalar started here", startMark)); return tuple(chomping, increment); } @@ -1299,8 +1277,8 @@ struct Scanner assert(increment < 10 && increment >= 0, "Digit has invalid value"); enforce(increment > 0, - new ScannerException("While scanning a block scalar", startMark, - expected("indentation indicator in range 1-9", "0"), reader_.mark)); + new ScannerException(expected("While scanning a block scalar, expected an indentation indicator in range 1-9", "0"), + reader_.mark, "scalar started here", startMark)); reader_.forward(); c = reader_.peek(); @@ -1314,8 +1292,8 @@ struct Scanner if(reader_.peekByte()== '#') { scanToNextBreak(); } enforce(reader_.peek().isBreak, - new ScannerException("While scanning a block scalar", startMark, - expected("comment or line break", reader_.peek()), reader_.mark)); + new ScannerException(expected("While scanning a block scalar, expected a comment or line break", reader_.peek()), + reader_.mark, "scalar started here", startMark)); scanLineBreak(); } @@ -1324,7 +1302,7 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - Tuple!(uint, Mark) scanBlockScalarIndentationToSlice() @safe + Tuple!(uint, Mark) scanBlockScalarIndentationToSlice(ref char[] slice) @safe { uint maxIndent; Mark endMark = reader_.mark; @@ -1333,7 +1311,7 @@ struct Scanner { if(reader_.peekByte() != ' ') { - reader_.sliceBuilder.write(scanLineBreak()); + slice ~= scanLineBreak(); endMark = reader_.mark; continue; } @@ -1348,7 +1326,7 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - Mark scanBlockScalarBreaksToSlice(const uint indent) @safe + Mark scanBlockScalarBreaksToSlice(ref char[] slice, const uint indent) @safe { Mark endMark = reader_.mark; @@ -1356,7 +1334,7 @@ struct Scanner { while(reader_.column < indent && reader_.peekByte() == ' ') { reader_.forward(); } if(!reader_.peek().among!('\n', '\r', '\u0085', '\u2028', '\u2029')) { break; } - reader_.sliceBuilder.write(scanLineBreak()); + slice ~= scanLineBreak(); endMark = reader_.mark; } @@ -1369,18 +1347,17 @@ struct Scanner const startMark = reader_.mark; const quote = reader_.get(); - reader_.sliceBuilder.begin(); + char[] slice; - scanFlowScalarNonSpacesToSlice(quotes, startMark); + scanFlowScalarNonSpacesToSlice(slice, quotes, startMark); while(reader_.peek() != quote) { - scanFlowScalarSpacesToSlice(startMark); - scanFlowScalarNonSpacesToSlice(quotes, startMark); + scanFlowScalarSpacesToSlice(slice, startMark); + scanFlowScalarNonSpacesToSlice(slice, quotes, startMark); } reader_.forward(); - auto slice = reader_.sliceBuilder.finish(); return scalarToken(startMark, reader_.mark, slice, quotes); } @@ -1388,7 +1365,7 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanFlowScalarNonSpacesToSlice(const ScalarStyle quotes, const Mark startMark) + void scanFlowScalarNonSpacesToSlice(ref char[] slice, const ScalarStyle quotes, const Mark startMark) @safe { for(;;) @@ -1398,19 +1375,19 @@ struct Scanner size_t numCodePoints; while(!reader_.peek(numCodePoints).isFlowScalarBreakSpace) { ++numCodePoints; } - if (numCodePoints > 0) { reader_.sliceBuilder.write(reader_.get(numCodePoints)); } + if (numCodePoints > 0) { slice ~= reader_.get(numCodePoints); } c = reader_.peek(); if(quotes == ScalarStyle.singleQuoted && c == '\'' && reader_.peek(1) == '\'') { reader_.forward(2); - reader_.sliceBuilder.write('\''); + slice ~= '\''; } else if((quotes == ScalarStyle.doubleQuoted && c == '\'') || (quotes == ScalarStyle.singleQuoted && c.among!('"', '\\'))) { reader_.forward(); - reader_.sliceBuilder.write(c); + slice ~= c; } else if(quotes == ScalarStyle.doubleQuoted && c == '\\') { @@ -1423,7 +1400,7 @@ struct Scanner // place (in a slice) in case of '\P' and '\L' (very uncommon, // but we don't want to break the spec) char[2] escapeSequence = ['\\', cast(char)c]; - reader_.sliceBuilder.write(escapeSequence); + slice ~= escapeSequence; } else if(c.among!(escapeHexCodeList)) { @@ -1432,32 +1409,27 @@ struct Scanner foreach(i; 0 .. hexLength) { enforce(reader_.peek(i).isHexDigit, - new ScannerException("While scanning a double quoted scalar", startMark, - expected("escape sequence of hexadecimal numbers", - reader_.peek(i)), reader_.mark)); + new ScannerException(expected("While scanning a double quoted scalar, expected an escape sequence of hexadecimal numbers", reader_.peek(i)), + reader_.mark, "scalar started here", startMark)); } char[] hex = reader_.get(hexLength); - enforce((hex.length > 0) && (hex.length <= 8), - new ScannerException("While scanning a double quoted scalar", startMark, - "overflow when parsing an escape sequence of " ~ - "hexadecimal numbers.", reader_.mark)); + assert((hex.length > 0) && (hex.length <= 8), "Hex escape overflow"); char[2] escapeStart = ['\\', cast(char) c]; - reader_.sliceBuilder.write(escapeStart); - reader_.sliceBuilder.write(hex); + slice ~= escapeStart; + slice ~= hex; } else if(c.among!('\n', '\r', '\u0085', '\u2028', '\u2029')) { scanLineBreak(); - scanFlowScalarBreaksToSlice(startMark); + scanFlowScalarBreaksToSlice(slice, startMark); } else { - throw new ScannerException("While scanning a double quoted scalar", startMark, - text("found unsupported escape character ", c), - reader_.mark); + throw new ScannerException(text("While scanning a double quoted scalar, found unsupported escape character ", c), + reader_.mark, "scalar started here", startMark); } } else { return; } @@ -1468,7 +1440,7 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// spaces into that slice. - void scanFlowScalarSpacesToSlice(const Mark startMark) @safe + void scanFlowScalarSpacesToSlice(ref char[] slice, const Mark startMark) @safe { // Increase length as long as we see whitespace. size_t length; @@ -1478,14 +1450,14 @@ struct Scanner // Can check the last byte without striding because '\0' is ASCII const c = reader_.peek(length); enforce(c != '\0', - new ScannerException("While scanning a quoted scalar", startMark, - "found unexpected end of buffer", reader_.mark)); + new ScannerException("While scanning a quoted scalar, found unexpected end of buffer", + reader_.mark, "scalar started here", startMark)); // Spaces not followed by a line break. if(!c.among!('\n', '\r', '\u0085', '\u2028', '\u2029')) { reader_.forward(length); - reader_.sliceBuilder.write(whitespaces); + slice ~= whitespaces; return; } @@ -1493,21 +1465,21 @@ struct Scanner reader_.forward(length); const lineBreak = scanLineBreak(); - if(lineBreak != '\n') { reader_.sliceBuilder.write(lineBreak); } + if(lineBreak != '\n') { slice ~= lineBreak; } // If we have extra line breaks after the first, scan them into the // slice. - const bool extraBreaks = scanFlowScalarBreaksToSlice(startMark); + const bool extraBreaks = scanFlowScalarBreaksToSlice(slice, startMark); // No extra breaks, one normal line break. Replace it with a space. - if(lineBreak == '\n' && !extraBreaks) { reader_.sliceBuilder.write(' '); } + if(lineBreak == '\n' && !extraBreaks) { slice ~= ' '; } } /// Scan line breaks in a flow scalar. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// line breaks into that slice. - bool scanFlowScalarBreaksToSlice(const Mark startMark) @safe + bool scanFlowScalarBreaksToSlice(ref char[] slice, const Mark startMark) @safe { // True if at least one line break was found. bool anyBreaks; @@ -1517,8 +1489,8 @@ struct Scanner const prefix = reader_.prefix(3); enforce(!(prefix == "---" || prefix == "...") || !reader_.peek(3).isWhiteSpace, - new ScannerException("While scanning a quoted scalar", startMark, - "found unexpected document separator", reader_.mark)); + new ScannerException("While scanning a quoted scalar, found unexpected document separator", + reader_.mark, "scalar started here", startMark)); // Skip any whitespaces. while(reader_.peekByte().among!(' ', '\t')) { reader_.forward(); } @@ -1528,7 +1500,7 @@ struct Scanner const lineBreak = scanLineBreak(); anyBreaks = true; - reader_.sliceBuilder.write(lineBreak); + slice ~= lineBreak; } return anyBreaks; } @@ -1546,10 +1518,9 @@ struct Scanner // document separators at the beginning of the line. // if(indent == 0) { indent = 1; } - reader_.sliceBuilder.begin(); + char[] slice; - alias Transaction = SliceBuilder.Transaction; - Transaction spacesTransaction; + char[] newSpacesSlice; // Stop at a comment. while(reader_.peekByte() != '#') { @@ -1561,7 +1532,8 @@ struct Scanner const cNext = reader_.peek(length + 1); if(c.isWhiteSpace || (flowLevel_ == 0 && c == ':' && cNext.isWhiteSpace) || - (flowLevel_ > 0 && c.among!(',', ':', '?', '[', ']', '{', '}'))) + (flowLevel_ > 0 && c == ':' && (cNext.isWhiteSpace || cNext.among!(',', '[', ']', '{', '}'))) || + (flowLevel_ > 0 && c.among!(',', '[', ']', '{', '}'))) { break; } @@ -1569,38 +1541,26 @@ struct Scanner c = cNext; } - // It's not clear what we should do with ':' in the flow context. - enforce(flowLevel_ == 0 || c != ':' || - reader_.peek(length + 1).isWhiteSpace || - reader_.peek(length + 1).among!(',', '[', ']', '{', '}'), - new ScannerException("While scanning a plain scalar", startMark, - "found unexpected ':' . Please check " ~ - "http://pyyaml.org/wiki/YAMLColonInFlowContext for details.", - reader_.mark)); - if(length == 0) { break; } allowSimpleKey_ = false; - reader_.sliceBuilder.write(reader_.get(length)); + newSpacesSlice ~= reader_.get(length); endMark = reader_.mark; - spacesTransaction.commit(); - spacesTransaction = Transaction(&reader_.sliceBuilder); + slice ~= newSpacesSlice; + newSpacesSlice = []; - const startLength = reader_.sliceBuilder.length; - scanPlainSpacesToSlice(); - if(startLength == reader_.sliceBuilder.length || + const startLength = slice.length; + scanPlainSpacesToSlice(newSpacesSlice); + if(startLength == slice.length + newSpacesSlice.length || (flowLevel_ == 0 && reader_.column < indent)) { break; } } - spacesTransaction.end(); - char[] slice = reader_.sliceBuilder.finish(); - return scalarToken(startMark, endMark, slice, ScalarStyle.plain); } @@ -1608,7 +1568,7 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the spaces /// into that slice. - void scanPlainSpacesToSlice() @safe + void scanPlainSpacesToSlice(ref char[] slice) @trusted { // The specification is really confusing about tabs in plain scalars. // We just forbid them completely. Do not use tabs in YAML! @@ -1623,7 +1583,7 @@ struct Scanner if(!c.isNSChar) { // We have spaces, but no newline. - if(whitespaces.length > 0) { reader_.sliceBuilder.write(whitespaces); } + if(whitespaces.length > 0) { slice ~= whitespaces; } return; } @@ -1642,9 +1602,8 @@ struct Scanner bool extraBreaks; - alias Transaction = SliceBuilder.Transaction; - auto transaction = Transaction(&reader_.sliceBuilder); - if(lineBreak != '\n') { reader_.sliceBuilder.write(lineBreak); } + char[] newSlice; + if(lineBreak != '\n') { newSlice ~= lineBreak; } while(reader_.peek().isNSChar) { if(reader_.peekByte() == ' ') { reader_.forward(); } @@ -1652,27 +1611,28 @@ struct Scanner { const lBreak = scanLineBreak(); extraBreaks = true; - reader_.sliceBuilder.write(lBreak); + newSlice ~= lBreak; if(end(reader_)) { return; } } } - transaction.commit(); + slice ~= newSlice; // No line breaks, only a space. - if(lineBreak == '\n' && !extraBreaks) { reader_.sliceBuilder.write(' '); } + if(lineBreak == '\n' && !extraBreaks) { slice ~= ' '; } } /// Scan handle of a tag token. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanTagHandleToSlice(string name)(const Mark startMark) + void scanTagHandleToSlice(string name)(ref char[] slice, const Mark startMark) { dchar c = reader_.peek(); - enum contextMsg = "While scanning a " ~ name; + enum contextMsg = "While scanning a " ~ name ~ ", expected a !"; + // should this be an assert? enforce(c == '!', - new ScannerException(contextMsg, startMark, expected("'!'", c), reader_.mark)); + new ScannerException(expected(contextMsg, c), reader_.mark, "tag started here", startMark)); uint length = 1; c = reader_.peek(length); @@ -1684,22 +1644,22 @@ struct Scanner c = reader_.peek(length); } enforce(c == '!', - new ScannerException(contextMsg, startMark, expected("'!'", c), reader_.mark)); + new ScannerException(expected(contextMsg, c), reader_.mark(length), "tag started here", startMark)); ++length; } - reader_.sliceBuilder.write(reader_.get(length)); + slice ~= reader_.get(length); } /// Scan URI in a tag token. /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanTagURIToSlice(string name)(const Mark startMark) + void scanTagURIToSlice(string name)(ref char[] slice, const Mark startMark) { // Note: we do not check if URI is well-formed. dchar c = reader_.peek(); - const startLen = reader_.sliceBuilder.length; + const startLen = slice.length; { uint length; while(c.isAlphaNum || c.isURIChar) @@ -1707,9 +1667,9 @@ struct Scanner if(c == '%') { auto chars = reader_.get(length); - reader_.sliceBuilder.write(chars); + slice ~= chars; length = 0; - scanURIEscapesToSlice!name(startMark); + scanURIEscapesToSlice!name(slice, startMark); } else { ++length; } c = reader_.peek(length); @@ -1717,14 +1677,14 @@ struct Scanner if(length > 0) { auto chars = reader_.get(length); - reader_.sliceBuilder.write(chars); + slice ~= chars; length = 0; } } // OK if we scanned something, error otherwise. - enum contextMsg = "While parsing a " ~ name; - enforce(reader_.sliceBuilder.length > startLen, - new ScannerException(contextMsg, startMark, expected("URI", c), reader_.mark)); + enum contextMsg = "While parsing a " ~ name ~ ", expected a URI"; + enforce(slice.length > startLen, + new ScannerException(expected(contextMsg, c), reader_.mark, "tag started here", startMark)); } // Not @nogc yet because std.utf.decode is not @nogc @@ -1732,7 +1692,7 @@ struct Scanner /// /// Assumes that the caller is building a slice in Reader, and puts the scanned /// characters into that slice. - void scanURIEscapesToSlice(string name)(const Mark startMark) + void scanURIEscapesToSlice(string name)(ref char[] slice, const Mark startMark) { import core.exception : UnicodeException; // URI escapes encode a UTF-8 string. We store UTF-8 code units here for @@ -1747,9 +1707,8 @@ struct Scanner char[2] nextByte = [reader_.peekByte(), reader_.peekByte(1)]; enforce(nextByte[0].isHexDigit && nextByte[1].isHexDigit, - new ScannerException(contextMsg, startMark, - expected("URI escape sequence of 2 hexadecimal " ~ - "numbers", nextByte), reader_.mark)); + new ScannerException(expected(contextMsg ~ ", expected a URI escape sequence of 2 hexadecimal numbers", nextByte), + reader_.mark, "tag started here", startMark)); buffer ~= nextByte[].to!ubyte(16); @@ -1759,14 +1718,13 @@ struct Scanner { foreach (dchar chr; buffer.data) { - reader_.sliceBuilder.write(chr); + slice ~= chr; } } catch (UnicodeException) { - throw new ScannerException(contextMsg, startMark, - "Invalid UTF-8 data encoded in URI escape sequence", - reader_.mark); + throw new ScannerException(contextMsg ~ ", found invalid UTF-8 data encoded in URI escape sequence", + reader_.mark, "tag started here", startMark); } } @@ -1811,16 +1769,22 @@ struct Scanner } } -// Issue 309 - https://github.com/dlang-community/D-YAML/issues/309 -@safe unittest +private void insert(ref char[] slice, const dchar c, const size_t position) @safe pure +in(position <= slice.length, text("Trying to insert after the end of the slice (", position, " > ", slice.length, ")")) { - enum str = q"EOS -exp: | - foobar -EOS".chomp; - - auto r = new Reader(cast(ubyte[])str.dup); - auto s = Scanner(r); - auto elems = s.map!"a.value".filter!"a.length > 0".array; - assert(elems[1] == "foobar"); + const point = position; + const movedLength = slice.length - point; + + // Encode c into UTF-8 + char[4] encodeBuf; + if(c < 0x80) { encodeBuf[0] = cast(char)c; } + const size_t bytes = c < 0x80 ? 1 : encode(encodeBuf, c); + + slice.length += bytes; + if(movedLength > 0) + { + copy(slice[point..point + movedLength * char.sizeof], + slice[point + bytes .. point + bytes + movedLength * char.sizeof]); + } + slice[point .. point + bytes] = encodeBuf[0 .. bytes]; } diff --git a/src/ext_depends/D-YAML/source/dyaml/test/common.d b/src/ext_depends/D-YAML/source/dyaml/test/common.d deleted file mode 100644 index a6bafa9..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/common.d +++ /dev/null @@ -1,223 +0,0 @@ - -// Copyright Ferdinand Majerech 2011. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.common; - -version(unittest) -{ - -import dyaml.node; -import dyaml.event; - -import core.exception; -import std.algorithm; -import std.array; -import std.conv; -import std.file; -import std.range; -import std.path; -import std.traits; -import std.typecons; - -package: - -/** -Run a test. - -Params: - testFunction = Unittest function. - unittestExt = Extensions of data files needed for the unittest. - skipExt = Extensions that must not be used for the unittest. - */ -void run(D)(D testFunction, string[] unittestExt, string[] skipExt = []) -{ - immutable string dataDir = __FILE_FULL_PATH__.dirName ~ "/../../../test/data"; - auto testFilenames = findTestFilenames(dataDir); - - if (unittestExt.length > 0) - { - outer: foreach (base, extensions; testFilenames) - { - string[] filenames; - foreach (ext; unittestExt) - { - if (!extensions.canFind(ext)) - { - continue outer; - } - filenames ~= base ~ '.' ~ ext; - } - foreach (ext; skipExt) - { - if (extensions.canFind(ext)) - { - continue outer; - } - } - - execute(testFunction, filenames); - } - } - else - { - execute(testFunction, string[].init); - } -} - -// TODO: remove when a @safe ubyte[] file read can be done. -/** -Reads a file as an array of bytes. - -Params: - filename = Full path to file to read. - -Returns: The file's data. -*/ -ubyte[] readData(string filename) @trusted -{ - import std.file : read; - return cast(ubyte[])read(filename); -} -void assertNodesEqual(const scope Node gotNode, const scope Node expectedNode) @safe -{ - import std.format : format; - assert(gotNode == expectedNode, format!"got %s, expected %s"(gotNode.debugString, expectedNode.debugString)); -} - -/** -Determine if events in events1 are equivalent to events in events2. - -Params: - events1 = A range of events to compare with. - events2 = A second range of events to compare. - -Returns: true if the events are equivalent, false otherwise. -*/ -bool compareEvents(T, U)(T events1, U events2) -if (isInputRange!T && isInputRange!U && is(ElementType!T == Event) && is(ElementType!U == Event)) -{ - foreach (e1, e2; zip(events1, events2)) - { - //Different event types. - if (e1.id != e2.id) - { - return false; - } - //Different anchor (if applicable). - if (e1.id.among!(EventID.sequenceStart, EventID.mappingStart, EventID.alias_, EventID.scalar) - && e1.anchor != e2.anchor) - { - return false; - } - //Different collection tag (if applicable). - if (e1.id.among!(EventID.sequenceStart, EventID.mappingStart) && e1.tag != e2.tag) - { - return false; - } - if (e1.id == EventID.scalar) - { - //Different scalar tag (if applicable). - if (!(e1.implicit || e2.implicit) && e1.tag != e2.tag) - { - return false; - } - //Different scalar value. - if (e1.value != e2.value) - { - return false; - } - } - } - return true; -} -/** -Throw an Error if events in events1 aren't equivalent to events in events2. - -Params: - events1 = First event array to compare. - events2 = Second event array to compare. -*/ -void assertEventsEqual(T, U)(T events1, U events2) -if (isInputRange!T && isInputRange!U && is(ElementType!T == Event) && is(ElementType!U == Event)) -{ - auto events1Copy = events1.array; - auto events2Copy = events2.array; - assert(compareEvents(events1Copy, events2Copy), text("Got '", events1Copy, "', expected '", events2Copy, "'")); -} - -private: - -/** -Find unittest input filenames. - -Params: dir = Directory to look in. - -Returns: Test input base filenames and their extensions. -*/ - //@trusted due to dirEntries -string[][string] findTestFilenames(const string dir) @trusted -{ - //Groups of extensions indexed by base names. - string[][string] names; - foreach (string name; dirEntries(dir, SpanMode.shallow)) - { - if (isFile(name)) - { - string base = name.stripExtension(); - string ext = name.extension(); - if (ext is null) - { - ext = ""; - } - if (ext[0] == '.') - { - ext = ext[1 .. $]; - } - - //If the base name doesn't exist yet, add it; otherwise add new extension. - names[base] = ((base in names) is null) ? [ext] : names[base] ~ ext; - } - } - return names; -} - -/** -Recursively copy an array of strings to a tuple to use for unittest function input. - -Params: - index = Current index in the array/tuple. - tuple = Tuple to copy to. - strings = Strings to copy. -*/ -void stringsToTuple(uint index, F ...)(ref F tuple, const string[] strings) -in(F.length == strings.length) -do -{ - tuple[index] = strings[index]; - static if (index > 0) - { - stringsToTuple!(index - 1, F)(tuple, strings); - } -} - -/** -Execute an unittest on specified files. - -Params: - testName = Name of the unittest. - testFunction = Unittest function. - filenames = Names of input files to test with. - */ -void execute(D)(D testFunction, string[] filenames) -{ - //Convert filenames to parameters tuple and call the test function. - alias F = Parameters!D[0..$]; - F parameters; - stringsToTuple!(F.length - 1, F)(parameters, filenames); - testFunction(parameters); -} - -} // version(unittest) diff --git a/src/ext_depends/D-YAML/source/dyaml/test/compare.d b/src/ext_depends/D-YAML/source/dyaml/test/compare.d deleted file mode 100644 index 5a37fd0..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/compare.d +++ /dev/null @@ -1,51 +0,0 @@ - -// Copyright Ferdinand Majerech 2011. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.compare; - -@safe unittest -{ - import dyaml : Loader; - import dyaml.test.common : assertNodesEqual, compareEvents, run; - - /** - Test parser by comparing output from parsing two equivalent YAML files. - - Params: - dataFilename = YAML file to parse. - canonicalFilename = Another file to parse, in canonical YAML format. - */ - static void testParser(string dataFilename, string canonicalFilename) @safe - { - auto dataEvents = Loader.fromFile(dataFilename).parse(); - auto canonicalEvents = Loader.fromFile(canonicalFilename).parse(); - - //BUG: the return value isn't checked! This test currently fails... - compareEvents(dataEvents, canonicalEvents); - } - - /** - Test loader by comparing output from loading two equivalent YAML files. - - Params: - dataFilename = YAML file to load. - canonicalFilename = Another file to load, in canonical YAML format. - */ - static void testLoader(string dataFilename, string canonicalFilename) @safe - { - import std.array : array; - auto data = Loader.fromFile(dataFilename).array; - auto canonical = Loader.fromFile(canonicalFilename).array; - - assert(data.length == canonical.length, "Unequal node count"); - foreach (n; 0 .. data.length) - { - assertNodesEqual(data[n], canonical[n]); - } - } - run(&testParser, ["data", "canonical"]); - run(&testLoader, ["data", "canonical"], ["test_loader_skip"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/constructor.d b/src/ext_depends/D-YAML/source/dyaml/test/constructor.d index aeb8653..ad9fb62 100644 --- a/src/ext_depends/D-YAML/source/dyaml/test/constructor.d +++ b/src/ext_depends/D-YAML/source/dyaml/test/constructor.d @@ -6,10 +6,9 @@ module dyaml.test.constructor; +package version(unittest): -version(unittest) -{ - +import std.algorithm; import std.conv; import std.datetime; import std.exception; @@ -920,38 +919,3 @@ struct TestStruct return Node(value.to!string, "!tag2"); } } - -} // version(unittest) - - -@safe unittest -{ - import dyaml.test.common : assertNodesEqual, run; - /** - Constructor unittest. - - Params: - dataFilename = File name to read from. - codeDummy = Dummy .code filename, used to determine that - .data file with the same name should be used in this test. - */ - static void testConstructor(string dataFilename, string codeDummy) @safe - { - string base = dataFilename.baseName.stripExtension; - assert((base in expected) !is null, "Unimplemented constructor test: " ~ base); - - auto loader = Loader.fromFile(dataFilename); - - Node[] exp = expected[base]; - - //Compare with expected results document by document. - size_t i; - foreach (node; loader) - { - assertNodesEqual(node, exp[i]); - ++i; - } - assert(i == exp.length); - } - run(&testConstructor, ["data", "code"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/emitter.d b/src/ext_depends/D-YAML/source/dyaml/test/emitter.d deleted file mode 100644 index 293f236..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/emitter.d +++ /dev/null @@ -1,132 +0,0 @@ - -// Copyright Ferdinand Majerech 2011-2014. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.emitter; - -@safe unittest -{ - import std.array : Appender; - import std.range : ElementType, isInputRange; - - import dyaml : CollectionStyle, LineBreak, Loader, Mark, ScalarStyle; - import dyaml.emitter : Emitter; - import dyaml.event : Event, EventID, mappingStartEvent, scalarEvent, sequenceStartEvent; - import dyaml.test.common : assertEventsEqual, run; - - // Try to emit an event range. - static void emitTestCommon(T)(ref Appender!string emitStream, T events, bool canonical = false) @safe - if (isInputRange!T && is(ElementType!T == Event)) - { - auto emitter = Emitter!(typeof(emitStream), char)(emitStream, canonical, 2, 80, LineBreak.unix); - foreach (ref event; events) - { - emitter.emit(event); - } - } - /** - Test emitter by getting events from parsing a file, emitting them, parsing - the emitted result and comparing events from parsing the emitted result with - originally parsed events. - - Params: - dataFilename = YAML file to parse. - canonicalFilename = Canonical YAML file used as dummy to determine - which data files to load. - */ - static void testEmitterOnData(string dataFilename, string canonicalFilename) @safe - { - //Must exist due to Anchor, Tags reference counts. - auto loader = Loader.fromFile(dataFilename); - auto events = loader.parse(); - auto emitStream = Appender!string(); - emitTestCommon(emitStream, events); - - auto loader2 = Loader.fromString(emitStream.data); - loader2.name = "TEST"; - auto newEvents = loader2.parse(); - assertEventsEqual(events, newEvents); - } - /** - Test emitter by getting events from parsing a canonical YAML file, emitting - them both in canonical and normal format, parsing the emitted results and - comparing events from parsing the emitted result with originally parsed events. - - Params: canonicalFilename = Canonical YAML file to parse. - */ - static void testEmitterOnCanonical(string canonicalFilename) @safe - { - //Must exist due to Anchor, Tags reference counts. - auto loader = Loader.fromFile(canonicalFilename); - auto events = loader.parse(); - foreach (canonical; [false, true]) - { - auto emitStream = Appender!string(); - emitTestCommon(emitStream, events, canonical); - - auto loader2 = Loader.fromString(emitStream.data); - loader2.name = "TEST"; - auto newEvents = loader2.parse(); - assertEventsEqual(events, newEvents); - } - } - /** - Test emitter by getting events from parsing a file, emitting them with all - possible scalar and collection styles, parsing the emitted results and - comparing events from parsing the emitted result with originally parsed events. - - Params: - dataFilename = YAML file to parse. - canonicalFilename = Canonical YAML file used as dummy to determine - which data files to load. - */ - static void testEmitterStyles(string dataFilename, string canonicalFilename) @safe - { - foreach (filename; [dataFilename, canonicalFilename]) - { - //must exist due to Anchor, Tags reference counts - auto loader = Loader.fromFile(canonicalFilename); - auto events = loader.parse(); - foreach (flowStyle; [CollectionStyle.block, CollectionStyle.flow]) - { - foreach (style; [ScalarStyle.literal, ScalarStyle.folded, - ScalarStyle.doubleQuoted, ScalarStyle.singleQuoted, - ScalarStyle.plain]) - { - Event[] styledEvents; - foreach (event; events) - { - if (event.id == EventID.scalar) - { - event = scalarEvent(Mark(), Mark(), event.anchor, event.tag, - event.implicit, - event.value, style); - } - else if (event.id == EventID.sequenceStart) - { - event = sequenceStartEvent(Mark(), Mark(), event.anchor, - event.tag, event.implicit, flowStyle); - } - else if (event.id == EventID.mappingStart) - { - event = mappingStartEvent(Mark(), Mark(), event.anchor, - event.tag, event.implicit, flowStyle); - } - styledEvents ~= event; - } - auto emitStream = Appender!string(); - emitTestCommon(emitStream, styledEvents); - auto loader2 = Loader.fromString(emitStream.data); - loader2.name = "TEST"; - auto newEvents = loader2.parse(); - assertEventsEqual(events, newEvents); - } - } - } - } - run(&testEmitterOnData, ["data", "canonical"]); - run(&testEmitterOnCanonical, ["canonical"]); - run(&testEmitterStyles, ["data", "canonical"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/errors.d b/src/ext_depends/D-YAML/source/dyaml/test/errors.d deleted file mode 100644 index 43b019c..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/errors.d +++ /dev/null @@ -1,64 +0,0 @@ - -// Copyright Ferdinand Majerech 2011-2014 -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.errors; - -@safe unittest -{ - import std.array : array; - import std.exception : assertThrown; - - import dyaml : Loader; - import dyaml.test.common : run; - - /** - Loader error unittest from file stream. - - Params: errorFilename = File name to read from. - */ - static void testLoaderError(string errorFilename) @safe - { - assertThrown(Loader.fromFile(errorFilename).array, - __FUNCTION__ ~ "(" ~ errorFilename ~ ") Expected an exception"); - } - - /** - Loader error unittest from string. - - Params: errorFilename = File name to read from. - */ - static void testLoaderErrorString(string errorFilename) @safe - { - assertThrown(Loader.fromFile(errorFilename).array, - __FUNCTION__ ~ "(" ~ errorFilename ~ ") Expected an exception"); - } - - /** - Loader error unittest from filename. - - Params: errorFilename = File name to read from. - */ - static void testLoaderErrorFilename(string errorFilename) @safe - { - assertThrown(Loader.fromFile(errorFilename).array, - __FUNCTION__ ~ "(" ~ errorFilename ~ ") Expected an exception"); - } - - /** - Loader error unittest loading a single document from a file. - - Params: errorFilename = File name to read from. - */ - static void testLoaderErrorSingle(string errorFilename) @safe - { - assertThrown(Loader.fromFile(errorFilename).load(), - __FUNCTION__ ~ "(" ~ errorFilename ~ ") Expected an exception"); - } - run(&testLoaderError, ["loader-error"]); - run(&testLoaderErrorString, ["loader-error"]); - run(&testLoaderErrorFilename, ["loader-error"]); - run(&testLoaderErrorSingle, ["single-loader-error"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/inputoutput.d b/src/ext_depends/D-YAML/source/dyaml/test/inputoutput.d deleted file mode 100644 index 758def8..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/inputoutput.d +++ /dev/null @@ -1,92 +0,0 @@ - -// Copyright Ferdinand Majerech 2011-2014. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.inputoutput; - -@safe unittest -{ - import std.array : join, split; - import std.conv : to; - import std.exception : assertThrown; - import std.file : readText; - import std.system : endian, Endian; - - import dyaml : Loader, Node, YAMLException; - import dyaml.test.common : run; - - /** - Get an UTF-16 byte order mark. - - Params: wrong = Get the incorrect BOM for this system. - - Returns: UTF-16 byte order mark. - */ - static wchar bom16(bool wrong = false) pure @safe - { - wchar little = '\uFEFF'; - wchar big = '\uFFFE'; - if (!wrong) - { - return endian == Endian.littleEndian ? little : big; - } - return endian == Endian.littleEndian ? big : little; - } - /** - Get an UTF-32 byte order mark. - - Params: wrong = Get the incorrect BOM for this system. - - Returns: UTF-32 byte order mark. - */ - static dchar bom32(bool wrong = false) pure @safe - { - dchar little = '\uFEFF'; - dchar big = '\uFFFE'; - if (!wrong) - { - return endian == Endian.littleEndian ? little : big; - } - return endian == Endian.littleEndian ? big : little; - } - /** - Unicode input unittest. Tests various encodings. - - Params: unicodeFilename = File name to read from. - */ - static void testUnicodeInput(string unicodeFilename) @safe - { - string data = readText(unicodeFilename); - string expected = data.split().join(" "); - - Node output = Loader.fromString(data).load(); - assert(output.as!string == expected); - - foreach (buffer; [cast(ubyte[]) (bom16() ~ data.to!(wchar[])), - cast(ubyte[]) (bom32() ~ data.to!(dchar[]))]) - { - output = Loader.fromBuffer(buffer).load(); - assert(output.as!string == expected); - } - } - /** - Unicode input error unittest. Tests various encodings with incorrect BOMs. - - Params: unicodeFilename = File name to read from. - */ - static void testUnicodeInputErrors(string unicodeFilename) @safe - { - string data = readText(unicodeFilename); - foreach (buffer; [cast(ubyte[]) (data.to!(wchar[])), - cast(ubyte[]) (data.to!(dchar[])), - cast(ubyte[]) (bom16(true) ~ data.to!(wchar[])), - cast(ubyte[]) (bom32(true) ~ data.to!(dchar[]))]) - { - assertThrown(Loader.fromBuffer(buffer).load()); - } - } - run(&testUnicodeInput, ["unicode"]); - run(&testUnicodeInputErrors, ["unicode"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/reader.d b/src/ext_depends/D-YAML/source/dyaml/test/reader.d deleted file mode 100644 index c20df6f..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/reader.d +++ /dev/null @@ -1,37 +0,0 @@ - -// Copyright Ferdinand Majerech 2011. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.reader; - -@safe unittest -{ - import std.exception :assertThrown; - - import dyaml.test.common : readData, run; - import dyaml.reader : Reader, ReaderException; - - /** - Try reading entire file through Reader, expecting an error (the file is invalid). - - Params: data = Stream to read. - */ - static void runReader(ubyte[] fileData) @safe - { - auto reader = new Reader(fileData); - while(reader.peek() != '\0') { reader.forward(); } - } - - /** - Stream error unittest. Tries to read invalid input files, expecting errors. - - Params: errorFilename = File name to read from. - */ - static void testStreamError(string errorFilename) @safe - { - assertThrown!ReaderException(runReader(readData(errorFilename))); - } - run(&testStreamError, ["stream-error"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/representer.d b/src/ext_depends/D-YAML/source/dyaml/test/representer.d index 4a1ae67..eeac157 100644 --- a/src/ext_depends/D-YAML/source/dyaml/test/representer.d +++ b/src/ext_depends/D-YAML/source/dyaml/test/representer.d @@ -9,12 +9,12 @@ module dyaml.test.representer; @safe unittest { import std.array : Appender, array; + import std.conv : text; import std.meta : AliasSeq; import std.path : baseName, stripExtension; import std.utf : toUTF8; import dyaml : dumper, Loader, Node; - import dyaml.test.common : assertNodesEqual, run; import dyaml.test.constructor : expected; /** @@ -38,13 +38,9 @@ module dyaml.test.representer; auto loader = Loader.fromString(emitStream.data.toUTF8); loader.name = "TEST"; - const readNodes = loader.array; + auto readNodes = loader.array; - assert(expectedNodes.length == readNodes.length); - foreach (n; 0 .. expectedNodes.length) - { - assertNodesEqual(expectedNodes[n], readNodes[n]); - } + assert(expectedNodes == readNodes, text("Got '", readNodes, "', expected '", expectedNodes, "'")); } } foreach (key, _; expected) diff --git a/src/ext_depends/D-YAML/source/dyaml/test/resolver.d b/src/ext_depends/D-YAML/source/dyaml/test/resolver.d deleted file mode 100644 index ea93720..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/resolver.d +++ /dev/null @@ -1,39 +0,0 @@ - -// Copyright Ferdinand Majerech 2011. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.resolver; - -@safe unittest -{ - import std.conv : text; - import std.file : readText; - import std.string : strip; - - import dyaml : Loader, Node, NodeID; - import dyaml.test.common : run; - - - /** - Implicit tag resolution unittest. - - Params: - dataFilename = File with unittest data. - detectFilename = Dummy filename used to specify which data filenames to use. - */ - static void testImplicitResolver(string dataFilename, string detectFilename) @safe - { - const correctTag = readText(detectFilename).strip(); - - auto node = Loader.fromFile(dataFilename).load(); - assert(node.nodeID == NodeID.sequence, text("Expected sequence when reading '", dataFilename, "', got ", node.nodeID)); - foreach (Node scalar; node) - { - assert(scalar.nodeID == NodeID.scalar, text("Expected sequence of scalars when reading '", dataFilename, "', got sequence of ", scalar.nodeID)); - assert(scalar.tag == correctTag, text("Expected tag '", correctTag, "' when reading '", dataFilename, "', got '", scalar.tag, "'")); - } - } - run(&testImplicitResolver, ["data", "detect"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/suite.d b/src/ext_depends/D-YAML/source/dyaml/test/suite.d new file mode 100644 index 0000000..2146c37 --- /dev/null +++ b/src/ext_depends/D-YAML/source/dyaml/test/suite.d @@ -0,0 +1,384 @@ +module dyaml.test.suite; + +import std.algorithm; +import std.conv; +import std.datetime.stopwatch; +import std.exception; +import std.file; +import std.format; +import std.meta; +import std.path; +import std.range; +import std.stdio; +import std.string; +import std.typecons; +import dyaml; +import dyaml.event; +import dyaml.parser; +import dyaml.reader; +import dyaml.scanner; +import dyaml.test.suitehelpers; + +private version(unittest): + +debug(verbose) +{ + enum alwaysPrintTestResults = true; +} +else +{ + enum alwaysPrintTestResults = false; +} + +struct TestResult { + string name; + Nullable!bool emitter; + Nullable!bool constructor; + Nullable!bool loaderError; + Nullable!bool mark1Error; + Nullable!bool mark2Error; + Nullable!bool implicitResolver; + Nullable!bool events; + Nullable!bool specificLoaderError; + Nullable!Mark mark1; + Nullable!Mark mark2; + Event[] parsedData; + Event[][2 * 2 * 5] parsedDataResult; + Node[] loadedData; + Exception nonDYAMLException; + MarkedYAMLException exception; + string eventsExpected; + string eventsGenerated; + string generatedLoadErrorMessage; + string expectedLoadErrorMessage; + string expectedTags; + string generatedTags; +} + +/// Pretty-print the differences between two arrays +auto prettyDifferencePrinter(alias eqPred = (a,b) => a == b, T)(string title, T[] expected, T[] got, bool trimWhitespace = false) @safe +{ + struct Result + { + void foo() { + toString(nullSink); + } + void toString(W)(ref W writer) const + { + import std.format : formattedWrite; + import std.range : put; + import std.string : lineSplitter; + size_t minWidth = 10; + foreach (line; chain(expected, got)) + { + if (line.text.length + 1 > minWidth) + { + minWidth = line.text.length + 1; + } + } + void writeSideBySide(ubyte colour, string a, string b) + { + if (trimWhitespace) + { + a = strip(a); + b = strip(b); + } + writer.formattedWrite!"%s%-(%s%)%s"(colourPrinter(colour, a), " ".repeat(minWidth - a.length), colourPrinter(colour, b)); + } + writefln!"%-(%s%)%s%-(%s%)"("=".repeat(max(0, minWidth * 2 - title.length) / 2), title, "=".repeat(max(0, minWidth * 2 - title.length) / 2)); + writeSideBySide(0, "Expected", "Got"); + put(writer, "\n"); + foreach (line1, line2; zip(StoppingPolicy.longest, expected, got)) + { + static if (is(T : const char[])) + { + if (trimWhitespace) + { + line1 = strip(line1); + line2 = strip(line2); + } + } + ubyte colour = (eqPred(line1, line2)) ? 32 : 31; + writeSideBySide(colour, line1.text, line2.text); + put(writer, "\n"); + } + } + } + return Result(); +} + +/** +Run a single test from the test suite. +Params: + name = The filename of the document to load, containing the test data +*/ +TestResult runTest(string name, Node doc) @safe +{ + TestResult result; + string[string] testData; + void tryLoadTestData(string what) + { + if (what in doc) + { + testData[what] = doc[what].as!string; + doc.removeAt(what); + } + } + string yamlPath(string testName, string section) + { + return format!"%s:%s"(testName, section); + } + tryLoadTestData("name"); + result.name = name~"#"~testData.get("name", "UNNAMED"); + Nullable!Mark getMark(string key) + { + if (auto node = key in doc) + { + Mark mark; + if ("name" in *node) + { + mark.name = (*node)["name"].as!string; + } + else // default to the test name + { + // if we ever have multiple yaml blocks to parse, be sure to change this + mark.name = yamlPath(result.name, "yaml"); + } + if ("line" in *node) + { + mark.line = cast(ushort)((*node)["line"].as!ushort - 1); + } + if ("column" in *node) + { + mark.column = cast(ushort)((*node)["column"].as!ushort - 1); + } + return Nullable!Mark(mark); + } + return Nullable!Mark.init; + } + tryLoadTestData("tags"); + tryLoadTestData("from"); + tryLoadTestData("yaml"); + tryLoadTestData("fail"); + tryLoadTestData("json"); //not yet implemented + tryLoadTestData("dump"); //not yet implemented + tryLoadTestData("detect"); + tryLoadTestData("tree"); + tryLoadTestData("error"); + tryLoadTestData("code"); + assert("yaml" in testData); + { + result.expectedLoadErrorMessage = testData.get("error", ""); + result.mark1 = getMark("mark"); + result.mark2 = getMark("mark2"); + try + { + result.parsedData = parseData(testData["yaml"], yamlPath(result.name, "yaml")).array; + result.loadedData = Loader.fromString(testData["yaml"], yamlPath(result.name, "yaml")).array; + result.emitter = testEmitterStyles(yamlPath(result.name, "canonical"), result.parsedData, result.parsedDataResult); + result.mark1Error = result.mark1.isNull; + result.mark2Error = result.mark2.isNull; + } + catch (MarkedYAMLException e) + { + result.exception = e; + result.generatedLoadErrorMessage = e.msg; + result.mark1Error = !result.mark1.isNull && (result.mark1.get() == e.mark); + result.mark2Error = result.mark2 == e.mark2; + if (testData.get("fail", "false") == "false") + { + result.loaderError = false; + } + else + { + result.loaderError = true; + } + } + catch (Exception e) + { + // all non-DYAML exceptions are failures. + result.nonDYAMLException = e; + result.generatedLoadErrorMessage = e.msg; + result.loaderError = false; + } + result.specificLoaderError = strip(result.generatedLoadErrorMessage) == strip(result.expectedLoadErrorMessage); + } + if (result.loaderError.get(false)) + { + // skip other tests if loading failure was expected, because we don't + // have a way to run them yet + return result; + } + if ("tree" in testData) + { + result.eventsGenerated = result.parsedData.map!(x => strip(x.text)).join("\n"); + result.eventsExpected = testData["tree"].lineSplitter.map!(x => strip(x)).join("\n"); + result.events = result.eventsGenerated == result.eventsExpected; + } + if ("code" in testData) + { + result.constructor = testConstructor(testData["yaml"], testData["code"]); + } + if ("detect" in testData) + { + result.implicitResolver = testImplicitResolver(yamlPath(result.name, "yaml"), testData["yaml"], testData["detect"], result.generatedTags, result.expectedTags); + } + foreach (string remaining, Node _; doc) + { + writeln("Warning: Unhandled section '", remaining, "' in ", result.name); + } + return result; +} + +enum goodColour = 32; +enum badColour = 31; +/** +Print something to the console in colour. +Params: + colour = The id of the colour to print, using the 256-colour palette + data = Something to print +*/ +private auto colourPrinter(T)(ubyte colour, T data) @safe pure +{ + struct Printer + { + void toString(S)(ref S sink) + { + sink.formattedWrite!"\033[%s;1m%s\033[0m"(colour, data); + } + } + return Printer(); +} + +/** +Run all tests in the test suite and print relevant results. The test docs are +all found in the ./test/data dir. +*/ +bool runTests() +{ + auto stopWatch = StopWatch(AutoStart.yes); + bool failed; + uint testsRun, testSetsRun, testsFailed; + foreach (string name; dirEntries(buildNormalizedPath("test"), "*.yaml", SpanMode.depth)/*.chain(dirEntries(buildNormalizedPath("yaml-test-suite/src"), "*.yaml", SpanMode.depth))*/) + { + Node doc; + try + { + doc = Loader.fromFile(name).load(); + } + catch (Exception e) + { + writefln!"[%s] %s"(colourPrinter(badColour, "FAIL"), name); + writeln(colourPrinter(badColour, e)); + assert(0, "Could not load test doc '"~name~"', bailing"); + } + assert (doc.nodeID == NodeID.sequence, name~"'s root node is not a sequence!"); + foreach (Node test; doc) + { + testSetsRun++; + bool resultPrinted; + // make sure the paths are normalized on windows by replacing backslashes with slashes + TestResult result = runTest(name.replace("\\", "/"), test); + void printResult(string label, Nullable!bool value) + { + if (!value.isNull) + { + if (!value.get) + { + testsFailed++; + } + testsRun++; + } + if (alwaysPrintTestResults && value.get(false)) + { + resultPrinted = true; + writef!"[%s]"(colourPrinter(goodColour, label)); + } + else if (!value.get(true)) + { + resultPrinted = true; + failed = true; + writef!"[%s]"(colourPrinter(badColour, label)); + } + } + printResult("Emitter", result.emitter); + printResult("Constructor", result.constructor); + printResult("Mark", result.mark1Error); + printResult("Context mark", result.mark2Error); + printResult("LoaderError", result.loaderError); + printResult("Resolver", result.implicitResolver); + printResult("Events", result.events); + printResult("SpecificLoaderError", result.specificLoaderError); + if (resultPrinted) + { + writeln(" ", result.name); + } + if (!result.loaderError.get(true)) + { + if (result.exception is null && result.nonDYAMLException is null) + { + writeln("\tNo Exception thrown"); + } + else if (result.nonDYAMLException !is null) + { + writeln(result.nonDYAMLException); + } + else if (result.exception !is null) + { + writeln(result.exception); + } + } + else + { + if (!result.mark1Error.get(true)) + { + writeln(prettyDifferencePrinter("Mark mismatch", [result.mark1.text], [result.exception.mark.text])); + } + if (!result.mark2Error.get(true)) + { + writeln(prettyDifferencePrinter("Context mark mismatch", [result.mark2.text], [result.exception.mark2.text])); + } + } + if (!result.emitter.get(true)) + { + enum titles = [ "Normal", "Canonical" ]; + enum styleTitles = + [ + "Block literal", "Block folded", "Block double-quoted", "Block single-quoted", "Block plain", + "Flow literal", "Flow folded", "Flow double-quoted", "Flow single-quoted", "Flow plain", + "Block literal", "Block folded", "Block double-quoted", "Block single-quoted", "Block plain", + "Flow literal", "Flow folded", "Flow double-quoted", "Flow single-quoted", "Flow plain", + ]; + foreach (idx, parsed; result.parsedDataResult) + { + writeln(prettyDifferencePrinter!eventCompare(styleTitles[idx], result.parsedData, parsed)); + } + } + if (!result.events.get(true)) + { + writeln(prettyDifferencePrinter("Events", result.eventsExpected.splitLines, result.eventsGenerated.splitLines, true)); + } + if (!result.specificLoaderError.get(true)) + { + writeln(prettyDifferencePrinter("Expected error", result.expectedLoadErrorMessage.splitLines, result.generatedLoadErrorMessage.splitLines)); + } + if (!result.implicitResolver.get(true)) + { + writeln(prettyDifferencePrinter("Expected error", result.expectedTags.splitLines, result.generatedTags.splitLines)); + } + } + } + if (alwaysPrintTestResults || failed) + { + if (testsFailed > 0) + { + writeln(colourPrinter(badColour, "tests failed: "), testsFailed); + } + writeln(testSetsRun, " test sets (", testsRun, " tests total) completed successfully in ", stopWatch.peek()); + } + return failed; +} + +unittest { + assert(!runTests()); +} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/suitehelpers.d b/src/ext_depends/D-YAML/source/dyaml/test/suitehelpers.d new file mode 100644 index 0000000..3016843 --- /dev/null +++ b/src/ext_depends/D-YAML/source/dyaml/test/suitehelpers.d @@ -0,0 +1,224 @@ + +// Copyright Ferdinand Majerech 2011. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +module dyaml.test.suitehelpers; + +import dyaml; +import dyaml.emitter; +import dyaml.event; +import dyaml.parser; +import dyaml.reader; +import dyaml.scanner; +import dyaml.token; +import dyaml.test.constructor; + +import std.algorithm; +import std.array; +import std.conv; +import std.exception; +import std.file; +import std.range; +import std.string; + +package version(unittest): + +// Like other types, wchar and dchar use the system's endianness, so \uFEFF +// will always be the 'correct' BOM and '\uFFFE' will always be the 'wrong' one +enum wchar[] bom16 = ['\uFEFF', '\uFFFE']; +enum dchar[] bom32 = ['\uFEFF', '\uFFFE']; + +Parser parseData(string data, string name = "TEST") @safe +{ + auto reader = Reader(cast(ubyte[])data.dup, name); + auto scanner = Scanner(reader); + return new Parser(scanner); +} + +/** +Test scanner by scanning a document, expecting no errors. + +Params: + name = Name of the document being scanned + data = Data to scan. +*/ +void testScanner(string name, string data) @safe +{ + ubyte[] yamlData = cast(ubyte[])data.dup; + string[] tokens; + foreach (token; Scanner(Reader(yamlData, name))) + { + tokens ~= token.id.text; + } +} + +/** +Implicit tag resolution unittest. + +Params: + name = Name of the document being tested + data = Document to compare + detectData = The tag that each scalar should resolve to +*/ +bool testImplicitResolver(string name, string data, string detectData, out string generatedTags, out string expectedTags) @safe +{ + const correctTag = detectData.strip(); + + const node = Loader.fromString(data, name).load(); + if (node.nodeID != NodeID.sequence) + { + return false; + } + bool success = true; + foreach (const Node scalar; node) + { + generatedTags ~= scalar.tag ~ "\n"; + expectedTags ~= correctTag ~ "\n"; + if ((scalar.nodeID != NodeID.scalar) || (scalar.tag != correctTag)) + { + success = false; + } + } + return success; +} + +// Try to emit an event range. +Event[] emitTestCommon(string name, Event[] events, bool canonical) @safe +{ + auto emitStream = new Appender!string(); + auto emitter = Emitter!(typeof(emitStream), char)(emitStream, canonical, 2, 80, LineBreak.unix); + foreach (event; events) + { + emitter.emit(event); + } + return parseData(emitStream.data, name).array; +} +/** +Test emitter by checking if events remain equal after round-tripping, with and +without canonical output enabled. + +Params: + name = Name of the document being tested + events = Events to test + results = Events that were produced by round-tripping +*/ +bool testEmitter(string name, Event[] events, out Event[][2] results) @safe +{ + bool matching = true; + foreach (idx, canonicalOutput; [false, true]) + { + results[idx] = emitTestCommon(name, events, canonicalOutput); + + if (!equal!eventCompare(events, results[idx])) + { + matching = false; + } + } + return matching; +} +/** +Test emitter by checking if events remain equal after round-tripping, with all +combinations of styles. + +Params: + name = Name of the document being tested + events = Events to test + results = Events that were produced by round-tripping +*/ +bool testEmitterStyles(string name, Event[] events, out Event[][2 * 2 * 5] results) @safe +{ + size_t idx; + foreach (styles; cartesianProduct( + [CollectionStyle.block, CollectionStyle.flow], + [ScalarStyle.literal, ScalarStyle.folded, + ScalarStyle.doubleQuoted, ScalarStyle.singleQuoted, + ScalarStyle.plain], + [false, true])) + { + const collectionStyle = styles[0]; + const scalarStyle = styles[1]; + const canonical = styles[2]; + Event[] styledEvents; + foreach (event; events) + { + if (event.id == EventID.scalar) + { + event = scalarEvent(Mark(), Mark(), event.anchor, event.tag, + event.implicit, + event.value, scalarStyle); + } + else if (event.id == EventID.sequenceStart) + { + event = sequenceStartEvent(Mark(), Mark(), event.anchor, + event.tag, event.implicit, collectionStyle); + } + else if (event.id == EventID.mappingStart) + { + event = mappingStartEvent(Mark(), Mark(), event.anchor, + event.tag, event.implicit, collectionStyle); + } + styledEvents ~= event; + } + auto newEvents = emitTestCommon(name, styledEvents, canonical); + results[idx++] = newEvents; + if (!equal!eventCompare(events, newEvents)) + { + return false; + } + } + return true; +} + +/** +Constructor unittest. + +Params: + data = The document being tested + base = A unique id corresponding to one of the premade sequences in dyaml.test.constructor +*/ +bool testConstructor(string data, string base) @safe +{ + assert((base in expected) !is null, "Unimplemented constructor test: " ~ base); + auto loader = Loader.fromString(data); + + Node[] exp = expected[base]; + + //Compare with expected results document by document. + return equal(loader, exp); +} + +bool eventCompare(const Event a, const Event b) @safe pure +{ + //Different event types. + if (a.id != b.id) + { + return false; + } + //Different anchor (if applicable). + if (a.id.among!(EventID.sequenceStart, EventID.mappingStart, EventID.alias_, EventID.scalar) + && a.anchor != b.anchor) + { + return false; + } + //Different collection tag (if applicable). + if (a.id.among!(EventID.sequenceStart, EventID.mappingStart) && a.tag != b.tag) + { + return false; + } + if (a.id == EventID.scalar) + { + //Different scalar tag (if applicable). + if (!(a.implicit || b.implicit) && a.tag != b.tag) + { + return false; + } + //Different scalar value. + if (a.value != b.value) + { + return false; + } + } + return true; +} diff --git a/src/ext_depends/D-YAML/source/dyaml/test/tokens.d b/src/ext_depends/D-YAML/source/dyaml/test/tokens.d deleted file mode 100644 index d3dce6e..0000000 --- a/src/ext_depends/D-YAML/source/dyaml/test/tokens.d +++ /dev/null @@ -1,93 +0,0 @@ - -// Copyright Ferdinand Majerech 2011. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.test.tokens; - -@safe unittest -{ - import std.array : split; - import std.conv : text; - import std.file : readText; - - import dyaml.test.common : run; - import dyaml.reader : Reader; - import dyaml.scanner : Scanner; - import dyaml.token : TokenID; - - // Read and scan a YAML doc, returning a range of tokens. - static auto scanTestCommon(string filename) @safe - { - ubyte[] yamlData = cast(ubyte[])readText(filename).dup; - return Scanner(new Reader(yamlData, filename)); - } - - /** - Test tokens output by scanner. - - Params: - dataFilename = File to scan. - tokensFilename = File containing expected tokens. - */ - static void testTokens(string dataFilename, string tokensFilename) @safe - { - //representations of YAML tokens in tokens file. - auto replace = [ - TokenID.directive: "%", - TokenID.documentStart: "---", - TokenID.documentEnd: "...", - TokenID.alias_: "*", - TokenID.anchor: "&", - TokenID.tag: "!", - TokenID.scalar: "_", - TokenID.blockSequenceStart: "[[", - TokenID.blockMappingStart: "{{", - TokenID.blockEnd: "]}", - TokenID.flowSequenceStart: "[", - TokenID.flowSequenceEnd: "]", - TokenID.flowMappingStart: "{", - TokenID.flowMappingEnd: "}", - TokenID.blockEntry: ",", - TokenID.flowEntry: ",", - TokenID.key: "?", - TokenID.value: ":" - ]; - - string[] tokens; - string[] expectedTokens = readText(tokensFilename).split(); - - foreach (token; scanTestCommon(dataFilename)) - { - if (token.id != TokenID.streamStart && token.id != TokenID.streamEnd) - { - tokens ~= replace[token.id]; - } - } - - assert(tokens == expectedTokens, - text("In token test for '", tokensFilename, "', expected '", expectedTokens, "', got '", tokens, "'")); - } - - /** - Test scanner by scanning a file, expecting no errors. - - Params: - dataFilename = File to scan. - canonicalFilename = Another file to scan, in canonical YAML format. - */ - static void testScanner(string dataFilename, string canonicalFilename) @safe - { - foreach (filename; [dataFilename, canonicalFilename]) - { - string[] tokens; - foreach (token; scanTestCommon(filename)) - { - tokens ~= token.id.text; - } - } - } - run(&testTokens, ["data", "tokens"]); - run(&testScanner, ["data", "canonical"]); -} diff --git a/src/ext_depends/D-YAML/source/dyaml/token.d b/src/ext_depends/D-YAML/source/dyaml/token.d index 5400a3f..b67400f 100644 --- a/src/ext_depends/D-YAML/source/dyaml/token.d +++ b/src/ext_depends/D-YAML/source/dyaml/token.d @@ -62,8 +62,6 @@ enum DirectiveType : ubyte /// 32 bytes on 64-bit. struct Token { - @disable int opCmp(ref Token); - // 16B /// Value of the token, if any. /// |