From 02ca32ae0a5bc290918d2b2a3288e385b9cc6b11 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 19 Feb 2021 17:10:51 -0500 Subject: external & build dependences in src tree - external & build dependences boost licensed - ext_depends (external depends) - D-YAML - tinyendian - d2sqlite3 - imageformats - build_depends - dub2nix --- src/ext_depends/D-YAML/source/dyaml/token.d | 172 ++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 src/ext_depends/D-YAML/source/dyaml/token.d (limited to 'src/ext_depends/D-YAML/source/dyaml/token.d') diff --git a/src/ext_depends/D-YAML/source/dyaml/token.d b/src/ext_depends/D-YAML/source/dyaml/token.d new file mode 100644 index 0000000..5400a3f --- /dev/null +++ b/src/ext_depends/D-YAML/source/dyaml/token.d @@ -0,0 +1,172 @@ + +// Copyright Ferdinand Majerech 2011-2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +/// YAML tokens. +/// Code based on PyYAML: http://www.pyyaml.org +module dyaml.token; + + +import std.conv; + +import dyaml.encoding; +import dyaml.exception; +import dyaml.reader; +import dyaml.style; + + +package: + +/// Token types. +enum TokenID : ubyte +{ + // Invalid (uninitialized) token + invalid = 0, + directive, + documentStart, + documentEnd, + streamStart, + streamEnd, + blockSequenceStart, + blockMappingStart, + blockEnd, + flowSequenceStart, + flowMappingStart, + flowSequenceEnd, + flowMappingEnd, + key, + value, + blockEntry, + flowEntry, + alias_, + anchor, + tag, + scalar +} + +/// Specifies the type of a tag directive token. +enum DirectiveType : ubyte +{ + // YAML version directive. + yaml, + // Tag directive. + tag, + // Any other directive is "reserved" for future YAML versions. + reserved +} + +/// Token produced by scanner. +/// +/// 32 bytes on 64-bit. +struct Token +{ + @disable int opCmp(ref Token); + + // 16B + /// Value of the token, if any. + /// + /// Values are char[] instead of string, as Parser may still change them in a few + /// cases. Parser casts values to strings when producing Events. + char[] value; + // 4B + /// Start position of the token in file/stream. + Mark startMark; + // 4B + /// End position of the token in file/stream. + Mark endMark; + // 1B + /// Token type. + TokenID id; + // 1B + /// Style of scalar token, if this is a scalar token. + ScalarStyle style; + // 1B + /// Encoding, if this is a stream start token. + Encoding encoding; + // 1B + /// Type of directive for directiveToken. + DirectiveType directive; + // 4B + /// Used to split value into 2 substrings for tokens that need 2 values (tagToken) + uint valueDivider; + + /// Get string representation of the token ID. + @property string idString() @safe pure const {return id.to!string;} +} + +/// Construct a directive token. +/// +/// Params: start = Start position of the token. +/// end = End position of the token. +/// value = Value of the token. +/// directive = Directive type (YAML or TAG in YAML 1.1). +/// nameEnd = Position of the end of the name +Token directiveToken(const Mark start, const Mark end, char[] value, + DirectiveType directive, const uint nameEnd) @safe pure nothrow @nogc +{ + return Token(value, start, end, TokenID.directive, ScalarStyle.init, Encoding.init, + directive, nameEnd); +} + +/// Construct a simple (no value) token with specified type. +/// +/// Params: id = Type of the token. +/// start = Start position of the token. +/// end = End position of the token. +Token simpleToken(TokenID id)(const Mark start, const Mark end) +{ + return Token(null, start, end, id); +} + +/// Construct a stream start token. +/// +/// Params: start = Start position of the token. +/// end = End position of the token. +/// encoding = Encoding of the stream. +Token streamStartToken(const Mark start, const Mark end, const Encoding encoding) @safe pure nothrow @nogc +{ + return Token(null, start, end, TokenID.streamStart, ScalarStyle.invalid, encoding); +} + +/// Aliases for construction of simple token types. +alias streamEndToken = simpleToken!(TokenID.streamEnd); +alias blockSequenceStartToken = simpleToken!(TokenID.blockSequenceStart); +alias blockMappingStartToken = simpleToken!(TokenID.blockMappingStart); +alias blockEndToken = simpleToken!(TokenID.blockEnd); +alias keyToken = simpleToken!(TokenID.key); +alias valueToken = simpleToken!(TokenID.value); +alias blockEntryToken = simpleToken!(TokenID.blockEntry); +alias flowEntryToken = simpleToken!(TokenID.flowEntry); + +/// Construct a simple token with value with specified type. +/// +/// Params: id = Type of the token. +/// start = Start position of the token. +/// end = End position of the token. +/// value = Value of the token. +/// valueDivider = A hack for TagToken to store 2 values in value; the first +/// value goes up to valueDivider, the second after it. +Token simpleValueToken(TokenID id)(const Mark start, const Mark end, char[] value, + const uint valueDivider = uint.max) +{ + return Token(value, start, end, id, ScalarStyle.invalid, Encoding.init, + DirectiveType.init, valueDivider); +} + +/// Alias for construction of tag token. +alias tagToken = simpleValueToken!(TokenID.tag); +alias aliasToken = simpleValueToken!(TokenID.alias_); +alias anchorToken = simpleValueToken!(TokenID.anchor); + +/// Construct a scalar token. +/// +/// Params: start = Start position of the token. +/// end = End position of the token. +/// value = Value of the token. +/// style = Style of the token. +Token scalarToken(const Mark start, const Mark end, char[] value, const ScalarStyle style) @safe pure nothrow @nogc +{ + return Token(value, start, end, TokenID.scalar, style); +} -- cgit v1.2.3