From 22cea68385b57a1593f5751e49cfdd41d8067997 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Thu, 13 Jul 2017 07:44:03 -0400 Subject: 0.18.0 rename meta from ao (considered adr) --- org/meta_conf_make_meta.org | 738 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 738 insertions(+) create mode 100644 org/meta_conf_make_meta.org (limited to 'org/meta_conf_make_meta.org') diff --git a/org/meta_conf_make_meta.org b/org/meta_conf_make_meta.org new file mode 100644 index 0000000..5e4cb6d --- /dev/null +++ b/org/meta_conf_make_meta.org @@ -0,0 +1,738 @@ +#+TITLE: sdp header extract +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+DESCRIPTION for documents - structuring, publishing in multiple formats and search +#+KEYWORDS +#+LANGUAGE: en +#+STARTUP: indent content +#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t +#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc +#+OPTIONS: author:nil email:nil creator:nil timestamp:nil +#+PROPERTY: header-args :padline no :exports code :noweb yes +#+EXPORT_SELECT_TAGS: export +#+EXPORT_EXCLUDE_TAGS: noexport +#+FILETAGS: :sdp:rel:meta: +#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n) + +[[./sdp.org][sdp]] [[./][org/]] +* 0. header extract (native & sdlang) to AA :module:sdp:meta_conf_make_meta: +** module template + +#+BEGIN_SRC d :tangle ../src/sdp/meta/conf_make_meta.d +/++ + extract native/orig header return associative array
+ + the header is passed as text (lopped off top of a sisu markup file until the + required first heading ^A~), determine whether is a native header or sdlang one + with a regex check if whether it contains the "native header" required tag/field + @title: then process accordingly as a "native header" or "sdlang header" + converting the metadata and make instructions to a common json format used by + program internally. Moved to associative array. ++/ +module sdp.meta.conf_make_meta; +template docHeaderMakeAndMetaTupExtractAndConvertToAA() { + import + std.exception, + std.regex, + std.stdio, + std.traits, + std.typecons, + std.utf, + std.conv : to; + import sdlang; + import + sdp.meta.conf_make_meta_native, + sdp.meta.conf_make_meta_sdlang, + sdp.meta.rgx; + mixin SiSUrgxInit; + mixin SiSUheaderExtractNative; + mixin SiSUextractSDLang; + auto rgx = Rgx(); + auto docHeaderMakeAndMetaTupExtractAndConvertToAA(DocMake, Src)( + DocMake conf_doc_make_aa, + Src header_src, + ) { + debug(asserts){ + static assert(is(typeof(header_src) == char[])); + static assert(is(typeof(conf_doc_make_aa) == string[string][string])); + } + auto head_native = HeaderDocMetadataAndMakeNativeToAA(); + auto header_sdlang_tag = (!(header_src.match(rgx.native_header_meta_title))) + ? extractSDL().docHeaderSDLtagGet(header_src) // sdlang.ast.Tag + : null; + auto header_make_and_meta_tuple = (header_src.match(rgx.native_header_meta_title)) + ? (head_native.headerNativeToAA(header_src)) + : (extractSDL().docHeaderSDLtoAA(conf_doc_make_aa, header_sdlang_tag)); + static assert(!isTypeTuple!(header_make_and_meta_tuple)); + static assert(header_make_and_meta_tuple.length==2); + return header_make_and_meta_tuple; + } +} +#+END_SRC + +* A. module sdlang :module:sdp:meta_conf_make_meta_sdlang: +** 0. module template + +#+BEGIN_SRC d :tangle ../src/sdp/meta/conf_make_meta_sdlang.d +/++ + sdlang headers
+ extract sdlang header return sdlang ++/ +module sdp.meta.conf_make_meta_sdlang; +template SiSUextractSDLang() { + import + std.exception, + std.regex, + std.stdio, + std.string, + std.traits, + std.typecons, + std.utf, + std.conv : to; + import + sdp.meta.defaults, + sdp.meta.rgx; + struct extractSDL { + mixin SiSUregisters; + mixin SiSUrgxInit; + auto rgx = Rgx(); + <> + } +} +#+END_SRC + +** 1. sdlang header _extract root Tag_ :sdlang:root:tag: + +#+name: meta_conf_make_meta_sdl +#+BEGIN_SRC d +private auto docHeaderSDLtagGet(Hs)(Hs src_header) { + debug(asserts){ + static assert(is(typeof(src_header) == char[])); + } + char[][] source_header_arr = + (cast(char[]) src_header).split(rgx.newline_eol_delimiter); + char[] _src_header; + foreach(header_line; source_header_arr) { + if (!match(header_line, rgx.comments)) { + _src_header ~= header_line ~ "\n"; + } + } + scope(failure) { + stderr.writefln( + "%s\n%s\n%s:%s failed here:\n _src_header: %s", + __MODULE__, __FUNCTION__, + __FILE__, __LINE__, + _src_header, + ); + } + Tag sdl_root_header; + try { + sdl_root_header = parseSource(_src_header.to!string); + } + catch(ParseException e) { + stderr.writeln("SDLang problem with this document header:"); + stderr.writeln(_src_header); + // Error messages of the form: + // myFile.sdl(5:28): Error: Invalid integer suffix. + stderr.writeln(e.msg); + } + debug(sdlang) { + writeln("header SDL:"); + writeln(__LINE__, ": ", sdl_root_header.toSDLDocument()); + writeln(__LINE__, ": ", sdl_root_header.maybe.namespaces); + writeln("header make sdlang: ", sdl_root_header.toSDLDocument()); + writeln(__LINE__, ": ", sdl_root_header.getTagValues("title")); + writeln(__LINE__, ": ", sdl_root_header.getTagValues("creator")); + Tag creator = sdl_root_header.getTag("creator"); + if (creator !is null) { + if ("author" in creator.maybe.tags) { + writeln(__LINE__, ": ", creator.getTagValues("author")); + } else if ("author" in creator.maybe.attributes) { + writeln(__LINE__, ": ", creator.maybe.attributes["author"][0].value); + } + } + } + return sdl_root_header; // sdlang.ast.Tag +} +#+END_SRC + +** 2a. _sdlang to associative array_ + +#+name: meta_conf_make_meta_sdl +#+BEGIN_SRC d +private auto sdlangToAA(C,Tag)(C conf, Tag conf_sdlang) { + debug(asserts){ + static assert(is(typeof(conf) == string[string][string])); + } + foreach (maintag, subtags; conf) { + /+ writeln(__LINE__, ": ", maintag, ":- ", subtags); +/ + foreach (subtag, content; subtags) { + if (maintag in conf_sdlang.maybe.tags) { + Tag _maintag = conf_sdlang.getTag(maintag); + if ((subtag in _maintag.maybe.tags) + && (_maintag.getTagValues(subtag).length > 0)) { + debug(headersdlang) { + writeln(__LINE__, ": ", maintag, ":", subtag, ": ", _maintag.getTagValues(subtag)[0]); + } + if (_maintag.getTagValues(subtag).length == 1) { + conf[maintag][subtag] = + (_maintag.getTagValues(subtag)[0]).to!string; + } else if (_maintag.getTagValues(subtag).length > 1) { + foreach (st; _maintag.getTagValues(subtag)) { + conf[maintag][subtag] ~= + st.to!string ~ ";"; + } + } + } else if ((subtag in _maintag.maybe.attributes) + && (_maintag.maybe.attributes[subtag][0].value.length > 0)) { + debug(headersdlang) { + writeln(__LINE__, ": ", maintag, ":", subtag, ": ", conf_sdlang.tags[maintag][0].attributes[subtag][0].value); + } + conf[maintag][subtag] = + (conf_sdlang.tags[maintag][0].attributes[subtag][0].value).to!string; + } + } + } + } + return conf; +} +#+END_SRC + +** 2b. sdlang header to _associative array_ make sdlTag in :sdlang:aa: + +#+name: meta_conf_make_meta_sdl +#+BEGIN_SRC d +private auto docHeaderSDLtoAA(Ma, Tag)(Ma dochead_make, Tag header_sdlang) { + debug(asserts){ + static assert(is(typeof(dochead_make) == string[string][string])); + } + dochead_make = sdlangToAA(dochead_make, header_sdlang); + auto dochead_meta = sdlangToAA(meta_aa_empty, header_sdlang); + if (dochead_meta["title"]["main"].empty) { + { + Tag _maintag = header_sdlang.getTag("title"); + if (_maintag !is null) { + if ("main" in _maintag.maybe.tags) { + dochead_meta["title"]["main"] = + to!string(_maintag.getTagValues("main")); + } else if ("main" !in _maintag.maybe.attributes) { + writeln(__LINE__, ": ", _maintag.values[0]); + dochead_meta["title"]["main"] = + (_maintag.values[0]).to!string; // test that this exists + } + } + } + } + if (!(dochead_meta["title"]["subtitle"].empty) + && (dochead_meta["title"]["sub"].empty)) { + dochead_meta["title"]["sub"] ~= dochead_meta["title"]["subtitle"]; + } + dochead_meta["title"].remove("subtitle"); + if (dochead_meta["title"]["sub"].empty) { + dochead_meta["title"]["full"] ~= dochead_meta["title"]["main"]; + } else { + dochead_meta["title"]["full"] ~= format( + "%s - %s", + dochead_meta["title"]["main"], + dochead_meta["title"]["sub"], + ); + } + dochead_meta["creator"]["author_raw"] = dochead_meta["creator"]["author"]; + string[] authors_arr; + auto authors_raw_arr = dochead_meta["creator"]["author"].split(rgx.arr_delimiter); + foreach (author_raw; authors_raw_arr) { + authors_arr ~= author_raw.replace(rgx.raw_author_munge, "$2 $1"); + } + dochead_meta["creator"]["author"] = join(authors_arr, ", ").chomp.chomp; + auto t = tuple(dochead_make, dochead_meta); + static assert(t.length==2); + return t; +} +#+END_SRC + +* B. module native document header :module:sdp:meta_conf_make_meta_native: +** module template + +#+BEGIN_SRC d :tangle ../src/sdp/meta/conf_make_meta_native.d +/++ + native headers using
@title:
:subtitle:
type tags
+ extract native/orig header return associative array ++/ +module sdp.meta.conf_make_meta_native; +template SiSUheaderExtractNative() { + import + std.exception, + std.regex, + std.stdio, + std.string, + std.traits, + std.typecons, + std.utf, + std.conv : to; + import + sdp.meta.defaults, + sdp.meta.rgx; + struct HeaderDocMetadataAndMakeNativeToAA { + mixin SiSUregisters; + mixin SiSUrgxInitFlags; + mixin SiSUrgxInit; + auto rgx = Rgx(); + enum State { off, on } + string hm, hs; + <> + } +} +#+END_SRC + +** native header document metadata in associative array :aa: + +#+name: meta_markup_header_extract_native +#+BEGIN_SRC d +auto header_metadata_and_make_aa(H,Me,Ma)( + H header, + Me dochead_meta, + Ma dochead_make +) +in { + debug(asserts){ + static assert(is(typeof(header) == string)); + static assert(is(typeof(dochead_meta) == string[string][string])); + static assert(is(typeof(dochead_make) == string[string][string])); + } +} +body { + scope(exit) { + destroy(header); + destroy(dochead_meta); + destroy(dochead_make); + } + if (auto t = header.match(rgx.native_header_main)) { + char[][] header_obj_spl = + (cast(char[]) header).split(rgx.line_delimiter_ws_strip); + auto hm = to!string(t.captures[1]); + if (hm.match(rgx.main_headers)) { + foreach (line; header_obj_spl) { + if (auto m = line.match(rgx.native_header_main)) { + if (!empty(m.captures[2])) { + if (hm == "creator") { + dochead_meta[hm]["author"] = + to!string(m.captures[2]); + } else if (hm == "title") { + dochead_meta[hm]["main"] = + to!string(m.captures[2]); + } else if (hm == "publisher") { + dochead_meta[hm]["name"] = + to!string(m.captures[2]); + } + } + } else if (auto s = match(line, rgx.native_header_sub)) { + if (!empty(s.captures[2])) { + auto hs = to!string(s.captures[1]); + if ((hm == "make" ) + && (dochead_make[hm])) { + switch (hm) { + case "make": + if (hs.match(rgx.native_subhead_make)) { + if (dochead_make[hm][hs]) { + dochead_make[hm][hs] = to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + default: + break; + } + } else if (dochead_meta[hm]) { + switch (hm) { + case "creator": + if (hs.match(rgx.native_subhead_creator)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "title": + if (hs.match(rgx.native_subhead_title)) { + if ((hs == "subtitle") + && (dochead_meta[hm]["sub"])) { + dochead_meta[hm]["sub"] = + to!string(s.captures[2]); + } else if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "rights": + if (hs.match(rgx.native_subhead_rights)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "date": + if (hs.match(rgx.native_subhead_date)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "original": + if (hs.match(rgx.native_subhead_original)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "classify": + if (hs.match(rgx.native_subhead_classify)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "identifier": + if (hs.match(rgx.native_subhead_identifier)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "notes": + if (hs.match(rgx.native_subhead_notes)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "publisher": + if (hs.match(rgx.native_subhead_publisher)) { + if (dochead_meta[hm][hs]) { + dochead_meta[hm][hs] = + to!string(s.captures[2]); + } + } else { + writeln("not a valid header type:", hm, ":", hs); + destroy(hm); + destroy(hs); + } + break; + case "links": + destroy(hm); + destroy(hs); + break; + default: + break; + } + } + } + } + } + } else { + writeln("not a valid header type:", hm); + } + } + auto t = tuple(dochead_meta, dochead_make); + static assert(t.length==2); + return t; +} +#+END_SRC + +** native header extract to string object :string: + +#+name: meta_markup_header_extract_native +#+BEGIN_SRC d +private auto native_header_extract(L,Lo,O,T)( + L line, + return ref Lo line_occur, + return ref O an_object, + return ref T type +) { + debug(asserts){ + static assert(is(typeof(line) == char[])); + static assert(is(typeof(line_occur) == int[string])); + static assert(is(typeof(an_object) == string[string])); + static assert(is(typeof(type) == int[string])); + } + if (line.matchFirst(rgx.native_header_make)) { /+ matched header_make +/ + debug(header1) { /+ writeln(line); +/ } + type["header"] = State.on; + type["header_make"] = State.on; + type["header_meta"] = State.off; + ++line_occur["header_make"]; + an_object["body_nugget"] ~= line ~= "\n"; + } else if (line.matchFirst(rgx.native_header)) { /+ matched header_metadata +/ + /+ (generic header match and not previously caught by header_make) +/ + debug(header1) { /+ writeln(line); +/ } + type["header"] = State.on; + type["header_make"] = State.off; + type["header_meta"] = State.on; + ++line_occur["header_meta"]; + an_object["body_nugget"] ~= line ~= "\n"; + } else if (type["header_make"] == State.on + && (line_occur["header_make"] > State.off)) { /+ header_make flag set +/ + if (line.matchFirst(rgx.native_header_sub)) { /+ sub-header +/ + debug(header1) { /+ writeln(line); +/ } + ++line_occur["header_make"]; + an_object["body_nugget"] ~= line ~= "\n"; + } + } else if (type["header_meta"] == State.on + && (line_occur["header_meta"] > State.off)) { /+ header_metadata flag set +/ + if (line.matchFirst(rgx.native_header_sub)) { /+ sub-header +/ + debug(header1) { /+ writeln(line); +/ } + ++line_occur["header_meta"]; + an_object["body_nugget"] ~= line ~= "\n"; + } + } + return an_object; +} +#+END_SRC + +** native header reset states :reset: + +#+name: meta_markup_header_extract_native +#+BEGIN_SRC d +auto header_reset_states_common(Lo,O,T)( + return ref Lo line_occur, + return ref O an_object, + return ref T type +) { + debug(asserts){ + static assert(is(typeof(line_occur) == int[string])); + static assert(is(typeof(an_object) == string[string])); + static assert(is(typeof(type) == int[string])); + } + line_occur["header_make"] = State.off; + line_occur["header_meta"] = State.off; + type["header"] = State.off; + an_object.remove("body_nugget"); + an_object.remove("is"); + an_object.remove("attrib"); +} +#+END_SRC + +** hub: native header start :hub: + +#+name: meta_markup_header_extract_native +#+BEGIN_SRC d +private auto headerNativeToAA(Hn)(Hn src_header) { + debug(asserts){ + static assert(is(typeof(src_header) == char[])); + } + auto type = flags_type_init; + type = [ + "header" : State.off, + "header_make" : State.off, + "header_meta" : State.off, + ]; + string[string] an_object; + int[string] line_occur; + auto dochead_make = conf_aa_empty; + auto dochead_meta = meta_aa_empty; + auto set_header = HeaderDocMetadataAndMakeNativeToAA(); + char[][] source_header_arr = + (cast(char[]) src_header).split(rgx.newline_eol_delimiter); + foreach(header_line; source_header_arr) { + if (auto m = header_line.matchFirst(rgx.comment)) { + /+ matched comment +/ + debug(comment) { + } + header_reset_states_common(line_occur, an_object, type); + } else if ((header_line.matchFirst(rgx.native_header)) + || (type["header_make"] == State.on + && (line_occur["header_make"] > State.off)) + || (type["header_meta"] == State.on + && (line_occur["header_meta"] > State.off))) { + if (header_line.length == 0) { + /+ header_make instructions (current line empty) +/ + auto dochead_metadata_and_make = + set_header.header_metadata_and_make_aa(strip(an_object["body_nugget"]), dochead_meta, dochead_make); + static assert(!isTypeTuple!(dochead_metadata_and_make)); + dochead_meta = dochead_metadata_and_make[0]; + dochead_make = dochead_metadata_and_make[1]; + header_reset_states_common(line_occur, an_object, type); + type["header_make"] = State.off; + type["header_meta"] = State.off; + debug(headersdlang) { + writeln(dochead_metadata_and_make); + } + } else { + an_object = native_header_extract(header_line, line_occur, an_object, type); + } + } + } + auto t = tuple( + dochead_make, + dochead_meta, + ); + return t; +} +#+END_SRC + +* 0. composite make :module:sdp:meta_composite_make: +** TODO 0. template +*** composite make + +#+BEGIN_SRC d :tangle ../src/sdp/meta/composite_make.d +/++ + output hub
+ check & generate output types requested ++/ +module sdp.meta.composite_make; +template compositeMkCnf() { + <> + mixin SiSUrgxInit; + string[] _substitutions; + string[string][] _sub; + string _bold; + string _italics; + string _emphasis; + auto compositeMkCnf(Mks...)(Mks makes) { + foreach (make; makes) { + auto rgx = Rgx(); + if (auto z = "make" in make) { + if (auto x = "substitute" in *z) { + foreach (m; (*x).matchAll(rgx.make_simple_substitutions_d)) { + _sub ~= ["match": (m["match"]), "replace": (m["replace"])]; + _substitutions ~= format("`%s`,\"%s\"", + m["match"], + m["replace"], + ); + } + foreach (m; (*x).matchAll(rgx.make_simple_substitutions_rb)) { + _sub ~= ["match": (m["match"]), "replace": (m["replace"])]; + _substitutions ~= format("`%s`,\"%s\"", + m["match"], + m["replace"], + ); + } + } + if (auto x = "bold" in *z) { + _bold = (*x).to!string; + } + if (auto x = "italics" in *z) { + _italics = (*x).to!string; + } + if (auto x = "emphasis" in *z) { + _emphasis = (*x).to!string; + } + } + } + struct _composite_make { + auto substitutions() { + auto _k = _sub; + return _k; + } + auto substitute() { + auto _k = _substitutions; + return _k; + } + auto italics() { + auto _k = _italics; + return _k; + } + auto bold() { + auto _k = _bold; + return _k; + } + auto emphasis() { + auto _k = _emphasis; + return _k; + } + } + return _composite_make(); + } +} +#+END_SRC + +*** composite make aa +#+BEGIN_SRC d :tangle ../src/sdp/meta/composite_make.d +/++ + output hub
+ check & generate output types requested ++/ +template compositeMkCnfAA() { + <> + mixin SiSUrgxInit; + string[] _substitutions; + string[string][] _sub; + auto rgx = Rgx(); + auto compositeMkCnfAA(Mks...)(Mks makes) { + /+ + - skip.first_make which is the "composite make output" + - pass config files as associative arrays, + - skip.last_make which is getopt, treat separately + +/ + auto _composite_make = makes[0]; + auto _opts = makes[$-1]; + foreach (make; makes[1..$-1]) { + if (auto z = "make" in make) { // document head: make (part of individual document) + if (auto x = "substitute" in *z) { + _composite_make["make"]["substitute"] = *x; + } + if (auto x = "italics" in *z) { + _composite_make["make"]["italics"] = *x; + } + if (auto x = "bold" in *z) { + _composite_make["make"]["bold"] = *x; + } + if (auto x = "emphasis" in *z) { + _composite_make["make"]["emphasis"] = *x; + } + } + } + /+ logic for adding _opts make instructions to _composite make +/ + return _composite_make; + } +} +#+END_SRC + +** initialize / imports + +#+name: imports +#+BEGIN_SRC d +import sdp.meta; +import std.array; +#+END_SRC -- cgit v1.2.3