From c6a479b9bd8bdd34ee9699e10247ebdeb183e032 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 11 Sep 2026 21:45:24 -0400 Subject: org renames --- org/in_abstraction_artefacts.org | 1476 ++++++++++++++++++++++++++ org/in_markup_source_files.org | 801 +++++++++++++++ org/in_source_artefact.org | 392 ------- org/in_source_files.org | 1656 ------------------------------ org/ocda_get.org | 284 ----- org/out_ocda_peg_ssp.org | 736 +++++++++++++ org/out_ocda_sqlite_db.org | 666 ++++++++++++ org/out_src_abstraction_ocda_peg_ssp.org | 736 ------------- org/out_src_abstraction_sqlite_db.org | 666 ------------ 9 files changed, 3679 insertions(+), 3734 deletions(-) create mode 100644 org/in_abstraction_artefacts.org create mode 100644 org/in_markup_source_files.org delete mode 100644 org/in_source_artefact.org delete mode 100644 org/in_source_files.org delete mode 100644 org/ocda_get.org create mode 100644 org/out_ocda_peg_ssp.org create mode 100644 org/out_ocda_sqlite_db.org delete mode 100644 org/out_src_abstraction_ocda_peg_ssp.org delete mode 100644 org/out_src_abstraction_sqlite_db.org (limited to 'org') diff --git a/org/in_abstraction_artefacts.org b/org/in_abstraction_artefacts.org new file mode 100644 index 0000000..d8ce545 --- /dev/null +++ b/org/in_abstraction_artefacts.org @@ -0,0 +1,1476 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) markup source raw +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:sourcefile:read: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] + +* A. load abstraction - sisudoc.ocda.abstraction.load + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.ocda.abstraction.load; +@safe: +/+ ↓ one way in, whatever the document is being read from + spine's pipeline is markup -> abstraction -> output. once the abstraction + is serialised, there is more than one thing an abstraction can be read + from, and a consumer should not have to know which it was handed: + . + .sst / .ssm + images the markup source + pod (dir) + images the same, bundled + pod .zip the same, zipped + .ssp + images the abstraction, as text + .ocda.db the abstraction, sqlite, images inside + . + A source may also be remote. A URL ending in .zip or in .ocda.db is + downloaded to a temp file (downloadSourceUrl in + ocda/io_in/read_zip_pod.d, guarded by rgx_url_source and by + --allow-downloads) and the local path is processed in its place. + + Those two travel because each is self-sufficient: a pod holds its + source and images, a .ocda.db holds its abstraction and images. A .ssp + does not, describing its images by name and digest without carrying + them, so it would arrive without them and is not fetched. + . + This module names those sources, tells them apart, and loads the two + that are self-describing artefacts, returning the value the parser + produces. + . + The three source forms are deliberately *not* loaded here. They are the + parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs + the environment, the options and the configuration that spine.d + assembles, none of which belongs in a loader. What this module gives + that case is the dispatch and a plain statement of where to go. ++/ +template spineAbstractionLoad() { + import std.algorithm : endsWith; + import std.conv : to; + import std.file; + import std.path; + import std.stdio; + import std.string; + import sisudoc.ocda.abstraction.db_in; + mixin spineAbstractionDbRead; // brings the .ssp reader with it + + enum AbstractionSource { + unknown, + markup, // .sst or .ssm, with its images beside it + pod, // a directory holding pod.manifest + pod_zip, // that directory, zipped + ssp, // .ssp, the abstraction as text, images beside it + ocda_db, // .ocda.db, the abstraction as sqlite, images inside it + } + + string abstractionSourceName(AbstractionSource _s) { + final switch (_s) { + case AbstractionSource.unknown: return "unknown"; + case AbstractionSource.markup: return "markup source (.sst/.ssm)"; + case AbstractionSource.pod: return "pod directory"; + case AbstractionSource.pod_zip: return "pod zip"; + case AbstractionSource.ssp: return ".ssp (abstraction as text)"; + case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)"; + } + } + + /+ ↓ what is this? by name, and for a directory by what it holds +/ + AbstractionSource abstractionSourceOf(string _path) { + if (_path.length == 0) { return AbstractionSource.unknown; } + if (_path.isValidPath && _path.exists && _path.isDir) { + return (_path.chainPath("pod.manifest").array.exists) + ? AbstractionSource.pod : AbstractionSource.unknown; + } + if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } + if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } + if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } + if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } + if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } + return AbstractionSource.unknown; + } + + struct LoadedAbstraction { + AbstractionSource source; + string path; + bool loaded; // is .doc filled + string note; // why not, when it is not + SSPdocument doc; + } + + /+ ↓ load what can be loaded from the path alone. + + .ssp and .ocda.db come back filled. the three source forms come back + with loaded = false and a note saying where they are handled, because + reading them needs the manifest, environment and configuration that + spine.d builds, not a file path. + +/ + LoadedAbstraction abstractionLoad(string _path) { + LoadedAbstraction _out; + _out.path = _path; + _out.source = abstractionSourceOf(_path); + if (!_path.exists) { + _out.note = "no such file or directory"; + return _out; + } + final switch (_out.source) { + case AbstractionSource.ssp: + _out.doc = sspReadFile(_path); + _out.loaded = (_out.doc.section_order.length > 0); + if (!_out.loaded) { _out.note = "no object sections found"; } + break; + case AbstractionSource.ocda_db: + _out.doc = dbReadFile(_path); + _out.loaded = (_out.doc.section_order.length > 0); + if (!_out.loaded) { _out.note = "no object sections found"; } + break; + case AbstractionSource.markup: + case AbstractionSource.pod: + case AbstractionSource.pod_zip: + _out.note = "a source form: read by the parser," + ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the" + ~ " manifest, environment and configuration spine assembles"; + break; + case AbstractionSource.unknown: + _out.note = "not a document source spine knows"; + break; + } + return _out; + } + + /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a + given artefact really does hold what it should +/ + string[] abstractionLoadSummary(LoadedAbstraction _l) { + string[] _out; + _out ~= "source: " ~ abstractionSourceName(_l.source); + _out ~= "path: " ~ _l.path; + if (!_l.loaded) { + _out ~= "loaded: no"; + if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; } + return _out; + } + _out ~= "loaded: yes"; + if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; } + if ("title.main" in _l.doc.meta) { + _out ~= "title: " ~ _l.doc.meta["title.main"]; + } + if ("creator.author" in _l.doc.meta) { + _out ~= "author: " ~ _l.doc.meta["creator.author"]; + } + _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, " + ~ _l.doc.make.length.to!string ~ " make, " + ~ _l.doc.doc_has.length.to!string ~ " doc_has"; + int _objs, _citable, _headings; + string[] _sections; + foreach (section; _l.doc.section_order) { + int _n; + foreach (obj; _l.doc.abstraction[section]) { + ++_objs; ++_n; + if (obj.metainfo.ocn > 0) { ++_citable; } + if (obj.metainfo.is_a == "heading") { ++_headings; } + } + _sections ~= section ~ " " ~ _n.to!string; + } + _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string + ~ " citable, " ~ _headings.to!string ~ " headings)"; + _out ~= "sections: " ~ _sections.join(", "); + return _out; + } +} +#+END_SRC + +* B. read a .ssp file back into the document abstraction - sisudoc.ocda.abstraction.ssp_in + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp_in.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.ocda.abstraction.ssp_in; +@safe: +/+ ↓ read a .ssp file back into the document abstraction + . + the reverse of sisudoc.ocda.abstraction.ssp. what it returns is the same + value the parser produces, ObjGenericComposite[][string], so anything + that consumes the abstraction can be fed from a .ssp instead of from + markup. + . + the check that it is faithful is a round trip: load a .ssp, emit each + object again with sspObjectRecord (the writer's own definition of a + record) and require the result to be byte identical to the file read. ++/ +template spineAbstractionRead() { + import std.algorithm : startsWith, findSplit; + import std.array; + import std.conv : to; + import std.file; + import std.json : JSONValue; + import std.stdio; + import std.string; + import sisudoc.ocda.meta.metadoc_object_setter; + import sisudoc.ocda.abstraction.ssp : sspEscape; + mixin sspEscape; + mixin ObjectSetter; + /+ ↓ what a .ssp file holds: the three header blocks as key/value, and the + object sections in the order the file gives them + +/ + struct SSPdocument { + string format; // "% SiSU Document Abstraction v1.1" + string source; // "% Source: ..." + string[string] source_info; // the @source block + string[] source_info_order; + string ssp_digest; // sha256 of the .ssp read, hex + string[string] meta; + string[] meta_order; + string[string] make; + string[] make_order; + string[string] doc_has; + string[] doc_has_order; + string[] section_order; // as found in the file + ObjGenericComposite[][string] abstraction; + } + private int _levOfMarkedUp(string lev) { + switch (lev) { + case "A": return 0; + case "B": return 1; + case "C": return 2; + case "D": return 3; + case "1": return 4; + case "2": return 5; + case "3": return 6; + case "4": return 7; + default: return 9; + } + } + private int[] _ints(string val) { + int[] _out; + foreach (f; val.split(" ")) { + if (f.length > 0) { _out ~= f.to!int; } + } + return _out; + } + /+ ↓ the eight slot arrays are int[] with a literal default, so every + default constructed object shares one array; write into a fresh one + +/ + private int[8] _eight(string val) { + int[8] _out; + foreach (i, v; _ints(val)) { + if (i < _out.length) { _out[i] = v; } + } + return _out; + } + private ubyte[32] _hexToBytes(string hex) { + ubyte[32] _out; + if (hex.length < 64) { return _out; } + foreach (i; 0..32) { + _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); + } + return _out; + } + SSPdocument sspRead(string[] lines) { + SSPdocument doc; + string section; // the @section block currently open + string block; // "meta", "make", "doc_has", or "" for objects + bool in_object; + string[] text_lines; + ObjGenericComposite obj; + void _closeObject() { + if (!in_object) { return; } + obj.text = text_lines.join("\n"); + doc.abstraction[section] ~= obj; + obj = ObjGenericComposite(); + text_lines = []; + in_object = false; + } + foreach (line; lines) { + if (line.length == 0) { continue; } + if (line.startsWith("% SiSU Document Abstraction")) { + doc.format = line; continue; + } + if (line.startsWith("% Source: ")) { + doc.source = line["% Source: ".length .. $]; continue; + } + if (line.startsWith("@") && line.endsWith(" {")) { + _closeObject(); + string _name = line[1 .. $-2]; + switch (_name) { + case "source": case "meta": case "make": case "doc_has": + block = _name; section = ""; break; + default: + block = ""; section = _name; + doc.section_order ~= _name; + doc.abstraction[_name] = []; + break; + } + continue; + } + if (line == "}") { _closeObject(); block = ""; section = ""; continue; } + if (block.length > 0) { // a header block key: value line + auto _kv = line.stripLeft.findSplit(": "); // leading only: a value may end in a space + if (_kv[1].length > 0) { + final switch (block) { + case "source": + doc.source_info[_kv[0].to!string] = _kv[2].to!string; + doc.source_info_order ~= _kv[0].to!string; + break; + case "meta": + doc.meta[_kv[0].to!string] = _kv[2].to!string; + doc.meta_order ~= _kv[0].to!string; + break; + case "make": + doc.make[_kv[0].to!string] = _kv[2].to!string; + doc.make_order ~= _kv[0].to!string; + break; + case "doc_has": + doc.doc_has[_kv[0].to!string] = _kv[2].to!string; + doc.doc_has_order ~= _kv[0].to!string; + break; + } + } + continue; + } + if (line.startsWith("[")) { // object declaration line + _closeObject(); + in_object = true; + auto _close = line.indexOf("] "); + obj.metainfo.ocn = line[1 .. _close].to!int; + string _rest = line[_close + 2 .. $]; + if (_rest.startsWith("heading :")) { + obj.metainfo.is_a = "heading"; + string _tail = _rest["heading :".length .. $]; + auto _sp = _tail.indexOf(" "); + if (_sp > 0) { + obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail[0 .. _sp]); + obj.metainfo.identifier = _tail[_sp + 1 .. $]; + } else { + obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail); + /+ ↓ omitted means it equals the ocn, and for an object with no ocn + it was empty: "a"~N identifiers are always written + +/ + obj.metainfo.identifier = (obj.metainfo.ocn != 0) + ? obj.metainfo.ocn.to!string : ""; + } + } else { + obj.metainfo.is_a = _rest; + obj.metainfo.identifier = (obj.metainfo.ocn != 0) + ? obj.metainfo.ocn.to!string : ""; + } + obj.metainfo.is_of_section = section; + continue; + } + if (line.startsWith("| ")) { text_lines ~= line[2 .. $]; continue; } + if (!line.startsWith(".")) { continue; } + auto _p = line[1 .. $].findSplit(": "); + string _key = (_p[1].length > 0) ? _p[0].to!string : line[1 .. $].to!string; + string _val = (_p[1].length > 0) ? _p[2].to!string : ""; + switch (_key) { + case "identifier": obj.metainfo.identifier = _val; break; + case "part": obj.metainfo.is_of_part = _val; break; + case "section": obj.metainfo.is_of_section = _val; break; + case "parent": obj.metainfo.parent_ocn = _val.to!int; break; + case "last_descendant": obj.metainfo.last_descendant_ocn = _val.to!int; break; + case "children": obj.metainfo.children_headings = _ints(_val); break; + case "ancestors": + obj.metainfo.markedup_ancestors = _eight(_val); + break; + case "ancestors_collapsed": + obj.metainfo.collapsed_ancestors = _eight(_val); + break; + case "dom_status": + obj.metainfo.dom_structure_markedup_tags_status = _eight(_val); + break; + case "dom_status_collapsed": + obj.metainfo.dom_structure_collapsed_tags_status = _eight(_val); + break; + case "heading_lev_collapsed": obj.metainfo.heading_lev_collapsed = _val.to!int; break; + case "parent_lev": obj.metainfo.parent_lev_markup = _val.to!int; break; + case "dummy": obj.metainfo.dummy_heading = true; break; + case "ocn_off": obj.metainfo.object_number_off = true; break; + case "is_of_type": obj.metainfo.is_of_type = _val; break; + case "attrib": obj.metainfo.attrib = _val; break; + case "meta_lang": obj.metainfo.lang = _val; break; + case "meta_syntax": obj.metainfo.syntax = _val; break; + case "sha256": obj.metainfo.sha256.text = _hexToBytes(_val); break; + case "image": + { + ST_file_name_hash_size_ _img; + auto _fields = _val.split(" "); + if (_fields.length > 0) { _img.fileName = _fields[0].to!string; } + foreach (f; _fields[1 .. $]) { + if (f == "missing:true") { _img.fileMissing = true; } + else if (f.startsWith("sha256:")) { _img.fileHash_sha256 = _hexToBytes(f[7 .. $].to!string); } + else if (f.startsWith("bytes:")) { _img.fileSize = f[6 .. $].to!ulong; } + else if (f.startsWith("px:")) { + auto _wh = f[3 .. $].findSplit("x"); + if (_wh[1].length > 0) { + _img.imageWidth = _wh[0].to!int; + _img.imageHeight = _wh[2].to!int; + } + } + } + obj.metainfo.sha256.images ~= _img; + } + break; + case "indent": + { + auto _i = _ints(_val); + if (_i.length > 0) { obj.attrib.indent_base = _i[0]; } + if (_i.length > 1) { obj.attrib.indent_hang = _i[1]; } + } + break; + case "bullet": obj.attrib.bullet = true; break; + case "lang": obj.attrib.language = _val; break; + case "has": + foreach (f; _val.split(" ")) { + switch (f) { + case "links": obj.has.inline_links = true; break; + case "notes_reg": obj.has.inline_notes_reg = true; break; + case "notes_star": obj.has.inline_notes_star = true; break; + case "images": obj.has.images = true; break; + case "images_no_dim": obj.has.image_without_dimensions = true; break; + default: break; + } + } + break; + case "table_cols": obj.table.number_of_columns = _val.to!int; break; + case "table_widths": + foreach (w; _val.split(" ")) { obj.table.column_widths ~= w.to!double; } + break; + case "table_aligns": + foreach (a; _val.split(" ")) { obj.table.column_aligns ~= a.to!string; } + break; + case "table_header": obj.table.heading = true; break; + case "code_linenumbers": obj.code_block.linenumbers = true; break; + case "stow_link": obj.stow.link ~= _val; break; + case "segment": obj.tags.in_segment_html = _val; break; + case "anchor": obj.tags.anchor_tag_html = _val; break; + case "segment_prev": obj.tags.segname_prev = _val; break; + case "segment_next": obj.tags.segname_next = _val; break; + case "heading_lev_anchor": obj.tags.heading_lev_anchor_tag = _val; break; + case "segment_epub": obj.tags.segment_anchor_tag_epub = _val; break; + case "segment_html_is": obj.tags.html_segment_anchor_tag_is = _val; break; + case "segment_epub_is": obj.tags.epub_segment_anchor_tag_is = _val; break; + case "segment_lv4_is": obj.tags.segment_lv4_is = _val; break; + case "heading_ancestors_text": + { + string[8] _h; + foreach (i, h; sspUnescapeSplit(_val)) { + if (i < _h.length) { _h[i] = h; } + } + obj.tags.heading_ancestors_text = _h; + } + break; + case "lev4_subtoc": obj.tags.lev4_subtoc ~= _val; break; + case "anchor_tag": obj.tags.anchor_tags ~= _val; break; + default: break; + } + } + _closeObject(); + /+ ↓ .anchor is written only when it differs from .segment, and .section + only when it differs from the enclosing block; restore both + +/ + foreach (sect; doc.section_order) { + foreach (ref o; doc.abstraction[sect]) { + if (o.tags.anchor_tag_html.length == 0 + && o.tags.in_segment_html.length > 0) { + o.tags.anchor_tag_html = o.tags.in_segment_html; + } + } + } + return doc; + } + /+ ↓ the whole document as it survives a trip through the .ssp text: + emitted as lines by the writer, read straight back. what a consumer + built from this cannot see, the .ssp does not carry, which is what + makes a second artefact built from it unable to drift. the header + blocks come back with it, so the database is filled from these and + not from a second list read off doc_matters + +/ + SSPdocument sspRoundTripDocument(D)(D doc) { + import sisudoc.ocda.abstraction.ssp; + mixin spineAbstractionTxt; + string[] _lines = sspDocumentLines(doc); + auto _out = sspRead(_lines); + /+ ↓ the writer emits one line per entry, each with its newline, so this + is the digest of the file that is written beside the pod +/ + _out.ssp_digest = sspDigestOfText(_lines.join("\n") ~ "\n"); + return _out; + } + /+ ↓ sha256 of a .ssp, upper case hex, as digests.txt gives it +/ + string sspDigestOfText(string _text) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + return _text.sha256Of.toHexString.to!string; + } + auto sspRoundTripAbstraction(D)(D doc) { + return sspRoundTripDocument(doc).abstraction; + } + /+ ↓ the version a reader will accept. + + the major part must match: a format whose properties have changed + meaning is not readable by guesswork, and loading it half populated in + silence is the failure this check exists to stop. a newer minor + version is accepted, since a minor bump only adds properties and an + unknown property line is ignored. + +/ + bool sspFormatVersionOk(string _format_line, string _what) { + import sisudoc.ocda.abstraction.ssp : ssp_format_version, ssp_format_version_major; + enum _prefix = "% SiSU Document Abstraction v"; + if (_format_line.length == 0) { + writeln("ERROR: ", _what, ": no format line, not a .ssp"); + return false; + } + if (!_format_line.startsWith(_prefix)) { + writeln("ERROR: ", _what, ": unrecognised format line: ", _format_line); + return false; + } + string _ver = _format_line[_prefix.length .. $].strip; + string _major = _ver.findSplit(".")[0]; + if (_major != ssp_format_version_major) { + writeln("ERROR: ", _what, ": .ssp format v", _ver, + ", this spine reads v", ssp_format_version_major, ".x (v", + ssp_format_version, "). Regenerate the abstraction from its markup."); + return false; + } + return true; + } + /+ ↓ with_digest is off by default: hashing the file is only wanted where + the digest is going to be recorded, and on the largest document in the + sample set it is a tenth of what reading the file costs + +/ + @trusted SSPdocument sspReadFile(string file_path, bool with_digest = false) { + string _text; + string[] _lines; + try { + _text = (cast(char[]) file_path.read).to!string; + _lines = _text.split("\n"); + } catch (Exception ex) { + writeln("ERROR: could not read ", file_path, ": ", ex.msg); + return SSPdocument(); + } + auto _doc = sspRead(_lines); + if (!sspFormatVersionOk(_doc.format, file_path)) { return SSPdocument(); } + if (with_digest) { _doc.ssp_digest = sspDigestOfText(_text); } + return _doc; + } +} +#+END_SRC + +* C. read a .db file into the document abstraction - sisudoc.ocda.abstraction.db_in + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/db_in.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.ocda.abstraction.db_in; +@safe: +/+ ↓ read a .ocda.db back into the document abstraction + + the counterpart of sisudoc.outputs.io_out.create_ocda, and the + sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, + so a consumer does not have to know which of the two artefacts it was + handed. + + the check is the same one: read a database, emit it as .ssp with the + writer's own record definition, and require the result to equal the .ssp + written from the same document. ++/ +template spineAbstractionDbRead() { + import std.conv : to; + import std.file; + import std.stdio; + import std.string; + import std.array; + import std.json; + import d2sqlite3; + import sisudoc.ocda.abstraction.ssp_in : spineAbstractionRead; + mixin spineAbstractionRead; // for SSPdocument, and the shared unescape + @trusted private int[8] _jsonEight(string _s) { + int[8] _out; + if (_s.length == 0) { return _out; } + try { + auto _j = parseJSON(_s); + foreach (i, v; _j.array) { + if (i < _out.length) { _out[i] = v.integer.to!int; } + } + } catch (Exception ex) {} + return _out; + } + @trusted private int[] _jsonInts(string _s) { + int[] _out; + if (_s.length == 0) { return _out; } + try { + auto _j = parseJSON(_s); + foreach (v; _j.array) { _out ~= v.integer.to!int; } + } catch (Exception ex) {} + return _out; + } + @trusted private string[] _jsonStrs(string _s) { + string[] _out; + if (_s.length == 0) { return _out; } + try { + auto _j = parseJSON(_s); + foreach (v; _j.array) { _out ~= v.str; } + } catch (Exception ex) {} + return _out; + } + @trusted private double[] _jsonDoubles(string _s) { + double[] _out; + if (_s.length == 0) { return _out; } + try { + auto _j = parseJSON(_s); + foreach (v; _j.array) { + _out ~= (v.type == JSONType.integer) ? v.integer.to!double : v.floating; + } + } catch (Exception ex) {} + return _out; + } + private ubyte[32] _hex32(string hex) { + ubyte[32] _out; + if (hex.length < 64) { return _out; } + foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } + return _out; + } + private string _levMarkedUp(int lev) { + switch (lev) { + case 0: return "A"; + case 1: return "B"; + case 2: return "C"; + case 3: return "D"; + case 4: return "1"; + case 5: return "2"; + case 6: return "3"; + case 7: return "4"; + default: return ""; + } + } + @trusted SSPdocument dbReadFile(string db_file) { + SSPdocument doc; + if (!db_file.exists) { + writeln("ERROR: no such file: ", db_file); + return doc; + } + auto db = Database(db_file, SQLITE_OPEN_READONLY); + /+ ↓ the four header blocks, from the metadata table. the key prefix + says which block a row belongs to, and insertion order is kept +/ + foreach (row; db.execute("SELECT key, value FROM metadata ORDER BY rowid")) { + string _k = row["key"].as!string; + string _v = row["value"].as!string; + if (_k.startsWith("make.")) { + string _kk = _k["make.".length .. $]; + doc.make[_kk] = _v; + doc.make_order ~= _kk; + } else if (_k.startsWith("doc_has.")) { + string _kk = _k["doc_has.".length .. $]; + doc.doc_has[_kk] = _v; + doc.doc_has_order ~= _kk; + } else if (_k.startsWith("schema.")) { + if (_k == "schema.version") { + doc.format = "% SiSU Document Abstraction v" ~ _v; + } + } else if (_k == "source.filename") { + doc.source = _v; + } else if (_k == "source.ssp_digest") { + /+ ↓ this database's own note of the .ssp it was built from. it is + not part of the .ssp, so it does not go back into the block +/ + doc.ssp_digest = _v; + } else if (_k.startsWith("source.")) { + string _kk = _k["source.".length .. $]; + doc.source_info[_kk] = _v; + doc.source_info_order ~= _kk; + } else { + doc.meta[_k] = _v; + doc.meta_order ~= _k; + } + } + if (!sspFormatVersionOk(doc.format, db_file)) { return SSPdocument(); } + /+ ↓ the per object lists, gathered first, one sweep of each table. + + these four tables hold what a single object row cannot: the images + an object carries, the links stowed off it, its anchor tags and its + lev4 subtoc entries. They are read here in four ordered passes and + kept by object id, rather than queried per object as the objects are + built. The cost is then what the tables hold, not what the document + holds: war and peace has 12,135 objects and 138 rows across all + four, which as four statements per object was 48,540 statement + preparations and made reading a .ocda.db slower than parsing the + markup it came from. + +/ + ST_file_name_hash_size_[][long] _images_by_id; + string[][long] _links_by_id; + string[][long] _anchors_by_id; + string[][long] _subtoc_by_id; + foreach (r; db.execute( + "SELECT object_id, name, bytes, sha256, width, height, missing" + ~ " FROM object_images ORDER BY object_id, seq") + ) { + ST_file_name_hash_size_ _img; + _img.fileName = r["name"].as!string; + _img.fileSize = r["bytes"].as!ulong; + _img.fileHash_sha256 = _hex32(r["sha256"].as!string); + _img.imageWidth = r["width"].as!int; + _img.imageHeight = r["height"].as!int; + _img.fileMissing = (r["missing"].as!int == 1); + _images_by_id[r["object_id"].as!long] ~= _img; + } + foreach (r; db.execute( + "SELECT object_id, url FROM object_links ORDER BY object_id, seq") + ) { _links_by_id[r["object_id"].as!long] ~= r["url"].as!string; } + foreach (r; db.execute( + "SELECT object_id, anchor FROM object_anchors ORDER BY object_id, seq") + ) { _anchors_by_id[r["object_id"].as!long] ~= r["anchor"].as!string; } + foreach (r; db.execute( + "SELECT object_id, entry FROM object_subtoc ORDER BY object_id, seq") + ) { _subtoc_by_id[r["object_id"].as!long] ~= r["entry"].as!string; } + /+ ↓ the objects, section by section, in the order they were written +/ + string[] _sections; + foreach (row; db.execute( + "SELECT section FROM objects GROUP BY section ORDER BY MIN(id)") + ) { + _sections ~= row["section"].as!string; + } + foreach (section; _sections) { + doc.section_order ~= section; + doc.abstraction[section] = []; + /+ ↓ column name to index, resolved once for the statement. + + d2sqlite3's row["name"] is indexForName, a linear scan over the + columns that calls sqlite3_column_name and allocates a D string + for each one it passes. With some fifty named reads per object + over forty columns that is around a thousand of those per object, + and it was the larger half of what made reading a database slow. + Resolved once here, the reads below are an integer index. + +/ + int[string] _col; + foreach (row; db.execute( + "SELECT * FROM objects WHERE section = '" ~ section ~ "' ORDER BY seq") + ) { + if (_col.length == 0) { + foreach (_i; 0 .. row.length) { _col[row.columnName(_i)] = _i.to!int; } + } + ObjGenericComposite obj; + long _id = row[_col["id"]].as!long; + obj.metainfo.ocn = row[_col["ocn"]].as!int; + obj.metainfo.is_a = row[_col["is_a"]].as!string; + obj.metainfo.is_of_part = row[_col["is_of_part"]].as!string; + obj.metainfo.is_of_section = (row[_col["is_of_section"]].as!string.length > 0) + ? row[_col["is_of_section"]].as!string : section; + obj.metainfo.is_of_type = row[_col["is_of_type"]].as!string; + obj.metainfo.identifier = row[_col["identifier"]].as!string; + obj.metainfo.heading_lev_markup = (obj.metainfo.is_a == "heading") + ? row[_col["heading_level"]].as!int : 9; + obj.metainfo.heading_lev_collapsed = (row[_col["heading_lev_collapsed"]].as!string.length > 0) + ? row[_col["heading_lev_collapsed"]].as!int : 9; + obj.metainfo.parent_ocn = row[_col["parent_ocn"]].as!int; + obj.metainfo.parent_lev_markup = row[_col["parent_lev"]].as!int; + obj.metainfo.last_descendant_ocn = row[_col["last_descendant_ocn"]].as!int; + obj.metainfo.children_headings = _jsonInts(row[_col["children"]].as!string); + obj.metainfo.markedup_ancestors = _jsonEight(row[_col["ancestors"]].as!string); + obj.metainfo.collapsed_ancestors = _jsonEight(row[_col["ancestors_collapsed"]].as!string); + obj.metainfo.dom_structure_markedup_tags_status + = _jsonEight(row[_col["dom_status"]].as!string); + obj.metainfo.dom_structure_collapsed_tags_status + = _jsonEight(row[_col["dom_status_collapsed"]].as!string); + obj.metainfo.dummy_heading = (row[_col["dummy_heading"]].as!int == 1); + obj.metainfo.object_number_off = (row[_col["object_number_off"]].as!int == 1); + obj.metainfo.attrib = row[_col["attrib"]].as!string; + obj.metainfo.lang = row[_col["meta_lang"]].as!string; + obj.metainfo.syntax = row[_col["meta_syntax"]].as!string; + obj.metainfo.sha256.text = _hex32(row[_col["sha256"]].as!string); + obj.attrib.indent_base = row[_col["indent_base"]].as!int; + obj.attrib.indent_hang = row[_col["indent_hang"]].as!int; + obj.attrib.bullet = (row[_col["bullet"]].as!int == 1); + obj.attrib.language = row[_col["lang"]].as!string; + obj.has.inline_links = (row[_col["has_links"]].as!int == 1); + obj.has.inline_notes_reg = (row[_col["has_notes_reg"]].as!int == 1); + obj.has.inline_notes_star = (row[_col["has_notes_star"]].as!int == 1); + obj.has.images = (row[_col["has_images"]].as!int == 1); + obj.has.image_without_dimensions = (row[_col["has_images_no_dim"]].as!int == 1); + obj.tags.in_segment_html = row[_col["segment"]].as!string; + obj.tags.segname_prev = row[_col["segment_prev"]].as!string; + obj.tags.segname_next = row[_col["segment_next"]].as!string; + obj.tags.segment_anchor_tag_epub = row[_col["segment_epub"]].as!string; + obj.tags.html_segment_anchor_tag_is = row[_col["segment_html_is"]].as!string; + obj.tags.epub_segment_anchor_tag_is = row[_col["segment_epub_is"]].as!string; + obj.tags.segment_lv4_is = row[_col["segment_lv4_is"]].as!string; + obj.tags.anchor_tag_html = row[_col["anchor"]].as!string; + obj.tags.heading_lev_anchor_tag = row[_col["heading_lev_anchor"]].as!string; + { + string[8] _h; + foreach (i, h; _jsonStrs(row[_col["heading_ancestors_text"]].as!string)) { + if (i < _h.length) { _h[i] = h; } + } + obj.tags.heading_ancestors_text = _h; + } + if (obj.metainfo.is_a == "table") { + obj.table.number_of_columns = row[_col["table_cols"]].as!int; + obj.table.column_widths = _jsonDoubles(row[_col["table_widths"]].as!string); + obj.table.column_aligns = _jsonStrs(row[_col["table_aligns"]].as!string); + obj.table.heading = (row[_col["table_header"]].as!int == 1); + } + obj.code_block.linenumbers = (row[_col["code_linenumbers"]].as!int == 1); + obj.text = row[_col["text"]].as!string; + /+ ↓ the per object lists, from the sweeps above +/ + if (auto _v = _id in _images_by_id) { obj.metainfo.sha256.images = *_v; } + if (auto _v = _id in _links_by_id) { obj.stow.link = *_v; } + if (auto _v = _id in _anchors_by_id) { obj.tags.anchor_tags = *_v; } + if (auto _v = _id in _subtoc_by_id) { obj.tags.lev4_subtoc = *_v; } + doc.abstraction[section] ~= obj; + } + } + return doc; + } + /+ ↓ the files a database carries. + + a .ocda.db holds its images as blobs, which is what lets it travel on + its own where a .ssp cannot: a .ssp describes its images by name, + size, pixels and sha256, but does not carry them. The role column + says what a file is; today only 'image' is written. + +/ + struct ST_ArtefactFile { + string role; + string name; + ulong bytes; + string sha256; + ubyte[] data; + } + @trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") { + ST_ArtefactFile[] _out; + if (!db_file.exists) { return _out; } + auto db = Database(db_file, SQLITE_OPEN_READONLY); + foreach (row; db.execute( + "SELECT role, name, bytes, sha256, data FROM files WHERE role = '" ~ role + ~ "' ORDER BY name") + ) { + ST_ArtefactFile _f; + _f.role = row["role"].as!string; + _f.name = row["name"].as!string; + _f.bytes = row["bytes"].as!ulong; + _f.sha256 = row["sha256"].as!string; + _f.data = row["data"].as!(ubyte[]); + _out ~= _f; + } + return _out; + } +} +#+END_SRC + +* D. get doc from artefact - sisudoc.ocda.abstraction.doc_from_artefact + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.ocda.abstraction.doc_from_artefact; +@safe: +/+ ↓ a document, from an artefact rather than from markup + spineAbstraction() parses markup and hands the output writers a doc: + an abstraction and a doc_matters. This does the same from a .ssp or a + .ocda.db, filling the same docMattersMake() arguments, so that what + comes out is the value the writers already take and they never learn + which of the two made it. + . + Three of the seven arguments come from the run and are the same either + way: program_info, opt_action, and the conf half of conf_make_meta, + which is site and run scoped (urls, papersize, the search database) and + has no business in a document's own artefact. The other four come from + the artefact: + . + conf_make_meta meta and make from the @meta and @make blocks, over + the site config the run assembled + doc_has docHasFromAbstraction, over the objects + doc_digest the markup digest from @source + manifest synthesised from the artefact's path and the + document's own name and language + . + What is not here, deliberately: insert_file_list, which is read only by + source_pod.d for --source and --pod2. No artefact carries the markup, + so a pod cannot be rebuilt from one, and the empty list is the truthful + answer rather than a gap. ++/ +template spineDocFromArtefact() { + import std.algorithm : endsWith; + import std.array; + import std.conv : to; + import std.path; + import std.process : thisProcessID; + import std.regex; + import std.stdio; + import std.string; + import sisudoc.ocda.meta.conf_make_meta_structs; + import sisudoc.ocda.meta.doc_matters; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.abstraction.doc_has; + import sisudoc.ocda.abstraction.load; + import sisudoc.ocda.meta.topic_register; + mixin spineDocMatters; + mixin spineDocHasFromAbstraction; + mixin spineAbstractionLoad; + mixin spineTopicRegister; + /+ ↓ a .ssp header key from a field name: the first underscore becomes the + dot that separates the group, so title_main is title.main and + rights_copyright_text is rights.copyright_text. The writer's keys are + formed the same way, which is what lets this be read off the struct + rather than kept as a second list to fall out of step with it. + +/ + private string _sspKeyOf(string _member) { + foreach (i, c; _member) { + if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; } + } + return _member; + } + /+ ↓ the document's own header, over the site config the run assembled. + every string field of MetaComposite is looked for under its own key, + so a field added there and emitted by the writer arrives here without + anything being added. The arr fields are split from the string they + come from, as agreed: that is parsing a value, not deriving a + structure. + +/ + ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) { + ConfComposite _out = _site_conf; // conf: site and run scoped, kept + static foreach (_m; __traits(allMembers, MetaComposite)) { + static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{ + enum string _key = _sspKeyOf(_m); + if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; } + }} + } + /+ ↓ title_sub is a copy of title_subtitle, made where the header is + read, and is what epub3 puts in dc:title id="subtitle". It is not + emitted, being the same string twice; it is remade here the way the + yaml reader makes it +/ + _out.meta.title_sub = _out.meta.title_subtitle; + /+ ↓ creator.author is the author list joined with ", ", so splitting on + it gives the list back +/ + if (_out.meta.creator_author.length > 0) { + foreach (_a; _out.meta.creator_author.split(", ")) { + if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; } + } + } + /+ ↓ the topic register arrays, by the one rule the yaml reader uses: + the split is not a plain one and deriving it twice would be two + answers to the same question +/ + { + auto _tr = topicRegisterArrays(_out.meta.classify_topic_register); + _out.meta.classify_topic_register_arr = _tr.arr; + _out.meta.classify_topic_register_expanded_arr = _tr.expanded; + } + /+ ↓ make: the keys are the field names, with no group to separate +/ + if (auto _v = "doc_type" in _ssp.make) { + _out.make.doc_type = *_v; _out.make_str.doc_type = *_v; + } + if (auto _v = "auto_num_top_at_level" in _ssp.make) { + _out.make.auto_num_top_at_level = *_v; + } + if (auto _v = "auto_num_top_lv" in _ssp.make) { + try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {} + } + if (auto _v = "auto_num_depth" in _ssp.make) { + try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {} + } + if (auto _v = "breaks" in _ssp.make) { + _out.make.breaks = *_v; _out.make_str.breaks = *_v; + } + if (auto _v = "home_button_text" in _ssp.make) { + _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v; + } + /+ ↓ footer is a list, joined with the separator the format escapes +/ + if (auto _v = "footer" in _ssp.make) { + _out.make.footer = sspUnescapeSplit(*_v); + _out.make_str.footer = _out.make.footer; + } + return _out; + } + /+ ↓ where a document loaded from an artefact says it is. + an artefact names its document (% Source:) and its language (@source + language) but not where the pod holding its images sits, so that is + taken from where the artefact itself was found: + . + pod//media/abstraction/.ssp -> pod// + pod/.ocda.db -> pod// + . + which is where --pod2 puts them both. The images beside a .ssp are + then found where html.d already looks; a .ocda.db carries its own and + does not need the directory to exist. + +/ + struct ST_ArtefactPaths { + string pod_dir; + string src_file_with_path; + } + ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) { + ST_ArtefactPaths _out; + string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en"; + string _dir = _artefact.dirName; + if (_artefact.endsWith(".ssp")) { + /+ ↓ /media/abstraction/.ssp: two directories up is the pod +/ + _out.pod_dir = (_dir.baseName == "abstraction") + ? _dir.dirName.dirName : _dir; + } else { + /+ ↓ /.ocda.db, the pod tree beside it named for the document +/ + string _uid = _artefact.baseName; + foreach (_sfx; [".ocda.db", ".db"]) { + if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; } + } + if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; } + _out.pod_dir = (_dir.chainPath(_uid).array).to!string; + } + _out.src_file_with_path = (_out.pod_dir + .chainPath("media").chainPath("text").chainPath(_lang) + .chainPath(_ssp.source).array).to!string; + return _out; + } + /+ ↓ a database's images, written where the writers look for them. + returns the pod directory to use in place of the one beside the + artefact, or "" when the database carries no images and the pod + beside it should stand. Each blob is checked against the sha256 the + database recorded with it: the two were written together, so a + mismatch means the file is damaged, and saying so is the point of + having recorded the digest at all. + +/ + @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + import std.file : mkdirRecurse, write, tempDir; + import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; + mixin spineAbstractionDbRead _dbr; + auto _files = _dbr.dbReadFiles(_artefact, "image"); + if (_files.length == 0) { return ""; } + string _root = (tempDir.chainPath("spine-ocda-" + ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string; + string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string; + try { + _img_dir.mkdirRecurse; + } catch (Exception ex) { + stderr.writeln("WARNING: could not make an image directory for ", _artefact, + ": ", ex.msg); + return ""; + } + foreach (_f; _files) { + string _got = _f.data.sha256Of.toHexString.to!string; + if (_f.sha256.length > 0 && _got != _f.sha256) { + stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName, + " does not match the digest recorded with it (", _f.sha256, " expected, ", + _got, " found); it is written out as it stands"); + } + try { + (_img_dir.chainPath(_f.name).array).to!string.write(_f.data); + } catch (Exception ex) { + stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg); + } + } + if (_opt_action.vox_gt_1) { + writeln(" ", _files.length, " image(s) from the database, in ", _img_dir); + } + return _root; + } + /+ ↓ the images a .ssp describes, against the ones beside it. + a .ssp carries no image bytes, so what it can do is say whether the + files in the pod it sits in are the ones the abstraction was built + from. Nothing checked that before: a changed or wrong image in the + directory paired silently. + +/ + @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + import std.file : exists, read; + string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string; + int _checked, _missing, _wrong; + foreach (_section; _ssp.section_order) { + foreach (obj; _ssp.abstraction[_section]) { + foreach (_img; obj.metainfo.sha256.images) { + if (_img.fileName.length == 0) { continue; } + string _pth = (_img_dir.chainPath(_img.fileName).array).to!string; + if (!_pth.exists) { ++_missing; continue; } + ++_checked; + string _want = _img.fileHash_sha256.toHexString.to!string; + string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string; + if (_want != _got && _want != "00000000000000000000000000000000" + ~ "00000000000000000000000000000000") { + ++_wrong; + stderr.writeln("WARNING: image ", _img.fileName, " beside ", + _ssp.source, " is not the one the abstraction was built from"); + } + } + } + } + if (_missing > 0) { + stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are", + " not in ", _img_dir); + } + if (_opt_action.vox_gt_1 && _checked > 0) { + writeln(" ", _checked, " image(s) checked against the abstraction's digests", + (_wrong == 0) ? ", all matching" : ""); + } + } + /+ ↓ the doc the output writers take, from an artefact. + the reading is done here rather than by the caller, and that is not + only tidiness: main() is @system, so a template mixed in there infers + @system for everything it declares, and the output writers are @safe + and will not take objects of such a type. Doing it inside this module, + which is @safe:, is what makes the loaded objects the same kind of + thing the parser's are. Called as spineDocFromArtefact!()(...), the + way spineAbstraction!() is, rather than mixed in. + +/ + auto spineDocFromArtefact(P,O,E)( + string _artefact, + P program_info, + O _opt_action, + E _env, + ConfComposite _site_conf, + ) { + auto _loaded = abstractionLoad(_artefact); + auto _ssp = _loaded.doc; + auto _pths = artefactPaths(_artefact, _ssp); + /+ ↓ the images. + a .ocda.db carries its own, which is what lets it travel on its + own, so they are taken from it and written where the output writers + look for images: a directory of this run's making, not the pod + beside the artefact, which may be somebody's published tree. A .ssp + only describes its images, so those are the ones in the pod it sits + in, and are checked against the digests it recorded. + +/ + string _images_tmp; + if (_loaded.loaded) { + if (_artefact.endsWith(".ssp")) { + _imagesVerify(_pths.pod_dir, _ssp, _opt_action); + } else { + string _extracted = _imagesExtract(_artefact, _opt_action); + if (_extracted.length > 0) { + /+ ↓ the source path has to move with it: image_dir_path is + reached from the document's own file, not from the pod +/ + _images_tmp = _extracted; + _pths.pod_dir = _extracted; + _pths.src_file_with_path = (_extracted + .chainPath("media").chainPath("text") + .chainPath(("language" in _ssp.source_info) + ? _ssp.source_info["language"] : "en") + .chainPath(_ssp.source).array).to!string; + } + } + } + auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); + auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); + auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); + /+ ↓ the digests: the markup digest is what @source carries, and it is + the one that says which source this abstraction was made from. The + header and text digests are of parts an artefact does not keep + separately, and are left unset rather than guessed at + +/ + ST_ArtefactDigest _dig; + if (auto _v = "digest" in _ssp.source_info) { + _dig.markup_doc = _hexToDigest(*_v); + } + auto _abstraction = _ssp.abstraction; + auto _matters = docMattersMake( + program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init, + ); + /+ ↓ the same shape spineAbstraction() returns, built the same way: a + struct of accessors over the locals, with whether the artefact + could be read at all, which the caller has to know +/ + auto theDOC() { + struct ST_DOC { + const auto abstraction() { return _abstraction; } + auto matters() { return _matters; } + bool loaded() { return _loaded.loaded; } + string images_tmp() { return _images_tmp; } + string note() { return _loaded.note; } + string source_name() { return abstractionSourceName(_loaded.source); } + } + return ST_DOC(); + } + return theDOC(); + } + /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from + an artefact, @source having recorded it; the header and text digests + are of parts no artefact keeps separately and are left unset rather + than guessed at. Shaped to match ST_doc_digest, which is declared + inside spineRawMarkupContent and reachable only by parsing a file. + +/ + struct ST_ArtefactDigest { + ubyte[32] markup_doc; + ubyte[32] header; + ubyte[32] text; + } + private ubyte[32] _hexToDigest(string _hex) { + ubyte[32] _out; + if (_hex.length < 64) { return _out; } + foreach (i; 0..32) { + try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {} + } + return _out; + } +} +#+END_SRC + +* E. Get OCD + +get abstracted objects for downstream processing + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_has.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.ocda.abstraction.doc_has; +@safe: +/+ ↓ ST_DocHas from a loaded abstraction + + the parser builds ST_DocHas as it goes, with the markup in front of it. + A document loaded from a .ssp or a .ocda.db has to arrive at the same + value from what the artefact carries, and this is where that is done. + + Every one of these is an *index over properties the artefact already + holds*, not a re-derivation of something it does not: + + imagelist the .image records' filenames + segnames_lv4 .anchor of each body heading at level 4 + segnames_lv_0_to_4 .segment_epub of each heading at level 4 or less + tag_associations .anchor, .anchor_tag and .segment_* per object + section_keys_sequenced which sections are non-empty, plus the run's + own output flags + + That distinction is the line the .ssp skill draws: assembling an index + is fine and cannot drift, because a wrong index would not survive the + round trip that produced its inputs; recomputing dom_status or ancestors + would be re-derivation and is exactly what ocda exists to remove. If a + value is not in the file and cannot be indexed out of what is, it must + be added to the artefact instead of reconstructed here. + + The counts come from the @doc_has block rather than being recounted, for + the same reason. ++/ +template spineDocHasFromAbstraction() { + import std.algorithm : sort, uniq; + import std.array; + import std.conv : to; + import std.json : JSONValue; + import std.regex; + import sisudoc.ocda.meta.metadoc_object_setter; + import sisudoc.ocda.meta.rgx; + mixin ObjectSetter; + mixin spineRgxIn; + /+ ↓ the sections, in the order a document holds them +/ + enum string[] doc_sections = [ + "head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail", + ]; + /+ ↓ the tail sections that become segments of their own, in the order + after_doc_determine_segnames appends them +/ + enum string[] tail_sections = [ + "endnotes", "glossary", "bibliography", "bookindex", "blurb", + ]; + private uint _count(string[string] _doc_has, string _key) { + if (auto _v = _key in _doc_has) { + try { return (*_v).to!uint; } catch (Exception ex) { return 0; } + } + return 0; + } + private bool _has_section(A)(A _abst, string _section) { + if (auto _s = _section in _abst) { return (*_s).length > 1; } + return false; + } + /+ ↓ which document sections each output format walks, and in what order. + + the same rule the parser applies: the three that are always there, + then each tail section that the document actually has, then "tail" + for the formats that close with one. It depends on the run as well as + on the document, since only html and epub take the tail. + +/ + string[][string] sectionKeysSequenced(A,O)(A _abst, O _opt_action) { + string[][string] _keys = [ + "scroll": ["head", "toc", "body",], + "seg": ["head", "toc", "body",], + "sql": ["head", "body",], + "latex": ["head", "toc", "body",], + "text": ["head", "toc", "body",], + ]; + if (_has_section(_abst, "endnotes")) { + _keys["scroll"] ~= "endnotes"; + _keys["seg"] ~= "endnotes"; + _keys["latex"] ~= "endnotes"; + _keys["text"] ~= "endnotes"; + } + if (_has_section(_abst, "glossary")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "glossary"; } + } + if (_has_section(_abst, "bibliography")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bibliography"; } + } + if (_has_section(_abst, "bookindex")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bookindex"; } + } + if (_has_section(_abst, "blurb")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "blurb"; } + } + if (_opt_action.html_scroll || _opt_action.html_seg || _opt_action.epub) { + _keys["scroll"] ~= "tail"; + _keys["seg"] ~= "tail"; + } + return _keys; + } + /+ ↓ the whole of it +/ + ST_DocHas docHasFromAbstraction(A,O)( + A _abst, + string[string] _doc_has_block, + O _opt_action, + ) { + static auto rgx = RgxI(); + ST_DocHas _out; + _out.inline_links = _count(_doc_has_block, "inline_links"); + _out.inline_notes_reg = _count(_doc_has_block, "inline_notes_reg"); + _out.inline_notes_star = _count(_doc_has_block, "inline_notes_star"); + _out.codeblocks = _count(_doc_has_block, "codeblocks"); + _out.tables = _count(_doc_has_block, "tables"); + _out.blocks = _count(_doc_has_block, "blocks"); + _out.groups = _count(_doc_has_block, "groups"); + _out.poems = _count(_doc_has_block, "poems"); + _out.quotes = _count(_doc_has_block, "quotes"); + string[] _images; + string[string][string] _tag_assoc; + /+ ↓ segnames["html"] is seeded with the toc, which is a segment of its + own before any level 4 heading opens one +/ + _out.segnames_lv4 ~= "toc"; + foreach (_section; doc_sections) { + if (_section !in _abst) { continue; } + /+ ↓ head is always walked; every other section only when it holds + more than the placeholder object, which is the guard the parser + puts on each of its section loops + +/ + if (_section != "head" && !_has_section(_abst, _section)) { continue; } + foreach (obj; _abst[_section]) { + /+ ↓ the images this object names, as the abstraction recorded them +/ + foreach (_img; obj.metainfo.sha256.images) { _images ~= _img.fileName; } + /+ ↓ every object: its html anchor is in a segment, and its epub + segment anchor stands for itself +/ + if (obj.tags.anchor_tag_html.length > 0) { + _tag_assoc[obj.tags.anchor_tag_html]["seg_lv4"] = obj.tags.in_segment_html; + } + if (obj.tags.segment_anchor_tag_epub.length > 0) { + _tag_assoc[obj.tags.segment_anchor_tag_epub]["seg_lv1to4"] + = obj.tags.segment_anchor_tag_epub; + } + /+ ↓ a heading's own anchor tags, and the anchors declared inline in + an object's text, both point at the segment the object is in +/ + /+ ↓ a heading above level 4 opens no html segment of its own, so a + cross reference to it has to land on the level 4 segment that + follows it. .segment is exactly that (the segment a heading + opens or falls into), where .segment_html_is is the segment an + object sits in; epub segments go down to level 1, so there + .segment_epub stands for the heading itself + +/ + if (obj.tags.heading_lev_anchor_tag.length > 0 + && obj.tags.in_segment_html.length > 0 + && obj.metainfo.heading_lev_markup >= 4 + ) { + _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv4"] + = obj.tags.in_segment_html; + if (obj.tags.segment_anchor_tag_epub.length > 0) { + _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv1to4"] + = obj.tags.segment_anchor_tag_epub; + } + } + foreach (m; obj.text.matchAll(rgx.inline_link_anchor)) { + string _a = m["anchor"].to!string; + if (_a.length == 0 || _a in _tag_assoc) { continue; } + _tag_assoc[_a]["seg_lv4"] = obj.tags.html_segment_anchor_tag_is; + _tag_assoc[_a]["seg_lv1to4"] = obj.tags.epub_segment_anchor_tag_is; + } + /+ ↓ the segment name lists, in document order +/ + if (obj.metainfo.is_a == "heading") { + if (obj.metainfo.heading_lev_markup <= 4) { + _out.segnames_lv_0_to_4 ~= obj.tags.segment_anchor_tag_epub; + } + if (_section == "body" && obj.metainfo.heading_lev_markup == 4) { + _out.segnames_lv4 ~= obj.tags.anchor_tag_html; + } + /+ ↓ a heading above level 4 opens no html segment of its own, so + a cross reference to it lands on the level 4 segment that + follows. That is .segment_lv4_is, resolved in ocda where the + finished sections are to hand and carried by the artefact, so + there is nothing to work out here + +/ + if (obj.tags.segment_lv4_is.length > 0) { + /+ ↓ seg_lv4 only, and not through heading_lev_anchor_tag: that + names the level 4 heading itself, whose own entry says it + opens that segment rather than merely sits in it. epub + segments go down to level 1, so seg_lv1to4 is untouched + +/ + foreach (_k; [obj.metainfo.identifier, obj.tags.anchor_tag_html]) { + if (_k.length > 0) { _tag_assoc[_k]["seg_lv4"] = obj.tags.segment_lv4_is; } + } + } + } + } + } + foreach (_section; tail_sections) { + if (_has_section(_abst, _section)) { _out.segnames_lv4 ~= _section; } + } + /+ ↓ the body's objects, in a pass of their own after every anchor above + has been recorded, because the parser does it in that order and the + order is what decides two of these entries. + . + each body object's identifier maps to the segment it is in. seg_lv4 + is set only where it is not already, an anchor of the same name + having said it *names* a segment rather than sits in one; seg_lv1to4 + is set always. A heading's own anchor can be a bare number taken + from its text ("2. Shorter Terms" gives the anchor "2"), which then + collides with the ocn of an unrelated object, and this is the order + that resolves the collision the way the parser resolves it: in + free_culture, ocn 2 keeps seg_lv4 "them" from the anchor and takes + seg_lv1to4 "_part_1" from itself. + +/ + if (_has_section(_abst, "body")) { + foreach (obj; _abst["body"]) { + if (obj.metainfo.identifier.length == 0) { continue; } + if (!((obj.metainfo.identifier in _tag_assoc) + && ("seg_lv4" in _tag_assoc[obj.metainfo.identifier])) + ) { + _tag_assoc[obj.metainfo.identifier]["seg_lv4"] + = obj.tags.html_segment_anchor_tag_is; + } + _tag_assoc[obj.metainfo.identifier]["seg_lv1to4"] + = obj.tags.epub_segment_anchor_tag_is; + } + } + _out.imagelist = _images.sort.uniq.array; + _out.tag_associations = _tag_assoc; + _out.section_keys_sequenced = sectionKeysSequenced(_abst, _opt_action); + return _out; + } +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/in_markup_source_files.org b/org/in_markup_source_files.org new file mode 100644 index 0000000..1b8cee0 --- /dev/null +++ b/org/in_markup_source_files.org @@ -0,0 +1,801 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) markup source raw +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:sourcefile:read: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] + +* A. get _config file_, read in (.readText) [#A] +** _module template_ :module:config_files: + +#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +/++ + read configuration files
+ - read config files
+ meta_config_files.d ++/ +module sisudoc.ocda.io_in.read_config_files; +@safe: +import std.file; +import std.path; +import sisudoc.ocda.meta; +import sisudoc.ocda.io_in.paths_source; +import sisudoc.ocda.meta.rgx_files; +import sisudoc.ocda.meta.rgx; +<> +<> +<> +#+END_SRC + +*** read config files (config local site & dr document make) (yaml) +**** site configuration + +SEE NOTES on configuration hierarchy in spine.org + +#+NAME: meta_config_file_hub_read_site_config +#+BEGIN_SRC d +template readConfigSite() { + @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) { + mixin spineRgxIn; + static auto rgx = RgxI(); + string conf_filename = "NONE"; + string config_file_str; + string default_config_file_str = format(q"┃ +flag: + act0: "--html" + act1: "--html --epub" +output: + path: "%s" +default: + language: "en" + papersize: "a4" + text_wrap: "80" + digest: "sha256" +webserv: + http: "%s" + host: "%s" + data_http: "%s" + data_host: "%s" + data_root_url: "%s" + data_root_path: "%s" + data_root_part: "" + images_root_part: "image" + cgi_search_form_title: "%s" + cgi_http: "%s" + cgi_host: "%s" + cgi_bin_url: "%s" + cgi_bin_subpath: "%s" + cgi_bin_path: "%s" + cgi_search_script: "%s" + cgi_port: "" + cgi_user: "" + cgi_action: "%s" + db_sqlite_path: "%s" + db_sqlite_filename: "%s" + db_pg_table: "" + db_pg_user: "" +┃", + _cfg.processing_path_doc_root, // doc root + _cfg.http_request_type, // http + _cfg.http_host, // host / domain + _cfg.http_request_type, // data "http" or "https" + _cfg.http_host, // data domain "localhost" + _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org" + _cfg.processing_path_doc_root, // data root path + _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search" + _cfg.http_request_type, // cgi http + _cfg.http_host, // cgi host + _cfg.cgi_url_root, // cgi bin url + _cfg.cgi_bin_subpath, // cgi bin path + _cfg.cgi_bin_root, // cgi bin path + _cfg.cgi_filename, // cgi filename + _cfg.cgi_url_action, // cgi action + _cfg.db_sqlite_path, // sqlite db path + _cfg.db_sqlite_filename, // sqlite db filename +); + foreach(conf_fn; [_conf_file_details.config_filename_site]) { + foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) { + char[] conf_file; + conf_filename = conf_fn; + if (exists(pth)) { + auto f_attrib = pth.getLinkAttributes; + if ( + _conf_file_details.possible_config_path_locations.config_local_site.length == 1 + && f_attrib.attrIsFile + ) { + conf_file = pth.to!(char[]); + conf_filename = pth.baseName; + } else if (f_attrib.attrIsDir) { + conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; + conf_filename = conf_fn; + } + try { + if (exists(conf_file)) { + if (conf_file.getLinkAttributes.attrIsFile) { + if (_opt_action.vox_gt_2 || _opt_action.debug_do) { + writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)"); + } + config_file_str = conf_file.readText; + break; + } + } + } catch (ErrnoException ex) { + } catch (FileException ex) { + } + } + } + if (config_file_str.length > 0) { break; } + } + if (config_file_str.length > 0) { + import dyaml; + Node yaml_root; + try { + yaml_root = Loader.fromString(config_file_str).load(); + } catch (Throwable) { + import std.stdio; + writeln("ERROR failed to read config file content, not parsed as yaml, program default used"); + conf_filename = "VIRTUAL"; + config_file_str = default_config_file_str; + } + } + if (config_file_str.length == 0) { /+ use dummy default config file +/ + // writeln("WARNING config file NOT found, default provided"); + conf_filename = "VIRTUAL"; + config_file_str = default_config_file_str; + } + struct _ConfContent { + string filename() { + return conf_filename; + } + string filetype() { + string _ft = ""; + if (content.match(rgx.yaml_config)) { + _ft = "yaml"; + } + return _ft; + } + string content() { + return config_file_str; + } + } + return _ConfContent(); + } +} +#+END_SRC + +**** document make/config + +#+NAME: meta_config_file_hub_read_document_config +#+HEADER: :noweb yes +#+BEGIN_SRC d +static template readConfigDoc() { + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.meta.rgx_files; + import sisudoc.ocda.meta.rgx; + @system final auto readConfigDoc(M,E)(M _manifested, E _env) { + mixin spineRgxIn; + static auto rgx = RgxI(); + mixin spineRgxFiles; + static auto rgx_files = RgxFiles(); + string config_file_str; + string conf_filename = "NONE"; + auto _conf_file_details = configFilePaths!()(_manifested, _env); + string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; + foreach(conf_fn; [_conf_file_details.config_filename_document]) { + foreach(pth; possible_config_path_locations) { + char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; + conf_filename = conf_fn; + if (config_file_str.length > 0) { + break; + } + try { + if (exists(conf_file)) { + if (conf_file.getLinkAttributes.attrIsFile) { + config_file_str = conf_file.readText; + break; + } + } + } catch (ErrnoException ex) { + } catch (FileException ex) { + } + } + if (config_file_str.length > 0) { break; } + } + struct _ConfContent { + string filename() { + return conf_filename; + } + string content() { + return config_file_str; + } + string filetype() { + string _ft = ""; + if (content.match(rgx.yaml_config)) { + _ft = "yaml"; + } + return _ft; + } + } + return _ConfContent(); + } +} +#+END_SRC + +*** YAML config (config local site & dr document make) :file:config:hub: + +#+NAME: meta_config_file_hub_read_site_yaml +#+HEADER: :noweb yes +#+BEGIN_SRC d +static template configReadSiteYAML() { + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.meta.rgx_files; + import sisudoc.ocda.meta.rgx; + final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) { + string _configuration = configReadInSiteYAML!()(_manifested, _env); + auto _conf_file_details = configFilePaths!()(_manifested, _env); + string _conf_yaml_fn = _conf_file_details.config_filename_site; + YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); + return _yaml_conf; + } +} +static template configReadDocYAML() { + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) { + string _configuration = configReadInDocYAML!()(_manifested, _env); + auto _conf_file_details = configFilePaths!()(_manifested, _env); + string _conf_yaml_fn = _conf_file_details.config_filename_document; + YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); + return _yaml_conf; + } +} +#+END_SRC + +* B. get _markup source_, read file (.readText) [#A]:module:source_files: +** _module template_ + +#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +/++ + module source_read_source_files;
+ - open markup files
+ - if master file scan for addional files to import/insert ++/ +module sisudoc.ocda.io_in.read_source_files; +@safe: +template spineRawMarkupContent() { + import std.digest.sha; + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.meta.rgx_files; + import sisudoc.ocda.meta.rgx; + mixin spineRgxIn; + static auto rgx = RgxI(); + mixin spineRgxFiles; + static auto rgx_files = RgxFiles(); + string[] _images=[]; + /+ ↓ the images a markup source shows. + matched on the image markup itself ({ name.png }image, and the url and + ~^ forms), ensure a filename-shaped word in prose is not mistaken for + one, and over every match (so a line showing two images reports two). + +/ + string[] _extract_images(S)(S content_block) { + string[] images_; + string _content_block = content_block.to!string; + foreach (m; _content_block.matchAll(rgx.smid_image)) { + images_ ~= m["image"].to!string; + } + return images_; + } + auto rawsrc = RawMarkupContent(); + struct ST_contents_inserts_images { + char[][] contents; + string[] insert_files; + string[] images; + } + struct ST_header_content_inserts_images { + char[] header; + char[][] src_txt; + string[] insert_files; + string[] images; + } + struct ST_doc_digest { + ubyte[32] markup_doc; + ubyte[32] header; + ubyte[32] text; + } + struct ST_doc_parts { + char[] header_raw; + char[][] sourcefile_body_content; + string[] insert_file_list; + string[] images_list; + ST_doc_digest doc_digest; + } + ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) { + ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct + = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src); + return _0_header_1_body_content_2_insert_filelist_struct; + } + struct RawMarkupContent { + final sourceContent(in string fn_src) { + auto raw = MarkupRawUnit(); + string source_txt_str + = raw.markupSourceReadIn(fn_src); + return source_txt_str; + } + final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)( + O _opt_action, + in string source_txt_str, + in string fn_src="" + ) { + auto raw = MarkupRawUnit(); + string[] insert_file_list_get; + string[] images_list_get; + ST_header_content_inserts_images st + = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str); + char[] header_raw = st.header; + char[][] sourcefile_body_content = st.src_txt; + if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise + auto ins = Inserts(); + ST_contents_inserts_images _cii + = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); + sourcefile_body_content = _cii.contents; + insert_file_list_get = _cii.insert_files.dup; + images_list_get = _cii.images.dup; + } else if (_opt_action.source_or_pod) { + auto ins = Inserts(); + ST_contents_inserts_images _cii + = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); + images_list_get = _cii.images.dup; + } // image_list, if path could take sha256 digests already here? + string header_type = ""; + ST_doc_digest dig; + { + dig.markup_doc = source_txt_str.sha256Of; + dig.header = st.header.sha256Of; + dig.text = sourcefile_body_content.sha256Of; + } + ST_doc_parts ret; + { + ret.header_raw = st.header; + ret.sourcefile_body_content = sourcefile_body_content; + ret.insert_file_list = insert_file_list_get; + ret.images_list = images_list_get; + ret.doc_digest = dig; + } + return ret; + } + } + struct MarkupRawUnit { + import std.digest.sha; + import std.file; + <> + <> + <> + <> + <> + <> + } + struct Inserts { + struct ST_contents_and_images { + char[][] insert_contents; + string[] images; + } + ST_contents_and_images scan_subdoc_source(O)( + O _opt_action, + char[][] markup_sourcefile_insert_content, + string fn_src + ) { + <> + foreach (line; markup_sourcefile_insert_content) { + <> + } // end src subdoc (inserts) loop + <> + } + ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)( + O _opt_action, + char[][] sourcefile_body_content, + string fn_src + ) { + import std.algorithm; + <> + foreach (line; sourcefile_body_content) { + <> + } // end src doc loop + <> + } + } +} +#+END_SRC + +** get markup source, read file :source:markup: +*** read file, source string [#A] :string: + +#+NAME: meta_markup_source_raw_read_file_source_string +#+BEGIN_SRC d +final private string readInMarkupSource(in char[] fn_src) { + enforce( + exists(fn_src) != 0, + "file not found: «" ~ + fn_src ~ "»" + ); + string source_txt_str; + try { + if (exists(fn_src)) { + if (fn_src.getLinkAttributes.attrIsFile) { + source_txt_str = fn_src.readText; + } else { + } + } + } catch (ErrnoException ex) { + } catch (UTFException ex) { + // Handle validation errors + } catch (FileException ex) { + // Handle errors + } + std.utf.validate(source_txt_str); + return source_txt_str; +} +#+END_SRC + +*** document header & content, array.length == 2 [#A] :array: + +here you split document header and body, an array.length == 2 +split is on first match of level A~ (which is required) + +#+NAME: meta_markup_source_raw_doc_header_and_content_split +#+BEGIN_SRC d +@trusted final private char[][] header0Content1(in string src_text) { // cast(char[]) + /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ + char[][] header_and_content; + auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a); + header_and_content ~= m.pre; + header_and_content ~= m.hit ~ m.post; + assert(header_and_content.length == 2, + "document markup is broken, header body split == " + ~ header_and_content.length.to!string + ~ "; (header / body array split should == 2 (split is on level A~))" + ); + return header_and_content; +} +#+END_SRC + +*** source line array :array: + +#+NAME: meta_markup_source_raw_source_line_array +#+BEGIN_SRC d +@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[]) + char[][] source_line_arr + = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding); + return source_line_arr; +} +#+END_SRC + +*** source content raw line array :array: +- used for regular .sst files; master .ssm files and; .ssi inserts +- regex is passed for relevant enforce match +**** read in file + +#+NAME: meta_markup_source_raw_read_in_file +#+BEGIN_SRC d +string markupSourceReadIn(in string fn_src) { + static auto rgx_files = RgxFiles(); + enforce( + fn_src.match(rgx_files.src_pth_sst_or_ssm), + "not a dr markup filename: «" ~ + fn_src ~ "»" + ); + string source_txt_str = readInMarkupSource(fn_src); + return source_txt_str; +} +#+END_SRC + +**** tuple (a) header, (b) body content, (c) file insert list & (d) image list? + +- header +- body content +- file insert list +- [image list?] + +#+NAME: meta_markup_source_raw_struct_of_header_and_body +#+BEGIN_SRC d +ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) { + string[] file_insert_list = []; + string[] images_list = []; + char[][] hc = header0Content1(source_txt_str); + char[] header = hc[0]; + char[] source_txt = hc[1]; + char[][] source_line_arr = markupSourceLineArray(source_txt); + ST_header_content_inserts_images ret; + { + ret.header = header; + ret.src_txt = source_line_arr; + ret.insert_files = file_insert_list; + ret.images = images_list; + } + return ret; +} +#+END_SRC + +**** get insert source line array + +#+NAME: meta_markup_source_raw_get_insert_source_line_array +#+BEGIN_SRC d +final char[][] getInsertMarkupSourceContentRawLineArray( + in char[] fn_src_insert, + Regex!(char) rgx_file +) { + enforce( + fn_src_insert.match(rgx_file), + "not a dr markup filename: «" ~ + fn_src_insert ~ "»" + ); + string source_txt_str = readInMarkupSource(fn_src_insert); + char[][] source_line_arr = markupSourceLineArray(source_txt_str); + return source_line_arr; +} +#+END_SRC + +** get markup source, master file & inserts :masterfile:inserts: +*** scan inserts (sub-document) source :scan_insert_src: +**** scan subdoc source + +#+NAME: meta_inserts_scan +#+BEGIN_SRC d +char[][] contents_insert; +int code_block_status = 0; +enum codeBlock { off, curly, tic, quotemarks } +auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); +auto markup_src_file_path = fn_pth_full.captures[1]; +#+END_SRC + +**** loop insert (sub-document) + +#+NAME: meta_inserts_scan_loop +#+BEGIN_SRC d +if (code_block_status == codeBlock.curly) { + if (line.matchFirst(rgx.block_curly_code_close)) { + code_block_status = codeBlock.off; + } + contents_insert ~= line; +} else if (line.matchFirst(rgx.block_curly_code_open)) { + code_block_status = codeBlock.curly; + contents_insert ~= line; +} else if (code_block_status == codeBlock.quotemarks) { + if (line.matchFirst(rgx.block_quotemarks_close)) { + code_block_status = codeBlock.off; + } + contents_insert ~= line; +} else if (code_block_status == codeBlock.tic) { + if (line.matchFirst(rgx.block_tic_close)) { + code_block_status = codeBlock.off; + } + contents_insert ~= line; +} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { + code_block_status = codeBlock.quotemarks; + contents_insert ~= line; +} else if (line.matchFirst(rgx.block_tic_code_open)) { + code_block_status = codeBlock.tic; + contents_insert ~= line; +} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { + auto insert_fn = m.captures[2]; + auto insert_sub_pth = m.captures[1]; + auto fn_src_insert + = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; + auto raw = MarkupRawUnit(); + auto markup_sourcesubfile_insert_content + = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); + debug(insert_file) { + writeln(line); + writeln(fn_src_insert); + writeln( + " length contents insert array: ", + markup_sourcesubfile_insert_content.length + ); + } + if (_opt_action.source_or_pod) { + _images ~= _extract_images(markup_sourcesubfile_insert_content); + } + auto ins = Inserts(); + /+ + - 1. load file + - 2. read lines + - 3. scan lines + - a. if filename insert, and insert filename + - repeat 1 + - b. else + - add line to new array; + - build image list, search for any image files to add to image list + +/ +} else { + contents_insert ~= line; // images to extract for image list? + if (_opt_action.source_or_pod) { + string[] _image_linelist = _extract_images(line); + if (_image_linelist.length > 0) { + _images ~= _image_linelist; + } + } +} +#+END_SRC + +**** post loop + +#+NAME: meta_inserts_scan_post +#+BEGIN_SRC d +ST_contents_and_images ret; +{ + ret.insert_contents = contents_insert; + ret.images = _images; +} +return ret; +#+END_SRC + +*** scan document source :scan_src: +**** scan doc source + +#+NAME: meta_master_doc_scan_for_insert_filenames +#+BEGIN_SRC d +char[][] contents; +int code_block_status = 0; +enum codeBlock { off, curly, tic, quotemarks } +auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); +auto markup_src_file_path = fn_pth_full.captures[1]; +char[][] contents_insert; +string[] _images =[]; +string[] insert_file_list =[]; +#+END_SRC + +**** include inserts: _loop master_ scan for inserts (insert documents) + +#+NAME: meta_master_doc_scan_for_insert_filenames_loop +#+BEGIN_SRC d +if (code_block_status == codeBlock.curly) { + if (line.matchFirst(rgx.block_curly_code_close)) { + code_block_status = codeBlock.off; + } + contents ~= line; +} else if (line.matchFirst(rgx.block_curly_code_open)) { + code_block_status = codeBlock.curly; + contents ~= line; +} else if (code_block_status == codeBlock.quotemarks) { + if (line.matchFirst(rgx.block_quotemarks_close)) { + code_block_status = codeBlock.off; + } + contents ~= line; +} else if (code_block_status == codeBlock.tic) { + if (line.matchFirst(rgx.block_tic_close)) { + code_block_status = codeBlock.off; + } + contents ~= line; +} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { + code_block_status = codeBlock.quotemarks; + contents ~= line; +} else if (line.matchFirst(rgx.block_tic_code_open)) { + code_block_status = codeBlock.tic; + contents ~= line; +} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { + auto insert_fn = m.captures[2]; + auto insert_sub_pth = m.captures[1]; + auto fn_src_insert + = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; + insert_file_list ~= fn_src_insert.to!string; + auto raw = MarkupRawUnit(); + /+ TODO +/ + auto markup_sourcefile_insert_content + = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); + debug(insert_file) { + writeln(line); + writeln(fn_src_insert); + writeln( + " length contents insert array: ", + markup_sourcefile_insert_content.length + ); + } + auto ins = Inserts(); + ST_contents_and_images contents_insert_st = ins.scan_subdoc_source( + _opt_action, + markup_sourcefile_insert_content, + fn_src_insert.to!string + ); + contents ~= contents_insert_st.insert_contents; + if (_opt_action.source_or_pod) { + string[] _image_linelist = _extract_images(contents_insert_st.images); + if (_image_linelist.length > 0) { + _images ~= _image_linelist; + } + } + /+ + - 1. load file + - 2. read lines + - 3. scan lines + - a. if filename insert, and insert filename + - repeat 1 + - b. else + - add line to new array; + - build image list, search for any image files to add to image list + +/ +} else { + contents ~= line; + if (_opt_action.source_or_pod) { + string[] _image_linelist = _extract_images(line); + if (_image_linelist.length > 0) { + _images ~= _image_linelist; + } + } +} +#+END_SRC + +**** post loop + +#+NAME: meta_master_doc_scan_for_insert_filenames_post +#+BEGIN_SRC d +string[] images = []; +foreach(i; uniq(_images.sort())) { + images ~= i; +} +debug(insert_file) { + writeln(__LINE__); + writeln(contents.length); +} +ST_contents_inserts_images ret; +{ + ret.contents = contents; + ret.insert_files = insert_file_list; + ret.images = images; +} +return ret; +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/in_source_artefact.org b/org/in_source_artefact.org deleted file mode 100644 index 07c865f..0000000 --- a/org/in_source_artefact.org +++ /dev/null @@ -1,392 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) markup source raw -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:sourcefile:read: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* A. get doc from artefact -** _module template_ :module:config_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.ocda.abstraction.doc_from_artefact; -@safe: -/+ ↓ a document, from an artefact rather than from markup - spineAbstraction() parses markup and hands the output writers a doc: - an abstraction and a doc_matters. This does the same from a .ssp or a - .ocda.db, filling the same docMattersMake() arguments, so that what - comes out is the value the writers already take and they never learn - which of the two made it. - . - Three of the seven arguments come from the run and are the same either - way: program_info, opt_action, and the conf half of conf_make_meta, - which is site and run scoped (urls, papersize, the search database) and - has no business in a document's own artefact. The other four come from - the artefact: - . - conf_make_meta meta and make from the @meta and @make blocks, over - the site config the run assembled - doc_has docHasFromAbstraction, over the objects - doc_digest the markup digest from @source - manifest synthesised from the artefact's path and the - document's own name and language - . - What is not here, deliberately: insert_file_list, which is read only by - source_pod.d for --source and --pod2. No artefact carries the markup, - so a pod cannot be rebuilt from one, and the empty list is the truthful - answer rather than a gap. -+/ -template spineDocFromArtefact() { - import std.algorithm : endsWith; - import std.array; - import std.conv : to; - import std.path; - import std.process : thisProcessID; - import std.regex; - import std.stdio; - import std.string; - import sisudoc.ocda.meta.conf_make_meta_structs; - import sisudoc.ocda.meta.doc_matters; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.abstraction.doc_has; - import sisudoc.ocda.abstraction.load; - import sisudoc.ocda.meta.topic_register; - mixin spineDocMatters; - mixin spineDocHasFromAbstraction; - mixin spineAbstractionLoad; - mixin spineTopicRegister; - /+ ↓ a .ssp header key from a field name: the first underscore becomes the - dot that separates the group, so title_main is title.main and - rights_copyright_text is rights.copyright_text. The writer's keys are - formed the same way, which is what lets this be read off the struct - rather than kept as a second list to fall out of step with it. - +/ - private string _sspKeyOf(string _member) { - foreach (i, c; _member) { - if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; } - } - return _member; - } - /+ ↓ the document's own header, over the site config the run assembled. - every string field of MetaComposite is looked for under its own key, - so a field added there and emitted by the writer arrives here without - anything being added. The arr fields are split from the string they - come from, as agreed: that is parsing a value, not deriving a - structure. - +/ - ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) { - ConfComposite _out = _site_conf; // conf: site and run scoped, kept - static foreach (_m; __traits(allMembers, MetaComposite)) { - static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{ - enum string _key = _sspKeyOf(_m); - if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; } - }} - } - /+ ↓ title_sub is a copy of title_subtitle, made where the header is - read, and is what epub3 puts in dc:title id="subtitle". It is not - emitted, being the same string twice; it is remade here the way the - yaml reader makes it +/ - _out.meta.title_sub = _out.meta.title_subtitle; - /+ ↓ creator.author is the author list joined with ", ", so splitting on - it gives the list back +/ - if (_out.meta.creator_author.length > 0) { - foreach (_a; _out.meta.creator_author.split(", ")) { - if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; } - } - } - /+ ↓ the topic register arrays, by the one rule the yaml reader uses: - the split is not a plain one and deriving it twice would be two - answers to the same question +/ - { - auto _tr = topicRegisterArrays(_out.meta.classify_topic_register); - _out.meta.classify_topic_register_arr = _tr.arr; - _out.meta.classify_topic_register_expanded_arr = _tr.expanded; - } - /+ ↓ make: the keys are the field names, with no group to separate +/ - if (auto _v = "doc_type" in _ssp.make) { - _out.make.doc_type = *_v; _out.make_str.doc_type = *_v; - } - if (auto _v = "auto_num_top_at_level" in _ssp.make) { - _out.make.auto_num_top_at_level = *_v; - } - if (auto _v = "auto_num_top_lv" in _ssp.make) { - try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {} - } - if (auto _v = "auto_num_depth" in _ssp.make) { - try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {} - } - if (auto _v = "breaks" in _ssp.make) { - _out.make.breaks = *_v; _out.make_str.breaks = *_v; - } - if (auto _v = "home_button_text" in _ssp.make) { - _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v; - } - /+ ↓ footer is a list, joined with the separator the format escapes +/ - if (auto _v = "footer" in _ssp.make) { - _out.make.footer = sspUnescapeSplit(*_v); - _out.make_str.footer = _out.make.footer; - } - return _out; - } - /+ ↓ where a document loaded from an artefact says it is. - an artefact names its document (% Source:) and its language (@source - language) but not where the pod holding its images sits, so that is - taken from where the artefact itself was found: - . - pod//media/abstraction/.ssp -> pod// - pod/.ocda.db -> pod// - . - which is where --pod2 puts them both. The images beside a .ssp are - then found where html.d already looks; a .ocda.db carries its own and - does not need the directory to exist. - +/ - struct ST_ArtefactPaths { - string pod_dir; - string src_file_with_path; - } - ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) { - ST_ArtefactPaths _out; - string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en"; - string _dir = _artefact.dirName; - if (_artefact.endsWith(".ssp")) { - /+ ↓ /media/abstraction/.ssp: two directories up is the pod +/ - _out.pod_dir = (_dir.baseName == "abstraction") - ? _dir.dirName.dirName : _dir; - } else { - /+ ↓ /.ocda.db, the pod tree beside it named for the document +/ - string _uid = _artefact.baseName; - foreach (_sfx; [".ocda.db", ".db"]) { - if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; } - } - if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; } - _out.pod_dir = (_dir.chainPath(_uid).array).to!string; - } - _out.src_file_with_path = (_out.pod_dir - .chainPath("media").chainPath("text").chainPath(_lang) - .chainPath(_ssp.source).array).to!string; - return _out; - } - /+ ↓ a database's images, written where the writers look for them. - returns the pod directory to use in place of the one beside the - artefact, or "" when the database carries no images and the pod - beside it should stand. Each blob is checked against the sha256 the - database recorded with it: the two were written together, so a - mismatch means the file is damaged, and saying so is the point of - having recorded the digest at all. - +/ - @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - import std.file : mkdirRecurse, write, tempDir; - import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; - mixin spineAbstractionDbRead _dbr; - auto _files = _dbr.dbReadFiles(_artefact, "image"); - if (_files.length == 0) { return ""; } - string _root = (tempDir.chainPath("spine-ocda-" - ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string; - string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string; - try { - _img_dir.mkdirRecurse; - } catch (Exception ex) { - stderr.writeln("WARNING: could not make an image directory for ", _artefact, - ": ", ex.msg); - return ""; - } - foreach (_f; _files) { - string _got = _f.data.sha256Of.toHexString.to!string; - if (_f.sha256.length > 0 && _got != _f.sha256) { - stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName, - " does not match the digest recorded with it (", _f.sha256, " expected, ", - _got, " found); it is written out as it stands"); - } - try { - (_img_dir.chainPath(_f.name).array).to!string.write(_f.data); - } catch (Exception ex) { - stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg); - } - } - if (_opt_action.vox_gt_1) { - writeln(" ", _files.length, " image(s) from the database, in ", _img_dir); - } - return _root; - } - /+ ↓ the images a .ssp describes, against the ones beside it. - a .ssp carries no image bytes, so what it can do is say whether the - files in the pod it sits in are the ones the abstraction was built - from. Nothing checked that before: a changed or wrong image in the - directory paired silently. - +/ - @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - import std.file : exists, read; - string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string; - int _checked, _missing, _wrong; - foreach (_section; _ssp.section_order) { - foreach (obj; _ssp.abstraction[_section]) { - foreach (_img; obj.metainfo.sha256.images) { - if (_img.fileName.length == 0) { continue; } - string _pth = (_img_dir.chainPath(_img.fileName).array).to!string; - if (!_pth.exists) { ++_missing; continue; } - ++_checked; - string _want = _img.fileHash_sha256.toHexString.to!string; - string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string; - if (_want != _got && _want != "00000000000000000000000000000000" - ~ "00000000000000000000000000000000") { - ++_wrong; - stderr.writeln("WARNING: image ", _img.fileName, " beside ", - _ssp.source, " is not the one the abstraction was built from"); - } - } - } - } - if (_missing > 0) { - stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are", - " not in ", _img_dir); - } - if (_opt_action.vox_gt_1 && _checked > 0) { - writeln(" ", _checked, " image(s) checked against the abstraction's digests", - (_wrong == 0) ? ", all matching" : ""); - } - } - /+ ↓ the doc the output writers take, from an artefact. - the reading is done here rather than by the caller, and that is not - only tidiness: main() is @system, so a template mixed in there infers - @system for everything it declares, and the output writers are @safe - and will not take objects of such a type. Doing it inside this module, - which is @safe:, is what makes the loaded objects the same kind of - thing the parser's are. Called as spineDocFromArtefact!()(...), the - way spineAbstraction!() is, rather than mixed in. - +/ - auto spineDocFromArtefact(P,O,E)( - string _artefact, - P program_info, - O _opt_action, - E _env, - ConfComposite _site_conf, - ) { - auto _loaded = abstractionLoad(_artefact); - auto _ssp = _loaded.doc; - auto _pths = artefactPaths(_artefact, _ssp); - /+ ↓ the images. - a .ocda.db carries its own, which is what lets it travel on its - own, so they are taken from it and written where the output writers - look for images: a directory of this run's making, not the pod - beside the artefact, which may be somebody's published tree. A .ssp - only describes its images, so those are the ones in the pod it sits - in, and are checked against the digests it recorded. - +/ - string _images_tmp; - if (_loaded.loaded) { - if (_artefact.endsWith(".ssp")) { - _imagesVerify(_pths.pod_dir, _ssp, _opt_action); - } else { - string _extracted = _imagesExtract(_artefact, _opt_action); - if (_extracted.length > 0) { - /+ ↓ the source path has to move with it: image_dir_path is - reached from the document's own file, not from the pod +/ - _images_tmp = _extracted; - _pths.pod_dir = _extracted; - _pths.src_file_with_path = (_extracted - .chainPath("media").chainPath("text") - .chainPath(("language" in _ssp.source_info) - ? _ssp.source_info["language"] : "en") - .chainPath(_ssp.source).array).to!string; - } - } - } - auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); - auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); - auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); - /+ ↓ the digests: the markup digest is what @source carries, and it is - the one that says which source this abstraction was made from. The - header and text digests are of parts an artefact does not keep - separately, and are left unset rather than guessed at - +/ - ST_ArtefactDigest _dig; - if (auto _v = "digest" in _ssp.source_info) { - _dig.markup_doc = _hexToDigest(*_v); - } - auto _abstraction = _ssp.abstraction; - auto _matters = docMattersMake( - program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init, - ); - /+ ↓ the same shape spineAbstraction() returns, built the same way: a - struct of accessors over the locals, with whether the artefact - could be read at all, which the caller has to know +/ - auto theDOC() { - struct ST_DOC { - const auto abstraction() { return _abstraction; } - auto matters() { return _matters; } - bool loaded() { return _loaded.loaded; } - string images_tmp() { return _images_tmp; } - string note() { return _loaded.note; } - string source_name() { return abstractionSourceName(_loaded.source); } - } - return ST_DOC(); - } - return theDOC(); - } - /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from - an artefact, @source having recorded it; the header and text digests - are of parts no artefact keeps separately and are left unset rather - than guessed at. Shaped to match ST_doc_digest, which is declared - inside spineRawMarkupContent and reachable only by parsing a file. - +/ - struct ST_ArtefactDigest { - ubyte[32] markup_doc; - ubyte[32] header; - ubyte[32] text; - } - private ubyte[32] _hexToDigest(string _hex) { - ubyte[32] _out; - if (_hex.length < 64) { return _out; } - foreach (i; 0..32) { - try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {} - } - return _out; - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/in_source_files.org b/org/in_source_files.org deleted file mode 100644 index 78141b7..0000000 --- a/org/in_source_files.org +++ /dev/null @@ -1,1656 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) markup source raw -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:sourcefile:read: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* A. get _config file_, read in (.readText) [#A] -** _module template_ :module:config_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -/++ - read configuration files
- - read config files
- meta_config_files.d -+/ -module sisudoc.ocda.io_in.read_config_files; -@safe: -import std.file; -import std.path; -import sisudoc.ocda.meta; -import sisudoc.ocda.io_in.paths_source; -import sisudoc.ocda.meta.rgx_files; -import sisudoc.ocda.meta.rgx; -<> -<> -<> -#+END_SRC - -*** read config files (config local site & dr document make) (yaml) -**** site configuration - -SEE NOTES on configuration hierarchy in spine.org - -#+NAME: meta_config_file_hub_read_site_config -#+BEGIN_SRC d -template readConfigSite() { - @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) { - mixin spineRgxIn; - static auto rgx = RgxI(); - string conf_filename = "NONE"; - string config_file_str; - string default_config_file_str = format(q"┃ -flag: - act0: "--html" - act1: "--html --epub" -output: - path: "%s" -default: - language: "en" - papersize: "a4" - text_wrap: "80" - digest: "sha256" -webserv: - http: "%s" - host: "%s" - data_http: "%s" - data_host: "%s" - data_root_url: "%s" - data_root_path: "%s" - data_root_part: "" - images_root_part: "image" - cgi_search_form_title: "%s" - cgi_http: "%s" - cgi_host: "%s" - cgi_bin_url: "%s" - cgi_bin_subpath: "%s" - cgi_bin_path: "%s" - cgi_search_script: "%s" - cgi_port: "" - cgi_user: "" - cgi_action: "%s" - db_sqlite_path: "%s" - db_sqlite_filename: "%s" - db_pg_table: "" - db_pg_user: "" -┃", - _cfg.processing_path_doc_root, // doc root - _cfg.http_request_type, // http - _cfg.http_host, // host / domain - _cfg.http_request_type, // data "http" or "https" - _cfg.http_host, // data domain "localhost" - _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org" - _cfg.processing_path_doc_root, // data root path - _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search" - _cfg.http_request_type, // cgi http - _cfg.http_host, // cgi host - _cfg.cgi_url_root, // cgi bin url - _cfg.cgi_bin_subpath, // cgi bin path - _cfg.cgi_bin_root, // cgi bin path - _cfg.cgi_filename, // cgi filename - _cfg.cgi_url_action, // cgi action - _cfg.db_sqlite_path, // sqlite db path - _cfg.db_sqlite_filename, // sqlite db filename -); - foreach(conf_fn; [_conf_file_details.config_filename_site]) { - foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) { - char[] conf_file; - conf_filename = conf_fn; - if (exists(pth)) { - auto f_attrib = pth.getLinkAttributes; - if ( - _conf_file_details.possible_config_path_locations.config_local_site.length == 1 - && f_attrib.attrIsFile - ) { - conf_file = pth.to!(char[]); - conf_filename = pth.baseName; - } else if (f_attrib.attrIsDir) { - conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; - conf_filename = conf_fn; - } - try { - if (exists(conf_file)) { - if (conf_file.getLinkAttributes.attrIsFile) { - if (_opt_action.vox_gt_2 || _opt_action.debug_do) { - writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)"); - } - config_file_str = conf_file.readText; - break; - } - } - } catch (ErrnoException ex) { - } catch (FileException ex) { - } - } - } - if (config_file_str.length > 0) { break; } - } - if (config_file_str.length > 0) { - import dyaml; - Node yaml_root; - try { - yaml_root = Loader.fromString(config_file_str).load(); - } catch (Throwable) { - import std.stdio; - writeln("ERROR failed to read config file content, not parsed as yaml, program default used"); - conf_filename = "VIRTUAL"; - config_file_str = default_config_file_str; - } - } - if (config_file_str.length == 0) { /+ use dummy default config file +/ - // writeln("WARNING config file NOT found, default provided"); - conf_filename = "VIRTUAL"; - config_file_str = default_config_file_str; - } - struct _ConfContent { - string filename() { - return conf_filename; - } - string filetype() { - string _ft = ""; - if (content.match(rgx.yaml_config)) { - _ft = "yaml"; - } - return _ft; - } - string content() { - return config_file_str; - } - } - return _ConfContent(); - } -} -#+END_SRC - -**** document make/config - -#+NAME: meta_config_file_hub_read_document_config -#+HEADER: :noweb yes -#+BEGIN_SRC d -static template readConfigDoc() { - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.meta.rgx_files; - import sisudoc.ocda.meta.rgx; - @system final auto readConfigDoc(M,E)(M _manifested, E _env) { - mixin spineRgxIn; - static auto rgx = RgxI(); - mixin spineRgxFiles; - static auto rgx_files = RgxFiles(); - string config_file_str; - string conf_filename = "NONE"; - auto _conf_file_details = configFilePaths!()(_manifested, _env); - string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; - foreach(conf_fn; [_conf_file_details.config_filename_document]) { - foreach(pth; possible_config_path_locations) { - char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; - conf_filename = conf_fn; - if (config_file_str.length > 0) { - break; - } - try { - if (exists(conf_file)) { - if (conf_file.getLinkAttributes.attrIsFile) { - config_file_str = conf_file.readText; - break; - } - } - } catch (ErrnoException ex) { - } catch (FileException ex) { - } - } - if (config_file_str.length > 0) { break; } - } - struct _ConfContent { - string filename() { - return conf_filename; - } - string content() { - return config_file_str; - } - string filetype() { - string _ft = ""; - if (content.match(rgx.yaml_config)) { - _ft = "yaml"; - } - return _ft; - } - } - return _ConfContent(); - } -} -#+END_SRC - -*** YAML config (config local site & dr document make) :file:config:hub: - -#+NAME: meta_config_file_hub_read_site_yaml -#+HEADER: :noweb yes -#+BEGIN_SRC d -static template configReadSiteYAML() { - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.meta.rgx_files; - import sisudoc.ocda.meta.rgx; - final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) { - string _configuration = configReadInSiteYAML!()(_manifested, _env); - auto _conf_file_details = configFilePaths!()(_manifested, _env); - string _conf_yaml_fn = _conf_file_details.config_filename_site; - YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); - return _yaml_conf; - } -} -static template configReadDocYAML() { - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) { - string _configuration = configReadInDocYAML!()(_manifested, _env); - auto _conf_file_details = configFilePaths!()(_manifested, _env); - string _conf_yaml_fn = _conf_file_details.config_filename_document; - YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); - return _yaml_conf; - } -} -#+END_SRC - -* B. get _markup source_, read file (.readText) [#A]:module:source_files: -** _module template_ - -#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -/++ - module source_read_source_files;
- - open markup files
- - if master file scan for addional files to import/insert -+/ -module sisudoc.ocda.io_in.read_source_files; -@safe: -template spineRawMarkupContent() { - import std.digest.sha; - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.meta.rgx_files; - import sisudoc.ocda.meta.rgx; - mixin spineRgxIn; - static auto rgx = RgxI(); - mixin spineRgxFiles; - static auto rgx_files = RgxFiles(); - string[] _images=[]; - /+ ↓ the images a markup source shows. - matched on the image markup itself ({ name.png }image, and the url and - ~^ forms), ensure a filename-shaped word in prose is not mistaken for - one, and over every match (so a line showing two images reports two). - +/ - string[] _extract_images(S)(S content_block) { - string[] images_; - string _content_block = content_block.to!string; - foreach (m; _content_block.matchAll(rgx.smid_image)) { - images_ ~= m["image"].to!string; - } - return images_; - } - auto rawsrc = RawMarkupContent(); - struct ST_contents_inserts_images { - char[][] contents; - string[] insert_files; - string[] images; - } - struct ST_header_content_inserts_images { - char[] header; - char[][] src_txt; - string[] insert_files; - string[] images; - } - struct ST_doc_digest { - ubyte[32] markup_doc; - ubyte[32] header; - ubyte[32] text; - } - struct ST_doc_parts { - char[] header_raw; - char[][] sourcefile_body_content; - string[] insert_file_list; - string[] images_list; - ST_doc_digest doc_digest; - } - ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) { - ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct - = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src); - return _0_header_1_body_content_2_insert_filelist_struct; - } - struct RawMarkupContent { - final sourceContent(in string fn_src) { - auto raw = MarkupRawUnit(); - string source_txt_str - = raw.markupSourceReadIn(fn_src); - return source_txt_str; - } - final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)( - O _opt_action, - in string source_txt_str, - in string fn_src="" - ) { - auto raw = MarkupRawUnit(); - string[] insert_file_list_get; - string[] images_list_get; - ST_header_content_inserts_images st - = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str); - char[] header_raw = st.header; - char[][] sourcefile_body_content = st.src_txt; - if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise - auto ins = Inserts(); - ST_contents_inserts_images _cii - = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); - sourcefile_body_content = _cii.contents; - insert_file_list_get = _cii.insert_files.dup; - images_list_get = _cii.images.dup; - } else if (_opt_action.source_or_pod) { - auto ins = Inserts(); - ST_contents_inserts_images _cii - = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); - images_list_get = _cii.images.dup; - } // image_list, if path could take sha256 digests already here? - string header_type = ""; - ST_doc_digest dig; - { - dig.markup_doc = source_txt_str.sha256Of; - dig.header = st.header.sha256Of; - dig.text = sourcefile_body_content.sha256Of; - } - ST_doc_parts ret; - { - ret.header_raw = st.header; - ret.sourcefile_body_content = sourcefile_body_content; - ret.insert_file_list = insert_file_list_get; - ret.images_list = images_list_get; - ret.doc_digest = dig; - } - return ret; - } - } - struct MarkupRawUnit { - import std.digest.sha; - import std.file; - <> - <> - <> - <> - <> - <> - } - struct Inserts { - struct ST_contents_and_images { - char[][] insert_contents; - string[] images; - } - ST_contents_and_images scan_subdoc_source(O)( - O _opt_action, - char[][] markup_sourcefile_insert_content, - string fn_src - ) { - <> - foreach (line; markup_sourcefile_insert_content) { - <> - } // end src subdoc (inserts) loop - <> - } - ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)( - O _opt_action, - char[][] sourcefile_body_content, - string fn_src - ) { - import std.algorithm; - <> - foreach (line; sourcefile_body_content) { - <> - } // end src doc loop - <> - } - } -} -#+END_SRC - -** get markup source, read file :source:markup: -*** read file, source string [#A] :string: - -#+NAME: meta_markup_source_raw_read_file_source_string -#+BEGIN_SRC d -final private string readInMarkupSource(in char[] fn_src) { - enforce( - exists(fn_src) != 0, - "file not found: «" ~ - fn_src ~ "»" - ); - string source_txt_str; - try { - if (exists(fn_src)) { - if (fn_src.getLinkAttributes.attrIsFile) { - source_txt_str = fn_src.readText; - } else { - } - } - } catch (ErrnoException ex) { - } catch (UTFException ex) { - // Handle validation errors - } catch (FileException ex) { - // Handle errors - } - std.utf.validate(source_txt_str); - return source_txt_str; -} -#+END_SRC - -*** document header & content, array.length == 2 [#A] :array: - -here you split document header and body, an array.length == 2 -split is on first match of level A~ (which is required) - -#+NAME: meta_markup_source_raw_doc_header_and_content_split -#+BEGIN_SRC d -@trusted final private char[][] header0Content1(in string src_text) { // cast(char[]) - /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ - char[][] header_and_content; - auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a); - header_and_content ~= m.pre; - header_and_content ~= m.hit ~ m.post; - assert(header_and_content.length == 2, - "document markup is broken, header body split == " - ~ header_and_content.length.to!string - ~ "; (header / body array split should == 2 (split is on level A~))" - ); - return header_and_content; -} -#+END_SRC - -*** source line array :array: - -#+NAME: meta_markup_source_raw_source_line_array -#+BEGIN_SRC d -@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[]) - char[][] source_line_arr - = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding); - return source_line_arr; -} -#+END_SRC - -*** source content raw line array :array: -- used for regular .sst files; master .ssm files and; .ssi inserts -- regex is passed for relevant enforce match - -**** read in file - -#+NAME: meta_markup_source_raw_read_in_file -#+BEGIN_SRC d -string markupSourceReadIn(in string fn_src) { - static auto rgx_files = RgxFiles(); - enforce( - fn_src.match(rgx_files.src_pth_sst_or_ssm), - "not a dr markup filename: «" ~ - fn_src ~ "»" - ); - string source_txt_str = readInMarkupSource(fn_src); - return source_txt_str; -} -#+END_SRC - -**** tuple (a) header, (b) body content, (c) file insert list & (d) image list? - -- header -- body content -- file insert list -- [image list?] - -#+NAME: meta_markup_source_raw_struct_of_header_and_body -#+BEGIN_SRC d -ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) { - string[] file_insert_list = []; - string[] images_list = []; - char[][] hc = header0Content1(source_txt_str); - char[] header = hc[0]; - char[] source_txt = hc[1]; - char[][] source_line_arr = markupSourceLineArray(source_txt); - ST_header_content_inserts_images ret; - { - ret.header = header; - ret.src_txt = source_line_arr; - ret.insert_files = file_insert_list; - ret.images = images_list; - } - return ret; -} -#+END_SRC - -**** get insert source line array - -#+NAME: meta_markup_source_raw_get_insert_source_line_array -#+BEGIN_SRC d -final char[][] getInsertMarkupSourceContentRawLineArray( - in char[] fn_src_insert, - Regex!(char) rgx_file -) { - enforce( - fn_src_insert.match(rgx_file), - "not a dr markup filename: «" ~ - fn_src_insert ~ "»" - ); - string source_txt_str = readInMarkupSource(fn_src_insert); - char[][] source_line_arr = markupSourceLineArray(source_txt_str); - return source_line_arr; -} -#+END_SRC - -** get markup source, master file & inserts :masterfile:inserts: - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -*** scan inserts (sub-document) source :scan_insert_src: -**** scan subdoc source - -#+NAME: meta_inserts_scan -#+BEGIN_SRC d -char[][] contents_insert; -int code_block_status = 0; -enum codeBlock { off, curly, tic, quotemarks } -auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); -auto markup_src_file_path = fn_pth_full.captures[1]; -#+END_SRC - -**** loop insert (sub-document) - -#+NAME: meta_inserts_scan_loop -#+BEGIN_SRC d -if (code_block_status == codeBlock.curly) { - if (line.matchFirst(rgx.block_curly_code_close)) { - code_block_status = codeBlock.off; - } - contents_insert ~= line; -} else if (line.matchFirst(rgx.block_curly_code_open)) { - code_block_status = codeBlock.curly; - contents_insert ~= line; -} else if (code_block_status == codeBlock.quotemarks) { - if (line.matchFirst(rgx.block_quotemarks_close)) { - code_block_status = codeBlock.off; - } - contents_insert ~= line; -} else if (code_block_status == codeBlock.tic) { - if (line.matchFirst(rgx.block_tic_close)) { - code_block_status = codeBlock.off; - } - contents_insert ~= line; -} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { - code_block_status = codeBlock.quotemarks; - contents_insert ~= line; -} else if (line.matchFirst(rgx.block_tic_code_open)) { - code_block_status = codeBlock.tic; - contents_insert ~= line; -} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { - auto insert_fn = m.captures[2]; - auto insert_sub_pth = m.captures[1]; - auto fn_src_insert - = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; - auto raw = MarkupRawUnit(); - auto markup_sourcesubfile_insert_content - = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); - debug(insert_file) { - writeln(line); - writeln(fn_src_insert); - writeln( - " length contents insert array: ", - markup_sourcesubfile_insert_content.length - ); - } - if (_opt_action.source_or_pod) { - _images ~= _extract_images(markup_sourcesubfile_insert_content); - } - auto ins = Inserts(); - /+ - - 1. load file - - 2. read lines - - 3. scan lines - - a. if filename insert, and insert filename - - repeat 1 - - b. else - - add line to new array; - - build image list, search for any image files to add to image list - +/ -} else { - contents_insert ~= line; // images to extract for image list? - if (_opt_action.source_or_pod) { - string[] _image_linelist = _extract_images(line); - if (_image_linelist.length > 0) { - _images ~= _image_linelist; - } - } -} -#+END_SRC - -**** post loop - -#+NAME: meta_inserts_scan_post -#+BEGIN_SRC d -ST_contents_and_images ret; -{ - ret.insert_contents = contents_insert; - ret.images = _images; -} -return ret; -#+END_SRC - -*** scan document source :scan_src: -**** scan doc source - -#+NAME: meta_master_doc_scan_for_insert_filenames -#+BEGIN_SRC d -char[][] contents; -int code_block_status = 0; -enum codeBlock { off, curly, tic, quotemarks } -auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); -auto markup_src_file_path = fn_pth_full.captures[1]; -char[][] contents_insert; -string[] _images =[]; -string[] insert_file_list =[]; -#+END_SRC - -**** include inserts: _loop master_ scan for inserts (insert documents) - -#+NAME: meta_master_doc_scan_for_insert_filenames_loop -#+BEGIN_SRC d -if (code_block_status == codeBlock.curly) { - if (line.matchFirst(rgx.block_curly_code_close)) { - code_block_status = codeBlock.off; - } - contents ~= line; -} else if (line.matchFirst(rgx.block_curly_code_open)) { - code_block_status = codeBlock.curly; - contents ~= line; -} else if (code_block_status == codeBlock.quotemarks) { - if (line.matchFirst(rgx.block_quotemarks_close)) { - code_block_status = codeBlock.off; - } - contents ~= line; -} else if (code_block_status == codeBlock.tic) { - if (line.matchFirst(rgx.block_tic_close)) { - code_block_status = codeBlock.off; - } - contents ~= line; -} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { - code_block_status = codeBlock.quotemarks; - contents ~= line; -} else if (line.matchFirst(rgx.block_tic_code_open)) { - code_block_status = codeBlock.tic; - contents ~= line; -} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { - auto insert_fn = m.captures[2]; - auto insert_sub_pth = m.captures[1]; - auto fn_src_insert - = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; - insert_file_list ~= fn_src_insert.to!string; - auto raw = MarkupRawUnit(); - /+ TODO +/ - auto markup_sourcefile_insert_content - = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); - debug(insert_file) { - writeln(line); - writeln(fn_src_insert); - writeln( - " length contents insert array: ", - markup_sourcefile_insert_content.length - ); - } - auto ins = Inserts(); - ST_contents_and_images contents_insert_st = ins.scan_subdoc_source( - _opt_action, - markup_sourcefile_insert_content, - fn_src_insert.to!string - ); - contents ~= contents_insert_st.insert_contents; - if (_opt_action.source_or_pod) { - string[] _image_linelist = _extract_images(contents_insert_st.images); - if (_image_linelist.length > 0) { - _images ~= _image_linelist; - } - } - /+ - - 1. load file - - 2. read lines - - 3. scan lines - - a. if filename insert, and insert filename - - repeat 1 - - b. else - - add line to new array; - - build image list, search for any image files to add to image list - +/ -} else { - contents ~= line; - if (_opt_action.source_or_pod) { - string[] _image_linelist = _extract_images(line); - if (_image_linelist.length > 0) { - _images ~= _image_linelist; - } - } -} -#+END_SRC - -**** post loop - -#+NAME: meta_master_doc_scan_for_insert_filenames_post -#+BEGIN_SRC d -string[] images = []; -foreach(i; uniq(_images.sort())) { - images ~= i; -} -debug(insert_file) { - writeln(__LINE__); - writeln(contents.length); -} -ST_contents_inserts_images ret; -{ - ret.contents = contents; - ret.insert_files = insert_file_list; - ret.images = images; -} -return ret; -#+END_SRC - -* C. read a .ssp file back into the document abstraction -** _module template_ :module:ssp_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp_in.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.ocda.abstraction.ssp_in; -@safe: -/+ ↓ read a .ssp file back into the document abstraction - . - the reverse of sisudoc.ocda.abstraction.ssp. what it returns is the same - value the parser produces, ObjGenericComposite[][string], so anything - that consumes the abstraction can be fed from a .ssp instead of from - markup. - . - the check that it is faithful is a round trip: load a .ssp, emit each - object again with sspObjectRecord (the writer's own definition of a - record) and require the result to be byte identical to the file read. -+/ -template spineAbstractionRead() { - import std.algorithm : startsWith, findSplit; - import std.array; - import std.conv : to; - import std.file; - import std.json : JSONValue; - import std.stdio; - import std.string; - import sisudoc.ocda.meta.metadoc_object_setter; - import sisudoc.ocda.abstraction.ssp : sspEscape; - mixin sspEscape; - mixin ObjectSetter; - /+ ↓ what a .ssp file holds: the three header blocks as key/value, and the - object sections in the order the file gives them - +/ - struct SSPdocument { - string format; // "% SiSU Document Abstraction v1.1" - string source; // "% Source: ..." - string[string] source_info; // the @source block - string[] source_info_order; - string ssp_digest; // sha256 of the .ssp read, hex - string[string] meta; - string[] meta_order; - string[string] make; - string[] make_order; - string[string] doc_has; - string[] doc_has_order; - string[] section_order; // as found in the file - ObjGenericComposite[][string] abstraction; - } - private int _levOfMarkedUp(string lev) { - switch (lev) { - case "A": return 0; - case "B": return 1; - case "C": return 2; - case "D": return 3; - case "1": return 4; - case "2": return 5; - case "3": return 6; - case "4": return 7; - default: return 9; - } - } - private int[] _ints(string val) { - int[] _out; - foreach (f; val.split(" ")) { - if (f.length > 0) { _out ~= f.to!int; } - } - return _out; - } - /+ ↓ the eight slot arrays are int[] with a literal default, so every - default constructed object shares one array; write into a fresh one - +/ - private int[8] _eight(string val) { - int[8] _out; - foreach (i, v; _ints(val)) { - if (i < _out.length) { _out[i] = v; } - } - return _out; - } - private ubyte[32] _hexToBytes(string hex) { - ubyte[32] _out; - if (hex.length < 64) { return _out; } - foreach (i; 0..32) { - _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); - } - return _out; - } - SSPdocument sspRead(string[] lines) { - SSPdocument doc; - string section; // the @section block currently open - string block; // "meta", "make", "doc_has", or "" for objects - bool in_object; - string[] text_lines; - ObjGenericComposite obj; - void _closeObject() { - if (!in_object) { return; } - obj.text = text_lines.join("\n"); - doc.abstraction[section] ~= obj; - obj = ObjGenericComposite(); - text_lines = []; - in_object = false; - } - foreach (line; lines) { - if (line.length == 0) { continue; } - if (line.startsWith("% SiSU Document Abstraction")) { - doc.format = line; continue; - } - if (line.startsWith("% Source: ")) { - doc.source = line["% Source: ".length .. $]; continue; - } - if (line.startsWith("@") && line.endsWith(" {")) { - _closeObject(); - string _name = line[1 .. $-2]; - switch (_name) { - case "source": case "meta": case "make": case "doc_has": - block = _name; section = ""; break; - default: - block = ""; section = _name; - doc.section_order ~= _name; - doc.abstraction[_name] = []; - break; - } - continue; - } - if (line == "}") { _closeObject(); block = ""; section = ""; continue; } - if (block.length > 0) { // a header block key: value line - auto _kv = line.stripLeft.findSplit(": "); // leading only: a value may end in a space - if (_kv[1].length > 0) { - final switch (block) { - case "source": - doc.source_info[_kv[0].to!string] = _kv[2].to!string; - doc.source_info_order ~= _kv[0].to!string; - break; - case "meta": - doc.meta[_kv[0].to!string] = _kv[2].to!string; - doc.meta_order ~= _kv[0].to!string; - break; - case "make": - doc.make[_kv[0].to!string] = _kv[2].to!string; - doc.make_order ~= _kv[0].to!string; - break; - case "doc_has": - doc.doc_has[_kv[0].to!string] = _kv[2].to!string; - doc.doc_has_order ~= _kv[0].to!string; - break; - } - } - continue; - } - if (line.startsWith("[")) { // object declaration line - _closeObject(); - in_object = true; - auto _close = line.indexOf("] "); - obj.metainfo.ocn = line[1 .. _close].to!int; - string _rest = line[_close + 2 .. $]; - if (_rest.startsWith("heading :")) { - obj.metainfo.is_a = "heading"; - string _tail = _rest["heading :".length .. $]; - auto _sp = _tail.indexOf(" "); - if (_sp > 0) { - obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail[0 .. _sp]); - obj.metainfo.identifier = _tail[_sp + 1 .. $]; - } else { - obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail); - /+ ↓ omitted means it equals the ocn, and for an object with no ocn - it was empty: "a"~N identifiers are always written - +/ - obj.metainfo.identifier = (obj.metainfo.ocn != 0) - ? obj.metainfo.ocn.to!string : ""; - } - } else { - obj.metainfo.is_a = _rest; - obj.metainfo.identifier = (obj.metainfo.ocn != 0) - ? obj.metainfo.ocn.to!string : ""; - } - obj.metainfo.is_of_section = section; - continue; - } - if (line.startsWith("| ")) { text_lines ~= line[2 .. $]; continue; } - if (!line.startsWith(".")) { continue; } - auto _p = line[1 .. $].findSplit(": "); - string _key = (_p[1].length > 0) ? _p[0].to!string : line[1 .. $].to!string; - string _val = (_p[1].length > 0) ? _p[2].to!string : ""; - switch (_key) { - case "identifier": obj.metainfo.identifier = _val; break; - case "part": obj.metainfo.is_of_part = _val; break; - case "section": obj.metainfo.is_of_section = _val; break; - case "parent": obj.metainfo.parent_ocn = _val.to!int; break; - case "last_descendant": obj.metainfo.last_descendant_ocn = _val.to!int; break; - case "children": obj.metainfo.children_headings = _ints(_val); break; - case "ancestors": - obj.metainfo.markedup_ancestors = _eight(_val); - break; - case "ancestors_collapsed": - obj.metainfo.collapsed_ancestors = _eight(_val); - break; - case "dom_status": - obj.metainfo.dom_structure_markedup_tags_status = _eight(_val); - break; - case "dom_status_collapsed": - obj.metainfo.dom_structure_collapsed_tags_status = _eight(_val); - break; - case "heading_lev_collapsed": obj.metainfo.heading_lev_collapsed = _val.to!int; break; - case "parent_lev": obj.metainfo.parent_lev_markup = _val.to!int; break; - case "dummy": obj.metainfo.dummy_heading = true; break; - case "ocn_off": obj.metainfo.object_number_off = true; break; - case "is_of_type": obj.metainfo.is_of_type = _val; break; - case "attrib": obj.metainfo.attrib = _val; break; - case "meta_lang": obj.metainfo.lang = _val; break; - case "meta_syntax": obj.metainfo.syntax = _val; break; - case "sha256": obj.metainfo.sha256.text = _hexToBytes(_val); break; - case "image": - { - ST_file_name_hash_size_ _img; - auto _fields = _val.split(" "); - if (_fields.length > 0) { _img.fileName = _fields[0].to!string; } - foreach (f; _fields[1 .. $]) { - if (f == "missing:true") { _img.fileMissing = true; } - else if (f.startsWith("sha256:")) { _img.fileHash_sha256 = _hexToBytes(f[7 .. $].to!string); } - else if (f.startsWith("bytes:")) { _img.fileSize = f[6 .. $].to!ulong; } - else if (f.startsWith("px:")) { - auto _wh = f[3 .. $].findSplit("x"); - if (_wh[1].length > 0) { - _img.imageWidth = _wh[0].to!int; - _img.imageHeight = _wh[2].to!int; - } - } - } - obj.metainfo.sha256.images ~= _img; - } - break; - case "indent": - { - auto _i = _ints(_val); - if (_i.length > 0) { obj.attrib.indent_base = _i[0]; } - if (_i.length > 1) { obj.attrib.indent_hang = _i[1]; } - } - break; - case "bullet": obj.attrib.bullet = true; break; - case "lang": obj.attrib.language = _val; break; - case "has": - foreach (f; _val.split(" ")) { - switch (f) { - case "links": obj.has.inline_links = true; break; - case "notes_reg": obj.has.inline_notes_reg = true; break; - case "notes_star": obj.has.inline_notes_star = true; break; - case "images": obj.has.images = true; break; - case "images_no_dim": obj.has.image_without_dimensions = true; break; - default: break; - } - } - break; - case "table_cols": obj.table.number_of_columns = _val.to!int; break; - case "table_widths": - foreach (w; _val.split(" ")) { obj.table.column_widths ~= w.to!double; } - break; - case "table_aligns": - foreach (a; _val.split(" ")) { obj.table.column_aligns ~= a.to!string; } - break; - case "table_header": obj.table.heading = true; break; - case "code_linenumbers": obj.code_block.linenumbers = true; break; - case "stow_link": obj.stow.link ~= _val; break; - case "segment": obj.tags.in_segment_html = _val; break; - case "anchor": obj.tags.anchor_tag_html = _val; break; - case "segment_prev": obj.tags.segname_prev = _val; break; - case "segment_next": obj.tags.segname_next = _val; break; - case "heading_lev_anchor": obj.tags.heading_lev_anchor_tag = _val; break; - case "segment_epub": obj.tags.segment_anchor_tag_epub = _val; break; - case "segment_html_is": obj.tags.html_segment_anchor_tag_is = _val; break; - case "segment_epub_is": obj.tags.epub_segment_anchor_tag_is = _val; break; - case "segment_lv4_is": obj.tags.segment_lv4_is = _val; break; - case "heading_ancestors_text": - { - string[8] _h; - foreach (i, h; sspUnescapeSplit(_val)) { - if (i < _h.length) { _h[i] = h; } - } - obj.tags.heading_ancestors_text = _h; - } - break; - case "lev4_subtoc": obj.tags.lev4_subtoc ~= _val; break; - case "anchor_tag": obj.tags.anchor_tags ~= _val; break; - default: break; - } - } - _closeObject(); - /+ ↓ .anchor is written only when it differs from .segment, and .section - only when it differs from the enclosing block; restore both - +/ - foreach (sect; doc.section_order) { - foreach (ref o; doc.abstraction[sect]) { - if (o.tags.anchor_tag_html.length == 0 - && o.tags.in_segment_html.length > 0) { - o.tags.anchor_tag_html = o.tags.in_segment_html; - } - } - } - return doc; - } - /+ ↓ the whole document as it survives a trip through the .ssp text: - emitted as lines by the writer, read straight back. what a consumer - built from this cannot see, the .ssp does not carry, which is what - makes a second artefact built from it unable to drift. the header - blocks come back with it, so the database is filled from these and - not from a second list read off doc_matters - +/ - SSPdocument sspRoundTripDocument(D)(D doc) { - import sisudoc.ocda.abstraction.ssp; - mixin spineAbstractionTxt; - string[] _lines = sspDocumentLines(doc); - auto _out = sspRead(_lines); - /+ ↓ the writer emits one line per entry, each with its newline, so this - is the digest of the file that is written beside the pod +/ - _out.ssp_digest = sspDigestOfText(_lines.join("\n") ~ "\n"); - return _out; - } - /+ ↓ sha256 of a .ssp, upper case hex, as digests.txt gives it +/ - string sspDigestOfText(string _text) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - return _text.sha256Of.toHexString.to!string; - } - auto sspRoundTripAbstraction(D)(D doc) { - return sspRoundTripDocument(doc).abstraction; - } - /+ ↓ the version a reader will accept. - - the major part must match: a format whose properties have changed - meaning is not readable by guesswork, and loading it half populated in - silence is the failure this check exists to stop. a newer minor - version is accepted, since a minor bump only adds properties and an - unknown property line is ignored. - +/ - bool sspFormatVersionOk(string _format_line, string _what) { - import sisudoc.ocda.abstraction.ssp : ssp_format_version, ssp_format_version_major; - enum _prefix = "% SiSU Document Abstraction v"; - if (_format_line.length == 0) { - writeln("ERROR: ", _what, ": no format line, not a .ssp"); - return false; - } - if (!_format_line.startsWith(_prefix)) { - writeln("ERROR: ", _what, ": unrecognised format line: ", _format_line); - return false; - } - string _ver = _format_line[_prefix.length .. $].strip; - string _major = _ver.findSplit(".")[0]; - if (_major != ssp_format_version_major) { - writeln("ERROR: ", _what, ": .ssp format v", _ver, - ", this spine reads v", ssp_format_version_major, ".x (v", - ssp_format_version, "). Regenerate the abstraction from its markup."); - return false; - } - return true; - } - /+ ↓ with_digest is off by default: hashing the file is only wanted where - the digest is going to be recorded, and on the largest document in the - sample set it is a tenth of what reading the file costs - +/ - @trusted SSPdocument sspReadFile(string file_path, bool with_digest = false) { - string _text; - string[] _lines; - try { - _text = (cast(char[]) file_path.read).to!string; - _lines = _text.split("\n"); - } catch (Exception ex) { - writeln("ERROR: could not read ", file_path, ": ", ex.msg); - return SSPdocument(); - } - auto _doc = sspRead(_lines); - if (!sspFormatVersionOk(_doc.format, file_path)) { return SSPdocument(); } - if (with_digest) { _doc.ssp_digest = sspDigestOfText(_text); } - return _doc; - } -} -#+END_SRC - -* D. read a .db file into the document abstraction -** _module template_ :module:db_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/db_in.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.ocda.abstraction.db_in; -@safe: -/+ ↓ read a .ocda.db back into the document abstraction - - the counterpart of sisudoc.outputs.io_out.create_ocda, and the - sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, - so a consumer does not have to know which of the two artefacts it was - handed. - - the check is the same one: read a database, emit it as .ssp with the - writer's own record definition, and require the result to equal the .ssp - written from the same document. -+/ -template spineAbstractionDbRead() { - import std.conv : to; - import std.file; - import std.stdio; - import std.string; - import std.array; - import std.json; - import d2sqlite3; - import sisudoc.ocda.abstraction.ssp_in : spineAbstractionRead; - mixin spineAbstractionRead; // for SSPdocument, and the shared unescape - @trusted private int[8] _jsonEight(string _s) { - int[8] _out; - if (_s.length == 0) { return _out; } - try { - auto _j = parseJSON(_s); - foreach (i, v; _j.array) { - if (i < _out.length) { _out[i] = v.integer.to!int; } - } - } catch (Exception ex) {} - return _out; - } - @trusted private int[] _jsonInts(string _s) { - int[] _out; - if (_s.length == 0) { return _out; } - try { - auto _j = parseJSON(_s); - foreach (v; _j.array) { _out ~= v.integer.to!int; } - } catch (Exception ex) {} - return _out; - } - @trusted private string[] _jsonStrs(string _s) { - string[] _out; - if (_s.length == 0) { return _out; } - try { - auto _j = parseJSON(_s); - foreach (v; _j.array) { _out ~= v.str; } - } catch (Exception ex) {} - return _out; - } - @trusted private double[] _jsonDoubles(string _s) { - double[] _out; - if (_s.length == 0) { return _out; } - try { - auto _j = parseJSON(_s); - foreach (v; _j.array) { - _out ~= (v.type == JSONType.integer) ? v.integer.to!double : v.floating; - } - } catch (Exception ex) {} - return _out; - } - private ubyte[32] _hex32(string hex) { - ubyte[32] _out; - if (hex.length < 64) { return _out; } - foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } - return _out; - } - private string _levMarkedUp(int lev) { - switch (lev) { - case 0: return "A"; - case 1: return "B"; - case 2: return "C"; - case 3: return "D"; - case 4: return "1"; - case 5: return "2"; - case 6: return "3"; - case 7: return "4"; - default: return ""; - } - } - @trusted SSPdocument dbReadFile(string db_file) { - SSPdocument doc; - if (!db_file.exists) { - writeln("ERROR: no such file: ", db_file); - return doc; - } - auto db = Database(db_file, SQLITE_OPEN_READONLY); - /+ ↓ the four header blocks, from the metadata table. the key prefix - says which block a row belongs to, and insertion order is kept +/ - foreach (row; db.execute("SELECT key, value FROM metadata ORDER BY rowid")) { - string _k = row["key"].as!string; - string _v = row["value"].as!string; - if (_k.startsWith("make.")) { - string _kk = _k["make.".length .. $]; - doc.make[_kk] = _v; - doc.make_order ~= _kk; - } else if (_k.startsWith("doc_has.")) { - string _kk = _k["doc_has.".length .. $]; - doc.doc_has[_kk] = _v; - doc.doc_has_order ~= _kk; - } else if (_k.startsWith("schema.")) { - if (_k == "schema.version") { - doc.format = "% SiSU Document Abstraction v" ~ _v; - } - } else if (_k == "source.filename") { - doc.source = _v; - } else if (_k == "source.ssp_digest") { - /+ ↓ this database's own note of the .ssp it was built from. it is - not part of the .ssp, so it does not go back into the block +/ - doc.ssp_digest = _v; - } else if (_k.startsWith("source.")) { - string _kk = _k["source.".length .. $]; - doc.source_info[_kk] = _v; - doc.source_info_order ~= _kk; - } else { - doc.meta[_k] = _v; - doc.meta_order ~= _k; - } - } - if (!sspFormatVersionOk(doc.format, db_file)) { return SSPdocument(); } - /+ ↓ the per object lists, gathered first, one sweep of each table. - - these four tables hold what a single object row cannot: the images - an object carries, the links stowed off it, its anchor tags and its - lev4 subtoc entries. They are read here in four ordered passes and - kept by object id, rather than queried per object as the objects are - built. The cost is then what the tables hold, not what the document - holds: war and peace has 12,135 objects and 138 rows across all - four, which as four statements per object was 48,540 statement - preparations and made reading a .ocda.db slower than parsing the - markup it came from. - +/ - ST_file_name_hash_size_[][long] _images_by_id; - string[][long] _links_by_id; - string[][long] _anchors_by_id; - string[][long] _subtoc_by_id; - foreach (r; db.execute( - "SELECT object_id, name, bytes, sha256, width, height, missing" - ~ " FROM object_images ORDER BY object_id, seq") - ) { - ST_file_name_hash_size_ _img; - _img.fileName = r["name"].as!string; - _img.fileSize = r["bytes"].as!ulong; - _img.fileHash_sha256 = _hex32(r["sha256"].as!string); - _img.imageWidth = r["width"].as!int; - _img.imageHeight = r["height"].as!int; - _img.fileMissing = (r["missing"].as!int == 1); - _images_by_id[r["object_id"].as!long] ~= _img; - } - foreach (r; db.execute( - "SELECT object_id, url FROM object_links ORDER BY object_id, seq") - ) { _links_by_id[r["object_id"].as!long] ~= r["url"].as!string; } - foreach (r; db.execute( - "SELECT object_id, anchor FROM object_anchors ORDER BY object_id, seq") - ) { _anchors_by_id[r["object_id"].as!long] ~= r["anchor"].as!string; } - foreach (r; db.execute( - "SELECT object_id, entry FROM object_subtoc ORDER BY object_id, seq") - ) { _subtoc_by_id[r["object_id"].as!long] ~= r["entry"].as!string; } - /+ ↓ the objects, section by section, in the order they were written +/ - string[] _sections; - foreach (row; db.execute( - "SELECT section FROM objects GROUP BY section ORDER BY MIN(id)") - ) { - _sections ~= row["section"].as!string; - } - foreach (section; _sections) { - doc.section_order ~= section; - doc.abstraction[section] = []; - /+ ↓ column name to index, resolved once for the statement. - - d2sqlite3's row["name"] is indexForName, a linear scan over the - columns that calls sqlite3_column_name and allocates a D string - for each one it passes. With some fifty named reads per object - over forty columns that is around a thousand of those per object, - and it was the larger half of what made reading a database slow. - Resolved once here, the reads below are an integer index. - +/ - int[string] _col; - foreach (row; db.execute( - "SELECT * FROM objects WHERE section = '" ~ section ~ "' ORDER BY seq") - ) { - if (_col.length == 0) { - foreach (_i; 0 .. row.length) { _col[row.columnName(_i)] = _i.to!int; } - } - ObjGenericComposite obj; - long _id = row[_col["id"]].as!long; - obj.metainfo.ocn = row[_col["ocn"]].as!int; - obj.metainfo.is_a = row[_col["is_a"]].as!string; - obj.metainfo.is_of_part = row[_col["is_of_part"]].as!string; - obj.metainfo.is_of_section = (row[_col["is_of_section"]].as!string.length > 0) - ? row[_col["is_of_section"]].as!string : section; - obj.metainfo.is_of_type = row[_col["is_of_type"]].as!string; - obj.metainfo.identifier = row[_col["identifier"]].as!string; - obj.metainfo.heading_lev_markup = (obj.metainfo.is_a == "heading") - ? row[_col["heading_level"]].as!int : 9; - obj.metainfo.heading_lev_collapsed = (row[_col["heading_lev_collapsed"]].as!string.length > 0) - ? row[_col["heading_lev_collapsed"]].as!int : 9; - obj.metainfo.parent_ocn = row[_col["parent_ocn"]].as!int; - obj.metainfo.parent_lev_markup = row[_col["parent_lev"]].as!int; - obj.metainfo.last_descendant_ocn = row[_col["last_descendant_ocn"]].as!int; - obj.metainfo.children_headings = _jsonInts(row[_col["children"]].as!string); - obj.metainfo.markedup_ancestors = _jsonEight(row[_col["ancestors"]].as!string); - obj.metainfo.collapsed_ancestors = _jsonEight(row[_col["ancestors_collapsed"]].as!string); - obj.metainfo.dom_structure_markedup_tags_status - = _jsonEight(row[_col["dom_status"]].as!string); - obj.metainfo.dom_structure_collapsed_tags_status - = _jsonEight(row[_col["dom_status_collapsed"]].as!string); - obj.metainfo.dummy_heading = (row[_col["dummy_heading"]].as!int == 1); - obj.metainfo.object_number_off = (row[_col["object_number_off"]].as!int == 1); - obj.metainfo.attrib = row[_col["attrib"]].as!string; - obj.metainfo.lang = row[_col["meta_lang"]].as!string; - obj.metainfo.syntax = row[_col["meta_syntax"]].as!string; - obj.metainfo.sha256.text = _hex32(row[_col["sha256"]].as!string); - obj.attrib.indent_base = row[_col["indent_base"]].as!int; - obj.attrib.indent_hang = row[_col["indent_hang"]].as!int; - obj.attrib.bullet = (row[_col["bullet"]].as!int == 1); - obj.attrib.language = row[_col["lang"]].as!string; - obj.has.inline_links = (row[_col["has_links"]].as!int == 1); - obj.has.inline_notes_reg = (row[_col["has_notes_reg"]].as!int == 1); - obj.has.inline_notes_star = (row[_col["has_notes_star"]].as!int == 1); - obj.has.images = (row[_col["has_images"]].as!int == 1); - obj.has.image_without_dimensions = (row[_col["has_images_no_dim"]].as!int == 1); - obj.tags.in_segment_html = row[_col["segment"]].as!string; - obj.tags.segname_prev = row[_col["segment_prev"]].as!string; - obj.tags.segname_next = row[_col["segment_next"]].as!string; - obj.tags.segment_anchor_tag_epub = row[_col["segment_epub"]].as!string; - obj.tags.html_segment_anchor_tag_is = row[_col["segment_html_is"]].as!string; - obj.tags.epub_segment_anchor_tag_is = row[_col["segment_epub_is"]].as!string; - obj.tags.segment_lv4_is = row[_col["segment_lv4_is"]].as!string; - obj.tags.anchor_tag_html = row[_col["anchor"]].as!string; - obj.tags.heading_lev_anchor_tag = row[_col["heading_lev_anchor"]].as!string; - { - string[8] _h; - foreach (i, h; _jsonStrs(row[_col["heading_ancestors_text"]].as!string)) { - if (i < _h.length) { _h[i] = h; } - } - obj.tags.heading_ancestors_text = _h; - } - if (obj.metainfo.is_a == "table") { - obj.table.number_of_columns = row[_col["table_cols"]].as!int; - obj.table.column_widths = _jsonDoubles(row[_col["table_widths"]].as!string); - obj.table.column_aligns = _jsonStrs(row[_col["table_aligns"]].as!string); - obj.table.heading = (row[_col["table_header"]].as!int == 1); - } - obj.code_block.linenumbers = (row[_col["code_linenumbers"]].as!int == 1); - obj.text = row[_col["text"]].as!string; - /+ ↓ the per object lists, from the sweeps above +/ - if (auto _v = _id in _images_by_id) { obj.metainfo.sha256.images = *_v; } - if (auto _v = _id in _links_by_id) { obj.stow.link = *_v; } - if (auto _v = _id in _anchors_by_id) { obj.tags.anchor_tags = *_v; } - if (auto _v = _id in _subtoc_by_id) { obj.tags.lev4_subtoc = *_v; } - doc.abstraction[section] ~= obj; - } - } - return doc; - } - /+ ↓ the files a database carries. - - a .ocda.db holds its images as blobs, which is what lets it travel on - its own where a .ssp cannot: a .ssp describes its images by name, - size, pixels and sha256, but does not carry them. The role column - says what a file is; today only 'image' is written. - +/ - struct ST_ArtefactFile { - string role; - string name; - ulong bytes; - string sha256; - ubyte[] data; - } - @trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") { - ST_ArtefactFile[] _out; - if (!db_file.exists) { return _out; } - auto db = Database(db_file, SQLITE_OPEN_READONLY); - foreach (row; db.execute( - "SELECT role, name, bytes, sha256, data FROM files WHERE role = '" ~ role - ~ "' ORDER BY name") - ) { - ST_ArtefactFile _f; - _f.role = row["role"].as!string; - _f.name = row["name"].as!string; - _f.bytes = row["bytes"].as!ulong; - _f.sha256 = row["sha256"].as!string; - _f.data = row["data"].as!(ubyte[]); - _out ~= _f; - } - return _out; - } -} -#+END_SRC - -* E. read a .db file into the document abstraction -** _module template_ :module:db_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.ocda.abstraction.load; -@safe: -/+ ↓ one way in, whatever the document is being read from - spine's pipeline is markup -> abstraction -> output. once the abstraction - is serialised, there is more than one thing an abstraction can be read - from, and a consumer should not have to know which it was handed: - . - .sst / .ssm + images the markup source - pod (dir) + images the same, bundled - pod .zip the same, zipped - .ssp + images the abstraction, as text - .ocda.db the abstraction, sqlite, images inside - . - A source may also be remote. A URL ending in .zip or in .ocda.db is - downloaded to a temp file (downloadSourceUrl in - ocda/io_in/read_zip_pod.d, guarded by rgx_url_source and by - --allow-downloads) and the local path is processed in its place. - - Those two travel because each is self-sufficient: a pod holds its - source and images, a .ocda.db holds its abstraction and images. A .ssp - does not, describing its images by name and digest without carrying - them, so it would arrive without them and is not fetched. - . - This module names those sources, tells them apart, and loads the two - that are self-describing artefacts, returning the value the parser - produces. - . - The three source forms are deliberately *not* loaded here. They are the - parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs - the environment, the options and the configuration that spine.d - assembles, none of which belongs in a loader. What this module gives - that case is the dispatch and a plain statement of where to go. -+/ -template spineAbstractionLoad() { - import std.algorithm : endsWith; - import std.conv : to; - import std.file; - import std.path; - import std.stdio; - import std.string; - import sisudoc.ocda.abstraction.db_in; - mixin spineAbstractionDbRead; // brings the .ssp reader with it - - enum AbstractionSource { - unknown, - markup, // .sst or .ssm, with its images beside it - pod, // a directory holding pod.manifest - pod_zip, // that directory, zipped - ssp, // .ssp, the abstraction as text, images beside it - ocda_db, // .ocda.db, the abstraction as sqlite, images inside it - } - - string abstractionSourceName(AbstractionSource _s) { - final switch (_s) { - case AbstractionSource.unknown: return "unknown"; - case AbstractionSource.markup: return "markup source (.sst/.ssm)"; - case AbstractionSource.pod: return "pod directory"; - case AbstractionSource.pod_zip: return "pod zip"; - case AbstractionSource.ssp: return ".ssp (abstraction as text)"; - case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)"; - } - } - - /+ ↓ what is this? by name, and for a directory by what it holds +/ - AbstractionSource abstractionSourceOf(string _path) { - if (_path.length == 0) { return AbstractionSource.unknown; } - if (_path.isValidPath && _path.exists && _path.isDir) { - return (_path.chainPath("pod.manifest").array.exists) - ? AbstractionSource.pod : AbstractionSource.unknown; - } - if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } - if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } - if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } - if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } - if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } - return AbstractionSource.unknown; - } - - struct LoadedAbstraction { - AbstractionSource source; - string path; - bool loaded; // is .doc filled - string note; // why not, when it is not - SSPdocument doc; - } - - /+ ↓ load what can be loaded from the path alone. - - .ssp and .ocda.db come back filled. the three source forms come back - with loaded = false and a note saying where they are handled, because - reading them needs the manifest, environment and configuration that - spine.d builds, not a file path. - +/ - LoadedAbstraction abstractionLoad(string _path) { - LoadedAbstraction _out; - _out.path = _path; - _out.source = abstractionSourceOf(_path); - if (!_path.exists) { - _out.note = "no such file or directory"; - return _out; - } - final switch (_out.source) { - case AbstractionSource.ssp: - _out.doc = sspReadFile(_path); - _out.loaded = (_out.doc.section_order.length > 0); - if (!_out.loaded) { _out.note = "no object sections found"; } - break; - case AbstractionSource.ocda_db: - _out.doc = dbReadFile(_path); - _out.loaded = (_out.doc.section_order.length > 0); - if (!_out.loaded) { _out.note = "no object sections found"; } - break; - case AbstractionSource.markup: - case AbstractionSource.pod: - case AbstractionSource.pod_zip: - _out.note = "a source form: read by the parser," - ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the" - ~ " manifest, environment and configuration spine assembles"; - break; - case AbstractionSource.unknown: - _out.note = "not a document source spine knows"; - break; - } - return _out; - } - - /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a - given artefact really does hold what it should +/ - string[] abstractionLoadSummary(LoadedAbstraction _l) { - string[] _out; - _out ~= "source: " ~ abstractionSourceName(_l.source); - _out ~= "path: " ~ _l.path; - if (!_l.loaded) { - _out ~= "loaded: no"; - if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; } - return _out; - } - _out ~= "loaded: yes"; - if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; } - if ("title.main" in _l.doc.meta) { - _out ~= "title: " ~ _l.doc.meta["title.main"]; - } - if ("creator.author" in _l.doc.meta) { - _out ~= "author: " ~ _l.doc.meta["creator.author"]; - } - _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, " - ~ _l.doc.make.length.to!string ~ " make, " - ~ _l.doc.doc_has.length.to!string ~ " doc_has"; - int _objs, _citable, _headings; - string[] _sections; - foreach (section; _l.doc.section_order) { - int _n; - foreach (obj; _l.doc.abstraction[section]) { - ++_objs; ++_n; - if (obj.metainfo.ocn > 0) { ++_citable; } - if (obj.metainfo.is_a == "heading") { ++_headings; } - } - _sections ~= section ~ " " ~ _n.to!string; - } - _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string - ~ " citable, " ~ _headings.to!string ~ " headings)"; - _out ~= "sections: " ~ _sections.join(", "); - return _out; - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/ocda_get.org b/org/ocda_get.org deleted file mode 100644 index 40effc5..0000000 --- a/org/ocda_get.org +++ /dev/null @@ -1,284 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) object-centric document abstraction -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:abstraction: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* Get OCD - -get abstracted objects for downstream processing - -** _module template_ - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_has.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.ocda.abstraction.doc_has; -@safe: -/+ ↓ ST_DocHas from a loaded abstraction - - the parser builds ST_DocHas as it goes, with the markup in front of it. - A document loaded from a .ssp or a .ocda.db has to arrive at the same - value from what the artefact carries, and this is where that is done. - - Every one of these is an *index over properties the artefact already - holds*, not a re-derivation of something it does not: - - imagelist the .image records' filenames - segnames_lv4 .anchor of each body heading at level 4 - segnames_lv_0_to_4 .segment_epub of each heading at level 4 or less - tag_associations .anchor, .anchor_tag and .segment_* per object - section_keys_sequenced which sections are non-empty, plus the run's - own output flags - - That distinction is the line the .ssp skill draws: assembling an index - is fine and cannot drift, because a wrong index would not survive the - round trip that produced its inputs; recomputing dom_status or ancestors - would be re-derivation and is exactly what ocda exists to remove. If a - value is not in the file and cannot be indexed out of what is, it must - be added to the artefact instead of reconstructed here. - - The counts come from the @doc_has block rather than being recounted, for - the same reason. -+/ -template spineDocHasFromAbstraction() { - import std.algorithm : sort, uniq; - import std.array; - import std.conv : to; - import std.json : JSONValue; - import std.regex; - import sisudoc.ocda.meta.metadoc_object_setter; - import sisudoc.ocda.meta.rgx; - mixin ObjectSetter; - mixin spineRgxIn; - /+ ↓ the sections, in the order a document holds them +/ - enum string[] doc_sections = [ - "head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", "tail", - ]; - /+ ↓ the tail sections that become segments of their own, in the order - after_doc_determine_segnames appends them +/ - enum string[] tail_sections = [ - "endnotes", "glossary", "bibliography", "bookindex", "blurb", - ]; - private uint _count(string[string] _doc_has, string _key) { - if (auto _v = _key in _doc_has) { - try { return (*_v).to!uint; } catch (Exception ex) { return 0; } - } - return 0; - } - private bool _has_section(A)(A _abst, string _section) { - if (auto _s = _section in _abst) { return (*_s).length > 1; } - return false; - } - /+ ↓ which document sections each output format walks, and in what order. - - the same rule the parser applies: the three that are always there, - then each tail section that the document actually has, then "tail" - for the formats that close with one. It depends on the run as well as - on the document, since only html and epub take the tail. - +/ - string[][string] sectionKeysSequenced(A,O)(A _abst, O _opt_action) { - string[][string] _keys = [ - "scroll": ["head", "toc", "body",], - "seg": ["head", "toc", "body",], - "sql": ["head", "body",], - "latex": ["head", "toc", "body",], - "text": ["head", "toc", "body",], - ]; - if (_has_section(_abst, "endnotes")) { - _keys["scroll"] ~= "endnotes"; - _keys["seg"] ~= "endnotes"; - _keys["latex"] ~= "endnotes"; - _keys["text"] ~= "endnotes"; - } - if (_has_section(_abst, "glossary")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "glossary"; } - } - if (_has_section(_abst, "bibliography")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bibliography"; } - } - if (_has_section(_abst, "bookindex")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bookindex"; } - } - if (_has_section(_abst, "blurb")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "blurb"; } - } - if (_opt_action.html_scroll || _opt_action.html_seg || _opt_action.epub) { - _keys["scroll"] ~= "tail"; - _keys["seg"] ~= "tail"; - } - return _keys; - } - /+ ↓ the whole of it +/ - ST_DocHas docHasFromAbstraction(A,O)( - A _abst, - string[string] _doc_has_block, - O _opt_action, - ) { - static auto rgx = RgxI(); - ST_DocHas _out; - _out.inline_links = _count(_doc_has_block, "inline_links"); - _out.inline_notes_reg = _count(_doc_has_block, "inline_notes_reg"); - _out.inline_notes_star = _count(_doc_has_block, "inline_notes_star"); - _out.codeblocks = _count(_doc_has_block, "codeblocks"); - _out.tables = _count(_doc_has_block, "tables"); - _out.blocks = _count(_doc_has_block, "blocks"); - _out.groups = _count(_doc_has_block, "groups"); - _out.poems = _count(_doc_has_block, "poems"); - _out.quotes = _count(_doc_has_block, "quotes"); - string[] _images; - string[string][string] _tag_assoc; - /+ ↓ segnames["html"] is seeded with the toc, which is a segment of its - own before any level 4 heading opens one +/ - _out.segnames_lv4 ~= "toc"; - foreach (_section; doc_sections) { - if (_section !in _abst) { continue; } - /+ ↓ head is always walked; every other section only when it holds - more than the placeholder object, which is the guard the parser - puts on each of its section loops - +/ - if (_section != "head" && !_has_section(_abst, _section)) { continue; } - foreach (obj; _abst[_section]) { - /+ ↓ the images this object names, as the abstraction recorded them +/ - foreach (_img; obj.metainfo.sha256.images) { _images ~= _img.fileName; } - /+ ↓ every object: its html anchor is in a segment, and its epub - segment anchor stands for itself +/ - if (obj.tags.anchor_tag_html.length > 0) { - _tag_assoc[obj.tags.anchor_tag_html]["seg_lv4"] = obj.tags.in_segment_html; - } - if (obj.tags.segment_anchor_tag_epub.length > 0) { - _tag_assoc[obj.tags.segment_anchor_tag_epub]["seg_lv1to4"] - = obj.tags.segment_anchor_tag_epub; - } - /+ ↓ a heading's own anchor tags, and the anchors declared inline in - an object's text, both point at the segment the object is in +/ - /+ ↓ a heading above level 4 opens no html segment of its own, so a - cross reference to it has to land on the level 4 segment that - follows it. .segment is exactly that (the segment a heading - opens or falls into), where .segment_html_is is the segment an - object sits in; epub segments go down to level 1, so there - .segment_epub stands for the heading itself - +/ - if (obj.tags.heading_lev_anchor_tag.length > 0 - && obj.tags.in_segment_html.length > 0 - && obj.metainfo.heading_lev_markup >= 4 - ) { - _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv4"] - = obj.tags.in_segment_html; - if (obj.tags.segment_anchor_tag_epub.length > 0) { - _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv1to4"] - = obj.tags.segment_anchor_tag_epub; - } - } - foreach (m; obj.text.matchAll(rgx.inline_link_anchor)) { - string _a = m["anchor"].to!string; - if (_a.length == 0 || _a in _tag_assoc) { continue; } - _tag_assoc[_a]["seg_lv4"] = obj.tags.html_segment_anchor_tag_is; - _tag_assoc[_a]["seg_lv1to4"] = obj.tags.epub_segment_anchor_tag_is; - } - /+ ↓ the segment name lists, in document order +/ - if (obj.metainfo.is_a == "heading") { - if (obj.metainfo.heading_lev_markup <= 4) { - _out.segnames_lv_0_to_4 ~= obj.tags.segment_anchor_tag_epub; - } - if (_section == "body" && obj.metainfo.heading_lev_markup == 4) { - _out.segnames_lv4 ~= obj.tags.anchor_tag_html; - } - /+ ↓ a heading above level 4 opens no html segment of its own, so - a cross reference to it lands on the level 4 segment that - follows. That is .segment_lv4_is, resolved in ocda where the - finished sections are to hand and carried by the artefact, so - there is nothing to work out here - +/ - if (obj.tags.segment_lv4_is.length > 0) { - /+ ↓ seg_lv4 only, and not through heading_lev_anchor_tag: that - names the level 4 heading itself, whose own entry says it - opens that segment rather than merely sits in it. epub - segments go down to level 1, so seg_lv1to4 is untouched - +/ - foreach (_k; [obj.metainfo.identifier, obj.tags.anchor_tag_html]) { - if (_k.length > 0) { _tag_assoc[_k]["seg_lv4"] = obj.tags.segment_lv4_is; } - } - } - } - } - } - foreach (_section; tail_sections) { - if (_has_section(_abst, _section)) { _out.segnames_lv4 ~= _section; } - } - /+ ↓ the body's objects, in a pass of their own after every anchor above - has been recorded, because the parser does it in that order and the - order is what decides two of these entries. - . - each body object's identifier maps to the segment it is in. seg_lv4 - is set only where it is not already, an anchor of the same name - having said it *names* a segment rather than sits in one; seg_lv1to4 - is set always. A heading's own anchor can be a bare number taken - from its text ("2. Shorter Terms" gives the anchor "2"), which then - collides with the ocn of an unrelated object, and this is the order - that resolves the collision the way the parser resolves it: in - free_culture, ocn 2 keeps seg_lv4 "them" from the anchor and takes - seg_lv1to4 "_part_1" from itself. - +/ - if (_has_section(_abst, "body")) { - foreach (obj; _abst["body"]) { - if (obj.metainfo.identifier.length == 0) { continue; } - if (!((obj.metainfo.identifier in _tag_assoc) - && ("seg_lv4" in _tag_assoc[obj.metainfo.identifier])) - ) { - _tag_assoc[obj.metainfo.identifier]["seg_lv4"] - = obj.tags.html_segment_anchor_tag_is; - } - _tag_assoc[obj.metainfo.identifier]["seg_lv1to4"] - = obj.tags.epub_segment_anchor_tag_is; - } - } - _out.imagelist = _images.sort.uniq.array; - _out.tag_associations = _tag_assoc; - _out.section_keys_sequenced = sectionKeysSequenced(_abst, _opt_action); - return _out; - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/out_ocda_peg_ssp.org b/org/out_ocda_peg_ssp.org new file mode 100644 index 0000000..511c589 --- /dev/null +++ b/org/out_ocda_peg_ssp.org @@ -0,0 +1,736 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) output pod source abstraction peg text +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:output:source:pod: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] + +* (Object-Centric) Document Abstraction PEG text + +- Process markup document, create document abstraction + +** _module template_ :module:metadoc_from_src: + +rename source_abstraction_peg_txt.d + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.ocda.abstraction.ssp; +@safe: +/+ ↓ write document abstraction as human-readable .ssp text file +/ +/+ ↓ the format version, carried by both serialisations: the .ssp says it in + its "% SiSU Document Abstraction v" header line and the .ocda.db in its + metadata schema.version row, so that one bump moves both. + + a reader checks the major part and refuses what it does not know, rather + than loading an older artefact half populated in silence. bump the minor + part when a property is added (a reader of the newer format still reads + the older one), the major part when one changes meaning or goes away. + + 1.1 2026-09-10 @source block; the document metadata the output writers + read carried in full (14 meta and 3 make properties + added); .segment_lv4_is, the segment a cross + reference to a heading above level 4 lands on; + was 0.1 ++/ +enum ssp_format_version = "1.1"; +enum ssp_format_version_major = "1"; +/+ ↓ escaping for values that are folded onto one line + + a .ssp property value runs to the end of its line, and + .heading_ancestors_text joins its eight slots with "|", so a slot that + itself contains "|", a backslash or a line break has to be escaped or + the value cannot be read back. the pair here is the whole of the rule, + used by the writer and by the reader. ++/ +template sspEscape() { + import std.array : replace; + string sspEscapeValue(string _s) { + return _s + .replace("\\", "\\\\") + .replace("|", "\\|") + .replace("\n", "\\n") + .replace("\r", "\\r"); + } + string[] sspUnescapeSplit(string _s) { + string[] _out; + string _cur; + bool _esc; + foreach (c; _s) { + if (_esc) { + switch (c) { + case 'n': _cur ~= '\n'; break; + case 'r': _cur ~= '\r'; break; + default: _cur ~= c; break; // covers \\ and \| + } + _esc = false; + } else if (c == '\\') { + _esc = true; + } else if (c == '|') { + _out ~= _cur; + _cur = ""; + } else { + _cur ~= c; + } + } + _out ~= _cur; + return _out; + } +} +/+ ↓ one object record: the single definition of what an object looks + like in a .ssp file. the writer emits with it, and the reader is + checked against it by round trip, so the two cannot drift ++/ +template sspObjectRecord() { + import std.conv : to; + import std.digest : toHexString; + import std.stdio; + import std.string; + import std.array; + import sisudoc.ocda.io_in.paths_source; + mixin sspEscape; + string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) { + string[] output; + /+ ↓ object declaration line. a heading carries its identifier here, + on the declaration line, where it is not the ocn; every other + object carries it as .identifier below, on the same condition + +/ + string obj_decl = "[" ~ obj.metainfo.ocn.to!string ~ "] "; + if (obj.metainfo.is_a == "heading") { + string lev = obj.metainfo.marked_up_level; + obj_decl ~= "heading :" ~ lev; + if (obj.metainfo.identifier.length > 0 + && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { + obj_decl ~= " " ~ obj.metainfo.identifier; + } + } else { + obj_decl ~= obj.metainfo.is_a; + } + output ~= obj_decl; + /+ ↓ properties (only non-default values) +/ + if (obj.metainfo.is_a != "heading" + && obj.metainfo.identifier.length > 0 + && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { + output ~= ".identifier: " ~ obj.metainfo.identifier; + } + if (obj.metainfo.is_of_part.length > 0) { + output ~= ".part: " ~ obj.metainfo.is_of_part; + } + if (obj.metainfo.is_of_section.length > 0 + && obj.metainfo.is_of_section != section) { + output ~= ".section: " ~ obj.metainfo.is_of_section; + } + /+ ↓ parent: emitted for every object except: + - the document root, the one object that has no parent + - toc entries, generated furniture rather than authored objects, + whose place is given by their indent level; the toc heading + itself does carry its parent (the document root) + A parent ocn of 0 is not "no parent recorded", it is a parent + heading the author has made non-substantive (ocn suppressed, or a + dummy heading introduced to segment an output); it is structurally + truthful and is stated explicitly rather than omitted so that the + two cases remain distinguishable in the file + +/ + if (!(obj.metainfo.is_a == "heading" + && obj.metainfo.heading_lev_markup == 0) + && obj.metainfo.is_a != "toc" + ) { + output ~= ".parent: " ~ obj.metainfo.parent_ocn.to!string; + } + if (obj.metainfo.last_descendant_ocn != 0) { + output ~= ".last_descendant: " ~ obj.metainfo.last_descendant_ocn.to!string; + } + /+ ↓ child headings (from pre-computed map) +/ + if (obj.metainfo.children_headings.length > 0) { + string[] ch; + foreach (c; obj.metainfo.children_headings) { + ch ~= c.to!string; + } + output ~= ".children: " ~ ch.join(" "); + } + /+ ↓ ancestors (only if non-zero) +/ + { + bool has_anc = false; + foreach (a; obj.metainfo.markedup_ancestors) { + if (a != 0) { has_anc = true; break; } + } + if (has_anc) { + string anc; + foreach (i, a; obj.metainfo.markedup_ancestors) { + if (i > 0) { anc ~= " "; } + anc ~= a.to!string; + } + output ~= ".ancestors: " ~ anc; + } + } + /+ ↓ collapsed ancestors (only if non-zero) +/ + { + bool has_anc_c = false; + foreach (a; obj.metainfo.collapsed_ancestors) { + if (a != 0) { has_anc_c = true; break; } + } + if (has_anc_c) { + string anc; + foreach (i, a; obj.metainfo.collapsed_ancestors) { + if (i > 0) anc ~= " "; + anc ~= a.to!string; + } + output ~= ".ancestors_collapsed: " ~ anc; + } + } + /+ ↓ dom structure status (only if non-zero) +/ + { + bool has_dom = false; + foreach (d; obj.metainfo.dom_structure_markedup_tags_status) { + if (d != 0) { has_dom = true; break; } + } + if (has_dom) { + string ds; + foreach (i, d; obj.metainfo.dom_structure_markedup_tags_status) { + if (i > 0) { ds ~= " "; } + ds ~= d.to!string; + } + output ~= ".dom_status: " ~ ds; + } + } + { + bool has_dom_c = false; + foreach (d; obj.metainfo.dom_structure_collapsed_tags_status) { + if (d != 0) { has_dom_c = true; break; } + } + if (has_dom_c) { + string ds; + foreach (i, d; obj.metainfo.dom_structure_collapsed_tags_status) { + if (i > 0) { ds ~= " "; } + ds ~= d.to!string; + } + output ~= ".dom_status_collapsed: " ~ ds; + } + } + if (obj.metainfo.heading_lev_collapsed < 9) { + output ~= ".heading_lev_collapsed: " ~ obj.metainfo.heading_lev_collapsed.to!string; + } + if (obj.metainfo.parent_lev_markup != 0) { + output ~= ".parent_lev: " ~ obj.metainfo.parent_lev_markup.to!string; + } + if (obj.metainfo.dummy_heading) { + output ~= ".dummy: true"; + } + if (obj.metainfo.object_number_off) { + output ~= ".ocn_off: true"; + } + if (obj.metainfo.is_of_type.length > 0) { + output ~= ".is_of_type: " ~ obj.metainfo.is_of_type; + } + if (obj.metainfo.attrib.length > 0) { + output ~= ".attrib: " ~ obj.metainfo.attrib; + } + if (obj.metainfo.lang.length > 0) { + output ~= ".meta_lang: " ~ obj.metainfo.lang; + } + if (obj.metainfo.syntax.length > 0) { + output ~= ".meta_syntax: " ~ obj.metainfo.syntax; + } + /+ ↓ sha256 digest +/ + { + bool has_sha = false; + foreach (b; obj.metainfo.sha256.text) { + if (b != 0) { has_sha = true; break; } + } + if (has_sha) { + if (_vox) { + writeln(obj.metainfo.sha256.summary); + } + output ~= ".sha256: " ~ obj.metainfo.sha256.text.toHexString.to!string; + } + } + /+ ↓ images, repeatable record, one per image in order of appearance: + .image: sha256: bytes: px:x + .image: missing:true + filename first (the object's handle on the image), remaining fields + self-describing key:value tokens so that further fields can be + added later without invalidating existing readers. px: is the + file's own pixel size, not the display dimensions (w##h##) which + the object text already carries + +/ + if (obj.metainfo.sha256.images.length > 0) { + foreach (i; obj.metainfo.sha256.images) { + output ~= (i.fileMissing) + ? ".image: " ~ i.fileName + ~ " missing:true" + : ".image: " ~ i.fileName + ~ " sha256:" ~ i.fileHash_sha256.toHexString.to!string + ~ " bytes:" ~ i.fileSize.to!string + ~ ((i.imageWidth > 0 && i.imageHeight > 0) + ? " px:" ~ i.imageWidth.to!string + ~ "x" ~ i.imageHeight.to!string + : ""); + } + } + /+ ↓ text attributes +/ + if (obj.attrib.indent_base != 0 || obj.attrib.indent_hang != 0) { + output ~= ".indent: " ~ obj.attrib.indent_base.to!string + ~ " " ~ obj.attrib.indent_hang.to!string; + } + if (obj.attrib.bullet) { + output ~= ".bullet: true"; + } + if (obj.attrib.language.length > 0) { + output ~= ".lang: " ~ obj.attrib.language; + } + /+ ↓ has flags +/ + { + string[] has_flags; + if (obj.has.inline_links) { has_flags ~= "links"; } + if (obj.has.inline_notes_reg) { has_flags ~= "notes_reg"; } + if (obj.has.inline_notes_star) { has_flags ~= "notes_star"; } + if (obj.has.images) { has_flags ~= "images"; } + if (obj.has.image_without_dimensions) { has_flags ~= "images_no_dim"; } + if (has_flags.length > 0) { + output ~= ".has: " ~ has_flags.join(" "); + } + } + /+ ↓ table properties +/ + if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { + output ~= ".table_cols: " ~ obj.table.number_of_columns.to!string; + if (obj.table.column_widths.length > 0) { + string[] ws; + foreach (w; obj.table.column_widths) ws ~= w.to!string; + output ~= ".table_widths: " ~ ws.join(" "); + } + if (obj.table.column_aligns.length > 0) { + output ~= ".table_aligns: " ~ obj.table.column_aligns.join(" "); + } + if (obj.table.heading) { + output ~= ".table_header: true"; + } + } + /+ ↓ code block properties +/ + if (obj.metainfo.is_a == "code") { + if (obj.code_block.linenumbers) { + output ~= ".code_linenumbers: true"; + } + } + /+ ↓ stow (extracted links) +/ + if (obj.stow.link.length > 0) { + foreach (lnk; obj.stow.link) { + if (lnk.length > 0) { + output ~= ".stow_link: " ~ lnk; + } + } + } + /+ ↓ tag properties +/ + if (obj.tags.in_segment_html.length > 0) { + output ~= ".segment: " ~ obj.tags.in_segment_html; + } + if (obj.tags.anchor_tag_html.length > 0 + && obj.tags.anchor_tag_html != obj.tags.in_segment_html) { + output ~= ".anchor: " ~ obj.tags.anchor_tag_html; + } + if (obj.tags.segname_prev.length > 0) { + output ~= ".segment_prev: " ~ obj.tags.segname_prev; + } + if (obj.tags.segname_next.length > 0) { + output ~= ".segment_next: " ~ obj.tags.segname_next; + } + if (obj.tags.heading_lev_anchor_tag.length > 0) { + output ~= ".heading_lev_anchor: " ~ obj.tags.heading_lev_anchor_tag; + } + if (obj.tags.segment_anchor_tag_epub.length > 0) { + output ~= ".segment_epub: " ~ obj.tags.segment_anchor_tag_epub; + } + /+ ↓ the segment an object belongs to, as opposed to the segment a + heading opens. .segment (in_segment_html) is set on headings + only, so without these two a reader cannot tell which segment a + paragraph is in; the sqlite search writer reads the html one + +/ + if (obj.tags.html_segment_anchor_tag_is.length > 0) { + output ~= ".segment_html_is: " ~ obj.tags.html_segment_anchor_tag_is; + } + if (obj.tags.epub_segment_anchor_tag_is.length > 0) { + output ~= ".segment_epub_is: " ~ obj.tags.epub_segment_anchor_tag_is; + } + /+ ↓ the level 4 segment a cross reference to this heading lands on, + emitted only where it is not the segment the heading sits in, which + is every heading at level 4 or below +/ + if (obj.tags.segment_lv4_is.length > 0 + && obj.tags.segment_lv4_is != obj.tags.html_segment_anchor_tag_is + ) { + output ~= ".segment_lv4_is: " ~ obj.tags.segment_lv4_is; + } + /+ ↓ heading ancestors text +/ + { + bool has_hat = false; + foreach (h; obj.tags.heading_ancestors_text) { + if (h.length > 0) { has_hat = true; break; } + } + if (has_hat) { + string[] _hat; + foreach (h; obj.tags.heading_ancestors_text) { + _hat ~= sspEscapeValue(h.to!string); + } + output ~= ".heading_ancestors_text: " ~ _hat.join("|"); + } + } + /+ ↓ lev4 subtoc +/ + if (obj.tags.lev4_subtoc.length > 0) { + foreach (st; obj.tags.lev4_subtoc) { + if (st.length > 0) { + output ~= ".lev4_subtoc: " ~ st; + } + } + } + /+ ↓ anchor tags +/ + if (obj.tags.anchor_tags.length > 0) { + foreach (at; obj.tags.anchor_tags) { + if (at.length > 0) { + output ~= ".anchor_tag: " ~ at; + } + } + } + /+ ↓ text content +/ + if (obj.text.length > 0) { + foreach (line; obj.text.split("\n")) { + output ~= "| " ~ line; + } + } + output ~= ""; + return output; + } +} + +/+ ↓ abstraction directories whose stale .ssp have been cleared this run, one + entry per document. __gshared, and read and written only inside the + synchronized block in the writer below: a document's languages share one + abstraction directory and may be written in parallel ++/ +private __gshared bool[string] _abstraction_dir_prepared; +template spineAbstractionTxt() { + import std.conv : to; + import std.digest : toHexString; + import std.file; + import std.path; + import std.stdio; + import std.string; + import std.array; + import sisudoc.ocda.io_in.paths_source; + mixin sspObjectRecord; + /+ ↓ the whole .ssp for a document, as lines. the file writer below emits + these, and the abstraction db is built by reading them back, so the + database cannot carry anything the .ssp does not + +/ + string[] sspDocumentLines(D)(D doc) { + auto doc_abstraction = doc.abstraction; + auto doc_matters = doc.matters; + string[] output; + /+ ↓ header comment +/ + output ~= "% SiSU Document Abstraction v" ~ ssp_format_version; + output ~= "% Source: " ~ doc_matters.src.filename; + output ~= ""; + /+ ↓ @source block: what this abstraction was made from, so a reader can + say whether it is looking at the right images and whether the + markup has moved on since. the filename is in the % Source: line + above and is not repeated here + +/ + output ~= "@source {"; + output ~= " language: " ~ doc_matters.src.language; + if (doc_matters.pod.manifest_list_of_languages.length > 0) { + output ~= " languages: " ~ doc_matters.pod.manifest_list_of_languages.join(" "); + } + output ~= " digest: " ~ doc_matters.doc_digest.markup_doc.toHexString.to!string; + output ~= "}"; + output ~= ""; + /+ ↓ @meta block +/ + output ~= "@meta {"; + auto meta = doc_matters.conf_make_meta.meta; + if (meta.title_main.length > 0) { + output ~= " title.main: " ~ meta.title_main; + } + if (meta.title_subtitle.length > 0) { + output ~= " title.subtitle: " ~ meta.title_subtitle; + } + if (meta.title_full.length > 0) { + output ~= " title.full: " ~ meta.title_full; + } + if (meta.title_language.length > 0) { + output ~= " title.language: " ~ meta.title_language; + } + if (meta.title_edition.length > 0) { + output ~= " title.edition: " ~ meta.title_edition; + } + if (meta.creator_author.length > 0) { + output ~= " creator.author: " ~ meta.creator_author; + } + if (meta.creator_author_surname.length > 0) { + output ~= " creator.author_surname: " ~ meta.creator_author_surname; + } + if (meta.creator_author_surname_fn.length > 0) { + output ~= " creator.author_surname_fn: " ~ meta.creator_author_surname_fn; + } + if (meta.creator_author_email.length > 0) { + output ~= " creator.author_email: " ~ meta.creator_author_email; + } + if (meta.creator_illustrator.length > 0) { + output ~= " creator.illustrator: " ~ meta.creator_illustrator; + } + if (meta.creator_translator.length > 0) { + output ~= " creator.translator: " ~ meta.creator_translator; + } + if (meta.date_published.length > 0) { + output ~= " date.published: " ~ meta.date_published; + } + if (meta.date_created.length > 0) { + output ~= " date.created: " ~ meta.date_created; + } + if (meta.date_issued.length > 0) { + output ~= " date.issued: " ~ meta.date_issued; + } + if (meta.date_available.length > 0) { + output ~= " date.available: " ~ meta.date_available; + } + if (meta.date_modified.length > 0) { + output ~= " date.modified: " ~ meta.date_modified; + } + if (meta.date_valid.length > 0) { + output ~= " date.valid: " ~ meta.date_valid; + } + if (meta.date_added_to_site.length > 0) { + output ~= " date.added_to_site: " ~ meta.date_added_to_site; + } + if (meta.rights_copyright.length > 0) { + output ~= " rights.copyright: " ~ meta.rights_copyright; + } + if (meta.rights_copyright_text.length > 0) { + output ~= " rights.copyright_text: " ~ meta.rights_copyright_text; + } + if (meta.rights_copyright_translation.length > 0) { + output ~= " rights.copyright_translation: " ~ meta.rights_copyright_translation; + } + if (meta.rights_copyright_illustrations.length > 0) { + output ~= " rights.copyright_illustrations: " ~ meta.rights_copyright_illustrations; + } + if (meta.rights_copyright_photographs.length > 0) { + output ~= " rights.copyright_photographs: " ~ meta.rights_copyright_photographs; + } + if (meta.rights_copyright_cover.length > 0) { + output ~= " rights.copyright_cover: " ~ meta.rights_copyright_cover; + } + if (meta.rights_copyright_audio.length > 0) { + output ~= " rights.copyright_audio: " ~ meta.rights_copyright_audio; + } + if (meta.rights_copyright_video.length > 0) { + output ~= " rights.copyright_video: " ~ meta.rights_copyright_video; + } + if (meta.rights_license.length > 0) { + output ~= " rights.license: " ~ meta.rights_license; + } + if (meta.classify_topic_register.length > 0) { + output ~= " classify.topic_register: " ~ meta.classify_topic_register; + } + if (meta.classify_subject.length > 0) { + output ~= " classify.subject: " ~ meta.classify_subject; + } + if (meta.classify_keywords.length > 0) { + output ~= " classify.keywords: " ~ meta.classify_keywords; + } + if (meta.classify_loc.length > 0) { + output ~= " classify.loc: " ~ meta.classify_loc; + } + if (meta.classify_dewey.length > 0) { + output ~= " classify.dewey: " ~ meta.classify_dewey; + } + if (meta.identifier_isbn.length > 0) { + output ~= " identifier.isbn: " ~ meta.identifier_isbn; + } + if (meta.identifier_oclc.length > 0) { + output ~= " identifier.oclc: " ~ meta.identifier_oclc; + } + if (meta.language_document.length > 0) { + output ~= " language.document: " ~ meta.language_document; + } + if (meta.language_document_char.length > 0) { + output ~= " language.document_char: " ~ meta.language_document_char; + } + if (meta.original_title.length > 0) { + output ~= " original.title: " ~ meta.original_title; + } + if (meta.original_source.length > 0) { + output ~= " original.source: " ~ meta.original_source; + } + if (meta.original_language.length > 0) { + output ~= " original.language: " ~ meta.original_language; + } + if (meta.original_language_char.length > 0) { + output ~= " original.language_char: " ~ meta.original_language_char; + } + if (meta.notes_abstract.length > 0) { + output ~= " notes.abstract: " ~ meta.notes_abstract; + } + if (meta.notes_description.length > 0) { + output ~= " notes.description: " ~ meta.notes_description; + } + if (meta.notes_summary.length > 0) { + output ~= " notes.summary: " ~ meta.notes_summary; + } // check + output ~= "}"; + output ~= ""; + /+ ↓ @make block +/ + output ~= "@make {"; + auto make = doc_matters.conf_make_meta.make; + if (make.doc_type.length > 0) { + output ~= " doc_type: " ~ make.doc_type; + } + if (make.auto_num_top_at_level.length > 0) { + output ~= " auto_num_top_at_level: " ~ make.auto_num_top_at_level; + } // check + output ~= " auto_num_top_lv: " ~ make.auto_num_top_lv.to!string; + output ~= " auto_num_depth: " ~ make.auto_num_depth.to!string; + /+ ↓ the make settings the output writers read. the rest of make + (italics, bold, emphasis, substitute, headings) is read at parse + time and its effect is already in the objects, so it does not + travel; these three are read in outputs/io_out/ and must + +/ + if (make.breaks.length > 0) { + output ~= " breaks: " ~ make.breaks; + } + if (make.footer.length > 0) { // a list: joined, each slot escaped + string[] _ft; + foreach (_f; make.footer) { _ft ~= sspEscapeValue(_f); } + output ~= " footer: " ~ _ft.join("|"); + } + if (make.home_button_text.length > 0) { + output ~= " home_button_text: " ~ make.home_button_text; + } + output ~= "}"; + output ~= ""; + /+ ↓ @doc_has block +/ + output ~= "@doc_has {"; + output ~= " inline_links: " ~ doc_matters.has.inline_links.to!string; + output ~= " inline_notes_reg: " ~ doc_matters.has.inline_notes_reg.to!string; + output ~= " inline_notes_star: " ~ doc_matters.has.inline_notes_star.to!string; + output ~= " tables: " ~ doc_matters.has.tables.to!string; + output ~= " codeblocks: " ~ doc_matters.has.codeblocks.to!string; + output ~= " images: " ~ doc_matters.has.images.to!string; + output ~= " poems: " ~ doc_matters.has.poems.to!string; + output ~= " groups: " ~ doc_matters.has.groups.to!string; + output ~= " blocks: " ~ doc_matters.has.blocks.to!string; + output ~= " quotes: " ~ doc_matters.has.quotes.to!string; + output ~= "}"; + output ~= ""; + /+ ↓ document sections - every section the abstraction holds, in document order. + "tail" is the end-of-document used by some output writers + +/ + string[] section_order = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + foreach (section; section_order) { + if (section !in doc_abstraction) { continue; } + auto section_objs = doc_abstraction[section]; + if (section_objs.length == 0) { continue; } + output ~= "@" ~ section ~ " {"; + output ~= ""; + foreach (obj; section_objs) { + output ~= sspObjectRecord(obj, section, doc_matters.opt.action.vox_gt_3); + } + output ~= "}"; + output ~= ""; + } + return output; + } + @trusted void spineAbstractionTxt(D)(D doc) { + auto doc_matters = doc.matters; + string[] output = sspDocumentLines(doc); + /+ ↓ write to file, into the pod's own tree: + pod//media/abstraction/.ssp + . + where it sits beside pod//media/image/, which is what a .ssp + needs to be a document source: the file describes its images by + name, size, pixels and digest, and they have to be findable. Written + here once; --pod2 then digests and zips it where it stands rather + than copying it in from an output directory of its own. + +/ + auto pths_pod = spinePathsPods!()(doc_matters); + string base_pth + = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string; + /+ ↓ this directory is the one part of the pod written before the pod is + built, so the pod builder leaves it alone. It therefore clears itself + here, or a .ssp for a language the document no longer has would linger + and be bundled. + . + Every language of a document writes into this one directory, and the + languages may be running at once, so the clearing is done once per + directory per run, by whichever language reaches it first, with the + lock held across it. A language that finds the directory already + prepared passed through this lock before writing, so the clearing it + skipped had already finished, and no .ssp written by this run can be + removed by it. Keying the clearing on the manifest's first language + instead makes it wait for a thread that may be scheduled last, and + take the .ssp already written by the other languages with it. + +/ + synchronized { + if (base_pth !in _abstraction_dir_prepared) { + _abstraction_dir_prepared[base_pth] = true; + try { + if (!exists(base_pth)) { + base_pth.mkdirRecurse; + } else { + foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) { + _f.remove; + } + } + } catch (Exception ex) { + } + } + } + string out_file = ((base_pth.chainPath( + doc_matters.src.doc_uid_out ~ ".ssp")).asNormalizedPath).array; + if (doc_matters.opt.action.vox_gt_1) { + writeln(" ", out_file); + } + auto f = File(out_file, "w"); + foreach (line; output) { + f.writeln(line); + } + } +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/out_ocda_sqlite_db.org b/org/out_ocda_sqlite_db.org new file mode 100644 index 0000000..70c118a --- /dev/null +++ b/org/out_ocda_sqlite_db.org @@ -0,0 +1,666 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) output pod source sqlite db +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:output:source:pod: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] + +* (Object-Centric) Document Abstraction SQLite db + +- Process markup document, create document abstraction + +** _module template_ :module:metadoc_from_src: + +rename source_abstraction_peg_txt.d + +#+HEADER: :tangle "../src/sisudoc/outputs/io_out/create_ocda_db.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +module sisudoc.outputs.io_out.create_ocda_db; + +/+ ↓ write document abstraction as per-document sqlite3 database +/ +template spineAbstractionDb() { + import std.conv : to; + import std.digest : toHexString; + import std.file; + import std.path; + import std.stdio; + import std.string; + import std.array; + import d2sqlite3; + import sisudoc.outputs.io_out.paths_output; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.abstraction.ssp : ssp_format_version; + /+ ↓ the document is passed in rather than taken from doc, because the + caller hands over one that has been through the .ssp: written as text + and read back. so the database is a function of the .ssp and cannot + carry anything the .ssp does not, header blocks included: the metadata + table below is filled from what came back, not from a second list read + off doc_matters, which is how the two drifted apart before + +/ + void spineAbstractionDb(D,S)(D doc, S ssp_doc) { + auto doc_matters = doc.matters; + auto doc_abstraction = ssp_doc.abstraction; + /+ ↓ output path: beside the pod's zip and digests, not under + /abstraction/. the database carries its own images, so it is a + distributable artefact in its own right and belongs with the others: + pod// the source tree + pod/.zip the same, zipped + pod/.digests.txt what is in them + pod/.ocda.db the abstraction, one per language + +/ + auto pths_pod = spinePathsPods!()(doc_matters); + string base_pth = pths_pod.pod_dir_(); + try { + if (!exists(base_pth)) { + base_pth.mkdirRecurse; + } + } catch (Exception ex) { + } + string db_file = ((base_pth.chainPath( + doc_matters.src.doc_uid_out ~ ".ocda.db")).asNormalizedPath).array; + + /+ ↓ remove existing file to start fresh +/ + try { + if (exists(db_file)) { + remove(db_file); + } + } catch (Exception ex) { + } + + if (doc_matters.opt.action.vox_gt_1) { + writeln(" ", db_file); + } + + /+ ↓ open database and create schema +/ + auto db = Database(db_file); + /+ ↓ write once, read many: no WAL. + a WAL stamped file cannot be opened by a reader that cannot write + beside it (SQLITE_READONLY on a read-only directory, and inside a + zip or on read-only media), and this database has one writer and no + concurrency, so WAL buys nothing here. MEMORY journal keeps the bulk + load fast without leaving that mark on the file. + +/ + db.run("PRAGMA journal_mode=MEMORY"); + db.run("PRAGMA synchronous=OFF"); + + db.run(" + CREATE TABLE metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + -- objects: one row per abstraction object, in document order. + -- the fixed width arrays (ancestors, dom status, children, heading + -- ancestors text, table widths and aligns) are JSON arrays so that + -- json_extract, json_each and json_array_length can reach inside them; + -- the open ended lists (images, links, anchor tags, subtoc) are rows + -- in their own tables, not columns + CREATE TABLE objects ( + id INTEGER PRIMARY KEY, + section TEXT NOT NULL, + seq INTEGER NOT NULL, + ocn INTEGER DEFAULT 0, + is_a TEXT NOT NULL, + is_of_part TEXT, + is_of_section TEXT, + is_of_type TEXT, + heading_level INTEGER, + heading_lev_collapsed INTEGER, + identifier TEXT, + parent_ocn INTEGER DEFAULT 0, + parent_lev INTEGER DEFAULT 0, + last_descendant_ocn INTEGER DEFAULT 0, + children TEXT + CHECK(children IS NULL OR json_valid(children)), + ancestors TEXT + CHECK(ancestors IS NULL OR json_valid(ancestors)), + ancestors_collapsed TEXT + CHECK(ancestors_collapsed IS NULL OR json_valid(ancestors_collapsed)), + dom_status TEXT + CHECK(dom_status IS NULL OR json_valid(dom_status)), + dom_status_collapsed TEXT + CHECK(dom_status_collapsed IS NULL OR json_valid(dom_status_collapsed)), + dummy_heading INTEGER DEFAULT 0, + object_number_off INTEGER DEFAULT 0, + attrib TEXT, + meta_lang TEXT, + meta_syntax TEXT, + sha256 TEXT, + indent_base INTEGER DEFAULT 0, + indent_hang INTEGER DEFAULT 0, + bullet INTEGER DEFAULT 0, + lang TEXT, + has_links INTEGER DEFAULT 0, + has_notes_reg INTEGER DEFAULT 0, + has_notes_star INTEGER DEFAULT 0, + has_images INTEGER DEFAULT 0, + has_images_no_dim INTEGER DEFAULT 0, + segment TEXT, + segment_prev TEXT, + segment_next TEXT, + segment_epub TEXT, + segment_html_is TEXT, + segment_epub_is TEXT, + segment_lv4_is TEXT, + anchor TEXT, + heading_lev_anchor TEXT, + heading_ancestors_text TEXT + CHECK(heading_ancestors_text IS NULL OR json_valid(heading_ancestors_text)), + table_cols INTEGER, + table_widths TEXT + CHECK(table_widths IS NULL OR json_valid(table_widths)), + table_aligns TEXT + CHECK(table_aligns IS NULL OR json_valid(table_aligns)), + table_header INTEGER, + code_linenumbers INTEGER DEFAULT 0, + text TEXT, + UNIQUE(section, seq) + ); + + CREATE INDEX idx_objects_section ON objects(section); + CREATE INDEX idx_objects_ocn ON objects(ocn); + CREATE INDEX idx_objects_parent ON objects(parent_ocn); + CREATE INDEX idx_objects_is_a ON objects(is_a); + CREATE INDEX idx_objects_heading ON objects(heading_level) + WHERE heading_level IS NOT NULL; + + -- files: what this database carries with it, so that a reader needs + -- nothing beside it. role says what a file is; images are the ones the + -- output writers must have in order to produce a document. bytes are + -- stored exactly as read, so sha256 matches the digest the abstraction + -- was built with + CREATE TABLE files ( + id INTEGER PRIMARY KEY, + role TEXT NOT NULL, + name TEXT NOT NULL, + bytes INTEGER NOT NULL, + sha256 TEXT, + width INTEGER, + height INTEGER, + data BLOB, + UNIQUE(role, name) + ); + + -- the open ended per object lists, one row each, in order + CREATE TABLE object_images ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + name TEXT NOT NULL, + bytes INTEGER, + sha256 TEXT, + width INTEGER, + height INTEGER, + missing INTEGER DEFAULT 0, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_links ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + url TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_anchors ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + anchor TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_subtoc ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + entry TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + + CREATE INDEX idx_files_role ON files(role); + CREATE INDEX idx_object_images_name ON object_images(name); + + -- full text over the object text, external content so the text is not + -- stored twice. per document: this is for exploring one abstraction, + -- the collection wide search database is a different thing + CREATE VIRTUAL TABLE objects_fts USING fts5( + text, content='objects', content_rowid='id' + ); + + -- the shapes worth asking for, named + CREATE VIEW outline AS + SELECT section, seq, ocn, heading_level, heading_lev_collapsed, + last_descendant_ocn, identifier, text + FROM objects WHERE is_a = 'heading' ORDER BY id; + CREATE VIEW citable AS + SELECT * FROM objects WHERE ocn > 0 ORDER BY ocn; + CREATE VIEW document_files AS + SELECT role, name, bytes, sha256, width, height FROM files ORDER BY role, name; + "); + + /+ ↓ populate metadata, from the header blocks the .ssp gave back. + one list, in ssp.d, not two: a property added to the .ssp header + arrives here without anything being changed, and one that is not in + the .ssp cannot be here at all. the keys are the .ssp's own + (title.main, make.doc_type, doc_has.images), prefixed by their block + where the block name is not already part of the key. + +/ + db.run("BEGIN TRANSACTION"); + + auto meta_stmt = db.prepare( + "INSERT INTO metadata (key, value) VALUES (:key, :value)" + ); + + void insertMeta(string key, string value) { + if (value.length > 0) { + meta_stmt.bind(":key", key); + meta_stmt.bind(":value", value); + meta_stmt.execute(); + meta_stmt.reset(); + } + } + void insertBlock(string prefix, string[] keys_in_order, string[string] kv) { + foreach (k; keys_in_order) { + if (k in kv) { insertMeta(prefix ~ k, kv[k]); } + } + } + + /+ ↓ what this file is, so a reader can tell before it starts. the + version is the .ssp's own: the two serialisations are one format and + move together + +/ + insertMeta("schema.name", "sisu-abstraction-db"); + insertMeta("schema.version", ssp_format_version); + insertMeta("source.filename", ssp_doc.source); + insertBlock("source.", ssp_doc.source_info_order, ssp_doc.source_info); + /+ ↓ the digest of the .ssp this database was built from. a file cannot + hold its own hash, so this one is not in the .ssp and is taken here: + source.digest says which markup the abstraction came from, + source.ssp_digest says which abstraction this database came from, + and the chain .sst -> .ssp -> .ocda.db is checkable end to end + +/ + insertMeta("source.ssp_digest", ssp_doc.ssp_digest); + + insertBlock("", ssp_doc.meta_order, ssp_doc.meta); + insertBlock("make.", ssp_doc.make_order, ssp_doc.make); + insertBlock("doc_has.", ssp_doc.doc_has_order, ssp_doc.doc_has); + meta_stmt.finalize(); + + /+ ↓ populate objects +/ + auto obj_stmt = db.prepare( + "INSERT INTO objects (" + ~ "section, seq, ocn, is_a, is_of_part, is_of_section, is_of_type," + ~ "heading_level, heading_lev_collapsed, identifier," + ~ "parent_ocn, parent_lev, last_descendant_ocn," + ~ "children, ancestors, ancestors_collapsed," + ~ "dom_status, dom_status_collapsed," + ~ "dummy_heading, object_number_off," + ~ "attrib, meta_lang, meta_syntax, sha256," + ~ "indent_base, indent_hang, bullet, lang," + ~ "has_links, has_notes_reg, has_notes_star, has_images, has_images_no_dim," + ~ "segment, segment_prev, segment_next, segment_epub," + ~ "segment_html_is, segment_epub_is, segment_lv4_is," + ~ "anchor, heading_lev_anchor, heading_ancestors_text," + ~ "table_cols, table_widths, table_aligns, table_header," + ~ "code_linenumbers, text" + ~ ") VALUES (" + ~ ":section, :seq, :ocn, :is_a, :is_of_part, :is_of_section, :is_of_type," + ~ ":heading_level, :heading_lev_collapsed, :identifier," + ~ ":parent_ocn, :parent_lev, :last_descendant_ocn," + ~ ":children, :ancestors, :ancestors_collapsed," + ~ ":dom_status, :dom_status_collapsed," + ~ ":dummy_heading, :object_number_off," + ~ ":attrib, :meta_lang, :meta_syntax, :sha256," + ~ ":indent_base, :indent_hang, :bullet, :lang," + ~ ":has_links, :has_notes_reg, :has_notes_star, :has_images, :has_images_no_dim," + ~ ":segment, :segment_prev, :segment_next, :segment_epub," + ~ ":segment_html_is, :segment_epub_is, :segment_lv4_is," + ~ ":anchor, :heading_lev_anchor, :heading_ancestors_text," + ~ ":table_cols, :table_widths, :table_aligns, :table_header," + ~ ":code_linenumbers, :text" + ~ ")" + ); + + /+ ↓ the per object lists, one row each +/ + auto img_stmt = db.prepare( + "INSERT INTO object_images" + ~ " (object_id, seq, name, bytes, sha256, width, height, missing)" + ~ " VALUES (:object_id, :seq, :name, :bytes, :sha256, :width, :height, :missing)" + ); + auto lnk_stmt = db.prepare( + "INSERT INTO object_links (object_id, seq, url)" + ~ " VALUES (:object_id, :seq, :url)" + ); + auto anc_stmt = db.prepare( + "INSERT INTO object_anchors (object_id, seq, anchor)" + ~ " VALUES (:object_id, :seq, :anchor)" + ); + auto sub_stmt = db.prepare( + "INSERT INTO object_subtoc (object_id, seq, entry)" + ~ " VALUES (:object_id, :seq, :entry)" + ); + + string[] section_order = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + + foreach (section; section_order) { + if (section !in doc_abstraction) continue; + auto section_objs = doc_abstraction[section]; + if (section_objs.length == 0) continue; + + foreach (seq, obj; section_objs) { + obj_stmt.bind(":section", section); + obj_stmt.bind(":seq", cast(int) seq); + obj_stmt.bind(":ocn", obj.metainfo.ocn); + obj_stmt.bind(":is_a", obj.metainfo.is_a); + + /+ ↓ nullable string fields +/ + void bindStr(string param, string val) { + import std.typecons : Nullable; + if (val.length > 0) { + obj_stmt.bind(param, val); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + /+ ↓ nullable int fields +/ + void bindInt(string param, int val, bool is_set) { + import std.typecons : Nullable; + if (is_set) { + obj_stmt.bind(param, val); + } else { + obj_stmt.bind(param, Nullable!int()); + } + } + /+ ↓ fixed width int array as a JSON array, NULL when all zero +/ + void bindJsonInts(T)(string param, T arr) { + import std.typecons : Nullable; + bool any = false; + foreach (a; arr) { if (a != 0) { any = true; break; } } + if (any) { + string[] _j; + foreach (a; arr) { _j ~= a.to!string; } + obj_stmt.bind(param, "[" ~ _j.join(",") ~ "]"); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + /+ ↓ string array as a JSON array, NULL when every slot is empty +/ + void bindJsonStrs(T)(string param, T arr) { + import std.json : JSONValue; + import std.typecons : Nullable; + bool any = false; + foreach (s; arr) { if (s.length > 0) { any = true; break; } } + if (any) { + string[] _s; + foreach (s; arr) { _s ~= s.to!string; } + obj_stmt.bind(param, JSONValue(_s).toString); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + + bindStr(":is_of_part", obj.metainfo.is_of_part); + bindStr(":is_of_section", obj.metainfo.is_of_section); + bindStr(":is_of_type", obj.metainfo.is_of_type); + + /+ ↓ heading levels, marked up and collapsed, 9 meaning unset +/ + bindInt(":heading_level", obj.metainfo.heading_lev_markup, + (obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup < 9)); + bindInt(":heading_lev_collapsed", obj.metainfo.heading_lev_collapsed, + (obj.metainfo.heading_lev_collapsed < 9)); + + bindStr(":identifier", obj.metainfo.identifier); + obj_stmt.bind(":parent_ocn", obj.metainfo.parent_ocn); + obj_stmt.bind(":parent_lev", obj.metainfo.parent_lev_markup); + obj_stmt.bind(":last_descendant_ocn", obj.metainfo.last_descendant_ocn); + + bindJsonInts(":children", obj.metainfo.children_headings); + bindJsonInts(":ancestors", obj.metainfo.markedup_ancestors); + bindJsonInts(":ancestors_collapsed", obj.metainfo.collapsed_ancestors); + bindJsonInts(":dom_status", obj.metainfo.dom_structure_markedup_tags_status); + bindJsonInts(":dom_status_collapsed", obj.metainfo.dom_structure_collapsed_tags_status); + + obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); + obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); + bindStr(":attrib", obj.metainfo.attrib); + bindStr(":meta_lang", obj.metainfo.lang); + bindStr(":meta_syntax", obj.metainfo.syntax); + + /+ ↓ object digest, hex, absent when never taken +/ + { + import std.typecons : Nullable; + bool has_sha = false; + foreach (b; obj.metainfo.sha256.text) { + if (b != 0) { has_sha = true; break; } + } + if (has_sha) { + obj_stmt.bind(":sha256", obj.metainfo.sha256.text.toHexString.to!string); + } else { + obj_stmt.bind(":sha256", Nullable!string()); + } + } + + // obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); + // obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); + obj_stmt.bind(":indent_base", obj.attrib.indent_base); + obj_stmt.bind(":indent_hang", obj.attrib.indent_hang); + obj_stmt.bind(":bullet", obj.attrib.bullet ? 1 : 0); + bindStr(":lang", obj.attrib.language); + obj_stmt.bind(":has_links", obj.has.inline_links ? 1 : 0); + obj_stmt.bind(":has_notes_reg", obj.has.inline_notes_reg ? 1 : 0); + obj_stmt.bind(":has_notes_star", obj.has.inline_notes_star ? 1 : 0); + obj_stmt.bind(":has_images", obj.has.images ? 1 : 0); + obj_stmt.bind(":has_images_no_dim", obj.has.image_without_dimensions ? 1 : 0); + bindStr(":segment", obj.tags.in_segment_html); + bindStr(":segment_prev", obj.tags.segname_prev); + bindStr(":segment_next", obj.tags.segname_next); + bindStr(":segment_epub", obj.tags.segment_anchor_tag_epub); + bindStr(":segment_html_is", obj.tags.html_segment_anchor_tag_is); + bindStr(":segment_epub_is", obj.tags.epub_segment_anchor_tag_is); + bindStr(":segment_lv4_is", obj.tags.segment_lv4_is); + bindStr(":anchor", obj.tags.anchor_tag_html); + bindStr(":heading_lev_anchor", obj.tags.heading_lev_anchor_tag); + bindJsonStrs(":heading_ancestors_text", obj.tags.heading_ancestors_text); + + /+ ↓ table properties +/ + { + import std.typecons : Nullable; + if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { + obj_stmt.bind(":table_cols", obj.table.number_of_columns); + if (obj.table.column_widths.length > 0) { + string[] ws; + foreach (w; obj.table.column_widths) ws ~= w.to!string; + obj_stmt.bind(":table_widths", "[" ~ ws.join(",") ~ "]"); + } else { + obj_stmt.bind(":table_widths", Nullable!string()); + } + if (obj.table.column_aligns.length > 0) { + import std.json : JSONValue; + string[] _al; + foreach (a; obj.table.column_aligns) { _al ~= a.to!string; } + obj_stmt.bind(":table_aligns", JSONValue(_al).toString); + } else { + obj_stmt.bind(":table_aligns", Nullable!string()); + } + obj_stmt.bind(":table_header", obj.table.heading ? 1 : 0); + } else { + obj_stmt.bind(":table_cols", Nullable!int()); + obj_stmt.bind(":table_widths", Nullable!string()); + obj_stmt.bind(":table_aligns", Nullable!string()); + obj_stmt.bind(":table_header", Nullable!int()); + } + } + + /+ ↓ code block properties +/ + obj_stmt.bind(":code_linenumbers", + ((obj.metainfo.is_a == "code") && obj.code_block.linenumbers) ? 1 : 0); + + /+ ↓ text content +/ + bindStr(":text", obj.text); + + obj_stmt.execute(); + obj_stmt.reset(); + + /+ ↓ the per object lists, keyed on the row just written +/ + long _obj_id = db.lastInsertRowid; + { + int _n = 0; + foreach (i; obj.metainfo.sha256.images) { + img_stmt.bind(":object_id", _obj_id); + img_stmt.bind(":seq", _n); + img_stmt.bind(":name", i.fileName); + img_stmt.bind(":bytes", cast(long) i.fileSize); + img_stmt.bind(":sha256", i.fileHash_sha256.toHexString.to!string); + img_stmt.bind(":width", i.imageWidth); + img_stmt.bind(":height", i.imageHeight); + img_stmt.bind(":missing", i.fileMissing ? 1 : 0); + img_stmt.execute(); + img_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (lnk; obj.stow.link) { + if (lnk.length == 0) { continue; } + lnk_stmt.bind(":object_id", _obj_id); + lnk_stmt.bind(":seq", _n); + lnk_stmt.bind(":url", lnk); + lnk_stmt.execute(); + lnk_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (at; obj.tags.anchor_tags) { + if (at.length == 0) { continue; } + anc_stmt.bind(":object_id", _obj_id); + anc_stmt.bind(":seq", _n); + anc_stmt.bind(":anchor", at); + anc_stmt.execute(); + anc_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (st; obj.tags.lev4_subtoc) { + if (st.length == 0) { continue; } + sub_stmt.bind(":object_id", _obj_id); + sub_stmt.bind(":seq", _n); + sub_stmt.bind(":entry", st); + sub_stmt.execute(); + sub_stmt.reset(); + ++_n; + } + } + } + } + + obj_stmt.finalize(); + img_stmt.finalize(); + lnk_stmt.finalize(); + anc_stmt.finalize(); + sub_stmt.finalize(); + + /+ ↓ the document's images, carried inside the file so that output can be + produced from this database alone. bytes exactly as read, and the + digest taken over them, so that it can be checked against the digest + the abstraction recorded +/ + { + import std.digest.sha : sha256Of; + auto file_stmt = db.prepare( + "INSERT OR IGNORE INTO files" + ~ " (role, name, bytes, sha256, width, height, data)" + ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" + ); + int[string] _w, _h; + foreach (section; section_order) { + if (section !in doc_abstraction) { continue; } + foreach (obj; doc_abstraction[section]) { + foreach (i; obj.metainfo.sha256.images) { + if (i.imageWidth > 0) { _w[i.fileName] = i.imageWidth; } + if (i.imageHeight > 0) { _h[i.fileName] = i.imageHeight; } + } + } + } + foreach (image; doc_matters.srcs.image_list) { + string _fn = doc_matters.src.image_dir_path ~ "/" ~ image; + ubyte[] _bytes; + try { + if (_fn.exists) { _bytes = cast(ubyte[]) _fn.read; } + } catch (Exception ex) { + _bytes = []; + } + if (_bytes.length == 0) { + if (doc_matters.opt.action.vox_gt_1) { + writeln(" image not carried into abstraction db: ", _fn); + } + continue; + } + file_stmt.bind(":role", "image"); + file_stmt.bind(":name", image); + file_stmt.bind(":bytes", cast(long) _bytes.length); + file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); + if (image in _w) { file_stmt.bind(":width", _w[image]); } + else { + import std.typecons : Nullable; + file_stmt.bind(":width", Nullable!int()); + } + if (image in _h) { file_stmt.bind(":height", _h[image]); } + else { + import std.typecons : Nullable; + file_stmt.bind(":height", Nullable!int()); + } + file_stmt.bind(":data", _bytes); + file_stmt.execute(); + file_stmt.reset(); + } + file_stmt.finalize(); + } + + /+ ↓ build the full text index from the rows just written +/ + db.run("INSERT INTO objects_fts(rowid, text)" + ~ " SELECT id, text FROM objects WHERE text IS NOT NULL"); + + db.run("COMMIT TRANSACTION"); + } +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/out_src_abstraction_ocda_peg_ssp.org b/org/out_src_abstraction_ocda_peg_ssp.org deleted file mode 100644 index 511c589..0000000 --- a/org/out_src_abstraction_ocda_peg_ssp.org +++ /dev/null @@ -1,736 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) output pod source abstraction peg text -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:output:source:pod: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* (Object-Centric) Document Abstraction PEG text - -- Process markup document, create document abstraction - -** _module template_ :module:metadoc_from_src: - -rename source_abstraction_peg_txt.d - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.ocda.abstraction.ssp; -@safe: -/+ ↓ write document abstraction as human-readable .ssp text file +/ -/+ ↓ the format version, carried by both serialisations: the .ssp says it in - its "% SiSU Document Abstraction v" header line and the .ocda.db in its - metadata schema.version row, so that one bump moves both. - - a reader checks the major part and refuses what it does not know, rather - than loading an older artefact half populated in silence. bump the minor - part when a property is added (a reader of the newer format still reads - the older one), the major part when one changes meaning or goes away. - - 1.1 2026-09-10 @source block; the document metadata the output writers - read carried in full (14 meta and 3 make properties - added); .segment_lv4_is, the segment a cross - reference to a heading above level 4 lands on; - was 0.1 -+/ -enum ssp_format_version = "1.1"; -enum ssp_format_version_major = "1"; -/+ ↓ escaping for values that are folded onto one line - - a .ssp property value runs to the end of its line, and - .heading_ancestors_text joins its eight slots with "|", so a slot that - itself contains "|", a backslash or a line break has to be escaped or - the value cannot be read back. the pair here is the whole of the rule, - used by the writer and by the reader. -+/ -template sspEscape() { - import std.array : replace; - string sspEscapeValue(string _s) { - return _s - .replace("\\", "\\\\") - .replace("|", "\\|") - .replace("\n", "\\n") - .replace("\r", "\\r"); - } - string[] sspUnescapeSplit(string _s) { - string[] _out; - string _cur; - bool _esc; - foreach (c; _s) { - if (_esc) { - switch (c) { - case 'n': _cur ~= '\n'; break; - case 'r': _cur ~= '\r'; break; - default: _cur ~= c; break; // covers \\ and \| - } - _esc = false; - } else if (c == '\\') { - _esc = true; - } else if (c == '|') { - _out ~= _cur; - _cur = ""; - } else { - _cur ~= c; - } - } - _out ~= _cur; - return _out; - } -} -/+ ↓ one object record: the single definition of what an object looks - like in a .ssp file. the writer emits with it, and the reader is - checked against it by round trip, so the two cannot drift -+/ -template sspObjectRecord() { - import std.conv : to; - import std.digest : toHexString; - import std.stdio; - import std.string; - import std.array; - import sisudoc.ocda.io_in.paths_source; - mixin sspEscape; - string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) { - string[] output; - /+ ↓ object declaration line. a heading carries its identifier here, - on the declaration line, where it is not the ocn; every other - object carries it as .identifier below, on the same condition - +/ - string obj_decl = "[" ~ obj.metainfo.ocn.to!string ~ "] "; - if (obj.metainfo.is_a == "heading") { - string lev = obj.metainfo.marked_up_level; - obj_decl ~= "heading :" ~ lev; - if (obj.metainfo.identifier.length > 0 - && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { - obj_decl ~= " " ~ obj.metainfo.identifier; - } - } else { - obj_decl ~= obj.metainfo.is_a; - } - output ~= obj_decl; - /+ ↓ properties (only non-default values) +/ - if (obj.metainfo.is_a != "heading" - && obj.metainfo.identifier.length > 0 - && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { - output ~= ".identifier: " ~ obj.metainfo.identifier; - } - if (obj.metainfo.is_of_part.length > 0) { - output ~= ".part: " ~ obj.metainfo.is_of_part; - } - if (obj.metainfo.is_of_section.length > 0 - && obj.metainfo.is_of_section != section) { - output ~= ".section: " ~ obj.metainfo.is_of_section; - } - /+ ↓ parent: emitted for every object except: - - the document root, the one object that has no parent - - toc entries, generated furniture rather than authored objects, - whose place is given by their indent level; the toc heading - itself does carry its parent (the document root) - A parent ocn of 0 is not "no parent recorded", it is a parent - heading the author has made non-substantive (ocn suppressed, or a - dummy heading introduced to segment an output); it is structurally - truthful and is stated explicitly rather than omitted so that the - two cases remain distinguishable in the file - +/ - if (!(obj.metainfo.is_a == "heading" - && obj.metainfo.heading_lev_markup == 0) - && obj.metainfo.is_a != "toc" - ) { - output ~= ".parent: " ~ obj.metainfo.parent_ocn.to!string; - } - if (obj.metainfo.last_descendant_ocn != 0) { - output ~= ".last_descendant: " ~ obj.metainfo.last_descendant_ocn.to!string; - } - /+ ↓ child headings (from pre-computed map) +/ - if (obj.metainfo.children_headings.length > 0) { - string[] ch; - foreach (c; obj.metainfo.children_headings) { - ch ~= c.to!string; - } - output ~= ".children: " ~ ch.join(" "); - } - /+ ↓ ancestors (only if non-zero) +/ - { - bool has_anc = false; - foreach (a; obj.metainfo.markedup_ancestors) { - if (a != 0) { has_anc = true; break; } - } - if (has_anc) { - string anc; - foreach (i, a; obj.metainfo.markedup_ancestors) { - if (i > 0) { anc ~= " "; } - anc ~= a.to!string; - } - output ~= ".ancestors: " ~ anc; - } - } - /+ ↓ collapsed ancestors (only if non-zero) +/ - { - bool has_anc_c = false; - foreach (a; obj.metainfo.collapsed_ancestors) { - if (a != 0) { has_anc_c = true; break; } - } - if (has_anc_c) { - string anc; - foreach (i, a; obj.metainfo.collapsed_ancestors) { - if (i > 0) anc ~= " "; - anc ~= a.to!string; - } - output ~= ".ancestors_collapsed: " ~ anc; - } - } - /+ ↓ dom structure status (only if non-zero) +/ - { - bool has_dom = false; - foreach (d; obj.metainfo.dom_structure_markedup_tags_status) { - if (d != 0) { has_dom = true; break; } - } - if (has_dom) { - string ds; - foreach (i, d; obj.metainfo.dom_structure_markedup_tags_status) { - if (i > 0) { ds ~= " "; } - ds ~= d.to!string; - } - output ~= ".dom_status: " ~ ds; - } - } - { - bool has_dom_c = false; - foreach (d; obj.metainfo.dom_structure_collapsed_tags_status) { - if (d != 0) { has_dom_c = true; break; } - } - if (has_dom_c) { - string ds; - foreach (i, d; obj.metainfo.dom_structure_collapsed_tags_status) { - if (i > 0) { ds ~= " "; } - ds ~= d.to!string; - } - output ~= ".dom_status_collapsed: " ~ ds; - } - } - if (obj.metainfo.heading_lev_collapsed < 9) { - output ~= ".heading_lev_collapsed: " ~ obj.metainfo.heading_lev_collapsed.to!string; - } - if (obj.metainfo.parent_lev_markup != 0) { - output ~= ".parent_lev: " ~ obj.metainfo.parent_lev_markup.to!string; - } - if (obj.metainfo.dummy_heading) { - output ~= ".dummy: true"; - } - if (obj.metainfo.object_number_off) { - output ~= ".ocn_off: true"; - } - if (obj.metainfo.is_of_type.length > 0) { - output ~= ".is_of_type: " ~ obj.metainfo.is_of_type; - } - if (obj.metainfo.attrib.length > 0) { - output ~= ".attrib: " ~ obj.metainfo.attrib; - } - if (obj.metainfo.lang.length > 0) { - output ~= ".meta_lang: " ~ obj.metainfo.lang; - } - if (obj.metainfo.syntax.length > 0) { - output ~= ".meta_syntax: " ~ obj.metainfo.syntax; - } - /+ ↓ sha256 digest +/ - { - bool has_sha = false; - foreach (b; obj.metainfo.sha256.text) { - if (b != 0) { has_sha = true; break; } - } - if (has_sha) { - if (_vox) { - writeln(obj.metainfo.sha256.summary); - } - output ~= ".sha256: " ~ obj.metainfo.sha256.text.toHexString.to!string; - } - } - /+ ↓ images, repeatable record, one per image in order of appearance: - .image: sha256: bytes: px:x - .image: missing:true - filename first (the object's handle on the image), remaining fields - self-describing key:value tokens so that further fields can be - added later without invalidating existing readers. px: is the - file's own pixel size, not the display dimensions (w##h##) which - the object text already carries - +/ - if (obj.metainfo.sha256.images.length > 0) { - foreach (i; obj.metainfo.sha256.images) { - output ~= (i.fileMissing) - ? ".image: " ~ i.fileName - ~ " missing:true" - : ".image: " ~ i.fileName - ~ " sha256:" ~ i.fileHash_sha256.toHexString.to!string - ~ " bytes:" ~ i.fileSize.to!string - ~ ((i.imageWidth > 0 && i.imageHeight > 0) - ? " px:" ~ i.imageWidth.to!string - ~ "x" ~ i.imageHeight.to!string - : ""); - } - } - /+ ↓ text attributes +/ - if (obj.attrib.indent_base != 0 || obj.attrib.indent_hang != 0) { - output ~= ".indent: " ~ obj.attrib.indent_base.to!string - ~ " " ~ obj.attrib.indent_hang.to!string; - } - if (obj.attrib.bullet) { - output ~= ".bullet: true"; - } - if (obj.attrib.language.length > 0) { - output ~= ".lang: " ~ obj.attrib.language; - } - /+ ↓ has flags +/ - { - string[] has_flags; - if (obj.has.inline_links) { has_flags ~= "links"; } - if (obj.has.inline_notes_reg) { has_flags ~= "notes_reg"; } - if (obj.has.inline_notes_star) { has_flags ~= "notes_star"; } - if (obj.has.images) { has_flags ~= "images"; } - if (obj.has.image_without_dimensions) { has_flags ~= "images_no_dim"; } - if (has_flags.length > 0) { - output ~= ".has: " ~ has_flags.join(" "); - } - } - /+ ↓ table properties +/ - if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { - output ~= ".table_cols: " ~ obj.table.number_of_columns.to!string; - if (obj.table.column_widths.length > 0) { - string[] ws; - foreach (w; obj.table.column_widths) ws ~= w.to!string; - output ~= ".table_widths: " ~ ws.join(" "); - } - if (obj.table.column_aligns.length > 0) { - output ~= ".table_aligns: " ~ obj.table.column_aligns.join(" "); - } - if (obj.table.heading) { - output ~= ".table_header: true"; - } - } - /+ ↓ code block properties +/ - if (obj.metainfo.is_a == "code") { - if (obj.code_block.linenumbers) { - output ~= ".code_linenumbers: true"; - } - } - /+ ↓ stow (extracted links) +/ - if (obj.stow.link.length > 0) { - foreach (lnk; obj.stow.link) { - if (lnk.length > 0) { - output ~= ".stow_link: " ~ lnk; - } - } - } - /+ ↓ tag properties +/ - if (obj.tags.in_segment_html.length > 0) { - output ~= ".segment: " ~ obj.tags.in_segment_html; - } - if (obj.tags.anchor_tag_html.length > 0 - && obj.tags.anchor_tag_html != obj.tags.in_segment_html) { - output ~= ".anchor: " ~ obj.tags.anchor_tag_html; - } - if (obj.tags.segname_prev.length > 0) { - output ~= ".segment_prev: " ~ obj.tags.segname_prev; - } - if (obj.tags.segname_next.length > 0) { - output ~= ".segment_next: " ~ obj.tags.segname_next; - } - if (obj.tags.heading_lev_anchor_tag.length > 0) { - output ~= ".heading_lev_anchor: " ~ obj.tags.heading_lev_anchor_tag; - } - if (obj.tags.segment_anchor_tag_epub.length > 0) { - output ~= ".segment_epub: " ~ obj.tags.segment_anchor_tag_epub; - } - /+ ↓ the segment an object belongs to, as opposed to the segment a - heading opens. .segment (in_segment_html) is set on headings - only, so without these two a reader cannot tell which segment a - paragraph is in; the sqlite search writer reads the html one - +/ - if (obj.tags.html_segment_anchor_tag_is.length > 0) { - output ~= ".segment_html_is: " ~ obj.tags.html_segment_anchor_tag_is; - } - if (obj.tags.epub_segment_anchor_tag_is.length > 0) { - output ~= ".segment_epub_is: " ~ obj.tags.epub_segment_anchor_tag_is; - } - /+ ↓ the level 4 segment a cross reference to this heading lands on, - emitted only where it is not the segment the heading sits in, which - is every heading at level 4 or below +/ - if (obj.tags.segment_lv4_is.length > 0 - && obj.tags.segment_lv4_is != obj.tags.html_segment_anchor_tag_is - ) { - output ~= ".segment_lv4_is: " ~ obj.tags.segment_lv4_is; - } - /+ ↓ heading ancestors text +/ - { - bool has_hat = false; - foreach (h; obj.tags.heading_ancestors_text) { - if (h.length > 0) { has_hat = true; break; } - } - if (has_hat) { - string[] _hat; - foreach (h; obj.tags.heading_ancestors_text) { - _hat ~= sspEscapeValue(h.to!string); - } - output ~= ".heading_ancestors_text: " ~ _hat.join("|"); - } - } - /+ ↓ lev4 subtoc +/ - if (obj.tags.lev4_subtoc.length > 0) { - foreach (st; obj.tags.lev4_subtoc) { - if (st.length > 0) { - output ~= ".lev4_subtoc: " ~ st; - } - } - } - /+ ↓ anchor tags +/ - if (obj.tags.anchor_tags.length > 0) { - foreach (at; obj.tags.anchor_tags) { - if (at.length > 0) { - output ~= ".anchor_tag: " ~ at; - } - } - } - /+ ↓ text content +/ - if (obj.text.length > 0) { - foreach (line; obj.text.split("\n")) { - output ~= "| " ~ line; - } - } - output ~= ""; - return output; - } -} - -/+ ↓ abstraction directories whose stale .ssp have been cleared this run, one - entry per document. __gshared, and read and written only inside the - synchronized block in the writer below: a document's languages share one - abstraction directory and may be written in parallel -+/ -private __gshared bool[string] _abstraction_dir_prepared; -template spineAbstractionTxt() { - import std.conv : to; - import std.digest : toHexString; - import std.file; - import std.path; - import std.stdio; - import std.string; - import std.array; - import sisudoc.ocda.io_in.paths_source; - mixin sspObjectRecord; - /+ ↓ the whole .ssp for a document, as lines. the file writer below emits - these, and the abstraction db is built by reading them back, so the - database cannot carry anything the .ssp does not - +/ - string[] sspDocumentLines(D)(D doc) { - auto doc_abstraction = doc.abstraction; - auto doc_matters = doc.matters; - string[] output; - /+ ↓ header comment +/ - output ~= "% SiSU Document Abstraction v" ~ ssp_format_version; - output ~= "% Source: " ~ doc_matters.src.filename; - output ~= ""; - /+ ↓ @source block: what this abstraction was made from, so a reader can - say whether it is looking at the right images and whether the - markup has moved on since. the filename is in the % Source: line - above and is not repeated here - +/ - output ~= "@source {"; - output ~= " language: " ~ doc_matters.src.language; - if (doc_matters.pod.manifest_list_of_languages.length > 0) { - output ~= " languages: " ~ doc_matters.pod.manifest_list_of_languages.join(" "); - } - output ~= " digest: " ~ doc_matters.doc_digest.markup_doc.toHexString.to!string; - output ~= "}"; - output ~= ""; - /+ ↓ @meta block +/ - output ~= "@meta {"; - auto meta = doc_matters.conf_make_meta.meta; - if (meta.title_main.length > 0) { - output ~= " title.main: " ~ meta.title_main; - } - if (meta.title_subtitle.length > 0) { - output ~= " title.subtitle: " ~ meta.title_subtitle; - } - if (meta.title_full.length > 0) { - output ~= " title.full: " ~ meta.title_full; - } - if (meta.title_language.length > 0) { - output ~= " title.language: " ~ meta.title_language; - } - if (meta.title_edition.length > 0) { - output ~= " title.edition: " ~ meta.title_edition; - } - if (meta.creator_author.length > 0) { - output ~= " creator.author: " ~ meta.creator_author; - } - if (meta.creator_author_surname.length > 0) { - output ~= " creator.author_surname: " ~ meta.creator_author_surname; - } - if (meta.creator_author_surname_fn.length > 0) { - output ~= " creator.author_surname_fn: " ~ meta.creator_author_surname_fn; - } - if (meta.creator_author_email.length > 0) { - output ~= " creator.author_email: " ~ meta.creator_author_email; - } - if (meta.creator_illustrator.length > 0) { - output ~= " creator.illustrator: " ~ meta.creator_illustrator; - } - if (meta.creator_translator.length > 0) { - output ~= " creator.translator: " ~ meta.creator_translator; - } - if (meta.date_published.length > 0) { - output ~= " date.published: " ~ meta.date_published; - } - if (meta.date_created.length > 0) { - output ~= " date.created: " ~ meta.date_created; - } - if (meta.date_issued.length > 0) { - output ~= " date.issued: " ~ meta.date_issued; - } - if (meta.date_available.length > 0) { - output ~= " date.available: " ~ meta.date_available; - } - if (meta.date_modified.length > 0) { - output ~= " date.modified: " ~ meta.date_modified; - } - if (meta.date_valid.length > 0) { - output ~= " date.valid: " ~ meta.date_valid; - } - if (meta.date_added_to_site.length > 0) { - output ~= " date.added_to_site: " ~ meta.date_added_to_site; - } - if (meta.rights_copyright.length > 0) { - output ~= " rights.copyright: " ~ meta.rights_copyright; - } - if (meta.rights_copyright_text.length > 0) { - output ~= " rights.copyright_text: " ~ meta.rights_copyright_text; - } - if (meta.rights_copyright_translation.length > 0) { - output ~= " rights.copyright_translation: " ~ meta.rights_copyright_translation; - } - if (meta.rights_copyright_illustrations.length > 0) { - output ~= " rights.copyright_illustrations: " ~ meta.rights_copyright_illustrations; - } - if (meta.rights_copyright_photographs.length > 0) { - output ~= " rights.copyright_photographs: " ~ meta.rights_copyright_photographs; - } - if (meta.rights_copyright_cover.length > 0) { - output ~= " rights.copyright_cover: " ~ meta.rights_copyright_cover; - } - if (meta.rights_copyright_audio.length > 0) { - output ~= " rights.copyright_audio: " ~ meta.rights_copyright_audio; - } - if (meta.rights_copyright_video.length > 0) { - output ~= " rights.copyright_video: " ~ meta.rights_copyright_video; - } - if (meta.rights_license.length > 0) { - output ~= " rights.license: " ~ meta.rights_license; - } - if (meta.classify_topic_register.length > 0) { - output ~= " classify.topic_register: " ~ meta.classify_topic_register; - } - if (meta.classify_subject.length > 0) { - output ~= " classify.subject: " ~ meta.classify_subject; - } - if (meta.classify_keywords.length > 0) { - output ~= " classify.keywords: " ~ meta.classify_keywords; - } - if (meta.classify_loc.length > 0) { - output ~= " classify.loc: " ~ meta.classify_loc; - } - if (meta.classify_dewey.length > 0) { - output ~= " classify.dewey: " ~ meta.classify_dewey; - } - if (meta.identifier_isbn.length > 0) { - output ~= " identifier.isbn: " ~ meta.identifier_isbn; - } - if (meta.identifier_oclc.length > 0) { - output ~= " identifier.oclc: " ~ meta.identifier_oclc; - } - if (meta.language_document.length > 0) { - output ~= " language.document: " ~ meta.language_document; - } - if (meta.language_document_char.length > 0) { - output ~= " language.document_char: " ~ meta.language_document_char; - } - if (meta.original_title.length > 0) { - output ~= " original.title: " ~ meta.original_title; - } - if (meta.original_source.length > 0) { - output ~= " original.source: " ~ meta.original_source; - } - if (meta.original_language.length > 0) { - output ~= " original.language: " ~ meta.original_language; - } - if (meta.original_language_char.length > 0) { - output ~= " original.language_char: " ~ meta.original_language_char; - } - if (meta.notes_abstract.length > 0) { - output ~= " notes.abstract: " ~ meta.notes_abstract; - } - if (meta.notes_description.length > 0) { - output ~= " notes.description: " ~ meta.notes_description; - } - if (meta.notes_summary.length > 0) { - output ~= " notes.summary: " ~ meta.notes_summary; - } // check - output ~= "}"; - output ~= ""; - /+ ↓ @make block +/ - output ~= "@make {"; - auto make = doc_matters.conf_make_meta.make; - if (make.doc_type.length > 0) { - output ~= " doc_type: " ~ make.doc_type; - } - if (make.auto_num_top_at_level.length > 0) { - output ~= " auto_num_top_at_level: " ~ make.auto_num_top_at_level; - } // check - output ~= " auto_num_top_lv: " ~ make.auto_num_top_lv.to!string; - output ~= " auto_num_depth: " ~ make.auto_num_depth.to!string; - /+ ↓ the make settings the output writers read. the rest of make - (italics, bold, emphasis, substitute, headings) is read at parse - time and its effect is already in the objects, so it does not - travel; these three are read in outputs/io_out/ and must - +/ - if (make.breaks.length > 0) { - output ~= " breaks: " ~ make.breaks; - } - if (make.footer.length > 0) { // a list: joined, each slot escaped - string[] _ft; - foreach (_f; make.footer) { _ft ~= sspEscapeValue(_f); } - output ~= " footer: " ~ _ft.join("|"); - } - if (make.home_button_text.length > 0) { - output ~= " home_button_text: " ~ make.home_button_text; - } - output ~= "}"; - output ~= ""; - /+ ↓ @doc_has block +/ - output ~= "@doc_has {"; - output ~= " inline_links: " ~ doc_matters.has.inline_links.to!string; - output ~= " inline_notes_reg: " ~ doc_matters.has.inline_notes_reg.to!string; - output ~= " inline_notes_star: " ~ doc_matters.has.inline_notes_star.to!string; - output ~= " tables: " ~ doc_matters.has.tables.to!string; - output ~= " codeblocks: " ~ doc_matters.has.codeblocks.to!string; - output ~= " images: " ~ doc_matters.has.images.to!string; - output ~= " poems: " ~ doc_matters.has.poems.to!string; - output ~= " groups: " ~ doc_matters.has.groups.to!string; - output ~= " blocks: " ~ doc_matters.has.blocks.to!string; - output ~= " quotes: " ~ doc_matters.has.quotes.to!string; - output ~= "}"; - output ~= ""; - /+ ↓ document sections - every section the abstraction holds, in document order. - "tail" is the end-of-document used by some output writers - +/ - string[] section_order = ["head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", "tail"]; - foreach (section; section_order) { - if (section !in doc_abstraction) { continue; } - auto section_objs = doc_abstraction[section]; - if (section_objs.length == 0) { continue; } - output ~= "@" ~ section ~ " {"; - output ~= ""; - foreach (obj; section_objs) { - output ~= sspObjectRecord(obj, section, doc_matters.opt.action.vox_gt_3); - } - output ~= "}"; - output ~= ""; - } - return output; - } - @trusted void spineAbstractionTxt(D)(D doc) { - auto doc_matters = doc.matters; - string[] output = sspDocumentLines(doc); - /+ ↓ write to file, into the pod's own tree: - pod//media/abstraction/.ssp - . - where it sits beside pod//media/image/, which is what a .ssp - needs to be a document source: the file describes its images by - name, size, pixels and digest, and they have to be findable. Written - here once; --pod2 then digests and zips it where it stands rather - than copying it in from an output directory of its own. - +/ - auto pths_pod = spinePathsPods!()(doc_matters); - string base_pth - = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string; - /+ ↓ this directory is the one part of the pod written before the pod is - built, so the pod builder leaves it alone. It therefore clears itself - here, or a .ssp for a language the document no longer has would linger - and be bundled. - . - Every language of a document writes into this one directory, and the - languages may be running at once, so the clearing is done once per - directory per run, by whichever language reaches it first, with the - lock held across it. A language that finds the directory already - prepared passed through this lock before writing, so the clearing it - skipped had already finished, and no .ssp written by this run can be - removed by it. Keying the clearing on the manifest's first language - instead makes it wait for a thread that may be scheduled last, and - take the .ssp already written by the other languages with it. - +/ - synchronized { - if (base_pth !in _abstraction_dir_prepared) { - _abstraction_dir_prepared[base_pth] = true; - try { - if (!exists(base_pth)) { - base_pth.mkdirRecurse; - } else { - foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) { - _f.remove; - } - } - } catch (Exception ex) { - } - } - } - string out_file = ((base_pth.chainPath( - doc_matters.src.doc_uid_out ~ ".ssp")).asNormalizedPath).array; - if (doc_matters.opt.action.vox_gt_1) { - writeln(" ", out_file); - } - auto f = File(out_file, "w"); - foreach (line; output) { - f.writeln(line); - } - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/out_src_abstraction_sqlite_db.org b/org/out_src_abstraction_sqlite_db.org deleted file mode 100644 index 70c118a..0000000 --- a/org/out_src_abstraction_sqlite_db.org +++ /dev/null @@ -1,666 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) output pod source sqlite db -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:output:source:pod: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* (Object-Centric) Document Abstraction SQLite db - -- Process markup document, create document abstraction - -** _module template_ :module:metadoc_from_src: - -rename source_abstraction_peg_txt.d - -#+HEADER: :tangle "../src/sisudoc/outputs/io_out/create_ocda_db.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<> -module sisudoc.outputs.io_out.create_ocda_db; - -/+ ↓ write document abstraction as per-document sqlite3 database +/ -template spineAbstractionDb() { - import std.conv : to; - import std.digest : toHexString; - import std.file; - import std.path; - import std.stdio; - import std.string; - import std.array; - import d2sqlite3; - import sisudoc.outputs.io_out.paths_output; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.abstraction.ssp : ssp_format_version; - /+ ↓ the document is passed in rather than taken from doc, because the - caller hands over one that has been through the .ssp: written as text - and read back. so the database is a function of the .ssp and cannot - carry anything the .ssp does not, header blocks included: the metadata - table below is filled from what came back, not from a second list read - off doc_matters, which is how the two drifted apart before - +/ - void spineAbstractionDb(D,S)(D doc, S ssp_doc) { - auto doc_matters = doc.matters; - auto doc_abstraction = ssp_doc.abstraction; - /+ ↓ output path: beside the pod's zip and digests, not under - /abstraction/. the database carries its own images, so it is a - distributable artefact in its own right and belongs with the others: - pod// the source tree - pod/.zip the same, zipped - pod/.digests.txt what is in them - pod/.ocda.db the abstraction, one per language - +/ - auto pths_pod = spinePathsPods!()(doc_matters); - string base_pth = pths_pod.pod_dir_(); - try { - if (!exists(base_pth)) { - base_pth.mkdirRecurse; - } - } catch (Exception ex) { - } - string db_file = ((base_pth.chainPath( - doc_matters.src.doc_uid_out ~ ".ocda.db")).asNormalizedPath).array; - - /+ ↓ remove existing file to start fresh +/ - try { - if (exists(db_file)) { - remove(db_file); - } - } catch (Exception ex) { - } - - if (doc_matters.opt.action.vox_gt_1) { - writeln(" ", db_file); - } - - /+ ↓ open database and create schema +/ - auto db = Database(db_file); - /+ ↓ write once, read many: no WAL. - a WAL stamped file cannot be opened by a reader that cannot write - beside it (SQLITE_READONLY on a read-only directory, and inside a - zip or on read-only media), and this database has one writer and no - concurrency, so WAL buys nothing here. MEMORY journal keeps the bulk - load fast without leaving that mark on the file. - +/ - db.run("PRAGMA journal_mode=MEMORY"); - db.run("PRAGMA synchronous=OFF"); - - db.run(" - CREATE TABLE metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - -- objects: one row per abstraction object, in document order. - -- the fixed width arrays (ancestors, dom status, children, heading - -- ancestors text, table widths and aligns) are JSON arrays so that - -- json_extract, json_each and json_array_length can reach inside them; - -- the open ended lists (images, links, anchor tags, subtoc) are rows - -- in their own tables, not columns - CREATE TABLE objects ( - id INTEGER PRIMARY KEY, - section TEXT NOT NULL, - seq INTEGER NOT NULL, - ocn INTEGER DEFAULT 0, - is_a TEXT NOT NULL, - is_of_part TEXT, - is_of_section TEXT, - is_of_type TEXT, - heading_level INTEGER, - heading_lev_collapsed INTEGER, - identifier TEXT, - parent_ocn INTEGER DEFAULT 0, - parent_lev INTEGER DEFAULT 0, - last_descendant_ocn INTEGER DEFAULT 0, - children TEXT - CHECK(children IS NULL OR json_valid(children)), - ancestors TEXT - CHECK(ancestors IS NULL OR json_valid(ancestors)), - ancestors_collapsed TEXT - CHECK(ancestors_collapsed IS NULL OR json_valid(ancestors_collapsed)), - dom_status TEXT - CHECK(dom_status IS NULL OR json_valid(dom_status)), - dom_status_collapsed TEXT - CHECK(dom_status_collapsed IS NULL OR json_valid(dom_status_collapsed)), - dummy_heading INTEGER DEFAULT 0, - object_number_off INTEGER DEFAULT 0, - attrib TEXT, - meta_lang TEXT, - meta_syntax TEXT, - sha256 TEXT, - indent_base INTEGER DEFAULT 0, - indent_hang INTEGER DEFAULT 0, - bullet INTEGER DEFAULT 0, - lang TEXT, - has_links INTEGER DEFAULT 0, - has_notes_reg INTEGER DEFAULT 0, - has_notes_star INTEGER DEFAULT 0, - has_images INTEGER DEFAULT 0, - has_images_no_dim INTEGER DEFAULT 0, - segment TEXT, - segment_prev TEXT, - segment_next TEXT, - segment_epub TEXT, - segment_html_is TEXT, - segment_epub_is TEXT, - segment_lv4_is TEXT, - anchor TEXT, - heading_lev_anchor TEXT, - heading_ancestors_text TEXT - CHECK(heading_ancestors_text IS NULL OR json_valid(heading_ancestors_text)), - table_cols INTEGER, - table_widths TEXT - CHECK(table_widths IS NULL OR json_valid(table_widths)), - table_aligns TEXT - CHECK(table_aligns IS NULL OR json_valid(table_aligns)), - table_header INTEGER, - code_linenumbers INTEGER DEFAULT 0, - text TEXT, - UNIQUE(section, seq) - ); - - CREATE INDEX idx_objects_section ON objects(section); - CREATE INDEX idx_objects_ocn ON objects(ocn); - CREATE INDEX idx_objects_parent ON objects(parent_ocn); - CREATE INDEX idx_objects_is_a ON objects(is_a); - CREATE INDEX idx_objects_heading ON objects(heading_level) - WHERE heading_level IS NOT NULL; - - -- files: what this database carries with it, so that a reader needs - -- nothing beside it. role says what a file is; images are the ones the - -- output writers must have in order to produce a document. bytes are - -- stored exactly as read, so sha256 matches the digest the abstraction - -- was built with - CREATE TABLE files ( - id INTEGER PRIMARY KEY, - role TEXT NOT NULL, - name TEXT NOT NULL, - bytes INTEGER NOT NULL, - sha256 TEXT, - width INTEGER, - height INTEGER, - data BLOB, - UNIQUE(role, name) - ); - - -- the open ended per object lists, one row each, in order - CREATE TABLE object_images ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - name TEXT NOT NULL, - bytes INTEGER, - sha256 TEXT, - width INTEGER, - height INTEGER, - missing INTEGER DEFAULT 0, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_links ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - url TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_anchors ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - anchor TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_subtoc ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - entry TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - - CREATE INDEX idx_files_role ON files(role); - CREATE INDEX idx_object_images_name ON object_images(name); - - -- full text over the object text, external content so the text is not - -- stored twice. per document: this is for exploring one abstraction, - -- the collection wide search database is a different thing - CREATE VIRTUAL TABLE objects_fts USING fts5( - text, content='objects', content_rowid='id' - ); - - -- the shapes worth asking for, named - CREATE VIEW outline AS - SELECT section, seq, ocn, heading_level, heading_lev_collapsed, - last_descendant_ocn, identifier, text - FROM objects WHERE is_a = 'heading' ORDER BY id; - CREATE VIEW citable AS - SELECT * FROM objects WHERE ocn > 0 ORDER BY ocn; - CREATE VIEW document_files AS - SELECT role, name, bytes, sha256, width, height FROM files ORDER BY role, name; - "); - - /+ ↓ populate metadata, from the header blocks the .ssp gave back. - one list, in ssp.d, not two: a property added to the .ssp header - arrives here without anything being changed, and one that is not in - the .ssp cannot be here at all. the keys are the .ssp's own - (title.main, make.doc_type, doc_has.images), prefixed by their block - where the block name is not already part of the key. - +/ - db.run("BEGIN TRANSACTION"); - - auto meta_stmt = db.prepare( - "INSERT INTO metadata (key, value) VALUES (:key, :value)" - ); - - void insertMeta(string key, string value) { - if (value.length > 0) { - meta_stmt.bind(":key", key); - meta_stmt.bind(":value", value); - meta_stmt.execute(); - meta_stmt.reset(); - } - } - void insertBlock(string prefix, string[] keys_in_order, string[string] kv) { - foreach (k; keys_in_order) { - if (k in kv) { insertMeta(prefix ~ k, kv[k]); } - } - } - - /+ ↓ what this file is, so a reader can tell before it starts. the - version is the .ssp's own: the two serialisations are one format and - move together - +/ - insertMeta("schema.name", "sisu-abstraction-db"); - insertMeta("schema.version", ssp_format_version); - insertMeta("source.filename", ssp_doc.source); - insertBlock("source.", ssp_doc.source_info_order, ssp_doc.source_info); - /+ ↓ the digest of the .ssp this database was built from. a file cannot - hold its own hash, so this one is not in the .ssp and is taken here: - source.digest says which markup the abstraction came from, - source.ssp_digest says which abstraction this database came from, - and the chain .sst -> .ssp -> .ocda.db is checkable end to end - +/ - insertMeta("source.ssp_digest", ssp_doc.ssp_digest); - - insertBlock("", ssp_doc.meta_order, ssp_doc.meta); - insertBlock("make.", ssp_doc.make_order, ssp_doc.make); - insertBlock("doc_has.", ssp_doc.doc_has_order, ssp_doc.doc_has); - meta_stmt.finalize(); - - /+ ↓ populate objects +/ - auto obj_stmt = db.prepare( - "INSERT INTO objects (" - ~ "section, seq, ocn, is_a, is_of_part, is_of_section, is_of_type," - ~ "heading_level, heading_lev_collapsed, identifier," - ~ "parent_ocn, parent_lev, last_descendant_ocn," - ~ "children, ancestors, ancestors_collapsed," - ~ "dom_status, dom_status_collapsed," - ~ "dummy_heading, object_number_off," - ~ "attrib, meta_lang, meta_syntax, sha256," - ~ "indent_base, indent_hang, bullet, lang," - ~ "has_links, has_notes_reg, has_notes_star, has_images, has_images_no_dim," - ~ "segment, segment_prev, segment_next, segment_epub," - ~ "segment_html_is, segment_epub_is, segment_lv4_is," - ~ "anchor, heading_lev_anchor, heading_ancestors_text," - ~ "table_cols, table_widths, table_aligns, table_header," - ~ "code_linenumbers, text" - ~ ") VALUES (" - ~ ":section, :seq, :ocn, :is_a, :is_of_part, :is_of_section, :is_of_type," - ~ ":heading_level, :heading_lev_collapsed, :identifier," - ~ ":parent_ocn, :parent_lev, :last_descendant_ocn," - ~ ":children, :ancestors, :ancestors_collapsed," - ~ ":dom_status, :dom_status_collapsed," - ~ ":dummy_heading, :object_number_off," - ~ ":attrib, :meta_lang, :meta_syntax, :sha256," - ~ ":indent_base, :indent_hang, :bullet, :lang," - ~ ":has_links, :has_notes_reg, :has_notes_star, :has_images, :has_images_no_dim," - ~ ":segment, :segment_prev, :segment_next, :segment_epub," - ~ ":segment_html_is, :segment_epub_is, :segment_lv4_is," - ~ ":anchor, :heading_lev_anchor, :heading_ancestors_text," - ~ ":table_cols, :table_widths, :table_aligns, :table_header," - ~ ":code_linenumbers, :text" - ~ ")" - ); - - /+ ↓ the per object lists, one row each +/ - auto img_stmt = db.prepare( - "INSERT INTO object_images" - ~ " (object_id, seq, name, bytes, sha256, width, height, missing)" - ~ " VALUES (:object_id, :seq, :name, :bytes, :sha256, :width, :height, :missing)" - ); - auto lnk_stmt = db.prepare( - "INSERT INTO object_links (object_id, seq, url)" - ~ " VALUES (:object_id, :seq, :url)" - ); - auto anc_stmt = db.prepare( - "INSERT INTO object_anchors (object_id, seq, anchor)" - ~ " VALUES (:object_id, :seq, :anchor)" - ); - auto sub_stmt = db.prepare( - "INSERT INTO object_subtoc (object_id, seq, entry)" - ~ " VALUES (:object_id, :seq, :entry)" - ); - - string[] section_order = ["head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", "tail"]; - - foreach (section; section_order) { - if (section !in doc_abstraction) continue; - auto section_objs = doc_abstraction[section]; - if (section_objs.length == 0) continue; - - foreach (seq, obj; section_objs) { - obj_stmt.bind(":section", section); - obj_stmt.bind(":seq", cast(int) seq); - obj_stmt.bind(":ocn", obj.metainfo.ocn); - obj_stmt.bind(":is_a", obj.metainfo.is_a); - - /+ ↓ nullable string fields +/ - void bindStr(string param, string val) { - import std.typecons : Nullable; - if (val.length > 0) { - obj_stmt.bind(param, val); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - /+ ↓ nullable int fields +/ - void bindInt(string param, int val, bool is_set) { - import std.typecons : Nullable; - if (is_set) { - obj_stmt.bind(param, val); - } else { - obj_stmt.bind(param, Nullable!int()); - } - } - /+ ↓ fixed width int array as a JSON array, NULL when all zero +/ - void bindJsonInts(T)(string param, T arr) { - import std.typecons : Nullable; - bool any = false; - foreach (a; arr) { if (a != 0) { any = true; break; } } - if (any) { - string[] _j; - foreach (a; arr) { _j ~= a.to!string; } - obj_stmt.bind(param, "[" ~ _j.join(",") ~ "]"); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - /+ ↓ string array as a JSON array, NULL when every slot is empty +/ - void bindJsonStrs(T)(string param, T arr) { - import std.json : JSONValue; - import std.typecons : Nullable; - bool any = false; - foreach (s; arr) { if (s.length > 0) { any = true; break; } } - if (any) { - string[] _s; - foreach (s; arr) { _s ~= s.to!string; } - obj_stmt.bind(param, JSONValue(_s).toString); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - - bindStr(":is_of_part", obj.metainfo.is_of_part); - bindStr(":is_of_section", obj.metainfo.is_of_section); - bindStr(":is_of_type", obj.metainfo.is_of_type); - - /+ ↓ heading levels, marked up and collapsed, 9 meaning unset +/ - bindInt(":heading_level", obj.metainfo.heading_lev_markup, - (obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup < 9)); - bindInt(":heading_lev_collapsed", obj.metainfo.heading_lev_collapsed, - (obj.metainfo.heading_lev_collapsed < 9)); - - bindStr(":identifier", obj.metainfo.identifier); - obj_stmt.bind(":parent_ocn", obj.metainfo.parent_ocn); - obj_stmt.bind(":parent_lev", obj.metainfo.parent_lev_markup); - obj_stmt.bind(":last_descendant_ocn", obj.metainfo.last_descendant_ocn); - - bindJsonInts(":children", obj.metainfo.children_headings); - bindJsonInts(":ancestors", obj.metainfo.markedup_ancestors); - bindJsonInts(":ancestors_collapsed", obj.metainfo.collapsed_ancestors); - bindJsonInts(":dom_status", obj.metainfo.dom_structure_markedup_tags_status); - bindJsonInts(":dom_status_collapsed", obj.metainfo.dom_structure_collapsed_tags_status); - - obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); - obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); - bindStr(":attrib", obj.metainfo.attrib); - bindStr(":meta_lang", obj.metainfo.lang); - bindStr(":meta_syntax", obj.metainfo.syntax); - - /+ ↓ object digest, hex, absent when never taken +/ - { - import std.typecons : Nullable; - bool has_sha = false; - foreach (b; obj.metainfo.sha256.text) { - if (b != 0) { has_sha = true; break; } - } - if (has_sha) { - obj_stmt.bind(":sha256", obj.metainfo.sha256.text.toHexString.to!string); - } else { - obj_stmt.bind(":sha256", Nullable!string()); - } - } - - // obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); - // obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); - obj_stmt.bind(":indent_base", obj.attrib.indent_base); - obj_stmt.bind(":indent_hang", obj.attrib.indent_hang); - obj_stmt.bind(":bullet", obj.attrib.bullet ? 1 : 0); - bindStr(":lang", obj.attrib.language); - obj_stmt.bind(":has_links", obj.has.inline_links ? 1 : 0); - obj_stmt.bind(":has_notes_reg", obj.has.inline_notes_reg ? 1 : 0); - obj_stmt.bind(":has_notes_star", obj.has.inline_notes_star ? 1 : 0); - obj_stmt.bind(":has_images", obj.has.images ? 1 : 0); - obj_stmt.bind(":has_images_no_dim", obj.has.image_without_dimensions ? 1 : 0); - bindStr(":segment", obj.tags.in_segment_html); - bindStr(":segment_prev", obj.tags.segname_prev); - bindStr(":segment_next", obj.tags.segname_next); - bindStr(":segment_epub", obj.tags.segment_anchor_tag_epub); - bindStr(":segment_html_is", obj.tags.html_segment_anchor_tag_is); - bindStr(":segment_epub_is", obj.tags.epub_segment_anchor_tag_is); - bindStr(":segment_lv4_is", obj.tags.segment_lv4_is); - bindStr(":anchor", obj.tags.anchor_tag_html); - bindStr(":heading_lev_anchor", obj.tags.heading_lev_anchor_tag); - bindJsonStrs(":heading_ancestors_text", obj.tags.heading_ancestors_text); - - /+ ↓ table properties +/ - { - import std.typecons : Nullable; - if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { - obj_stmt.bind(":table_cols", obj.table.number_of_columns); - if (obj.table.column_widths.length > 0) { - string[] ws; - foreach (w; obj.table.column_widths) ws ~= w.to!string; - obj_stmt.bind(":table_widths", "[" ~ ws.join(",") ~ "]"); - } else { - obj_stmt.bind(":table_widths", Nullable!string()); - } - if (obj.table.column_aligns.length > 0) { - import std.json : JSONValue; - string[] _al; - foreach (a; obj.table.column_aligns) { _al ~= a.to!string; } - obj_stmt.bind(":table_aligns", JSONValue(_al).toString); - } else { - obj_stmt.bind(":table_aligns", Nullable!string()); - } - obj_stmt.bind(":table_header", obj.table.heading ? 1 : 0); - } else { - obj_stmt.bind(":table_cols", Nullable!int()); - obj_stmt.bind(":table_widths", Nullable!string()); - obj_stmt.bind(":table_aligns", Nullable!string()); - obj_stmt.bind(":table_header", Nullable!int()); - } - } - - /+ ↓ code block properties +/ - obj_stmt.bind(":code_linenumbers", - ((obj.metainfo.is_a == "code") && obj.code_block.linenumbers) ? 1 : 0); - - /+ ↓ text content +/ - bindStr(":text", obj.text); - - obj_stmt.execute(); - obj_stmt.reset(); - - /+ ↓ the per object lists, keyed on the row just written +/ - long _obj_id = db.lastInsertRowid; - { - int _n = 0; - foreach (i; obj.metainfo.sha256.images) { - img_stmt.bind(":object_id", _obj_id); - img_stmt.bind(":seq", _n); - img_stmt.bind(":name", i.fileName); - img_stmt.bind(":bytes", cast(long) i.fileSize); - img_stmt.bind(":sha256", i.fileHash_sha256.toHexString.to!string); - img_stmt.bind(":width", i.imageWidth); - img_stmt.bind(":height", i.imageHeight); - img_stmt.bind(":missing", i.fileMissing ? 1 : 0); - img_stmt.execute(); - img_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (lnk; obj.stow.link) { - if (lnk.length == 0) { continue; } - lnk_stmt.bind(":object_id", _obj_id); - lnk_stmt.bind(":seq", _n); - lnk_stmt.bind(":url", lnk); - lnk_stmt.execute(); - lnk_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (at; obj.tags.anchor_tags) { - if (at.length == 0) { continue; } - anc_stmt.bind(":object_id", _obj_id); - anc_stmt.bind(":seq", _n); - anc_stmt.bind(":anchor", at); - anc_stmt.execute(); - anc_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (st; obj.tags.lev4_subtoc) { - if (st.length == 0) { continue; } - sub_stmt.bind(":object_id", _obj_id); - sub_stmt.bind(":seq", _n); - sub_stmt.bind(":entry", st); - sub_stmt.execute(); - sub_stmt.reset(); - ++_n; - } - } - } - } - - obj_stmt.finalize(); - img_stmt.finalize(); - lnk_stmt.finalize(); - anc_stmt.finalize(); - sub_stmt.finalize(); - - /+ ↓ the document's images, carried inside the file so that output can be - produced from this database alone. bytes exactly as read, and the - digest taken over them, so that it can be checked against the digest - the abstraction recorded +/ - { - import std.digest.sha : sha256Of; - auto file_stmt = db.prepare( - "INSERT OR IGNORE INTO files" - ~ " (role, name, bytes, sha256, width, height, data)" - ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" - ); - int[string] _w, _h; - foreach (section; section_order) { - if (section !in doc_abstraction) { continue; } - foreach (obj; doc_abstraction[section]) { - foreach (i; obj.metainfo.sha256.images) { - if (i.imageWidth > 0) { _w[i.fileName] = i.imageWidth; } - if (i.imageHeight > 0) { _h[i.fileName] = i.imageHeight; } - } - } - } - foreach (image; doc_matters.srcs.image_list) { - string _fn = doc_matters.src.image_dir_path ~ "/" ~ image; - ubyte[] _bytes; - try { - if (_fn.exists) { _bytes = cast(ubyte[]) _fn.read; } - } catch (Exception ex) { - _bytes = []; - } - if (_bytes.length == 0) { - if (doc_matters.opt.action.vox_gt_1) { - writeln(" image not carried into abstraction db: ", _fn); - } - continue; - } - file_stmt.bind(":role", "image"); - file_stmt.bind(":name", image); - file_stmt.bind(":bytes", cast(long) _bytes.length); - file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); - if (image in _w) { file_stmt.bind(":width", _w[image]); } - else { - import std.typecons : Nullable; - file_stmt.bind(":width", Nullable!int()); - } - if (image in _h) { file_stmt.bind(":height", _h[image]); } - else { - import std.typecons : Nullable; - file_stmt.bind(":height", Nullable!int()); - } - file_stmt.bind(":data", _bytes); - file_stmt.execute(); - file_stmt.reset(); - } - file_stmt.finalize(); - } - - /+ ↓ build the full text index from the rows just written +/ - db.run("INSERT INTO objects_fts(rowid, text)" - ~ " SELECT id, text FROM objects WHERE text IS NOT NULL"); - - db.run("COMMIT TRANSACTION"); - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ -- cgit v1.2.3