From bf605ac48b231c5852986be2b9a91838c617bfb7 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 11 Sep 2026 22:01:51 -0400 Subject: some file renames --- src/sisudoc/outputs/io_out/create_ocda_db.d | 656 -------- src/sisudoc/outputs/io_out/create_zip_file.d | 67 - src/sisudoc/outputs/io_out/epub3.d | 2 +- src/sisudoc/outputs/io_out/html.d | 2 +- src/sisudoc/outputs/io_out/hub.d | 14 +- src/sisudoc/outputs/io_out/odt.d | 4 +- src/sisudoc/outputs/io_out/source_pod.d | 2 +- src/sisudoc/outputs/io_out/sqlite.d | 1811 --------------------- src/sisudoc/outputs/io_out/sqlite_collection_db.d | 1811 +++++++++++++++++++++ src/sisudoc/outputs/io_out/sqlite_ocda_db.d | 656 ++++++++ src/sisudoc/outputs/io_out/xmls.d | 2 +- src/sisudoc/outputs/io_out/zip_pod.d | 67 + 12 files changed, 2547 insertions(+), 2547 deletions(-) delete mode 100644 src/sisudoc/outputs/io_out/create_ocda_db.d delete mode 100644 src/sisudoc/outputs/io_out/create_zip_file.d delete mode 100644 src/sisudoc/outputs/io_out/sqlite.d create mode 100644 src/sisudoc/outputs/io_out/sqlite_collection_db.d create mode 100644 src/sisudoc/outputs/io_out/sqlite_ocda_db.d create mode 100644 src/sisudoc/outputs/io_out/zip_pod.d (limited to 'src/sisudoc/outputs') diff --git a/src/sisudoc/outputs/io_out/create_ocda_db.d b/src/sisudoc/outputs/io_out/create_ocda_db.d deleted file mode 100644 index b2cfd62..0000000 --- a/src/sisudoc/outputs/io_out/create_ocda_db.d +++ /dev/null @@ -1,656 +0,0 @@ -/+ -- Name: SisuDoc Spine, Doc Reform [a part of] - - Description: documents, structuring, processing, publishing, search - - static content generator - - - Author: Ralph Amissah - [ralph.amissah@gmail.com] - - - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - - - License: AGPL 3 or later: - - Spine (SiSU), a framework for document structuring, publishing and - search - - Copyright (C) Ralph Amissah - - This program is free software: you can redistribute it and/or modify it - under the terms of the GNU AFERO General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program. If not, see [https://www.gnu.org/licenses/]. - - If you have Internet connection, the latest version of the AGPL should be - available at these locations: - [https://www.fsf.org/licensing/licenses/agpl.html] - [https://www.gnu.org/licenses/agpl.html] - - - Spine (by Doc Reform, related to SiSU) uses standard: - - docReform markup syntax - - standard SiSU markup syntax with modified headers and minor modifications - - docReform object numbering - - standard SiSU object citation numbering & system - - - Homepages: - [https://www.sisudoc.org] - [https://www.doc-reform.org] - - - Git - [https://git.sisudoc.org/] - -+/ -module sisudoc.outputs.io_out.create_ocda_db; - -/+ ↓ write document abstraction as per-document sqlite3 database +/ -template spineAbstractionDb() { - import std.conv : to; - import std.digest : toHexString; - import std.file; - import std.path; - import std.stdio; - import std.string; - import std.array; - import d2sqlite3; - import sisudoc.outputs.io_out.paths_output; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.abstraction.ssp : ssp_format_version; - /+ ↓ the document is passed in rather than taken from doc, because the - caller hands over one that has been through the .ssp: written as text - and read back. so the database is a function of the .ssp and cannot - carry anything the .ssp does not, header blocks included: the metadata - table below is filled from what came back, not from a second list read - off doc_matters, which is how the two drifted apart before - +/ - void spineAbstractionDb(D,S)(D doc, S ssp_doc) { - auto doc_matters = doc.matters; - auto doc_abstraction = ssp_doc.abstraction; - /+ ↓ output path: beside the pod's zip and digests, not under - /abstraction/. the database carries its own images, so it is a - distributable artefact in its own right and belongs with the others: - pod// the source tree - pod/.zip the same, zipped - pod/.digests.txt what is in them - pod/.ocda.db the abstraction, one per language - +/ - auto pths_pod = spinePathsPods!()(doc_matters); - string base_pth = pths_pod.pod_dir_(); - try { - if (!exists(base_pth)) { - base_pth.mkdirRecurse; - } - } catch (Exception ex) { - } - string db_file = ((base_pth.chainPath( - doc_matters.src.doc_uid_out ~ ".ocda.db")).asNormalizedPath).array; - - /+ ↓ remove existing file to start fresh +/ - try { - if (exists(db_file)) { - remove(db_file); - } - } catch (Exception ex) { - } - - if (doc_matters.opt.action.vox_gt_1) { - writeln(" ", db_file); - } - - /+ ↓ open database and create schema +/ - auto db = Database(db_file); - /+ ↓ write once, read many: no WAL. - a WAL stamped file cannot be opened by a reader that cannot write - beside it (SQLITE_READONLY on a read-only directory, and inside a - zip or on read-only media), and this database has one writer and no - concurrency, so WAL buys nothing here. MEMORY journal keeps the bulk - load fast without leaving that mark on the file. - +/ - db.run("PRAGMA journal_mode=MEMORY"); - db.run("PRAGMA synchronous=OFF"); - - db.run(" - CREATE TABLE metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - -- objects: one row per abstraction object, in document order. - -- the fixed width arrays (ancestors, dom status, children, heading - -- ancestors text, table widths and aligns) are JSON arrays so that - -- json_extract, json_each and json_array_length can reach inside them; - -- the open ended lists (images, links, anchor tags, subtoc) are rows - -- in their own tables, not columns - CREATE TABLE objects ( - id INTEGER PRIMARY KEY, - section TEXT NOT NULL, - seq INTEGER NOT NULL, - ocn INTEGER DEFAULT 0, - is_a TEXT NOT NULL, - is_of_part TEXT, - is_of_section TEXT, - is_of_type TEXT, - heading_level INTEGER, - heading_lev_collapsed INTEGER, - identifier TEXT, - parent_ocn INTEGER DEFAULT 0, - parent_lev INTEGER DEFAULT 0, - last_descendant_ocn INTEGER DEFAULT 0, - children TEXT - CHECK(children IS NULL OR json_valid(children)), - ancestors TEXT - CHECK(ancestors IS NULL OR json_valid(ancestors)), - ancestors_collapsed TEXT - CHECK(ancestors_collapsed IS NULL OR json_valid(ancestors_collapsed)), - dom_status TEXT - CHECK(dom_status IS NULL OR json_valid(dom_status)), - dom_status_collapsed TEXT - CHECK(dom_status_collapsed IS NULL OR json_valid(dom_status_collapsed)), - dummy_heading INTEGER DEFAULT 0, - object_number_off INTEGER DEFAULT 0, - attrib TEXT, - meta_lang TEXT, - meta_syntax TEXT, - sha256 TEXT, - indent_base INTEGER DEFAULT 0, - indent_hang INTEGER DEFAULT 0, - bullet INTEGER DEFAULT 0, - lang TEXT, - has_links INTEGER DEFAULT 0, - has_notes_reg INTEGER DEFAULT 0, - has_notes_star INTEGER DEFAULT 0, - has_images INTEGER DEFAULT 0, - has_images_no_dim INTEGER DEFAULT 0, - segment TEXT, - segment_prev TEXT, - segment_next TEXT, - segment_epub TEXT, - segment_html_is TEXT, - segment_epub_is TEXT, - segment_lv4_is TEXT, - anchor TEXT, - heading_lev_anchor TEXT, - heading_ancestors_text TEXT - CHECK(heading_ancestors_text IS NULL OR json_valid(heading_ancestors_text)), - table_cols INTEGER, - table_widths TEXT - CHECK(table_widths IS NULL OR json_valid(table_widths)), - table_aligns TEXT - CHECK(table_aligns IS NULL OR json_valid(table_aligns)), - table_header INTEGER, - code_linenumbers INTEGER DEFAULT 0, - text TEXT, - UNIQUE(section, seq) - ); - - CREATE INDEX idx_objects_section ON objects(section); - CREATE INDEX idx_objects_ocn ON objects(ocn); - CREATE INDEX idx_objects_parent ON objects(parent_ocn); - CREATE INDEX idx_objects_is_a ON objects(is_a); - CREATE INDEX idx_objects_heading ON objects(heading_level) - WHERE heading_level IS NOT NULL; - - -- files: what this database carries with it, so that a reader needs - -- nothing beside it. role says what a file is; images are the ones the - -- output writers must have in order to produce a document. bytes are - -- stored exactly as read, so sha256 matches the digest the abstraction - -- was built with - CREATE TABLE files ( - id INTEGER PRIMARY KEY, - role TEXT NOT NULL, - name TEXT NOT NULL, - bytes INTEGER NOT NULL, - sha256 TEXT, - width INTEGER, - height INTEGER, - data BLOB, - UNIQUE(role, name) - ); - - -- the open ended per object lists, one row each, in order - CREATE TABLE object_images ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - name TEXT NOT NULL, - bytes INTEGER, - sha256 TEXT, - width INTEGER, - height INTEGER, - missing INTEGER DEFAULT 0, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_links ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - url TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_anchors ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - anchor TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_subtoc ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - entry TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - - CREATE INDEX idx_files_role ON files(role); - CREATE INDEX idx_object_images_name ON object_images(name); - - -- full text over the object text, external content so the text is not - -- stored twice. per document: this is for exploring one abstraction, - -- the collection wide search database is a different thing - CREATE VIRTUAL TABLE objects_fts USING fts5( - text, content='objects', content_rowid='id' - ); - - -- the shapes worth asking for, named - CREATE VIEW outline AS - SELECT section, seq, ocn, heading_level, heading_lev_collapsed, - last_descendant_ocn, identifier, text - FROM objects WHERE is_a = 'heading' ORDER BY id; - CREATE VIEW citable AS - SELECT * FROM objects WHERE ocn > 0 ORDER BY ocn; - CREATE VIEW document_files AS - SELECT role, name, bytes, sha256, width, height FROM files ORDER BY role, name; - "); - - /+ ↓ populate metadata, from the header blocks the .ssp gave back. - one list, in ssp.d, not two: a property added to the .ssp header - arrives here without anything being changed, and one that is not in - the .ssp cannot be here at all. the keys are the .ssp's own - (title.main, make.doc_type, doc_has.images), prefixed by their block - where the block name is not already part of the key. - +/ - db.run("BEGIN TRANSACTION"); - - auto meta_stmt = db.prepare( - "INSERT INTO metadata (key, value) VALUES (:key, :value)" - ); - - void insertMeta(string key, string value) { - if (value.length > 0) { - meta_stmt.bind(":key", key); - meta_stmt.bind(":value", value); - meta_stmt.execute(); - meta_stmt.reset(); - } - } - void insertBlock(string prefix, string[] keys_in_order, string[string] kv) { - foreach (k; keys_in_order) { - if (k in kv) { insertMeta(prefix ~ k, kv[k]); } - } - } - - /+ ↓ what this file is, so a reader can tell before it starts. the - version is the .ssp's own: the two serialisations are one format and - move together - +/ - insertMeta("schema.name", "sisu-abstraction-db"); - insertMeta("schema.version", ssp_format_version); - insertMeta("source.filename", ssp_doc.source); - insertBlock("source.", ssp_doc.source_info_order, ssp_doc.source_info); - /+ ↓ the digest of the .ssp this database was built from. a file cannot - hold its own hash, so this one is not in the .ssp and is taken here: - source.digest says which markup the abstraction came from, - source.ssp_digest says which abstraction this database came from, - and the chain .sst -> .ssp -> .ocda.db is checkable end to end - +/ - insertMeta("source.ssp_digest", ssp_doc.ssp_digest); - - insertBlock("", ssp_doc.meta_order, ssp_doc.meta); - insertBlock("make.", ssp_doc.make_order, ssp_doc.make); - insertBlock("doc_has.", ssp_doc.doc_has_order, ssp_doc.doc_has); - meta_stmt.finalize(); - - /+ ↓ populate objects +/ - auto obj_stmt = db.prepare( - "INSERT INTO objects (" - ~ "section, seq, ocn, is_a, is_of_part, is_of_section, is_of_type," - ~ "heading_level, heading_lev_collapsed, identifier," - ~ "parent_ocn, parent_lev, last_descendant_ocn," - ~ "children, ancestors, ancestors_collapsed," - ~ "dom_status, dom_status_collapsed," - ~ "dummy_heading, object_number_off," - ~ "attrib, meta_lang, meta_syntax, sha256," - ~ "indent_base, indent_hang, bullet, lang," - ~ "has_links, has_notes_reg, has_notes_star, has_images, has_images_no_dim," - ~ "segment, segment_prev, segment_next, segment_epub," - ~ "segment_html_is, segment_epub_is, segment_lv4_is," - ~ "anchor, heading_lev_anchor, heading_ancestors_text," - ~ "table_cols, table_widths, table_aligns, table_header," - ~ "code_linenumbers, text" - ~ ") VALUES (" - ~ ":section, :seq, :ocn, :is_a, :is_of_part, :is_of_section, :is_of_type," - ~ ":heading_level, :heading_lev_collapsed, :identifier," - ~ ":parent_ocn, :parent_lev, :last_descendant_ocn," - ~ ":children, :ancestors, :ancestors_collapsed," - ~ ":dom_status, :dom_status_collapsed," - ~ ":dummy_heading, :object_number_off," - ~ ":attrib, :meta_lang, :meta_syntax, :sha256," - ~ ":indent_base, :indent_hang, :bullet, :lang," - ~ ":has_links, :has_notes_reg, :has_notes_star, :has_images, :has_images_no_dim," - ~ ":segment, :segment_prev, :segment_next, :segment_epub," - ~ ":segment_html_is, :segment_epub_is, :segment_lv4_is," - ~ ":anchor, :heading_lev_anchor, :heading_ancestors_text," - ~ ":table_cols, :table_widths, :table_aligns, :table_header," - ~ ":code_linenumbers, :text" - ~ ")" - ); - - /+ ↓ the per object lists, one row each +/ - auto img_stmt = db.prepare( - "INSERT INTO object_images" - ~ " (object_id, seq, name, bytes, sha256, width, height, missing)" - ~ " VALUES (:object_id, :seq, :name, :bytes, :sha256, :width, :height, :missing)" - ); - auto lnk_stmt = db.prepare( - "INSERT INTO object_links (object_id, seq, url)" - ~ " VALUES (:object_id, :seq, :url)" - ); - auto anc_stmt = db.prepare( - "INSERT INTO object_anchors (object_id, seq, anchor)" - ~ " VALUES (:object_id, :seq, :anchor)" - ); - auto sub_stmt = db.prepare( - "INSERT INTO object_subtoc (object_id, seq, entry)" - ~ " VALUES (:object_id, :seq, :entry)" - ); - - string[] section_order = ["head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", "tail"]; - - foreach (section; section_order) { - if (section !in doc_abstraction) continue; - auto section_objs = doc_abstraction[section]; - if (section_objs.length == 0) continue; - - foreach (seq, obj; section_objs) { - obj_stmt.bind(":section", section); - obj_stmt.bind(":seq", cast(int) seq); - obj_stmt.bind(":ocn", obj.metainfo.ocn); - obj_stmt.bind(":is_a", obj.metainfo.is_a); - - /+ ↓ nullable string fields +/ - void bindStr(string param, string val) { - import std.typecons : Nullable; - if (val.length > 0) { - obj_stmt.bind(param, val); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - /+ ↓ nullable int fields +/ - void bindInt(string param, int val, bool is_set) { - import std.typecons : Nullable; - if (is_set) { - obj_stmt.bind(param, val); - } else { - obj_stmt.bind(param, Nullable!int()); - } - } - /+ ↓ fixed width int array as a JSON array, NULL when all zero +/ - void bindJsonInts(T)(string param, T arr) { - import std.typecons : Nullable; - bool any = false; - foreach (a; arr) { if (a != 0) { any = true; break; } } - if (any) { - string[] _j; - foreach (a; arr) { _j ~= a.to!string; } - obj_stmt.bind(param, "[" ~ _j.join(",") ~ "]"); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - /+ ↓ string array as a JSON array, NULL when every slot is empty +/ - void bindJsonStrs(T)(string param, T arr) { - import std.json : JSONValue; - import std.typecons : Nullable; - bool any = false; - foreach (s; arr) { if (s.length > 0) { any = true; break; } } - if (any) { - string[] _s; - foreach (s; arr) { _s ~= s.to!string; } - obj_stmt.bind(param, JSONValue(_s).toString); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - - bindStr(":is_of_part", obj.metainfo.is_of_part); - bindStr(":is_of_section", obj.metainfo.is_of_section); - bindStr(":is_of_type", obj.metainfo.is_of_type); - - /+ ↓ heading levels, marked up and collapsed, 9 meaning unset +/ - bindInt(":heading_level", obj.metainfo.heading_lev_markup, - (obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup < 9)); - bindInt(":heading_lev_collapsed", obj.metainfo.heading_lev_collapsed, - (obj.metainfo.heading_lev_collapsed < 9)); - - bindStr(":identifier", obj.metainfo.identifier); - obj_stmt.bind(":parent_ocn", obj.metainfo.parent_ocn); - obj_stmt.bind(":parent_lev", obj.metainfo.parent_lev_markup); - obj_stmt.bind(":last_descendant_ocn", obj.metainfo.last_descendant_ocn); - - bindJsonInts(":children", obj.metainfo.children_headings); - bindJsonInts(":ancestors", obj.metainfo.markedup_ancestors); - bindJsonInts(":ancestors_collapsed", obj.metainfo.collapsed_ancestors); - bindJsonInts(":dom_status", obj.metainfo.dom_structure_markedup_tags_status); - bindJsonInts(":dom_status_collapsed", obj.metainfo.dom_structure_collapsed_tags_status); - - obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); - obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); - bindStr(":attrib", obj.metainfo.attrib); - bindStr(":meta_lang", obj.metainfo.lang); - bindStr(":meta_syntax", obj.metainfo.syntax); - - /+ ↓ object digest, hex, absent when never taken +/ - { - import std.typecons : Nullable; - bool has_sha = false; - foreach (b; obj.metainfo.sha256.text) { - if (b != 0) { has_sha = true; break; } - } - if (has_sha) { - obj_stmt.bind(":sha256", obj.metainfo.sha256.text.toHexString.to!string); - } else { - obj_stmt.bind(":sha256", Nullable!string()); - } - } - - // obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); - // obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); - obj_stmt.bind(":indent_base", obj.attrib.indent_base); - obj_stmt.bind(":indent_hang", obj.attrib.indent_hang); - obj_stmt.bind(":bullet", obj.attrib.bullet ? 1 : 0); - bindStr(":lang", obj.attrib.language); - obj_stmt.bind(":has_links", obj.has.inline_links ? 1 : 0); - obj_stmt.bind(":has_notes_reg", obj.has.inline_notes_reg ? 1 : 0); - obj_stmt.bind(":has_notes_star", obj.has.inline_notes_star ? 1 : 0); - obj_stmt.bind(":has_images", obj.has.images ? 1 : 0); - obj_stmt.bind(":has_images_no_dim", obj.has.image_without_dimensions ? 1 : 0); - bindStr(":segment", obj.tags.in_segment_html); - bindStr(":segment_prev", obj.tags.segname_prev); - bindStr(":segment_next", obj.tags.segname_next); - bindStr(":segment_epub", obj.tags.segment_anchor_tag_epub); - bindStr(":segment_html_is", obj.tags.html_segment_anchor_tag_is); - bindStr(":segment_epub_is", obj.tags.epub_segment_anchor_tag_is); - bindStr(":segment_lv4_is", obj.tags.segment_lv4_is); - bindStr(":anchor", obj.tags.anchor_tag_html); - bindStr(":heading_lev_anchor", obj.tags.heading_lev_anchor_tag); - bindJsonStrs(":heading_ancestors_text", obj.tags.heading_ancestors_text); - - /+ ↓ table properties +/ - { - import std.typecons : Nullable; - if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { - obj_stmt.bind(":table_cols", obj.table.number_of_columns); - if (obj.table.column_widths.length > 0) { - string[] ws; - foreach (w; obj.table.column_widths) ws ~= w.to!string; - obj_stmt.bind(":table_widths", "[" ~ ws.join(",") ~ "]"); - } else { - obj_stmt.bind(":table_widths", Nullable!string()); - } - if (obj.table.column_aligns.length > 0) { - import std.json : JSONValue; - string[] _al; - foreach (a; obj.table.column_aligns) { _al ~= a.to!string; } - obj_stmt.bind(":table_aligns", JSONValue(_al).toString); - } else { - obj_stmt.bind(":table_aligns", Nullable!string()); - } - obj_stmt.bind(":table_header", obj.table.heading ? 1 : 0); - } else { - obj_stmt.bind(":table_cols", Nullable!int()); - obj_stmt.bind(":table_widths", Nullable!string()); - obj_stmt.bind(":table_aligns", Nullable!string()); - obj_stmt.bind(":table_header", Nullable!int()); - } - } - - /+ ↓ code block properties +/ - obj_stmt.bind(":code_linenumbers", - ((obj.metainfo.is_a == "code") && obj.code_block.linenumbers) ? 1 : 0); - - /+ ↓ text content +/ - bindStr(":text", obj.text); - - obj_stmt.execute(); - obj_stmt.reset(); - - /+ ↓ the per object lists, keyed on the row just written +/ - long _obj_id = db.lastInsertRowid; - { - int _n = 0; - foreach (i; obj.metainfo.sha256.images) { - img_stmt.bind(":object_id", _obj_id); - img_stmt.bind(":seq", _n); - img_stmt.bind(":name", i.fileName); - img_stmt.bind(":bytes", cast(long) i.fileSize); - img_stmt.bind(":sha256", i.fileHash_sha256.toHexString.to!string); - img_stmt.bind(":width", i.imageWidth); - img_stmt.bind(":height", i.imageHeight); - img_stmt.bind(":missing", i.fileMissing ? 1 : 0); - img_stmt.execute(); - img_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (lnk; obj.stow.link) { - if (lnk.length == 0) { continue; } - lnk_stmt.bind(":object_id", _obj_id); - lnk_stmt.bind(":seq", _n); - lnk_stmt.bind(":url", lnk); - lnk_stmt.execute(); - lnk_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (at; obj.tags.anchor_tags) { - if (at.length == 0) { continue; } - anc_stmt.bind(":object_id", _obj_id); - anc_stmt.bind(":seq", _n); - anc_stmt.bind(":anchor", at); - anc_stmt.execute(); - anc_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (st; obj.tags.lev4_subtoc) { - if (st.length == 0) { continue; } - sub_stmt.bind(":object_id", _obj_id); - sub_stmt.bind(":seq", _n); - sub_stmt.bind(":entry", st); - sub_stmt.execute(); - sub_stmt.reset(); - ++_n; - } - } - } - } - - obj_stmt.finalize(); - img_stmt.finalize(); - lnk_stmt.finalize(); - anc_stmt.finalize(); - sub_stmt.finalize(); - - /+ ↓ the document's images, carried inside the file so that output can be - produced from this database alone. bytes exactly as read, and the - digest taken over them, so that it can be checked against the digest - the abstraction recorded +/ - { - import std.digest.sha : sha256Of; - auto file_stmt = db.prepare( - "INSERT OR IGNORE INTO files" - ~ " (role, name, bytes, sha256, width, height, data)" - ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" - ); - int[string] _w, _h; - foreach (section; section_order) { - if (section !in doc_abstraction) { continue; } - foreach (obj; doc_abstraction[section]) { - foreach (i; obj.metainfo.sha256.images) { - if (i.imageWidth > 0) { _w[i.fileName] = i.imageWidth; } - if (i.imageHeight > 0) { _h[i.fileName] = i.imageHeight; } - } - } - } - foreach (image; doc_matters.srcs.image_list) { - string _fn = doc_matters.src.image_dir_path ~ "/" ~ image; - ubyte[] _bytes; - try { - if (_fn.exists) { _bytes = cast(ubyte[]) _fn.read; } - } catch (Exception ex) { - _bytes = []; - } - if (_bytes.length == 0) { - if (doc_matters.opt.action.vox_gt_1) { - writeln(" image not carried into abstraction db: ", _fn); - } - continue; - } - file_stmt.bind(":role", "image"); - file_stmt.bind(":name", image); - file_stmt.bind(":bytes", cast(long) _bytes.length); - file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); - if (image in _w) { file_stmt.bind(":width", _w[image]); } - else { - import std.typecons : Nullable; - file_stmt.bind(":width", Nullable!int()); - } - if (image in _h) { file_stmt.bind(":height", _h[image]); } - else { - import std.typecons : Nullable; - file_stmt.bind(":height", Nullable!int()); - } - file_stmt.bind(":data", _bytes); - file_stmt.execute(); - file_stmt.reset(); - } - file_stmt.finalize(); - } - - /+ ↓ build the full text index from the rows just written +/ - db.run("INSERT INTO objects_fts(rowid, text)" - ~ " SELECT id, text FROM objects WHERE text IS NOT NULL"); - - db.run("COMMIT TRANSACTION"); - } -} diff --git a/src/sisudoc/outputs/io_out/create_zip_file.d b/src/sisudoc/outputs/io_out/create_zip_file.d deleted file mode 100644 index d0ace54..0000000 --- a/src/sisudoc/outputs/io_out/create_zip_file.d +++ /dev/null @@ -1,67 +0,0 @@ -/+ -- Name: SisuDoc Spine, Doc Reform [a part of] - - Description: documents, structuring, processing, publishing, search - - static content generator - - - Author: Ralph Amissah - [ralph.amissah@gmail.com] - - - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - - - License: AGPL 3 or later: - - Spine (SiSU), a framework for document structuring, publishing and - search - - Copyright (C) Ralph Amissah - - This program is free software: you can redistribute it and/or modify it - under the terms of the GNU AFERO General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program. If not, see [https://www.gnu.org/licenses/]. - - If you have Internet connection, the latest version of the AGPL should be - available at these locations: - [https://www.fsf.org/licensing/licenses/agpl.html] - [https://www.gnu.org/licenses/agpl.html] - - - Spine (by Doc Reform, related to SiSU) uses standard: - - docReform markup syntax - - standard SiSU markup syntax with modified headers and minor modifications - - docReform object numbering - - standard SiSU object citation numbering & system - - - Homepages: - [https://www.sisudoc.org] - [https://www.doc-reform.org] - - - Git - [https://git.sisudoc.org/] - -+/ -module sisudoc.outputs.io_out.create_zip_file; -@safe: -template createZipFile() { - import std.file; - import std.outbuffer; - import std.string; - import std.zip; - void createZipFile( - string zip_file_name, - void[] compressed_zip_data, - ) { - try { - write(zip_file_name, compressed_zip_data); - } catch (ZipException ex) { - // Handle Errors - } - } -} diff --git a/src/sisudoc/outputs/io_out/epub3.d b/src/sisudoc/outputs/io_out/epub3.d index f3b4cd7..bc84053 100644 --- a/src/sisudoc/outputs/io_out/epub3.d +++ b/src/sisudoc/outputs/io_out/epub3.d @@ -58,7 +58,7 @@ template outputEPub3() { import sisudoc.outputs.io_out; import sisudoc.outputs.io_out.rgx; import sisudoc.outputs.io_out.rgx_xhtml; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; import sisudoc.outputs.io_out.xmls_css; mixin InternalMarkup; diff --git a/src/sisudoc/outputs/io_out/html.d b/src/sisudoc/outputs/io_out/html.d index efdedf8..e6cecb5 100644 --- a/src/sisudoc/outputs/io_out/html.d +++ b/src/sisudoc/outputs/io_out/html.d @@ -58,7 +58,7 @@ template outputHTML() { import sisudoc.outputs.io_out.rgx; import sisudoc.ocda.meta.rgx_files; import sisudoc.outputs.io_out.rgx_xhtml; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; import sisudoc.outputs.io_out.xmls_css; mixin outputXHTMLs; diff --git a/src/sisudoc/outputs/io_out/hub.d b/src/sisudoc/outputs/io_out/hub.d index 0aab81d..0973097 100644 --- a/src/sisudoc/outputs/io_out/hub.d +++ b/src/sisudoc/outputs/io_out/hub.d @@ -57,7 +57,7 @@ template outputHub() { import sisudoc.outputs.io_out; import sisudoc.outputs.io_out.metadata; import sisudoc.outputs.io_out.xmls; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.paths_output; @system void outputHub(D)(D doc) { mixin Msg; @@ -132,7 +132,7 @@ template outputHub() { } if (sched == outTask.sqlite) { msg.v("sqlite processing... "); - import sisudoc.outputs.io_out.sqlite; + import sisudoc.outputs.io_out.sqlite_collection_db; doc.SQLiteHubDiscreteBuildTablesAndPopulate!(); msg.vv("sqlite done"); } @@ -156,12 +156,12 @@ template outputHub() { } if (doc.matters.opt.action.sqlite_update) { msg.v("sqlite update processing..."); - import sisudoc.outputs.io_out.sqlite; + import sisudoc.outputs.io_out.sqlite_collection_db; doc.SQLiteHubBuildTablesAndPopulate!(); msg.vv("sqlite update done"); } else if (doc.matters.opt.action.sqlite_delete) { msg.v("sqlite delete processing..."); - import sisudoc.outputs.io_out.sqlite; + import sisudoc.outputs.io_out.sqlite_collection_db; doc.SQLiteHubBuildTablesAndPopulate!(); msg.vv("sqlite delete done"); } @@ -211,14 +211,14 @@ template outputHubOp() { import sisudoc.outputs.io_out; import sisudoc.outputs.io_out.metadata; import sisudoc.outputs.io_out.xmls; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.paths_output; @system void outputHubOp(E,O,C)(E env, O opt_action, C config) { if ((opt_action.sqlite_db_drop)) { if ((opt_action.vox_gt_2)) { writeln("sqlite drop db..."); } - import sisudoc.outputs.io_out.sqlite; + import sisudoc.outputs.io_out.sqlite_collection_db; SQLiteDbDrop!()(opt_action, config); if ((opt_action.vox_gt_3)) { writeln("sqlite drop db done"); @@ -229,7 +229,7 @@ template outputHubOp() { auto pth_sqlite_db = spinePathsSQLite!()(opt_action.cgi_sqlite_search_filename, opt_action.output_dir_set); writeln("sqlite create table..."); } - import sisudoc.outputs.io_out.sqlite; + import sisudoc.outputs.io_out.sqlite_collection_db; SQLiteTablesCreate!()(env, opt_action, config); if ((opt_action.vox_gt_3)) { writeln("sqlite create table done"); diff --git a/src/sisudoc/outputs/io_out/odt.d b/src/sisudoc/outputs/io_out/odt.d index ed288cb..073972c 100644 --- a/src/sisudoc/outputs/io_out/odt.d +++ b/src/sisudoc/outputs/io_out/odt.d @@ -58,7 +58,7 @@ template formatODT() { import std.uri; import std.zip; import std.conv : to; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; import sisudoc.outputs.io_out.xmls_css; mixin spineRgxOut; @@ -650,7 +650,7 @@ template outputODT() { import std.uri; import std.zip; import std.conv : to; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; import sisudoc.outputs.io_out.xmls_css; mixin InternalMarkup; diff --git a/src/sisudoc/outputs/io_out/source_pod.d b/src/sisudoc/outputs/io_out/source_pod.d index 0b014ca..76f841b 100644 --- a/src/sisudoc/outputs/io_out/source_pod.d +++ b/src/sisudoc/outputs/io_out/source_pod.d @@ -58,7 +58,7 @@ template spinePod() { import std.conv : to; import sisudoc.ocda.meta.rgx_files; import sisudoc.outputs.io_out; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; void spinePod(T)(T doc_matters) { debug(asserts) { diff --git a/src/sisudoc/outputs/io_out/sqlite.d b/src/sisudoc/outputs/io_out/sqlite.d deleted file mode 100644 index 1e1970e..0000000 --- a/src/sisudoc/outputs/io_out/sqlite.d +++ /dev/null @@ -1,1811 +0,0 @@ -/+ -- Name: SisuDoc Spine, Doc Reform [a part of] - - Description: documents, structuring, processing, publishing, search - - static content generator - - - Author: Ralph Amissah - [ralph.amissah@gmail.com] - - - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - - - License: AGPL 3 or later: - - Spine (SiSU), a framework for document structuring, publishing and - search - - Copyright (C) Ralph Amissah - - This program is free software: you can redistribute it and/or modify it - under the terms of the GNU AFERO General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program. If not, see [https://www.gnu.org/licenses/]. - - If you have Internet connection, the latest version of the AGPL should be - available at these locations: - [https://www.fsf.org/licensing/licenses/agpl.html] - [https://www.gnu.org/licenses/agpl.html] - - - Spine (by Doc Reform, related to SiSU) uses standard: - - docReform markup syntax - - standard SiSU markup syntax with modified headers and minor modifications - - docReform object numbering - - standard SiSU object citation numbering & system - - - Homepages: - [https://www.sisudoc.org] - [https://www.doc-reform.org] - - - Git - [https://git.sisudoc.org/] - -+/ -module sisudoc.outputs.io_out.sqlite; -import sisudoc.outputs.io_out; -import sisudoc.outputs.io_out.rgx; -import sisudoc.outputs.io_out.rgx_xhtml; -import std.file; -import std.uri; -// import std.digest.sha; -import std.conv : to; -import std.typecons : Nullable; -import d2sqlite3; -mixin spineRgxOut; -mixin spineRgxXHTML; -mixin InternalMarkup; -static auto rgx = RgxO(); -static auto rgx_xhtml = RgxXHTML(); -static auto mkup = InlineMarkup(); -long _metadata_tid_lastrowid; -/+ ↓ sqlite db schema version, stamped into the db as PRAGMA user_version +/ -/+ bump whenever the table or index definitions in this module change, and +/ -/+ bump the matching expectation in sisudoc-spine-search-cgi +/ -enum sqlite_db_schema_version = 1; -/+ ↓ tally of failed sqlite statements, shared: sqlite output may run in parallel +/ -private shared int _sqlite_failure_tally = 0; -/+ ↓ schema version mismatch is reported once per run, not once per document +/ -private shared int _sqlite_schema_mismatch_reported = 0; -/+ ↓ record a failed sqlite statement, reported on stderr, tallied for exit status +/ -void sqliteFailureRecord(string note, string msg) { - import core.atomic : atomicOp; - atomicOp!"+="(_sqlite_failure_tally, 1); - stderr.writeln("ERROR SQLite (", note, "): ", msg); -} -/+ ↓ count of failed sqlite statements this run, zero when all went well +/ -int sqliteFailureTally() { - import core.atomic : atomicLoad; - return atomicLoad(_sqlite_failure_tally); -} -/+ ↓ compare the schema version stamped in an existing db with this spine's +/ -void sqliteSchemaVersionCheck(Db)(Db db, string sqlite_file) { - import core.atomic : atomicOp; - int _found; - try { - _found = db.execute("PRAGMA user_version;").oneValue!int; - } catch (Exception) { - return; - } - if (_found == sqlite_db_schema_version - || _found == 0) { /+ ↓ zero: db predates the stamp, or has no tables yet +/ - return; - } - if (atomicOp!"+="(_sqlite_schema_mismatch_reported, 1) == 1) { - stderr.writeln("ERROR SQLite : db schema version mismatch: ", sqlite_file); - stderr.writeln(" db holds schema version ", _found, - ", this spine writes schema version ", sqlite_db_schema_version); - stderr.writeln(" recreate the db, e.g.: spine --sqlite-db-recreate", - " --sqlite-db-path=... --sqlite-db-filename=..."); - sqliteFailureRecord("schema version", - "db schema version " ~ _found.to!string - ~ " != spine schema version " ~ sqlite_db_schema_version.to!string); - } -} -template SQLiteHubBuildTablesAndPopulate() { - void SQLiteHubBuildTablesAndPopulate(D)(D doc) { - auto pth_sqlite = spinePathsSQLite!()(doc.matters.sqlite.filename, doc.matters.sqlite.path); - if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { - } else { - try { - pth_sqlite.base.mkdirRecurse; - } catch (FileException ex) { - stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); - stderr.writeln(" (", ex.msg, ")"); - import core.runtime; - core.runtime.Runtime.terminate(); - } - } - template SQLiteDbStatementComposite() { - void SQLiteDbStatementComposite(Db,D)( - Db db, - D doc - ) { - string _db_statement; - if ((doc.matters.opt.action.sqlite_db_create)) { - auto pth_sqlite = spinePathsSQLite!()(doc.matters.sqlite.filename, doc.matters.sqlite.path); - if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { - } else { - try { - pth_sqlite.base.mkdirRecurse; - } catch (FileException ex) { - stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); - stderr.writeln(" (", ex.msg, ")"); - import core.runtime; - core.runtime.Runtime.terminate(); - } - } - _db_statement ~= SQLiteTablesReCreate!()(); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "TABLE RE-CREATE"); - _db_statement = []; - } - if (doc.matters.opt.action.sqlite_delete) { - _db_statement ~= SQLiteDeleteDocument!()(doc.matters); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "DELETE Document"); - _db_statement = []; - } - if (doc.matters.opt.action.sqlite_update) { - _db_statement ~= SQLiteDeleteDocument!()(doc.matters); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "DELETE Document"); - _db_statement = []; - _db_statement ~= SQLiteInsertMetadata!()(doc.matters); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT MetaData"); - _db_statement = []; - /+ get tid (lastrowid or max) for use in doc.objects table +/ - _db_statement ~= doc.SQLiteInsertDocObjectsLoop!(); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT DocObjects"); - _db_statement = []; - _db_statement ~= SQLiteInsertMetadataTopics!()(doc.matters); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT MetaDataTopics"); - _db_statement = []; - } - db.close; - if (doc.matters.opt.action.vox_gt_1) { - writeln(" ", pth_sqlite.sqlite_file); - } - } - } - try { - auto db = Database(pth_sqlite.sqlite_file); - sqliteSchemaVersionCheck(db, pth_sqlite.sqlite_file); - SQLiteDbStatementComposite!()(db, doc); - } - catch (FileException e) { - writeln("Failed (FileException): ", e.msg, " ", pth_sqlite.sqlite_file); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (ErrnoException e) { - writeln("Failed (ErrnoException): ", e.msg, " ", pth_sqlite.sqlite_file); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (Exception e) { - writeln("Failed (Exception): ", e.msg, " ", pth_sqlite.sqlite_file); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (Throwable) { - writeln("Failed (Trowable): ", pth_sqlite.sqlite_file); - import core.runtime; - core.runtime.Runtime.terminate(); - } - } -} -template SQLiteHubDiscreteBuildTablesAndPopulate() { - void SQLiteHubDiscreteBuildTablesAndPopulate(D)(D doc) { - auto url_html = spineUrlsHTML!()(doc.matters.conf_make_meta.conf.w_srv_data_root_url_html, doc.matters.src.language); - auto pth_sqlite = spinePathsSQLiteDiscrete!()(doc.matters.output_path, doc.matters.src.language); // doc.matters.db_path - if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { - } else { - try { - pth_sqlite.base.mkdirRecurse; - } catch (FileException ex) { - stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); - stderr.writeln(" (", ex.msg, ")"); - import core.runtime; - core.runtime.Runtime.terminate(); - } - } - auto db = Database(pth_sqlite.sqlite_file(doc.matters.src.filename)); - sqliteSchemaVersionCheck(db, pth_sqlite.sqlite_file(doc.matters.src.filename)); - template SQLiteDiscreteDbStatementComposite() { - void SQLiteDiscreteDbStatementComposite(Db,D)( - Db db, - D doc - ) { - try { - { - string _db_statement; - _db_statement ~= SQLiteTablesReCreate!()(); - _db_statement ~= SQLiteInsertMetadata!()(doc.matters); - _db_statement ~= SQLiteInsertMetadataTopics!()(doc.matters); - _db_statement ~= doc.SQLiteInsertDocObjectsLoop!(); - SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "table CREATE Tables, INSERT DocObjects"); - } - db.close; - } - catch (FileException e) { - writeln("Failed (FileException): ", e.msg); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (ErrnoException e) { - writeln("Failed (ErrnoException): ", e.msg); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (Exception e) { - writeln("Failed (Exception): ", e.msg); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (Throwable) { - import core.runtime; - core.runtime.Runtime.terminate(); - } - if (doc.matters.opt.action.vox_gt_1) { - writeln(" ", pth_sqlite.sqlite_file(doc.matters.src.filename)); - } - } - } - SQLiteDiscreteDbStatementComposite!()(db, doc); - } -} -template SQLiteDbRun() { - void SQLiteDbRun(Db,St,O)( - Db db, - St db_statement, - O opt_action, - string note, - ) { - debug(sql_statement) { - writeln(db_statement); - } - try { - db.run( - "\nBEGIN TRANSACTION;\n" ~ - db_statement ~ - "\nCOMMIT TRANSACTION;\n" - ); - } catch (ErrnoException ex) { - sqliteFailureRecord(note, ex.msg); - } catch (Exception ex) { - sqliteFailureRecord(note, ex.msg); - } - { /+ debug +/ - if (opt_action.debug_do_sqlite) { - writeln(note); - if (opt_action.vox_gt_3) { - writeln(db_statement); - } - } - } - } -} -template SQLinsertDelimiter() { - string SQLinsertDelimiter(string _txt) { - _txt = _txt - .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); - return _txt; - } -} -template SQLiteFormatAndLoadObject() { - auto SQLiteFormatAndLoadObject(M)(M doc_matters) { - mixin spineRgxOut; - mixin spineRgxXHTML; - struct sqlite_format_and_load_objects { - string generic_munge_sanitize_text_for_search( - string _txt, - ) { - string _notes; - string _urls; - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - if (m["link"].match(rgx.url)) { - _urls ~= "\n" ~ m["link"]; - } - } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (_urls.length > 0) { - _txt ~= _urls; - } - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - writeln(_txt, "\n"); - } - } - debug(sql_text_clean) { - writeln(_txt); - } - return _txt; - } - string munge_html(M,O)( - M doc_matters, - const O obj, - ) { - string _html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx_xhtml.ampersand, "&") - .replaceAll(rgx_xhtml.quotation, """) - .replaceAll(rgx_xhtml.less_than, "<") - .replaceAll(rgx_xhtml.greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.br_line_inline, "
") - .replaceAll(rgx.br_line, "
") - .replaceAll(rgx.br_line_spaced, "

") - .replaceAll(rgx_xhtml.line_break, "
"); - return _txt; - } - string _html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; - } - string _notes; - string _urls; - string _txt = _html_font_face(_html_special_characters(obj.text)); - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - writeln(_txt, "\n"); - } - } - return _txt; - } - string html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx_xhtml.ampersand, "&") - .replaceAll(rgx_xhtml.quotation, """) - .replaceAll(rgx_xhtml.less_than, "<") - .replaceAll(rgx_xhtml.greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.br_line_inline, "
") - .replaceAll(rgx.br_line, "
") - .replaceAll(rgx.br_line_spaced, "

") - .replaceAll(rgx_xhtml.line_break, "
"); - return _txt; - } - string html_special_characters_code(string _txt){ - _txt = _txt - .replaceAll(rgx_xhtml.ampersand, "&") - .replaceAll(rgx_xhtml.quotation, """) - .replaceAll(rgx_xhtml.less_than, "<") - .replaceAll(rgx_xhtml.greater_than, ">") - .replaceAll(rgx.nbsp_char, " "); - return _txt; - } - string html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; - } - string inline_grouped_text_bullets_indents(M,O)( - M doc_matters, - const O obj, - string _txt, - string _suffix = ".html", - string _xml_type = "seg", - ) { - static auto rgx = RgxO(); - static auto rgx_xhtml = RgxXHTML(); - if (obj.metainfo.is_a == "group") { - _txt = (_txt) - .replaceAll(rgx.grouped_para_indent_1, - "  ") - .replaceAll(rgx.grouped_para_indent_2, - "    ") - .replaceAll(rgx.grouped_para_indent_3, - "      ") - .replaceAll(rgx.grouped_para_indent_4, - "        ") - .replaceAll(rgx.grouped_para_indent_5, - "          ") - .replaceAll(rgx.grouped_para_indent_6, - "            ") - .replaceAll(rgx.grouped_para_indent_7, - "              ") - .replaceAll(rgx.grouped_para_indent_8, - "                ") - .replaceAll(rgx.grouped_para_indent_9, - "                  ") - .replaceAll(rgx.grouped_para_indent_hang, "  ") - .replaceAll(rgx.grouped_para_bullet, "●  ") - .replaceAll(rgx.grouped_para_bullet_indent_1, - "  ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_2, - "    ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_3, - "      ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_4, - "        ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_5, - "          ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_6, - "            ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_7, - "              ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_8, - "                ●  ") - .replaceAll(rgx.grouped_para_bullet_indent_9, - "                  ●  "); - } - return _txt; - } - string inline_images(M,O)( - M doc_matters, - const O obj, - string _txt, - string _suffix = ".html", - string _xml_type = "seg", - ) { - string _img_pth; - if (_xml_type == "epub") { - _img_pth = "image/"; - } else if (_xml_type == "scroll") { - _img_pth = "../../image/"; - } else if (_xml_type == "seg") { - _img_pth = "../../../image/"; - } - if (_txt.match(rgx.inline_image)) { - _txt = _txt.replaceAll( // TODO bug where image dimensions (w or h) not given & consequently set to 0; should not be used (calculate earlier, abstraction) - rgx.inline_image, - ("$1 $6")); - } - return _txt; - } - string inline_links(M,O)( - M doc_matters, - const O obj, - string _txt, - string _xml_type = "seg", - ) { - if (obj.has.inline_links) { - if (obj.metainfo.is_a != "code") { - _txt = replaceAll!(m => - m["linked_text"] ~ "┤" ~ to!string((obj.stow.link[m["num"].to!ulong])).encode ~ "├" - )(_txt, rgx.inline_link_number_only); - } - if ((_txt.match(rgx.mark_internal_site_lnk)) - && (_xml_type == "scroll")) { // conditions reversed to avoid: gdc compiled program run segfault - _txt = _txt.replaceAll( - rgx.inline_seg_link, - "$1"); - } - auto pth_html = spinePathsHTML!()(doc_matters.output_path, doc_matters.src.language); - if (_xml_type == "seg") { - foreach (m; _txt.matchAll(rgx.inline_link_seg_and_hash)) { - if (m.captures["hash"] in doc_matters.has.tag_associations) { - if (m.captures["hash"] == doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"]) { - _txt = _txt.replaceFirst( - rgx.inline_link_seg_and_hash, - "┥$1┝┤" - ~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html - ~ "/" - ~ pth_html.tail_fn_seg(doc_matters.src.filename, "$2.html") - ~ "├" - ); - } else { - _txt = _txt.replaceFirst( - rgx.inline_link_seg_and_hash, - "┥$1┝┤" - ~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html - ~ "/" - ~ doc_matters.src.filename_base - ~ "/" - ~ doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"] - ~ ".html" - ~ "#" ~ m.captures["hash"] - ~ "├" - ); - } - } else { - if (doc_matters.opt.action.vox_gt_1) { - writeln( - "WARNING on internal document links, anchor to link <<" - ~ m.captures["hash"] - ~ ">> not found in document, " - ~ "anchor: " ~ m.captures["hash"] - ~ " document: " ~ doc_matters.src.filename - ); - } - } - } - } else { - if (auto m = _txt.match(rgx.inline_link_seg_and_hash)) { - _txt = _txt.replaceFirst( - rgx.inline_link_seg_and_hash, - "┥$1┝┤" - ~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html - ~ "/" - ~ pth_html.tail_fn_scroll(doc_matters.src.filename) - ~ "#" ~ m.captures["hash"] - ~ "├" - ); - } - } - _txt = _txt - .replaceAll( - rgx.inline_link_fn_suffix, - ("$1.html")) - .replaceAll( - rgx.inline_link, - ("$1")) - .replaceAll( - rgx.mark_internal_site_lnk, - ""); - } - debug(markup_links) { - if (_txt.match(rgx.inline_link)) { - writeln(__LINE__, - " (missed) markup link identified (", - obj.has.inline_links, - "): ", obj.metainfo.is_a, ": ", - obj.text - ); - } - // if (obj.metainfo.is_a == "bookindex") { // DEBUG LINE - // if (_txt.match(regex(r"" - ~ "" ~ m.captures["num"] ~ "." - ~ m.captures["note"] - ~ "

"; - } - _txt = replaceAll!(m => - (" " ~ "" ~ m["num"] ~ "")) - (_txt, rgx.inline_notes_al_regular_number_note) - ~ _endnotes.join("\n"); - } - debug(markup_endnotes) { - if (_txt.match(rgx.inline_notes_al_regular_number_note)) { - writeln(__LINE__, " (missed) markup endnote: ", obj.metainfo.is_a, ": ", obj.text); - } - } - debug(markup) { - if (_txt.match(rgx.inline_notes_al_regular_number_note)) { - writeln(__LINE__, " (missed) markup endnote: ", obj.metainfo.is_a, ": ", obj.text); - } - } - return _txt; - } - string xml_type="seg"; /+ set html document type to be linked to here (seg|scroll) +/ - string inline_markup(M,O)( - M doc_matters, - const O obj, - string _txt, - ) { - if (obj.metainfo.is_a == "group") { - _txt = inline_grouped_text_bullets_indents(doc_matters, obj, _txt, xml_type); - } - _txt = inline_images(doc_matters, obj, _txt, xml_type); - _txt = inline_links(doc_matters, obj, _txt, xml_type); - _txt = inline_notes_scroll(doc_matters, obj, _txt); - return _txt; - } - string html_heading(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter"); - assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb"); - assert(obj.metainfo.is_of_type == "para"); - assert(obj.metainfo.is_a == "heading"); - string _txt = munge_html(doc_matters, obj); - _txt = inline_markup(doc_matters, obj, _txt); - string o = format(q"┃

- %s -

┃", - obj.metainfo.is_a, - _txt, - ); - return o; - } - string html_para(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter"); - assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb"); - assert(obj.metainfo.is_of_type == "para"); - assert(obj.metainfo.is_a == "para" || "toc" || "endnote" || "glossary" || "bibliography" || "bookindex" || "blurb"); - string _txt = munge_html(doc_matters, obj); - _txt = (obj.attrib.bullet) ? ("●  " ~ _txt) : _txt; - _txt = inline_markup(doc_matters, obj, _txt); - string o = format(q"┃

- %s -

┃", - obj.metainfo.is_a, - obj.attrib.indent_hang, - obj.attrib.indent_base, - _txt - ); - return o; - } - string html_quote(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body"); - assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); - assert(obj.metainfo.is_of_type == "block"); - assert(obj.metainfo.is_a == "quote"); - string _txt = munge_html(doc_matters, obj); - string o = format(q"┃

- %s -

┃", - obj.metainfo.is_a, - _txt - ); - return o; - } - string html_group(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body"); - assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); - assert(obj.metainfo.is_of_type == "block"); - assert(obj.metainfo.is_a == "group"); - string _txt = munge_html(doc_matters, obj); - _txt = inline_markup(doc_matters, obj, _txt); - string o = format(q"┃

- %s -

┃", - obj.metainfo.is_a, - _txt - ); - return o; - } - string html_block(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body"); - assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); - assert(obj.metainfo.is_of_type == "block"); - assert(obj.metainfo.is_a == "block"); - string _txt = munge_html(doc_matters, obj); - _txt = inline_markup(doc_matters, obj, _txt); - string o = format(q"┃ -

%s

┃", - obj.metainfo.is_a, - _txt.stripRight - ); - return o; - } - string html_verse(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body"); - assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); - assert(obj.metainfo.is_of_type == "block"); - assert(obj.metainfo.is_a == "verse"); - string _txt = munge_html(doc_matters, obj); - string o = format(q"┃

%s

┃", - obj.metainfo.is_a, - _txt - ); - return o; - } - string html_code(O)( - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body"); - assert(obj.metainfo.is_of_section == "body"); - assert(obj.metainfo.is_of_type == "block"); - assert(obj.metainfo.is_a == "code"); - string _txt = html_special_characters_code(obj.text); - string o = format(q"┃

%s

┃", - obj.metainfo.is_a, - _txt - ); - return o; - } - string html_table(M,O)( - M doc_matters, - const O obj, - ) { - assert(obj.metainfo.is_of_part == "body"); - assert(obj.metainfo.is_of_section == "body"); - assert(obj.metainfo.is_of_type == "block"); - assert(obj.metainfo.is_a == "table"); - Tuple!(string, string) _tablarize(O)( - const O obj, - string _txt, - ) { - string[] _table_rows = _txt.split(rgx.table_delimiter_row); - string[] _table_cols; - string _table; - string _tablenote; - foreach(row_idx, row; _table_rows) { - _table_cols = row.split(rgx.table_delimiter_col); - _table ~= ""; - foreach(col_idx, cell; _table_cols) { - if ((_table_cols.length == 1) - && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) - _tablenote ~= cell; - } else { - string _col_is = (row_idx == 0 && obj.table.heading) ? "th" : "td"; - string _align = ("style=\"text-align:" - ~ ((obj.table.column_aligns[col_idx] == "l") - ? "left\"" : "right\"")); - _table ~= "<" - ~ _col_is - ~ " width=\"" - ~ obj.table.column_widths[col_idx].to!string - ~ "%\" " - ~ _align - ~ ">"; - _table ~= cell; - _table ~= ""; - } - } - _table ~= ""; - } - Tuple!(string, string) t = tuple( - _table, - _tablenote, - ); - return t; - } - string _txt = munge_html(doc_matters, obj); - Tuple!(string, string) t = _tablarize(obj, _txt); - _txt = t[0]; - string _note = t[1]; - string o = format(q"┃

- - %s -
- %s -

┃", - obj.metainfo.is_a, - _txt, - _note - ); - return o; - } - string sqlite_load_string(M,O)( - M doc_matters, - const O obj, - ) { - string o; - return o; - } - string sqlite_statement(O)( - const O obj, - string _txt, - string _html, - ) { - void _sql_exe(O)( - string _sql, - ) { - writeln(_html); - writeln(_sql); - } - string _sql; - return _sql; - } - string[string] heading(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_heading(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] para(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_para(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] quote(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_quote(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] group(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_group(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] block(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_block(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] verse(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_verse(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] code(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_code(obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - string[string] table(M,O)( - M doc_matters, - const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_table(doc_matters, obj) - ]; - { /+ debug +/ - if (doc_matters.opt.action.debug_do_sqlite) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - } - return obj_txt; - } - } - return sqlite_format_and_load_objects(); - } -} -template SQLiteTablesReCreate() { - string SQLiteTablesReCreate()() { - string _sql_instruct; - _sql_instruct = format(q"┃ - DROP INDEX IF EXISTS idx_ocn; - DROP INDEX IF EXISTS idx_uid; - DROP INDEX IF EXISTS idx_digest_clean; - DROP INDEX IF EXISTS idx_digest_all; - DROP INDEX IF EXISTS idx_clean; - DROP INDEX IF EXISTS idx_title; - DROP INDEX IF EXISTS idx_author; - DROP INDEX IF EXISTS src_filename_base; - DROP INDEX IF EXISTS idx_language_document_char; - DROP INDEX IF EXISTS idx_classify_topic_register; - DROP INDEX IF EXISTS idx_topic_list; - DROP TABLE IF EXISTS metadata_and_text; - DROP TABLE IF EXISTS topic_register; - DROP TABLE IF EXISTS doc_objects; - DROP TABLE IF EXISTS urls; - CREATE TABLE IF NOT EXISTS metadata_and_text ( - uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) - src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code - src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename - title VARCHAR(800) NOT NULL, - title_main VARCHAR(400) NOT NULL, - title_sub VARCHAR(400) NULL, - title_short VARCHAR(400) NULL, - title_edition VARCHAR(10) NULL, - title_language VARCHAR(100) NULL, - title_language_char VARCHAR(6) NULL, - creator_author VARCHAR(600) NOT NULL, - creator_author_last_first VARCHAR(600) NOT NULL, - creator_author_email VARCHAR(100) NULL, - creator_author_hon VARCHAR(100) NULL, - creator_author_nationality VARCHAR(100) NULL, - creator_editor VARCHAR(600) NULL, - creator_contributor VARCHAR(600) NULL, - creator_illustrator VARCHAR(600) NULL, - creator_photographer VARCHAR(600) NULL, - creator_translator VARCHAR(600) NULL, - creator_prepared_by VARCHAR(600) NULL, - creator_digitized_by VARCHAR(600) NULL, - creator_audio VARCHAR(600) NULL, - creator_video VARCHAR(600) NULL, - language_document VARCHAR(100) NULL, - language_document_char VARCHAR(6) NOT NULL, - language_original VARCHAR(100) NULL, - language_original_char VARCHAR(6) NULL, - date_added_to_site VARCHAR(10) NULL, - date_available VARCHAR(10) NULL, - date_created VARCHAR(10) NULL, - date_issued VARCHAR(10) NULL, - date_modified VARCHAR(10) NULL, - date_published VARCHAR(10) NULL, - date_valid VARCHAR(10) NULL, - date_translated VARCHAR(10) NULL, - date_original_publication VARCHAR(10) NULL, - date_generated VARCHAR(10) NULL, - original_title VARCHAR(800) NULL, - original_publisher VARCHAR(600) NULL, - original_language VARCHAR(100) NULL, - original_language_char VARCHAR(6) NULL, - original_source VARCHAR(600) NULL, - original_institution VARCHAR(600) NULL, - original_nationality VARCHAR(100) NULL, - rights_copyright VARCHAR(2500) NULL, - rights_copyright_audio VARCHAR(2500) NULL, - rights_copyright_cover VARCHAR(2500) NULL, - rights_copyright_illustrations VARCHAR(2500) NULL, - rights_copyright_photographs VARCHAR(2500) NULL, - rights_copyright_text VARCHAR(2500) NULL, - rights_copyright_translation VARCHAR(2500) NULL, - rights_copyright_video VARCHAR(2500) NULL, - rights_license VARCHAR(2500) NULL, - identifier_oclc VARCHAR(30) NULL, - identifier_isbn VARCHAR(16) NULL, - classify_topic_register VARCHAR(2500) NULL, - classify_subject VARCHAR(600) NULL, - classify_loc VARCHAR(30) NULL, - classify_dewey VARCHAR(30) NULL, - classify_keywords VARCHAR(600) NULL, - notes_abstract TEXT NULL, - notes_description TEXT NULL, - notes_comment TEXT NULL, - notes_coverage VARCHAR(200) NULL, - notes_relation VARCHAR(200) NULL, - notes_history VARCHAR(600) NULL, - notes_type VARCHAR(600) NULL, - notes_format VARCHAR(600) NULL, - notes_prefix TEXT NULL, - notes_prefix_a TEXT NULL, - notes_prefix_b TEXT NULL, - notes_suffix TEXT NULL, - publisher VARCHAR(600) NULL, - src_filename_base VARCHAR(256) NOT NULL, - src_filename_suffix VARCHAR(6) NOT NULL, - src_fingerprint VARCHAR(256) NULL, - src_filesize VARCHAR(10) NULL, - src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here - pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here - pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here - site_url_doc_root VARCHAR(256) NULL, -- url path to doc root - site_url_html_toc VARCHAR(256) NULL, - site_url_html_scroll VARCHAR(256) NULL, - site_url_epub VARCHAR(256) NULL, - links TEXT NULL - ); - CREATE TABLE IF NOT EXISTS topic_register ( - -- tid BIGINT PRIMARY KEY, - uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, - -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code - -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename - topic_register VARCHAR(250) NOT NULL, - site_url_doc_root VARCHAR(256) NULL, -- url path to doc root - site_url_html_toc VARCHAR(256) NULL, - site_url_html_scroll VARCHAR(256) NULL - ); - CREATE TABLE IF NOT EXISTS doc_objects ( - lid BIGINT PRIMARY KEY, - uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, - ocn SMALLINT, - obj_id VARCHAR(6) NULL, - clean TEXT NULL, - body TEXT NULL, - seg VARCHAR(256) NULL, - lev_an VARCHAR(1), - is_of_type VARCHAR(16), - is_a VARCHAR(16), - lev SMALLINT NULL, - node VARCHAR(16) NULL, -- not written to - parent VARCHAR(16) NULL, - last_descendant VARCHAR(16) NULL, -- headings only - digest_clean CHAR(256), - digest_all CHAR(256), - seg_name CHAR(256), - types CHAR(1) NULL - ); - CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn); - CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean); - CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all); - CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean); - CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title); - CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first); - CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid); - CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base); - CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char); - CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register); - CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register); - PRAGMA user_version = %s; - ┃", - sqlite_db_schema_version, - ); - return _sql_instruct; - } -} -template SQLiteDeleteDocument() { - string SQLiteDeleteDocument(M)(M doc_matters) { - string _uid = doc_matters.src.doc_uid; - string _delete_uid = format(q"┃ - DELETE FROM metadata_and_text - WHERE uid = '%s'; - DELETE FROM doc_objects - WHERE uid_metadata_and_text = '%s'; - ┃", - _uid, - _uid, - ); - return _delete_uid; - } -} -template SQLiteInsertMetadata() { - string SQLiteInsertMetadata(M)(M doc_matters) { - string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); - string _insert_metadata = format(q"┃ - INSERT INTO metadata_and_text ( - uid, - src_filename_base, - src_filename_suffix, - src_composite_id_per_txt, - src_composite_id_per_pod, - title, - title_main, - title_sub, - title_short, - title_edition, - title_language, - creator_author, - creator_author_last_first, - creator_author_email, - creator_illustrator, - creator_translator, - language_document, - language_document_char, - date_added_to_site, - date_available, - date_created, - date_issued, - date_modified, - date_published, - date_valid, - rights_copyright, - rights_copyright_audio, - rights_copyright_cover, - rights_copyright_illustrations, - rights_copyright_photographs, - rights_copyright_text, - rights_copyright_translation, - rights_copyright_video, - rights_license, - identifier_oclc, - identifier_isbn, - classify_dewey, - classify_keywords, - classify_loc, - classify_subject, - classify_topic_register, - original_title, - original_publisher, - original_language, - original_language_char, - original_source, - notes_abstract, - notes_description, - publisher, - site_url_doc_root - ) - VALUES ( - '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' - ); - ┃", - _uid, - SQLinsertDelimiter!()(doc_matters.src.filename_base), - SQLinsertDelimiter!()(doc_matters.src.filename_extension), - SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), - SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_surname_fn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.conf.w_srv_data_root_url_html) - ); - return _insert_metadata; - } -} -template SQLiteInsertMetadataTopics() { - string SQLiteInsertMetadataTopics(M)(M doc_matters) { - string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); - string[] _insert_topics; - foreach (topic_line; doc_matters.conf_make_meta.meta.classify_topic_register_expanded_arr) { - // writeln(topic_line); - _insert_topics ~= format(q"┃ - INSERT INTO topic_register ( - uid_metadata_and_text, - topic_register - ) - VALUES ( - '%s', - '%s' - ); - ┃", - _uid, - SQLinsertDelimiter!()(topic_line) - ); - } - return _insert_topics.join.to!(char[]).toUTF8; - } -} -template SQLiteInsertDocObjectsLoop() { - string SQLiteInsertDocObjectsLoop(D)(D doc) { - string _uid = SQLinsertDelimiter!()(doc.matters.src.doc_uid); - auto url_html = spineUrlsHTML!()(doc.matters.conf_make_meta.conf.w_srv_data_root_url_html, doc.matters.src.language); - string insertDocObjectsRow(O)(O obj) { - string _insert_doc_objects_row = format(q"┃ - INSERT INTO doc_objects ( - uid_metadata_and_text, - ocn, - obj_id, - clean, - body, - lev, - is_of_type, - is_a, - seg_name - ) - VALUES ( - '%s', %s, '%s', '%s', '%s', %s, '%s', '%s', '%s' - ); - ┃", - _uid, - obj.metainfo.ocn, - obj.metainfo.identifier, - SQLinsertDelimiter!()(obj_txt["text"]), - SQLinsertDelimiter!()(obj_txt["html"]), - obj.metainfo.heading_lev_markup, - obj.metainfo.is_of_type, - obj.metainfo.is_a, - obj.tags.html_segment_anchor_tag_is - ); - return _insert_doc_objects_row; - } - auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc.matters); - string[string] obj_txt; - string doc_text; - string[] _insert_doc_objects; - foreach (part; doc.matters.has.keys_seq.sql) { - foreach (obj; doc.abstraction[part]) { - switch (obj.metainfo.is_of_part) { - case "frontmatter": assert(part == "head", part); - switch (obj.metainfo.is_of_type) { - case "para": - switch (obj.metainfo.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(doc.matters, obj); - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); - } - } - break; - } - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); - } - } - break; - } - break; - case "body": // assert(part == "body", part); - switch (obj.metainfo.is_of_type) { - case "para": - switch (obj.metainfo.is_a) { - case "heading": - debug (asserts) { - if (part != "body") { - writeln(__LINE__, ": ", obj.text); - } - } - obj_txt = format_and_sqlite_load.heading(doc.matters, obj); - break; - case "para": - obj_txt = format_and_sqlite_load.para(doc.matters, obj); - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); - } - } - break; - } - break; - case "block": - switch (obj.metainfo.is_a) { - case "quote": - obj_txt = format_and_sqlite_load.quote(doc.matters, obj); - break; - case "group": - obj_txt = format_and_sqlite_load.group(doc.matters, obj); - break; - case "block": - obj_txt = format_and_sqlite_load.block(doc.matters, obj); - break; - case "poem": // double check on keeping both poem & verse - break; - case "verse": - obj_txt = format_and_sqlite_load.verse(doc.matters, obj); - break; - case "code": - obj_txt = format_and_sqlite_load.code(doc.matters, obj); - break; - case "table": - obj_txt = format_and_sqlite_load.table(doc.matters, obj); - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); - } - } - break; - } - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); - } - } - break; - } - break; - case "backmatter": - assert(part == "glossary" || "bibliography" || "bookindex" || "blurb" || "tail", part); - switch (obj.metainfo.is_of_type) { - case "para": - switch (obj.metainfo.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(doc.matters, obj); - break; - case "glossary": assert(part == "glossary", part); - obj_txt = format_and_sqlite_load.para(doc.matters, obj); - break; - case "bibliography": assert(part == "bibliography", part); - obj_txt = format_and_sqlite_load.para(doc.matters, obj); - break; - case "bookindex": assert(part == "bookindex", part); - obj_txt = format_and_sqlite_load.para(doc.matters, obj); - break; - case "blurb": assert(part == "blurb", part); - obj_txt = format_and_sqlite_load.para(doc.matters, obj); - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); - } - } - break; - } - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); - } - } - break; - } - break; - case "comment": - break; - default: - { /+ debug +/ - if (doc.matters.opt.action.debug_do_sqlite) { - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_part); // check where empty value could come from - writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); - writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from - } - } - break; - } - if (obj.metainfo.is_a == "heading") { - if (doc.matters.opt.action.show_sqlite) { - if (obj.metainfo.heading_lev_markup == 0) { - writeln(doc.matters.src.filename); - } - writeln( - "markup: ", obj.metainfo.heading_lev_markup, - "> ", obj.metainfo.dom_structure_markedup_tags_status, - "; collapsed: ", obj.metainfo.heading_lev_collapsed, - "> ", obj.metainfo.dom_structure_collapsed_tags_status, - "; ocn: ", obj.metainfo.ocn, - "; parent: ", obj.metainfo.parent_lev_markup, - "; ocn: ", obj.metainfo.parent_ocn, - "; ", - ); - } - } - if (!(obj.metainfo.is_a == "comment")) { - _insert_doc_objects ~= insertDocObjectsRow(obj); - } - } // loop closes - } - return _insert_doc_objects.join.to!(char[]).toUTF8; - } -} -template SQLiteTablesCreate() { - void SQLiteTablesCreate(E,O,C)(E env, O opt_action, C config) { - import d2sqlite3; - template SQLiteTablesReCreate() { - string SQLiteTablesReCreate()() { - string _sql_instruct; - _sql_instruct = format(q"┃ - DROP INDEX IF EXISTS idx_ocn; - DROP INDEX IF EXISTS idx_uid; - DROP INDEX IF EXISTS idx_digest_clean; - DROP INDEX IF EXISTS idx_digest_all; - DROP INDEX IF EXISTS idx_clean; - DROP INDEX IF EXISTS idx_title; - DROP INDEX IF EXISTS idx_author; - DROP INDEX IF EXISTS src_filename_base; - DROP INDEX IF EXISTS idx_language_document_char; - DROP INDEX IF EXISTS idx_classify_topic_register; - DROP INDEX IF EXISTS idx_topic_list; - DROP TABLE IF EXISTS metadata_and_text; - DROP TABLE IF EXISTS topic_register; - DROP TABLE IF EXISTS doc_objects; - DROP TABLE IF EXISTS urls; - CREATE TABLE IF NOT EXISTS metadata_and_text ( - uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) - src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code - src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename - title VARCHAR(800) NOT NULL, - title_main VARCHAR(400) NOT NULL, - title_sub VARCHAR(400) NULL, - title_short VARCHAR(400) NULL, - title_edition VARCHAR(10) NULL, - title_language VARCHAR(100) NULL, - title_language_char VARCHAR(6) NULL, - creator_author VARCHAR(600) NOT NULL, - creator_author_last_first VARCHAR(600) NOT NULL, - creator_author_email VARCHAR(100) NULL, - creator_author_hon VARCHAR(100) NULL, - creator_author_nationality VARCHAR(100) NULL, - creator_editor VARCHAR(600) NULL, - creator_contributor VARCHAR(600) NULL, - creator_illustrator VARCHAR(600) NULL, - creator_photographer VARCHAR(600) NULL, - creator_translator VARCHAR(600) NULL, - creator_prepared_by VARCHAR(600) NULL, - creator_digitized_by VARCHAR(600) NULL, - creator_audio VARCHAR(600) NULL, - creator_video VARCHAR(600) NULL, - language_document VARCHAR(100) NULL, - language_document_char VARCHAR(6) NOT NULL, - language_original VARCHAR(100) NULL, - language_original_char VARCHAR(6) NULL, - date_added_to_site VARCHAR(10) NULL, - date_available VARCHAR(10) NULL, - date_created VARCHAR(10) NULL, - date_issued VARCHAR(10) NULL, - date_modified VARCHAR(10) NULL, - date_published VARCHAR(10) NULL, - date_valid VARCHAR(10) NULL, - date_translated VARCHAR(10) NULL, - date_original_publication VARCHAR(10) NULL, - date_generated VARCHAR(10) NULL, - original_title VARCHAR(800) NULL, - original_publisher VARCHAR(600) NULL, - original_language VARCHAR(100) NULL, - original_language_char VARCHAR(6) NULL, - original_source VARCHAR(600) NULL, - original_institution VARCHAR(600) NULL, - original_nationality VARCHAR(100) NULL, - rights_copyright VARCHAR(2500) NULL, - rights_copyright_audio VARCHAR(2500) NULL, - rights_copyright_cover VARCHAR(2500) NULL, - rights_copyright_illustrations VARCHAR(2500) NULL, - rights_copyright_photographs VARCHAR(2500) NULL, - rights_copyright_text VARCHAR(2500) NULL, - rights_copyright_translation VARCHAR(2500) NULL, - rights_copyright_video VARCHAR(2500) NULL, - rights_license VARCHAR(2500) NULL, - identifier_oclc VARCHAR(30) NULL, - identifier_isbn VARCHAR(16) NULL, - classify_topic_register VARCHAR(2500) NULL, - classify_subject VARCHAR(600) NULL, - classify_loc VARCHAR(30) NULL, - classify_dewey VARCHAR(30) NULL, - classify_keywords VARCHAR(600) NULL, - notes_abstract TEXT NULL, - notes_description TEXT NULL, - notes_comment TEXT NULL, - notes_coverage VARCHAR(200) NULL, - notes_relation VARCHAR(200) NULL, - notes_history VARCHAR(600) NULL, - notes_type VARCHAR(600) NULL, - notes_format VARCHAR(600) NULL, - notes_prefix TEXT NULL, - notes_prefix_a TEXT NULL, - notes_prefix_b TEXT NULL, - notes_suffix TEXT NULL, - publisher VARCHAR(600) NULL, - src_filename_base VARCHAR(256) NOT NULL, - src_filename_suffix VARCHAR(6) NOT NULL, - src_fingerprint VARCHAR(256) NULL, - src_filesize VARCHAR(10) NULL, - src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here - pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here - pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here - site_url_doc_root VARCHAR(256) NULL, -- url path to doc root - site_url_html_toc VARCHAR(256) NULL, - site_url_html_scroll VARCHAR(256) NULL, - site_url_epub VARCHAR(256) NULL, - links TEXT NULL - ); - CREATE TABLE IF NOT EXISTS topic_register ( - -- tid BIGINT PRIMARY KEY, - uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, - -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code - -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename - topic_register VARCHAR(250) NOT NULL, - site_url_doc_root VARCHAR(256) NULL, -- url path to doc root - site_url_html_toc VARCHAR(256) NULL, - site_url_html_scroll VARCHAR(256) NULL - ); - CREATE TABLE IF NOT EXISTS doc_objects ( - lid BIGINT PRIMARY KEY, - uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, - ocn SMALLINT, - obj_id VARCHAR(6) NULL, - clean TEXT NULL, - body TEXT NULL, - seg VARCHAR(256) NULL, - lev_an VARCHAR(1), - is_of_type VARCHAR(16), - is_a VARCHAR(16), - lev SMALLINT NULL, - node VARCHAR(16) NULL, -- not written to - parent VARCHAR(16) NULL, - last_descendant VARCHAR(16) NULL, -- headings only - digest_clean CHAR(256), - digest_all CHAR(256), - seg_name CHAR(256), - types CHAR(1) NULL - ); - CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn); - CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean); - CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all); - CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean); - CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title); - CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first); - CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid); - CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base); - CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char); - CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register); - CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register); - PRAGMA user_version = %s; - ┃", - sqlite_db_schema_version, - ); - return _sql_instruct; - } - } - try { - if (opt_action.sqlite_db_create) { - string _db_statement; - string db_filename = (opt_action.sqliteDB_filename.length > 0) - ? opt_action.sqliteDB_filename - : (config.conf.w_srv_db_sqlite_filename.length > 0) - ? config.conf.w_srv_db_sqlite_filename - : ""; - string db_path = (opt_action.sqliteDB_path.length > 0) - ? opt_action.sqliteDB_path - : (config.conf.w_srv_db_sqlite_path.length > 0) - ? config.conf.w_srv_db_sqlite_path - : ""; - if (db_filename.length > 0 && db_path.length > 0) { - if (opt_action.vox_gt_3) { - writeln("db name: ", db_filename); - writeln("db path: ", db_path); - writeln("db name & path: ", db_path, "/", db_filename); - } - if (opt_action.vox_gt_2) { - writeln("attempting to create db: ", db_path, "/", db_filename); - } - auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path); - if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { - } else { - try { - pth_sqlite.base.mkdirRecurse; - } catch (FileException ex) { - stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); - stderr.writeln(" (", ex.msg, ")"); - import core.runtime; - core.runtime.Runtime.terminate(); - } - } - auto db = Database(pth_sqlite.sqlite_file); - { - _db_statement ~= SQLiteTablesReCreate!()(); - } - SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE"); - } else { - writeln("must provide db name & output root path either on the command line or in configuration file"); - writeln("db name: ", db_filename); - writeln("db path: ", db_path); - } - } - } - catch (FileException e) { - writeln("Failed (FileException): ", e.msg); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (ErrnoException e) { - writeln("Failed (ErrnoException): ", e.msg); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (Exception e) { - writeln("Failed (Exception): ", e.msg); - writeln(e.file, " line: ", e.line); - import core.runtime; - core.runtime.Runtime.terminate(); - } - catch (Throwable) { - import core.runtime; - core.runtime.Runtime.terminate(); - } - } -} -template SQLiteDbDrop() { - void SQLiteDbDrop(O,C)(O opt_action, C config) { - writeln("db drop"); - if ((opt_action.sqlite_db_drop)) { - string db_filename = (opt_action.sqliteDB_filename.length > 0) - ? opt_action.sqliteDB_filename - : (config.conf.w_srv_db_sqlite_filename.length > 0) - ? config.conf.w_srv_db_sqlite_filename - : ""; - string db_path = (opt_action.sqliteDB_path.length > 0) // - ? opt_action.sqliteDB_path - : (config.conf.w_srv_db_sqlite_path.length > 0) - ? config.conf.w_srv_db_sqlite_path - : ""; - if (db_filename.length > 0 && db_path.length > 0) { - auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path); - writeln("remove(", pth_sqlite.sqlite_file, ")"); - try { - remove(pth_sqlite.sqlite_file); - } catch (FileException ex) { - // handle error - } - } else { - writeln("must provide db name & output root path either on the command line or in configuration file"); - writeln("db name: ", db_filename); - writeln("db path: ", db_path); - } - } - } -} diff --git a/src/sisudoc/outputs/io_out/sqlite_collection_db.d b/src/sisudoc/outputs/io_out/sqlite_collection_db.d new file mode 100644 index 0000000..727c844 --- /dev/null +++ b/src/sisudoc/outputs/io_out/sqlite_collection_db.d @@ -0,0 +1,1811 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ +module sisudoc.outputs.io_out.sqlite_collection_db; +import sisudoc.outputs.io_out; +import sisudoc.outputs.io_out.rgx; +import sisudoc.outputs.io_out.rgx_xhtml; +import std.file; +import std.uri; +// import std.digest.sha; +import std.conv : to; +import std.typecons : Nullable; +import d2sqlite3; +mixin spineRgxOut; +mixin spineRgxXHTML; +mixin InternalMarkup; +static auto rgx = RgxO(); +static auto rgx_xhtml = RgxXHTML(); +static auto mkup = InlineMarkup(); +long _metadata_tid_lastrowid; +/+ ↓ sqlite db schema version, stamped into the db as PRAGMA user_version +/ +/+ bump whenever the table or index definitions in this module change, and +/ +/+ bump the matching expectation in sisudoc-spine-search-cgi +/ +enum sqlite_db_schema_version = 1; +/+ ↓ tally of failed sqlite statements, shared: sqlite output may run in parallel +/ +private shared int _sqlite_failure_tally = 0; +/+ ↓ schema version mismatch is reported once per run, not once per document +/ +private shared int _sqlite_schema_mismatch_reported = 0; +/+ ↓ record a failed sqlite statement, reported on stderr, tallied for exit status +/ +void sqliteFailureRecord(string note, string msg) { + import core.atomic : atomicOp; + atomicOp!"+="(_sqlite_failure_tally, 1); + stderr.writeln("ERROR SQLite (", note, "): ", msg); +} +/+ ↓ count of failed sqlite statements this run, zero when all went well +/ +int sqliteFailureTally() { + import core.atomic : atomicLoad; + return atomicLoad(_sqlite_failure_tally); +} +/+ ↓ compare the schema version stamped in an existing db with this spine's +/ +void sqliteSchemaVersionCheck(Db)(Db db, string sqlite_file) { + import core.atomic : atomicOp; + int _found; + try { + _found = db.execute("PRAGMA user_version;").oneValue!int; + } catch (Exception) { + return; + } + if (_found == sqlite_db_schema_version + || _found == 0) { /+ ↓ zero: db predates the stamp, or has no tables yet +/ + return; + } + if (atomicOp!"+="(_sqlite_schema_mismatch_reported, 1) == 1) { + stderr.writeln("ERROR SQLite : db schema version mismatch: ", sqlite_file); + stderr.writeln(" db holds schema version ", _found, + ", this spine writes schema version ", sqlite_db_schema_version); + stderr.writeln(" recreate the db, e.g.: spine --sqlite-db-recreate", + " --sqlite-db-path=... --sqlite-db-filename=..."); + sqliteFailureRecord("schema version", + "db schema version " ~ _found.to!string + ~ " != spine schema version " ~ sqlite_db_schema_version.to!string); + } +} +template SQLiteHubBuildTablesAndPopulate() { + void SQLiteHubBuildTablesAndPopulate(D)(D doc) { + auto pth_sqlite = spinePathsSQLite!()(doc.matters.sqlite.filename, doc.matters.sqlite.path); + if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { + } else { + try { + pth_sqlite.base.mkdirRecurse; + } catch (FileException ex) { + stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); + stderr.writeln(" (", ex.msg, ")"); + import core.runtime; + core.runtime.Runtime.terminate(); + } + } + template SQLiteDbStatementComposite() { + void SQLiteDbStatementComposite(Db,D)( + Db db, + D doc + ) { + string _db_statement; + if ((doc.matters.opt.action.sqlite_db_create)) { + auto pth_sqlite = spinePathsSQLite!()(doc.matters.sqlite.filename, doc.matters.sqlite.path); + if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { + } else { + try { + pth_sqlite.base.mkdirRecurse; + } catch (FileException ex) { + stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); + stderr.writeln(" (", ex.msg, ")"); + import core.runtime; + core.runtime.Runtime.terminate(); + } + } + _db_statement ~= SQLiteTablesReCreate!()(); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "TABLE RE-CREATE"); + _db_statement = []; + } + if (doc.matters.opt.action.sqlite_delete) { + _db_statement ~= SQLiteDeleteDocument!()(doc.matters); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "DELETE Document"); + _db_statement = []; + } + if (doc.matters.opt.action.sqlite_update) { + _db_statement ~= SQLiteDeleteDocument!()(doc.matters); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "DELETE Document"); + _db_statement = []; + _db_statement ~= SQLiteInsertMetadata!()(doc.matters); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT MetaData"); + _db_statement = []; + /+ get tid (lastrowid or max) for use in doc.objects table +/ + _db_statement ~= doc.SQLiteInsertDocObjectsLoop!(); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT DocObjects"); + _db_statement = []; + _db_statement ~= SQLiteInsertMetadataTopics!()(doc.matters); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT MetaDataTopics"); + _db_statement = []; + } + db.close; + if (doc.matters.opt.action.vox_gt_1) { + writeln(" ", pth_sqlite.sqlite_file); + } + } + } + try { + auto db = Database(pth_sqlite.sqlite_file); + sqliteSchemaVersionCheck(db, pth_sqlite.sqlite_file); + SQLiteDbStatementComposite!()(db, doc); + } + catch (FileException e) { + writeln("Failed (FileException): ", e.msg, " ", pth_sqlite.sqlite_file); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (ErrnoException e) { + writeln("Failed (ErrnoException): ", e.msg, " ", pth_sqlite.sqlite_file); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (Exception e) { + writeln("Failed (Exception): ", e.msg, " ", pth_sqlite.sqlite_file); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (Throwable) { + writeln("Failed (Trowable): ", pth_sqlite.sqlite_file); + import core.runtime; + core.runtime.Runtime.terminate(); + } + } +} +template SQLiteHubDiscreteBuildTablesAndPopulate() { + void SQLiteHubDiscreteBuildTablesAndPopulate(D)(D doc) { + auto url_html = spineUrlsHTML!()(doc.matters.conf_make_meta.conf.w_srv_data_root_url_html, doc.matters.src.language); + auto pth_sqlite = spinePathsSQLiteDiscrete!()(doc.matters.output_path, doc.matters.src.language); // doc.matters.db_path + if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { + } else { + try { + pth_sqlite.base.mkdirRecurse; + } catch (FileException ex) { + stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); + stderr.writeln(" (", ex.msg, ")"); + import core.runtime; + core.runtime.Runtime.terminate(); + } + } + auto db = Database(pth_sqlite.sqlite_file(doc.matters.src.filename)); + sqliteSchemaVersionCheck(db, pth_sqlite.sqlite_file(doc.matters.src.filename)); + template SQLiteDiscreteDbStatementComposite() { + void SQLiteDiscreteDbStatementComposite(Db,D)( + Db db, + D doc + ) { + try { + { + string _db_statement; + _db_statement ~= SQLiteTablesReCreate!()(); + _db_statement ~= SQLiteInsertMetadata!()(doc.matters); + _db_statement ~= SQLiteInsertMetadataTopics!()(doc.matters); + _db_statement ~= doc.SQLiteInsertDocObjectsLoop!(); + SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "table CREATE Tables, INSERT DocObjects"); + } + db.close; + } + catch (FileException e) { + writeln("Failed (FileException): ", e.msg); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (ErrnoException e) { + writeln("Failed (ErrnoException): ", e.msg); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (Exception e) { + writeln("Failed (Exception): ", e.msg); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (Throwable) { + import core.runtime; + core.runtime.Runtime.terminate(); + } + if (doc.matters.opt.action.vox_gt_1) { + writeln(" ", pth_sqlite.sqlite_file(doc.matters.src.filename)); + } + } + } + SQLiteDiscreteDbStatementComposite!()(db, doc); + } +} +template SQLiteDbRun() { + void SQLiteDbRun(Db,St,O)( + Db db, + St db_statement, + O opt_action, + string note, + ) { + debug(sql_statement) { + writeln(db_statement); + } + try { + db.run( + "\nBEGIN TRANSACTION;\n" ~ + db_statement ~ + "\nCOMMIT TRANSACTION;\n" + ); + } catch (ErrnoException ex) { + sqliteFailureRecord(note, ex.msg); + } catch (Exception ex) { + sqliteFailureRecord(note, ex.msg); + } + { /+ debug +/ + if (opt_action.debug_do_sqlite) { + writeln(note); + if (opt_action.vox_gt_3) { + writeln(db_statement); + } + } + } + } +} +template SQLinsertDelimiter() { + string SQLinsertDelimiter(string _txt) { + _txt = _txt + .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); + return _txt; + } +} +template SQLiteFormatAndLoadObject() { + auto SQLiteFormatAndLoadObject(M)(M doc_matters) { + mixin spineRgxOut; + mixin spineRgxXHTML; + struct sqlite_format_and_load_objects { + string generic_munge_sanitize_text_for_search( + string _txt, + ) { + string _notes; + string _urls; + if (_txt.matchFirst(rgx.inline_notes_al_gen)) { + foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { + _notes ~= "\n" ~ m["text"]; + } + _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); + } + if (_txt.matchFirst(rgx.inline_link)) { + foreach (m; _txt.matchAll(rgx.inline_link)) { + if (m["link"].match(rgx.url)) { + _urls ~= "\n" ~ m["link"]; + } + } + _txt = _txt.replaceAll(rgx.inline_link_clean, ""); + } + if (_notes.length > 0) { + _txt ~= _notes; + } + if (_urls.length > 0) { + _txt ~= _urls; + } + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + writeln(_txt, "\n"); + } + } + debug(sql_text_clean) { + writeln(_txt); + } + return _txt; + } + string munge_html(M,O)( + M doc_matters, + const O obj, + ) { + string _html_special_characters(string _txt){ + _txt = _txt + .replaceAll(rgx_xhtml.ampersand, "&") + .replaceAll(rgx_xhtml.quotation, """) + .replaceAll(rgx_xhtml.less_than, "<") + .replaceAll(rgx_xhtml.greater_than, ">") + .replaceAll(rgx.nbsp_char, " ") + .replaceAll(rgx.br_line_inline, "
") + .replaceAll(rgx.br_line, "
") + .replaceAll(rgx.br_line_spaced, "

") + .replaceAll(rgx_xhtml.line_break, "
"); + return _txt; + } + string _html_font_face(string _txt){ + _txt = _txt + .replaceAll(rgx.inline_emphasis, "$1") + .replaceAll(rgx.inline_bold, "$1") + .replaceAll(rgx.inline_underscore, "$1") + .replaceAll(rgx.inline_italics, "$1") + .replaceAll(rgx.inline_superscript, "$1") + .replaceAll(rgx.inline_subscript, "$1") + .replaceAll(rgx.inline_strike, "$1") + .replaceAll(rgx.inline_insert, "$1") + .replaceAll(rgx.inline_mono, "$1") + .replaceAll(rgx.inline_cite, "$1"); + return _txt; + } + string _notes; + string _urls; + string _txt = _html_font_face(_html_special_characters(obj.text)); + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + writeln(_txt, "\n"); + } + } + return _txt; + } + string html_special_characters(string _txt){ + _txt = _txt + .replaceAll(rgx_xhtml.ampersand, "&") + .replaceAll(rgx_xhtml.quotation, """) + .replaceAll(rgx_xhtml.less_than, "<") + .replaceAll(rgx_xhtml.greater_than, ">") + .replaceAll(rgx.nbsp_char, " ") + .replaceAll(rgx.br_line_inline, "
") + .replaceAll(rgx.br_line, "
") + .replaceAll(rgx.br_line_spaced, "

") + .replaceAll(rgx_xhtml.line_break, "
"); + return _txt; + } + string html_special_characters_code(string _txt){ + _txt = _txt + .replaceAll(rgx_xhtml.ampersand, "&") + .replaceAll(rgx_xhtml.quotation, """) + .replaceAll(rgx_xhtml.less_than, "<") + .replaceAll(rgx_xhtml.greater_than, ">") + .replaceAll(rgx.nbsp_char, " "); + return _txt; + } + string html_font_face(string _txt){ + _txt = _txt + .replaceAll(rgx.inline_emphasis, "$1") + .replaceAll(rgx.inline_bold, "$1") + .replaceAll(rgx.inline_underscore, "$1") + .replaceAll(rgx.inline_italics, "$1") + .replaceAll(rgx.inline_superscript, "$1") + .replaceAll(rgx.inline_subscript, "$1") + .replaceAll(rgx.inline_strike, "$1") + .replaceAll(rgx.inline_insert, "$1") + .replaceAll(rgx.inline_mono, "$1") + .replaceAll(rgx.inline_cite, "$1"); + return _txt; + } + string inline_grouped_text_bullets_indents(M,O)( + M doc_matters, + const O obj, + string _txt, + string _suffix = ".html", + string _xml_type = "seg", + ) { + static auto rgx = RgxO(); + static auto rgx_xhtml = RgxXHTML(); + if (obj.metainfo.is_a == "group") { + _txt = (_txt) + .replaceAll(rgx.grouped_para_indent_1, + "  ") + .replaceAll(rgx.grouped_para_indent_2, + "    ") + .replaceAll(rgx.grouped_para_indent_3, + "      ") + .replaceAll(rgx.grouped_para_indent_4, + "        ") + .replaceAll(rgx.grouped_para_indent_5, + "          ") + .replaceAll(rgx.grouped_para_indent_6, + "            ") + .replaceAll(rgx.grouped_para_indent_7, + "              ") + .replaceAll(rgx.grouped_para_indent_8, + "                ") + .replaceAll(rgx.grouped_para_indent_9, + "                  ") + .replaceAll(rgx.grouped_para_indent_hang, "  ") + .replaceAll(rgx.grouped_para_bullet, "●  ") + .replaceAll(rgx.grouped_para_bullet_indent_1, + "  ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_2, + "    ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_3, + "      ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_4, + "        ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_5, + "          ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_6, + "            ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_7, + "              ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_8, + "                ●  ") + .replaceAll(rgx.grouped_para_bullet_indent_9, + "                  ●  "); + } + return _txt; + } + string inline_images(M,O)( + M doc_matters, + const O obj, + string _txt, + string _suffix = ".html", + string _xml_type = "seg", + ) { + string _img_pth; + if (_xml_type == "epub") { + _img_pth = "image/"; + } else if (_xml_type == "scroll") { + _img_pth = "../../image/"; + } else if (_xml_type == "seg") { + _img_pth = "../../../image/"; + } + if (_txt.match(rgx.inline_image)) { + _txt = _txt.replaceAll( // TODO bug where image dimensions (w or h) not given & consequently set to 0; should not be used (calculate earlier, abstraction) + rgx.inline_image, + ("$1 $6")); + } + return _txt; + } + string inline_links(M,O)( + M doc_matters, + const O obj, + string _txt, + string _xml_type = "seg", + ) { + if (obj.has.inline_links) { + if (obj.metainfo.is_a != "code") { + _txt = replaceAll!(m => + m["linked_text"] ~ "┤" ~ to!string((obj.stow.link[m["num"].to!ulong])).encode ~ "├" + )(_txt, rgx.inline_link_number_only); + } + if ((_txt.match(rgx.mark_internal_site_lnk)) + && (_xml_type == "scroll")) { // conditions reversed to avoid: gdc compiled program run segfault + _txt = _txt.replaceAll( + rgx.inline_seg_link, + "$1"); + } + auto pth_html = spinePathsHTML!()(doc_matters.output_path, doc_matters.src.language); + if (_xml_type == "seg") { + foreach (m; _txt.matchAll(rgx.inline_link_seg_and_hash)) { + if (m.captures["hash"] in doc_matters.has.tag_associations) { + if (m.captures["hash"] == doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"]) { + _txt = _txt.replaceFirst( + rgx.inline_link_seg_and_hash, + "┥$1┝┤" + ~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html + ~ "/" + ~ pth_html.tail_fn_seg(doc_matters.src.filename, "$2.html") + ~ "├" + ); + } else { + _txt = _txt.replaceFirst( + rgx.inline_link_seg_and_hash, + "┥$1┝┤" + ~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html + ~ "/" + ~ doc_matters.src.filename_base + ~ "/" + ~ doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"] + ~ ".html" + ~ "#" ~ m.captures["hash"] + ~ "├" + ); + } + } else { + if (doc_matters.opt.action.vox_gt_1) { + writeln( + "WARNING on internal document links, anchor to link <<" + ~ m.captures["hash"] + ~ ">> not found in document, " + ~ "anchor: " ~ m.captures["hash"] + ~ " document: " ~ doc_matters.src.filename + ); + } + } + } + } else { + if (auto m = _txt.match(rgx.inline_link_seg_and_hash)) { + _txt = _txt.replaceFirst( + rgx.inline_link_seg_and_hash, + "┥$1┝┤" + ~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html + ~ "/" + ~ pth_html.tail_fn_scroll(doc_matters.src.filename) + ~ "#" ~ m.captures["hash"] + ~ "├" + ); + } + } + _txt = _txt + .replaceAll( + rgx.inline_link_fn_suffix, + ("$1.html")) + .replaceAll( + rgx.inline_link, + ("$1")) + .replaceAll( + rgx.mark_internal_site_lnk, + ""); + } + debug(markup_links) { + if (_txt.match(rgx.inline_link)) { + writeln(__LINE__, + " (missed) markup link identified (", + obj.has.inline_links, + "): ", obj.metainfo.is_a, ": ", + obj.text + ); + } + // if (obj.metainfo.is_a == "bookindex") { // DEBUG LINE + // if (_txt.match(regex(r"" + ~ "" ~ m.captures["num"] ~ "." + ~ m.captures["note"] + ~ "

"; + } + _txt = replaceAll!(m => + (" " ~ "" ~ m["num"] ~ "")) + (_txt, rgx.inline_notes_al_regular_number_note) + ~ _endnotes.join("\n"); + } + debug(markup_endnotes) { + if (_txt.match(rgx.inline_notes_al_regular_number_note)) { + writeln(__LINE__, " (missed) markup endnote: ", obj.metainfo.is_a, ": ", obj.text); + } + } + debug(markup) { + if (_txt.match(rgx.inline_notes_al_regular_number_note)) { + writeln(__LINE__, " (missed) markup endnote: ", obj.metainfo.is_a, ": ", obj.text); + } + } + return _txt; + } + string xml_type="seg"; /+ set html document type to be linked to here (seg|scroll) +/ + string inline_markup(M,O)( + M doc_matters, + const O obj, + string _txt, + ) { + if (obj.metainfo.is_a == "group") { + _txt = inline_grouped_text_bullets_indents(doc_matters, obj, _txt, xml_type); + } + _txt = inline_images(doc_matters, obj, _txt, xml_type); + _txt = inline_links(doc_matters, obj, _txt, xml_type); + _txt = inline_notes_scroll(doc_matters, obj, _txt); + return _txt; + } + string html_heading(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter"); + assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb"); + assert(obj.metainfo.is_of_type == "para"); + assert(obj.metainfo.is_a == "heading"); + string _txt = munge_html(doc_matters, obj); + _txt = inline_markup(doc_matters, obj, _txt); + string o = format(q"┃

+ %s +

┃", + obj.metainfo.is_a, + _txt, + ); + return o; + } + string html_para(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter"); + assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb"); + assert(obj.metainfo.is_of_type == "para"); + assert(obj.metainfo.is_a == "para" || "toc" || "endnote" || "glossary" || "bibliography" || "bookindex" || "blurb"); + string _txt = munge_html(doc_matters, obj); + _txt = (obj.attrib.bullet) ? ("●  " ~ _txt) : _txt; + _txt = inline_markup(doc_matters, obj, _txt); + string o = format(q"┃

+ %s +

┃", + obj.metainfo.is_a, + obj.attrib.indent_hang, + obj.attrib.indent_base, + _txt + ); + return o; + } + string html_quote(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body"); + assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); + assert(obj.metainfo.is_of_type == "block"); + assert(obj.metainfo.is_a == "quote"); + string _txt = munge_html(doc_matters, obj); + string o = format(q"┃

+ %s +

┃", + obj.metainfo.is_a, + _txt + ); + return o; + } + string html_group(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body"); + assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); + assert(obj.metainfo.is_of_type == "block"); + assert(obj.metainfo.is_a == "group"); + string _txt = munge_html(doc_matters, obj); + _txt = inline_markup(doc_matters, obj, _txt); + string o = format(q"┃

+ %s +

┃", + obj.metainfo.is_a, + _txt + ); + return o; + } + string html_block(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body"); + assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); + assert(obj.metainfo.is_of_type == "block"); + assert(obj.metainfo.is_a == "block"); + string _txt = munge_html(doc_matters, obj); + _txt = inline_markup(doc_matters, obj, _txt); + string o = format(q"┃ +

%s

┃", + obj.metainfo.is_a, + _txt.stripRight + ); + return o; + } + string html_verse(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body"); + assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); + assert(obj.metainfo.is_of_type == "block"); + assert(obj.metainfo.is_a == "verse"); + string _txt = munge_html(doc_matters, obj); + string o = format(q"┃

%s

┃", + obj.metainfo.is_a, + _txt + ); + return o; + } + string html_code(O)( + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body"); + assert(obj.metainfo.is_of_section == "body"); + assert(obj.metainfo.is_of_type == "block"); + assert(obj.metainfo.is_a == "code"); + string _txt = html_special_characters_code(obj.text); + string o = format(q"┃

%s

┃", + obj.metainfo.is_a, + _txt + ); + return o; + } + string html_table(M,O)( + M doc_matters, + const O obj, + ) { + assert(obj.metainfo.is_of_part == "body"); + assert(obj.metainfo.is_of_section == "body"); + assert(obj.metainfo.is_of_type == "block"); + assert(obj.metainfo.is_a == "table"); + Tuple!(string, string) _tablarize(O)( + const O obj, + string _txt, + ) { + string[] _table_rows = _txt.split(rgx.table_delimiter_row); + string[] _table_cols; + string _table; + string _tablenote; + foreach(row_idx, row; _table_rows) { + _table_cols = row.split(rgx.table_delimiter_col); + _table ~= ""; + foreach(col_idx, cell; _table_cols) { + if ((_table_cols.length == 1) + && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) + _tablenote ~= cell; + } else { + string _col_is = (row_idx == 0 && obj.table.heading) ? "th" : "td"; + string _align = ("style=\"text-align:" + ~ ((obj.table.column_aligns[col_idx] == "l") + ? "left\"" : "right\"")); + _table ~= "<" + ~ _col_is + ~ " width=\"" + ~ obj.table.column_widths[col_idx].to!string + ~ "%\" " + ~ _align + ~ ">"; + _table ~= cell; + _table ~= ""; + } + } + _table ~= ""; + } + Tuple!(string, string) t = tuple( + _table, + _tablenote, + ); + return t; + } + string _txt = munge_html(doc_matters, obj); + Tuple!(string, string) t = _tablarize(obj, _txt); + _txt = t[0]; + string _note = t[1]; + string o = format(q"┃

+ + %s +
+ %s +

┃", + obj.metainfo.is_a, + _txt, + _note + ); + return o; + } + string sqlite_load_string(M,O)( + M doc_matters, + const O obj, + ) { + string o; + return o; + } + string sqlite_statement(O)( + const O obj, + string _txt, + string _html, + ) { + void _sql_exe(O)( + string _sql, + ) { + writeln(_html); + writeln(_sql); + } + string _sql; + return _sql; + } + string[string] heading(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_heading(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] para(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_para(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] quote(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_quote(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] group(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_group(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] block(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_block(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] verse(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_verse(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] code(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_code(obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + string[string] table(M,O)( + M doc_matters, + const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_table(doc_matters, obj) + ]; + { /+ debug +/ + if (doc_matters.opt.action.debug_do_sqlite) { + debug(sql_txt) { + writeln(obj_txt["text"]); + } + debug(sql_html) { + writeln(obj_txt["html"]); + } + } else { + // load sql + } + } + return obj_txt; + } + } + return sqlite_format_and_load_objects(); + } +} +template SQLiteTablesReCreate() { + string SQLiteTablesReCreate()() { + string _sql_instruct; + _sql_instruct = format(q"┃ + DROP INDEX IF EXISTS idx_ocn; + DROP INDEX IF EXISTS idx_uid; + DROP INDEX IF EXISTS idx_digest_clean; + DROP INDEX IF EXISTS idx_digest_all; + DROP INDEX IF EXISTS idx_clean; + DROP INDEX IF EXISTS idx_title; + DROP INDEX IF EXISTS idx_author; + DROP INDEX IF EXISTS src_filename_base; + DROP INDEX IF EXISTS idx_language_document_char; + DROP INDEX IF EXISTS idx_classify_topic_register; + DROP INDEX IF EXISTS idx_topic_list; + DROP TABLE IF EXISTS metadata_and_text; + DROP TABLE IF EXISTS topic_register; + DROP TABLE IF EXISTS doc_objects; + DROP TABLE IF EXISTS urls; + CREATE TABLE IF NOT EXISTS metadata_and_text ( + uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) + src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code + src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename + title VARCHAR(800) NOT NULL, + title_main VARCHAR(400) NOT NULL, + title_sub VARCHAR(400) NULL, + title_short VARCHAR(400) NULL, + title_edition VARCHAR(10) NULL, + title_language VARCHAR(100) NULL, + title_language_char VARCHAR(6) NULL, + creator_author VARCHAR(600) NOT NULL, + creator_author_last_first VARCHAR(600) NOT NULL, + creator_author_email VARCHAR(100) NULL, + creator_author_hon VARCHAR(100) NULL, + creator_author_nationality VARCHAR(100) NULL, + creator_editor VARCHAR(600) NULL, + creator_contributor VARCHAR(600) NULL, + creator_illustrator VARCHAR(600) NULL, + creator_photographer VARCHAR(600) NULL, + creator_translator VARCHAR(600) NULL, + creator_prepared_by VARCHAR(600) NULL, + creator_digitized_by VARCHAR(600) NULL, + creator_audio VARCHAR(600) NULL, + creator_video VARCHAR(600) NULL, + language_document VARCHAR(100) NULL, + language_document_char VARCHAR(6) NOT NULL, + language_original VARCHAR(100) NULL, + language_original_char VARCHAR(6) NULL, + date_added_to_site VARCHAR(10) NULL, + date_available VARCHAR(10) NULL, + date_created VARCHAR(10) NULL, + date_issued VARCHAR(10) NULL, + date_modified VARCHAR(10) NULL, + date_published VARCHAR(10) NULL, + date_valid VARCHAR(10) NULL, + date_translated VARCHAR(10) NULL, + date_original_publication VARCHAR(10) NULL, + date_generated VARCHAR(10) NULL, + original_title VARCHAR(800) NULL, + original_publisher VARCHAR(600) NULL, + original_language VARCHAR(100) NULL, + original_language_char VARCHAR(6) NULL, + original_source VARCHAR(600) NULL, + original_institution VARCHAR(600) NULL, + original_nationality VARCHAR(100) NULL, + rights_copyright VARCHAR(2500) NULL, + rights_copyright_audio VARCHAR(2500) NULL, + rights_copyright_cover VARCHAR(2500) NULL, + rights_copyright_illustrations VARCHAR(2500) NULL, + rights_copyright_photographs VARCHAR(2500) NULL, + rights_copyright_text VARCHAR(2500) NULL, + rights_copyright_translation VARCHAR(2500) NULL, + rights_copyright_video VARCHAR(2500) NULL, + rights_license VARCHAR(2500) NULL, + identifier_oclc VARCHAR(30) NULL, + identifier_isbn VARCHAR(16) NULL, + classify_topic_register VARCHAR(2500) NULL, + classify_subject VARCHAR(600) NULL, + classify_loc VARCHAR(30) NULL, + classify_dewey VARCHAR(30) NULL, + classify_keywords VARCHAR(600) NULL, + notes_abstract TEXT NULL, + notes_description TEXT NULL, + notes_comment TEXT NULL, + notes_coverage VARCHAR(200) NULL, + notes_relation VARCHAR(200) NULL, + notes_history VARCHAR(600) NULL, + notes_type VARCHAR(600) NULL, + notes_format VARCHAR(600) NULL, + notes_prefix TEXT NULL, + notes_prefix_a TEXT NULL, + notes_prefix_b TEXT NULL, + notes_suffix TEXT NULL, + publisher VARCHAR(600) NULL, + src_filename_base VARCHAR(256) NOT NULL, + src_filename_suffix VARCHAR(6) NOT NULL, + src_fingerprint VARCHAR(256) NULL, + src_filesize VARCHAR(10) NULL, + src_wordcount VARCHAR(10) NULL, + pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL, + site_url_epub VARCHAR(256) NULL, + links TEXT NULL + ); + CREATE TABLE IF NOT EXISTS topic_register ( + -- tid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code + -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename + topic_register VARCHAR(250) NOT NULL, + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL + ); + CREATE TABLE IF NOT EXISTS doc_objects ( + lid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + ocn SMALLINT, + obj_id VARCHAR(6) NULL, + clean TEXT NULL, + body TEXT NULL, + seg VARCHAR(256) NULL, + lev_an VARCHAR(1), + is_of_type VARCHAR(16), + is_a VARCHAR(16), + lev SMALLINT NULL, + node VARCHAR(16) NULL, -- not written to + parent VARCHAR(16) NULL, + last_descendant VARCHAR(16) NULL, -- headings only + digest_clean CHAR(256), + digest_all CHAR(256), + seg_name CHAR(256), + types CHAR(1) NULL + ); + CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn); + CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean); + CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all); + CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean); + CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title); + CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first); + CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid); + CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base); + CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char); + CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register); + CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register); + PRAGMA user_version = %s; + ┃", + sqlite_db_schema_version, + ); + return _sql_instruct; + } +} +template SQLiteDeleteDocument() { + string SQLiteDeleteDocument(M)(M doc_matters) { + string _uid = doc_matters.src.doc_uid; + string _delete_uid = format(q"┃ + DELETE FROM metadata_and_text + WHERE uid = '%s'; + DELETE FROM doc_objects + WHERE uid_metadata_and_text = '%s'; + ┃", + _uid, + _uid, + ); + return _delete_uid; + } +} +template SQLiteInsertMetadata() { + string SQLiteInsertMetadata(M)(M doc_matters) { + string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); + string _insert_metadata = format(q"┃ + INSERT INTO metadata_and_text ( + uid, + src_filename_base, + src_filename_suffix, + src_composite_id_per_txt, + src_composite_id_per_pod, + title, + title_main, + title_sub, + title_short, + title_edition, + title_language, + creator_author, + creator_author_last_first, + creator_author_email, + creator_illustrator, + creator_translator, + language_document, + language_document_char, + date_added_to_site, + date_available, + date_created, + date_issued, + date_modified, + date_published, + date_valid, + rights_copyright, + rights_copyright_audio, + rights_copyright_cover, + rights_copyright_illustrations, + rights_copyright_photographs, + rights_copyright_text, + rights_copyright_translation, + rights_copyright_video, + rights_license, + identifier_oclc, + identifier_isbn, + classify_dewey, + classify_keywords, + classify_loc, + classify_subject, + classify_topic_register, + original_title, + original_publisher, + original_language, + original_language_char, + original_source, + notes_abstract, + notes_description, + publisher, + site_url_doc_root + ) + VALUES ( + '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' + ); + ┃", + _uid, + SQLinsertDelimiter!()(doc_matters.src.filename_base), + SQLinsertDelimiter!()(doc_matters.src.filename_extension), + SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), + SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_surname_fn), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.conf.w_srv_data_root_url_html) + ); + return _insert_metadata; + } +} +template SQLiteInsertMetadataTopics() { + string SQLiteInsertMetadataTopics(M)(M doc_matters) { + string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); + string[] _insert_topics; + foreach (topic_line; doc_matters.conf_make_meta.meta.classify_topic_register_expanded_arr) { + // writeln(topic_line); + _insert_topics ~= format(q"┃ + INSERT INTO topic_register ( + uid_metadata_and_text, + topic_register + ) + VALUES ( + '%s', + '%s' + ); + ┃", + _uid, + SQLinsertDelimiter!()(topic_line) + ); + } + return _insert_topics.join.to!(char[]).toUTF8; + } +} +template SQLiteInsertDocObjectsLoop() { + string SQLiteInsertDocObjectsLoop(D)(D doc) { + string _uid = SQLinsertDelimiter!()(doc.matters.src.doc_uid); + auto url_html = spineUrlsHTML!()(doc.matters.conf_make_meta.conf.w_srv_data_root_url_html, doc.matters.src.language); + string insertDocObjectsRow(O)(O obj) { + string _insert_doc_objects_row = format(q"┃ + INSERT INTO doc_objects ( + uid_metadata_and_text, + ocn, + obj_id, + clean, + body, + lev, + is_of_type, + is_a, + seg_name + ) + VALUES ( + '%s', %s, '%s', '%s', '%s', %s, '%s', '%s', '%s' + ); + ┃", + _uid, + obj.metainfo.ocn, + obj.metainfo.identifier, + SQLinsertDelimiter!()(obj_txt["text"]), + SQLinsertDelimiter!()(obj_txt["html"]), + obj.metainfo.heading_lev_markup, + obj.metainfo.is_of_type, + obj.metainfo.is_a, + obj.tags.html_segment_anchor_tag_is + ); + return _insert_doc_objects_row; + } + auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc.matters); + string[string] obj_txt; + string doc_text; + string[] _insert_doc_objects; + foreach (part; doc.matters.has.keys_seq.sql) { + foreach (obj; doc.abstraction[part]) { + switch (obj.metainfo.is_of_part) { + case "frontmatter": assert(part == "head", part); + switch (obj.metainfo.is_of_type) { + case "para": + switch (obj.metainfo.is_a) { + case "heading": + obj_txt = format_and_sqlite_load.heading(doc.matters, obj); + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); + } + } + break; + } + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); + } + } + break; + } + break; + case "body": // assert(part == "body", part); + switch (obj.metainfo.is_of_type) { + case "para": + switch (obj.metainfo.is_a) { + case "heading": + debug (asserts) { + if (part != "body") { + writeln(__LINE__, ": ", obj.text); + } + } + obj_txt = format_and_sqlite_load.heading(doc.matters, obj); + break; + case "para": + obj_txt = format_and_sqlite_load.para(doc.matters, obj); + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); + } + } + break; + } + break; + case "block": + switch (obj.metainfo.is_a) { + case "quote": + obj_txt = format_and_sqlite_load.quote(doc.matters, obj); + break; + case "group": + obj_txt = format_and_sqlite_load.group(doc.matters, obj); + break; + case "block": + obj_txt = format_and_sqlite_load.block(doc.matters, obj); + break; + case "poem": // double check on keeping both poem & verse + break; + case "verse": + obj_txt = format_and_sqlite_load.verse(doc.matters, obj); + break; + case "code": + obj_txt = format_and_sqlite_load.code(doc.matters, obj); + break; + case "table": + obj_txt = format_and_sqlite_load.table(doc.matters, obj); + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); + } + } + break; + } + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); + } + } + break; + } + break; + case "backmatter": + assert(part == "glossary" || "bibliography" || "bookindex" || "blurb" || "tail", part); + switch (obj.metainfo.is_of_type) { + case "para": + switch (obj.metainfo.is_a) { + case "heading": + obj_txt = format_and_sqlite_load.heading(doc.matters, obj); + break; + case "glossary": assert(part == "glossary", part); + obj_txt = format_and_sqlite_load.para(doc.matters, obj); + break; + case "bibliography": assert(part == "bibliography", part); + obj_txt = format_and_sqlite_load.para(doc.matters, obj); + break; + case "bookindex": assert(part == "bookindex", part); + obj_txt = format_and_sqlite_load.para(doc.matters, obj); + break; + case "blurb": assert(part == "blurb", part); + obj_txt = format_and_sqlite_load.para(doc.matters, obj); + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); + } + } + break; + } + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); + } + } + break; + } + break; + case "comment": + break; + default: + { /+ debug +/ + if (doc.matters.opt.action.debug_do_sqlite) { + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_part); // check where empty value could come from + writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); + writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from + } + } + break; + } + if (obj.metainfo.is_a == "heading") { + if (doc.matters.opt.action.show_sqlite) { + if (obj.metainfo.heading_lev_markup == 0) { + writeln(doc.matters.src.filename); + } + writeln( + "markup: ", obj.metainfo.heading_lev_markup, + "> ", obj.metainfo.dom_structure_markedup_tags_status, + "; collapsed: ", obj.metainfo.heading_lev_collapsed, + "> ", obj.metainfo.dom_structure_collapsed_tags_status, + "; ocn: ", obj.metainfo.ocn, + "; parent: ", obj.metainfo.parent_lev_markup, + "; ocn: ", obj.metainfo.parent_ocn, + "; ", + ); + } + } + if (!(obj.metainfo.is_a == "comment")) { + _insert_doc_objects ~= insertDocObjectsRow(obj); + } + } // loop closes + } + return _insert_doc_objects.join.to!(char[]).toUTF8; + } +} +template SQLiteTablesCreate() { + void SQLiteTablesCreate(E,O,C)(E env, O opt_action, C config) { + import d2sqlite3; + template SQLiteTablesReCreate() { + string SQLiteTablesReCreate()() { + string _sql_instruct; + _sql_instruct = format(q"┃ + DROP INDEX IF EXISTS idx_ocn; + DROP INDEX IF EXISTS idx_uid; + DROP INDEX IF EXISTS idx_digest_clean; + DROP INDEX IF EXISTS idx_digest_all; + DROP INDEX IF EXISTS idx_clean; + DROP INDEX IF EXISTS idx_title; + DROP INDEX IF EXISTS idx_author; + DROP INDEX IF EXISTS src_filename_base; + DROP INDEX IF EXISTS idx_language_document_char; + DROP INDEX IF EXISTS idx_classify_topic_register; + DROP INDEX IF EXISTS idx_topic_list; + DROP TABLE IF EXISTS metadata_and_text; + DROP TABLE IF EXISTS topic_register; + DROP TABLE IF EXISTS doc_objects; + DROP TABLE IF EXISTS urls; + CREATE TABLE IF NOT EXISTS metadata_and_text ( + uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) + src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code + src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename + title VARCHAR(800) NOT NULL, + title_main VARCHAR(400) NOT NULL, + title_sub VARCHAR(400) NULL, + title_short VARCHAR(400) NULL, + title_edition VARCHAR(10) NULL, + title_language VARCHAR(100) NULL, + title_language_char VARCHAR(6) NULL, + creator_author VARCHAR(600) NOT NULL, + creator_author_last_first VARCHAR(600) NOT NULL, + creator_author_email VARCHAR(100) NULL, + creator_author_hon VARCHAR(100) NULL, + creator_author_nationality VARCHAR(100) NULL, + creator_editor VARCHAR(600) NULL, + creator_contributor VARCHAR(600) NULL, + creator_illustrator VARCHAR(600) NULL, + creator_photographer VARCHAR(600) NULL, + creator_translator VARCHAR(600) NULL, + creator_prepared_by VARCHAR(600) NULL, + creator_digitized_by VARCHAR(600) NULL, + creator_audio VARCHAR(600) NULL, + creator_video VARCHAR(600) NULL, + language_document VARCHAR(100) NULL, + language_document_char VARCHAR(6) NOT NULL, + language_original VARCHAR(100) NULL, + language_original_char VARCHAR(6) NULL, + date_added_to_site VARCHAR(10) NULL, + date_available VARCHAR(10) NULL, + date_created VARCHAR(10) NULL, + date_issued VARCHAR(10) NULL, + date_modified VARCHAR(10) NULL, + date_published VARCHAR(10) NULL, + date_valid VARCHAR(10) NULL, + date_translated VARCHAR(10) NULL, + date_original_publication VARCHAR(10) NULL, + date_generated VARCHAR(10) NULL, + original_title VARCHAR(800) NULL, + original_publisher VARCHAR(600) NULL, + original_language VARCHAR(100) NULL, + original_language_char VARCHAR(6) NULL, + original_source VARCHAR(600) NULL, + original_institution VARCHAR(600) NULL, + original_nationality VARCHAR(100) NULL, + rights_copyright VARCHAR(2500) NULL, + rights_copyright_audio VARCHAR(2500) NULL, + rights_copyright_cover VARCHAR(2500) NULL, + rights_copyright_illustrations VARCHAR(2500) NULL, + rights_copyright_photographs VARCHAR(2500) NULL, + rights_copyright_text VARCHAR(2500) NULL, + rights_copyright_translation VARCHAR(2500) NULL, + rights_copyright_video VARCHAR(2500) NULL, + rights_license VARCHAR(2500) NULL, + identifier_oclc VARCHAR(30) NULL, + identifier_isbn VARCHAR(16) NULL, + classify_topic_register VARCHAR(2500) NULL, + classify_subject VARCHAR(600) NULL, + classify_loc VARCHAR(30) NULL, + classify_dewey VARCHAR(30) NULL, + classify_keywords VARCHAR(600) NULL, + notes_abstract TEXT NULL, + notes_description TEXT NULL, + notes_comment TEXT NULL, + notes_coverage VARCHAR(200) NULL, + notes_relation VARCHAR(200) NULL, + notes_history VARCHAR(600) NULL, + notes_type VARCHAR(600) NULL, + notes_format VARCHAR(600) NULL, + notes_prefix TEXT NULL, + notes_prefix_a TEXT NULL, + notes_prefix_b TEXT NULL, + notes_suffix TEXT NULL, + publisher VARCHAR(600) NULL, + src_filename_base VARCHAR(256) NOT NULL, + src_filename_suffix VARCHAR(6) NOT NULL, + src_fingerprint VARCHAR(256) NULL, + src_filesize VARCHAR(10) NULL, + src_wordcount VARCHAR(10) NULL, + pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL, + site_url_epub VARCHAR(256) NULL, + links TEXT NULL + ); + CREATE TABLE IF NOT EXISTS topic_register ( + -- tid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code + -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename + topic_register VARCHAR(250) NOT NULL, + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL + ); + CREATE TABLE IF NOT EXISTS doc_objects ( + lid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + ocn SMALLINT, + obj_id VARCHAR(6) NULL, + clean TEXT NULL, + body TEXT NULL, + seg VARCHAR(256) NULL, + lev_an VARCHAR(1), + is_of_type VARCHAR(16), + is_a VARCHAR(16), + lev SMALLINT NULL, + node VARCHAR(16) NULL, -- not written to + parent VARCHAR(16) NULL, + last_descendant VARCHAR(16) NULL, -- headings only + digest_clean CHAR(256), + digest_all CHAR(256), + seg_name CHAR(256), + types CHAR(1) NULL + ); + CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn); + CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean); + CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all); + CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean); + CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title); + CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first); + CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid); + CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base); + CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char); + CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register); + CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register); + PRAGMA user_version = %s; + ┃", + sqlite_db_schema_version, + ); + return _sql_instruct; + } + } + try { + if (opt_action.sqlite_db_create) { + string _db_statement; + string db_filename = (opt_action.sqliteDB_filename.length > 0) + ? opt_action.sqliteDB_filename + : (config.conf.w_srv_db_sqlite_filename.length > 0) + ? config.conf.w_srv_db_sqlite_filename + : ""; + string db_path = (opt_action.sqliteDB_path.length > 0) + ? opt_action.sqliteDB_path + : (config.conf.w_srv_db_sqlite_path.length > 0) + ? config.conf.w_srv_db_sqlite_path + : ""; + if (db_filename.length > 0 && db_path.length > 0) { + if (opt_action.vox_gt_3) { + writeln("db name: ", db_filename); + writeln("db path: ", db_path); + writeln("db name & path: ", db_path, "/", db_filename); + } + if (opt_action.vox_gt_2) { + writeln("attempting to create db: ", db_path, "/", db_filename); + } + auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path); + if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { + } else { + try { + pth_sqlite.base.mkdirRecurse; + } catch (FileException ex) { + stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); + stderr.writeln(" (", ex.msg, ")"); + import core.runtime; + core.runtime.Runtime.terminate(); + } + } + auto db = Database(pth_sqlite.sqlite_file); + { + _db_statement ~= SQLiteTablesReCreate!()(); + } + SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE"); + } else { + writeln("must provide db name & output root path either on the command line or in configuration file"); + writeln("db name: ", db_filename); + writeln("db path: ", db_path); + } + } + } + catch (FileException e) { + writeln("Failed (FileException): ", e.msg); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (ErrnoException e) { + writeln("Failed (ErrnoException): ", e.msg); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (Exception e) { + writeln("Failed (Exception): ", e.msg); + writeln(e.file, " line: ", e.line); + import core.runtime; + core.runtime.Runtime.terminate(); + } + catch (Throwable) { + import core.runtime; + core.runtime.Runtime.terminate(); + } + } +} +template SQLiteDbDrop() { + void SQLiteDbDrop(O,C)(O opt_action, C config) { + writeln("db drop"); + if ((opt_action.sqlite_db_drop)) { + string db_filename = (opt_action.sqliteDB_filename.length > 0) + ? opt_action.sqliteDB_filename + : (config.conf.w_srv_db_sqlite_filename.length > 0) + ? config.conf.w_srv_db_sqlite_filename + : ""; + string db_path = (opt_action.sqliteDB_path.length > 0) // + ? opt_action.sqliteDB_path + : (config.conf.w_srv_db_sqlite_path.length > 0) + ? config.conf.w_srv_db_sqlite_path + : ""; + if (db_filename.length > 0 && db_path.length > 0) { + auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path); + writeln("remove(", pth_sqlite.sqlite_file, ")"); + try { + remove(pth_sqlite.sqlite_file); + } catch (FileException ex) { + // handle error + } + } else { + writeln("must provide db name & output root path either on the command line or in configuration file"); + writeln("db name: ", db_filename); + writeln("db path: ", db_path); + } + } + } +} diff --git a/src/sisudoc/outputs/io_out/sqlite_ocda_db.d b/src/sisudoc/outputs/io_out/sqlite_ocda_db.d new file mode 100644 index 0000000..79be05e --- /dev/null +++ b/src/sisudoc/outputs/io_out/sqlite_ocda_db.d @@ -0,0 +1,656 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ +module sisudoc.outputs.io_out.sqlite_ocda_db; + +/+ ↓ write document abstraction as per-document sqlite3 database +/ +template spineAbstractionDb() { + import std.conv : to; + import std.digest : toHexString; + import std.file; + import std.path; + import std.stdio; + import std.string; + import std.array; + import d2sqlite3; + import sisudoc.outputs.io_out.paths_output; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.abstraction.ssp : ssp_format_version; + /+ ↓ the document is passed in rather than taken from doc, because the + caller hands over one that has been through the .ssp: written as text + and read back. so the database is a function of the .ssp and cannot + carry anything the .ssp does not, header blocks included: the metadata + table below is filled from what came back, not from a second list read + off doc_matters, which is how the two drifted apart before + +/ + void spineAbstractionDb(D,S)(D doc, S ssp_doc) { + auto doc_matters = doc.matters; + auto doc_abstraction = ssp_doc.abstraction; + /+ ↓ output path: beside the pod's zip and digests, not under + /abstraction/. the database carries its own images, so it is a + distributable artefact in its own right and belongs with the others: + pod// the source tree + pod/.zip the same, zipped + pod/.digests.txt what is in them + pod/.ocda.db the abstraction, one per language + +/ + auto pths_pod = spinePathsPods!()(doc_matters); + string base_pth = pths_pod.pod_dir_(); + try { + if (!exists(base_pth)) { + base_pth.mkdirRecurse; + } + } catch (Exception ex) { + } + string db_file = ((base_pth.chainPath( + doc_matters.src.doc_uid_out ~ ".ocda.db")).asNormalizedPath).array; + + /+ ↓ remove existing file to start fresh +/ + try { + if (exists(db_file)) { + remove(db_file); + } + } catch (Exception ex) { + } + + if (doc_matters.opt.action.vox_gt_1) { + writeln(" ", db_file); + } + + /+ ↓ open database and create schema +/ + auto db = Database(db_file); + /+ ↓ write once, read many: no WAL. + a WAL stamped file cannot be opened by a reader that cannot write + beside it (SQLITE_READONLY on a read-only directory, and inside a + zip or on read-only media), and this database has one writer and no + concurrency, so WAL buys nothing here. MEMORY journal keeps the bulk + load fast without leaving that mark on the file. + +/ + db.run("PRAGMA journal_mode=MEMORY"); + db.run("PRAGMA synchronous=OFF"); + + db.run(" + CREATE TABLE metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + -- objects: one row per abstraction object, in document order. + -- the fixed width arrays (ancestors, dom status, children, heading + -- ancestors text, table widths and aligns) are JSON arrays so that + -- json_extract, json_each and json_array_length can reach inside them; + -- the open ended lists (images, links, anchor tags, subtoc) are rows + -- in their own tables, not columns + CREATE TABLE objects ( + id INTEGER PRIMARY KEY, + section TEXT NOT NULL, + seq INTEGER NOT NULL, + ocn INTEGER DEFAULT 0, + is_a TEXT NOT NULL, + is_of_part TEXT, + is_of_section TEXT, + is_of_type TEXT, + heading_level INTEGER, + heading_lev_collapsed INTEGER, + identifier TEXT, + parent_ocn INTEGER DEFAULT 0, + parent_lev INTEGER DEFAULT 0, + last_descendant_ocn INTEGER DEFAULT 0, + children TEXT + CHECK(children IS NULL OR json_valid(children)), + ancestors TEXT + CHECK(ancestors IS NULL OR json_valid(ancestors)), + ancestors_collapsed TEXT + CHECK(ancestors_collapsed IS NULL OR json_valid(ancestors_collapsed)), + dom_status TEXT + CHECK(dom_status IS NULL OR json_valid(dom_status)), + dom_status_collapsed TEXT + CHECK(dom_status_collapsed IS NULL OR json_valid(dom_status_collapsed)), + dummy_heading INTEGER DEFAULT 0, + object_number_off INTEGER DEFAULT 0, + attrib TEXT, + meta_lang TEXT, + meta_syntax TEXT, + sha256 TEXT, + indent_base INTEGER DEFAULT 0, + indent_hang INTEGER DEFAULT 0, + bullet INTEGER DEFAULT 0, + lang TEXT, + has_links INTEGER DEFAULT 0, + has_notes_reg INTEGER DEFAULT 0, + has_notes_star INTEGER DEFAULT 0, + has_images INTEGER DEFAULT 0, + has_images_no_dim INTEGER DEFAULT 0, + segment TEXT, + segment_prev TEXT, + segment_next TEXT, + segment_epub TEXT, + segment_html_is TEXT, + segment_epub_is TEXT, + segment_lv4_is TEXT, + anchor TEXT, + heading_lev_anchor TEXT, + heading_ancestors_text TEXT + CHECK(heading_ancestors_text IS NULL OR json_valid(heading_ancestors_text)), + table_cols INTEGER, + table_widths TEXT + CHECK(table_widths IS NULL OR json_valid(table_widths)), + table_aligns TEXT + CHECK(table_aligns IS NULL OR json_valid(table_aligns)), + table_header INTEGER, + code_linenumbers INTEGER DEFAULT 0, + text TEXT, + UNIQUE(section, seq) + ); + + CREATE INDEX idx_objects_section ON objects(section); + CREATE INDEX idx_objects_ocn ON objects(ocn); + CREATE INDEX idx_objects_parent ON objects(parent_ocn); + CREATE INDEX idx_objects_is_a ON objects(is_a); + CREATE INDEX idx_objects_heading ON objects(heading_level) + WHERE heading_level IS NOT NULL; + + -- files: what this database carries with it, so that a reader needs + -- nothing beside it. role says what a file is; images are the ones the + -- output writers must have in order to produce a document. bytes are + -- stored exactly as read, so sha256 matches the digest the abstraction + -- was built with + CREATE TABLE files ( + id INTEGER PRIMARY KEY, + role TEXT NOT NULL, + name TEXT NOT NULL, + bytes INTEGER NOT NULL, + sha256 TEXT, + width INTEGER, + height INTEGER, + data BLOB, + UNIQUE(role, name) + ); + + -- the open ended per object lists, one row each, in order + CREATE TABLE object_images ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + name TEXT NOT NULL, + bytes INTEGER, + sha256 TEXT, + width INTEGER, + height INTEGER, + missing INTEGER DEFAULT 0, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_links ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + url TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_anchors ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + anchor TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_subtoc ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + entry TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + + CREATE INDEX idx_files_role ON files(role); + CREATE INDEX idx_object_images_name ON object_images(name); + + -- full text over the object text, external content so the text is not + -- stored twice. per document: this is for exploring one abstraction, + -- the collection wide search database is a different thing + CREATE VIRTUAL TABLE objects_fts USING fts5( + text, content='objects', content_rowid='id' + ); + + -- the shapes worth asking for, named + CREATE VIEW outline AS + SELECT section, seq, ocn, heading_level, heading_lev_collapsed, + last_descendant_ocn, identifier, text + FROM objects WHERE is_a = 'heading' ORDER BY id; + CREATE VIEW citable AS + SELECT * FROM objects WHERE ocn > 0 ORDER BY ocn; + CREATE VIEW document_files AS + SELECT role, name, bytes, sha256, width, height FROM files ORDER BY role, name; + "); + + /+ ↓ populate metadata, from the header blocks the .ssp gave back. + one list, in ssp.d, not two: a property added to the .ssp header + arrives here without anything being changed, and one that is not in + the .ssp cannot be here at all. the keys are the .ssp's own + (title.main, make.doc_type, doc_has.images), prefixed by their block + where the block name is not already part of the key. + +/ + db.run("BEGIN TRANSACTION"); + + auto meta_stmt = db.prepare( + "INSERT INTO metadata (key, value) VALUES (:key, :value)" + ); + + void insertMeta(string key, string value) { + if (value.length > 0) { + meta_stmt.bind(":key", key); + meta_stmt.bind(":value", value); + meta_stmt.execute(); + meta_stmt.reset(); + } + } + void insertBlock(string prefix, string[] keys_in_order, string[string] kv) { + foreach (k; keys_in_order) { + if (k in kv) { insertMeta(prefix ~ k, kv[k]); } + } + } + + /+ ↓ what this file is, so a reader can tell before it starts. the + version is the .ssp's own: the two serialisations are one format and + move together + +/ + insertMeta("schema.name", "sisu-abstraction-db"); + insertMeta("schema.version", ssp_format_version); + insertMeta("source.filename", ssp_doc.source); + insertBlock("source.", ssp_doc.source_info_order, ssp_doc.source_info); + /+ ↓ the digest of the .ssp this database was built from. a file cannot + hold its own hash, so this one is not in the .ssp and is taken here: + source.digest says which markup the abstraction came from, + source.ssp_digest says which abstraction this database came from, + and the chain .sst -> .ssp -> .ocda.db is checkable end to end + +/ + insertMeta("source.ssp_digest", ssp_doc.ssp_digest); + + insertBlock("", ssp_doc.meta_order, ssp_doc.meta); + insertBlock("make.", ssp_doc.make_order, ssp_doc.make); + insertBlock("doc_has.", ssp_doc.doc_has_order, ssp_doc.doc_has); + meta_stmt.finalize(); + + /+ ↓ populate objects +/ + auto obj_stmt = db.prepare( + "INSERT INTO objects (" + ~ "section, seq, ocn, is_a, is_of_part, is_of_section, is_of_type," + ~ "heading_level, heading_lev_collapsed, identifier," + ~ "parent_ocn, parent_lev, last_descendant_ocn," + ~ "children, ancestors, ancestors_collapsed," + ~ "dom_status, dom_status_collapsed," + ~ "dummy_heading, object_number_off," + ~ "attrib, meta_lang, meta_syntax, sha256," + ~ "indent_base, indent_hang, bullet, lang," + ~ "has_links, has_notes_reg, has_notes_star, has_images, has_images_no_dim," + ~ "segment, segment_prev, segment_next, segment_epub," + ~ "segment_html_is, segment_epub_is, segment_lv4_is," + ~ "anchor, heading_lev_anchor, heading_ancestors_text," + ~ "table_cols, table_widths, table_aligns, table_header," + ~ "code_linenumbers, text" + ~ ") VALUES (" + ~ ":section, :seq, :ocn, :is_a, :is_of_part, :is_of_section, :is_of_type," + ~ ":heading_level, :heading_lev_collapsed, :identifier," + ~ ":parent_ocn, :parent_lev, :last_descendant_ocn," + ~ ":children, :ancestors, :ancestors_collapsed," + ~ ":dom_status, :dom_status_collapsed," + ~ ":dummy_heading, :object_number_off," + ~ ":attrib, :meta_lang, :meta_syntax, :sha256," + ~ ":indent_base, :indent_hang, :bullet, :lang," + ~ ":has_links, :has_notes_reg, :has_notes_star, :has_images, :has_images_no_dim," + ~ ":segment, :segment_prev, :segment_next, :segment_epub," + ~ ":segment_html_is, :segment_epub_is, :segment_lv4_is," + ~ ":anchor, :heading_lev_anchor, :heading_ancestors_text," + ~ ":table_cols, :table_widths, :table_aligns, :table_header," + ~ ":code_linenumbers, :text" + ~ ")" + ); + + /+ ↓ the per object lists, one row each +/ + auto img_stmt = db.prepare( + "INSERT INTO object_images" + ~ " (object_id, seq, name, bytes, sha256, width, height, missing)" + ~ " VALUES (:object_id, :seq, :name, :bytes, :sha256, :width, :height, :missing)" + ); + auto lnk_stmt = db.prepare( + "INSERT INTO object_links (object_id, seq, url)" + ~ " VALUES (:object_id, :seq, :url)" + ); + auto anc_stmt = db.prepare( + "INSERT INTO object_anchors (object_id, seq, anchor)" + ~ " VALUES (:object_id, :seq, :anchor)" + ); + auto sub_stmt = db.prepare( + "INSERT INTO object_subtoc (object_id, seq, entry)" + ~ " VALUES (:object_id, :seq, :entry)" + ); + + string[] section_order = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + + foreach (section; section_order) { + if (section !in doc_abstraction) continue; + auto section_objs = doc_abstraction[section]; + if (section_objs.length == 0) continue; + + foreach (seq, obj; section_objs) { + obj_stmt.bind(":section", section); + obj_stmt.bind(":seq", cast(int) seq); + obj_stmt.bind(":ocn", obj.metainfo.ocn); + obj_stmt.bind(":is_a", obj.metainfo.is_a); + + /+ ↓ nullable string fields +/ + void bindStr(string param, string val) { + import std.typecons : Nullable; + if (val.length > 0) { + obj_stmt.bind(param, val); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + /+ ↓ nullable int fields +/ + void bindInt(string param, int val, bool is_set) { + import std.typecons : Nullable; + if (is_set) { + obj_stmt.bind(param, val); + } else { + obj_stmt.bind(param, Nullable!int()); + } + } + /+ ↓ fixed width int array as a JSON array, NULL when all zero +/ + void bindJsonInts(T)(string param, T arr) { + import std.typecons : Nullable; + bool any = false; + foreach (a; arr) { if (a != 0) { any = true; break; } } + if (any) { + string[] _j; + foreach (a; arr) { _j ~= a.to!string; } + obj_stmt.bind(param, "[" ~ _j.join(",") ~ "]"); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + /+ ↓ string array as a JSON array, NULL when every slot is empty +/ + void bindJsonStrs(T)(string param, T arr) { + import std.json : JSONValue; + import std.typecons : Nullable; + bool any = false; + foreach (s; arr) { if (s.length > 0) { any = true; break; } } + if (any) { + string[] _s; + foreach (s; arr) { _s ~= s.to!string; } + obj_stmt.bind(param, JSONValue(_s).toString); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + + bindStr(":is_of_part", obj.metainfo.is_of_part); + bindStr(":is_of_section", obj.metainfo.is_of_section); + bindStr(":is_of_type", obj.metainfo.is_of_type); + + /+ ↓ heading levels, marked up and collapsed, 9 meaning unset +/ + bindInt(":heading_level", obj.metainfo.heading_lev_markup, + (obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup < 9)); + bindInt(":heading_lev_collapsed", obj.metainfo.heading_lev_collapsed, + (obj.metainfo.heading_lev_collapsed < 9)); + + bindStr(":identifier", obj.metainfo.identifier); + obj_stmt.bind(":parent_ocn", obj.metainfo.parent_ocn); + obj_stmt.bind(":parent_lev", obj.metainfo.parent_lev_markup); + obj_stmt.bind(":last_descendant_ocn", obj.metainfo.last_descendant_ocn); + + bindJsonInts(":children", obj.metainfo.children_headings); + bindJsonInts(":ancestors", obj.metainfo.markedup_ancestors); + bindJsonInts(":ancestors_collapsed", obj.metainfo.collapsed_ancestors); + bindJsonInts(":dom_status", obj.metainfo.dom_structure_markedup_tags_status); + bindJsonInts(":dom_status_collapsed", obj.metainfo.dom_structure_collapsed_tags_status); + + obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); + obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); + bindStr(":attrib", obj.metainfo.attrib); + bindStr(":meta_lang", obj.metainfo.lang); + bindStr(":meta_syntax", obj.metainfo.syntax); + + /+ ↓ object digest, hex, absent when never taken +/ + { + import std.typecons : Nullable; + bool has_sha = false; + foreach (b; obj.metainfo.sha256.text) { + if (b != 0) { has_sha = true; break; } + } + if (has_sha) { + obj_stmt.bind(":sha256", obj.metainfo.sha256.text.toHexString.to!string); + } else { + obj_stmt.bind(":sha256", Nullable!string()); + } + } + + // obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); + // obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); + obj_stmt.bind(":indent_base", obj.attrib.indent_base); + obj_stmt.bind(":indent_hang", obj.attrib.indent_hang); + obj_stmt.bind(":bullet", obj.attrib.bullet ? 1 : 0); + bindStr(":lang", obj.attrib.language); + obj_stmt.bind(":has_links", obj.has.inline_links ? 1 : 0); + obj_stmt.bind(":has_notes_reg", obj.has.inline_notes_reg ? 1 : 0); + obj_stmt.bind(":has_notes_star", obj.has.inline_notes_star ? 1 : 0); + obj_stmt.bind(":has_images", obj.has.images ? 1 : 0); + obj_stmt.bind(":has_images_no_dim", obj.has.image_without_dimensions ? 1 : 0); + bindStr(":segment", obj.tags.in_segment_html); + bindStr(":segment_prev", obj.tags.segname_prev); + bindStr(":segment_next", obj.tags.segname_next); + bindStr(":segment_epub", obj.tags.segment_anchor_tag_epub); + bindStr(":segment_html_is", obj.tags.html_segment_anchor_tag_is); + bindStr(":segment_epub_is", obj.tags.epub_segment_anchor_tag_is); + bindStr(":segment_lv4_is", obj.tags.segment_lv4_is); + bindStr(":anchor", obj.tags.anchor_tag_html); + bindStr(":heading_lev_anchor", obj.tags.heading_lev_anchor_tag); + bindJsonStrs(":heading_ancestors_text", obj.tags.heading_ancestors_text); + + /+ ↓ table properties +/ + { + import std.typecons : Nullable; + if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { + obj_stmt.bind(":table_cols", obj.table.number_of_columns); + if (obj.table.column_widths.length > 0) { + string[] ws; + foreach (w; obj.table.column_widths) ws ~= w.to!string; + obj_stmt.bind(":table_widths", "[" ~ ws.join(",") ~ "]"); + } else { + obj_stmt.bind(":table_widths", Nullable!string()); + } + if (obj.table.column_aligns.length > 0) { + import std.json : JSONValue; + string[] _al; + foreach (a; obj.table.column_aligns) { _al ~= a.to!string; } + obj_stmt.bind(":table_aligns", JSONValue(_al).toString); + } else { + obj_stmt.bind(":table_aligns", Nullable!string()); + } + obj_stmt.bind(":table_header", obj.table.heading ? 1 : 0); + } else { + obj_stmt.bind(":table_cols", Nullable!int()); + obj_stmt.bind(":table_widths", Nullable!string()); + obj_stmt.bind(":table_aligns", Nullable!string()); + obj_stmt.bind(":table_header", Nullable!int()); + } + } + + /+ ↓ code block properties +/ + obj_stmt.bind(":code_linenumbers", + ((obj.metainfo.is_a == "code") && obj.code_block.linenumbers) ? 1 : 0); + + /+ ↓ text content +/ + bindStr(":text", obj.text); + + obj_stmt.execute(); + obj_stmt.reset(); + + /+ ↓ the per object lists, keyed on the row just written +/ + long _obj_id = db.lastInsertRowid; + { + int _n = 0; + foreach (i; obj.metainfo.sha256.images) { + img_stmt.bind(":object_id", _obj_id); + img_stmt.bind(":seq", _n); + img_stmt.bind(":name", i.fileName); + img_stmt.bind(":bytes", cast(long) i.fileSize); + img_stmt.bind(":sha256", i.fileHash_sha256.toHexString.to!string); + img_stmt.bind(":width", i.imageWidth); + img_stmt.bind(":height", i.imageHeight); + img_stmt.bind(":missing", i.fileMissing ? 1 : 0); + img_stmt.execute(); + img_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (lnk; obj.stow.link) { + if (lnk.length == 0) { continue; } + lnk_stmt.bind(":object_id", _obj_id); + lnk_stmt.bind(":seq", _n); + lnk_stmt.bind(":url", lnk); + lnk_stmt.execute(); + lnk_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (at; obj.tags.anchor_tags) { + if (at.length == 0) { continue; } + anc_stmt.bind(":object_id", _obj_id); + anc_stmt.bind(":seq", _n); + anc_stmt.bind(":anchor", at); + anc_stmt.execute(); + anc_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (st; obj.tags.lev4_subtoc) { + if (st.length == 0) { continue; } + sub_stmt.bind(":object_id", _obj_id); + sub_stmt.bind(":seq", _n); + sub_stmt.bind(":entry", st); + sub_stmt.execute(); + sub_stmt.reset(); + ++_n; + } + } + } + } + + obj_stmt.finalize(); + img_stmt.finalize(); + lnk_stmt.finalize(); + anc_stmt.finalize(); + sub_stmt.finalize(); + + /+ ↓ the document's images, carried inside the file so that output can be + produced from this database alone. bytes exactly as read, and the + digest taken over them, so that it can be checked against the digest + the abstraction recorded +/ + { + import std.digest.sha : sha256Of; + auto file_stmt = db.prepare( + "INSERT OR IGNORE INTO files" + ~ " (role, name, bytes, sha256, width, height, data)" + ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" + ); + int[string] _w, _h; + foreach (section; section_order) { + if (section !in doc_abstraction) { continue; } + foreach (obj; doc_abstraction[section]) { + foreach (i; obj.metainfo.sha256.images) { + if (i.imageWidth > 0) { _w[i.fileName] = i.imageWidth; } + if (i.imageHeight > 0) { _h[i.fileName] = i.imageHeight; } + } + } + } + foreach (image; doc_matters.srcs.image_list) { + string _fn = doc_matters.src.image_dir_path ~ "/" ~ image; + ubyte[] _bytes; + try { + if (_fn.exists) { _bytes = cast(ubyte[]) _fn.read; } + } catch (Exception ex) { + _bytes = []; + } + if (_bytes.length == 0) { + if (doc_matters.opt.action.vox_gt_1) { + writeln(" image not carried into abstraction db: ", _fn); + } + continue; + } + file_stmt.bind(":role", "image"); + file_stmt.bind(":name", image); + file_stmt.bind(":bytes", cast(long) _bytes.length); + file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); + if (image in _w) { file_stmt.bind(":width", _w[image]); } + else { + import std.typecons : Nullable; + file_stmt.bind(":width", Nullable!int()); + } + if (image in _h) { file_stmt.bind(":height", _h[image]); } + else { + import std.typecons : Nullable; + file_stmt.bind(":height", Nullable!int()); + } + file_stmt.bind(":data", _bytes); + file_stmt.execute(); + file_stmt.reset(); + } + file_stmt.finalize(); + } + + /+ ↓ build the full text index from the rows just written +/ + db.run("INSERT INTO objects_fts(rowid, text)" + ~ " SELECT id, text FROM objects WHERE text IS NOT NULL"); + + db.run("COMMIT TRANSACTION"); + } +} diff --git a/src/sisudoc/outputs/io_out/xmls.d b/src/sisudoc/outputs/io_out/xmls.d index a672a40..907bf90 100644 --- a/src/sisudoc/outputs/io_out/xmls.d +++ b/src/sisudoc/outputs/io_out/xmls.d @@ -58,7 +58,7 @@ template outputXHTMLs() { import sisudoc.outputs.io_out.rgx; import sisudoc.ocda.meta.rgx_files; import sisudoc.outputs.io_out.rgx_xhtml; - import sisudoc.outputs.io_out.create_zip_file; + import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; import sisudoc.outputs.io_out.xmls_css; mixin spineRgxOut; diff --git a/src/sisudoc/outputs/io_out/zip_pod.d b/src/sisudoc/outputs/io_out/zip_pod.d new file mode 100644 index 0000000..80f0890 --- /dev/null +++ b/src/sisudoc/outputs/io_out/zip_pod.d @@ -0,0 +1,67 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ +module sisudoc.outputs.io_out.zip_pod; +@safe: +template createZipFile() { + import std.file; + import std.outbuffer; + import std.string; + import std.zip; + void createZipFile( + string zip_file_name, + void[] compressed_zip_data, + ) { + try { + write(zip_file_name, compressed_zip_data); + } catch (ZipException ex) { + // Handle Errors + } + } +} -- cgit v1.2.3