From 557810df54fb5411c362528b459648611e8267ba Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Tue, 8 Sep 2026 17:30:00 -0400 Subject: --ocda-db replaces --show-abstraction-db ocda (object centric document abstraction) flag --ocda-db (or --abstraction-db) replaces --show-abstraction-db rename results in consequently large diff --- org/in_source_files.org | 2 +- org/out_metadata.org | 2 +- org/out_src_abstraction_sqlite_db.org | 4 +- org/out_src_pod.org | 28 +- org/spine.org | 32 +- ...ests_for_document_abstraction_shell_scripts.org | 13 +- src/sisudoc/ocda/abstraction/db_in.d | 2 +- src/sisudoc/outputs/io_out/create_abstraction_db.d | 676 --------------------- src/sisudoc/outputs/io_out/create_ocda_db.d | 676 +++++++++++++++++++++ src/sisudoc/outputs/io_out/metadata.d | 2 +- src/sisudoc/outputs/io_out/source_pod.d | 28 +- src/sisudoc/spine.d | 32 +- test/run-tests.sh | 9 +- test/test-abstraction-db-roundtrip.sh | 2 +- test/test-abstraction-db.sh | 2 +- 15 files changed, 782 insertions(+), 728 deletions(-) delete mode 100644 src/sisudoc/outputs/io_out/create_abstraction_db.d create mode 100644 src/sisudoc/outputs/io_out/create_ocda_db.d mode change 100644 => 100755 test/test-abstraction-db-roundtrip.sh mode change 100644 => 100755 test/test-abstraction-db.sh diff --git a/org/in_source_files.org b/org/in_source_files.org index dc6571e..2f12196 100644 --- a/org/in_source_files.org +++ b/org/in_source_files.org @@ -1098,7 +1098,7 @@ module sisudoc.ocda.abstraction.db_in; @safe: /+ ↓ read a .ocda.db back into the document abstraction - the counterpart of sisudoc.outputs.io_out.create_abstraction_db, and the + the counterpart of sisudoc.outputs.io_out.create_ocda, and the sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, so a consumer does not have to know which of the two artefacts it was handed. diff --git a/org/out_metadata.org b/org/out_metadata.org index 71bfe9f..19bc02e 100644 --- a/org/out_metadata.org +++ b/org/out_metadata.org @@ -195,7 +195,7 @@ if (doc_matters.opt.action.html_link_markup_source) { writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); } } - if (doc_matters.opt.action.show_abstraction_db) { + if (doc_matters.opt.action.ocda_db) { try { // sha digest and size for the abstraction database, beside the pod string _db_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.doc_uid_out ~ ".ocda.db"; diff --git a/org/out_src_abstraction_sqlite_db.org b/org/out_src_abstraction_sqlite_db.org index 9144fdd..6befb4f 100644 --- a/org/out_src_abstraction_sqlite_db.org +++ b/org/out_src_abstraction_sqlite_db.org @@ -25,11 +25,11 @@ rename source_abstraction_peg_txt.d -#+HEADER: :tangle "../src/sisudoc/outputs/io_out/create_abstraction_db.d" +#+HEADER: :tangle "../src/sisudoc/outputs/io_out/create_ocda_db.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> -module sisudoc.outputs.io_out.create_abstraction_db; +module sisudoc.outputs.io_out.create_ocda_db; /+ ↓ write document abstraction as per-document sqlite3 database +/ template spineAbstractionDb() { diff --git a/org/out_src_pod.org b/org/out_src_pod.org index b987151..94094ae 100644 --- a/org/out_src_pod.org +++ b/org/out_src_pod.org @@ -37,11 +37,15 @@ template spinePod() { regeneration, but only on the first language of a multi-language document, so that subsequent languages' files are not wiped. . - media/abstraction/ is left alone: the .ssp files there are written - by the abstraction stage, which runs before this, and for a - multi-language document they accumulate one language at a time. - That directory clears itself, in the .ssp writer, on the first - language of a run. + media/abstraction/ is left standing: the .ssp files there are + written by the abstraction stage, which runs before this, and for + a multi-language document they accumulate one language at a time. + What is pruned from it instead is any .ssp that does not belong to + a language this document has, which would otherwise linger from an + earlier run and be bundled. That is done here rather than in the + writer because pod actions run serially and the manifest's list of + languages is to hand, where the writer may be one of several + languages running at once. +/ if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0]) { string doc_pod_dir = pths_pod.base_filesystem_(doc_matters.src.filename); @@ -63,6 +67,20 @@ template spinePod() { } } } + if (exists(keep_dir) && keep_dir.isDir) { + string _doc_uid_base + = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length]; + foreach (string _f; dirEntries(keep_dir, "*.ssp", SpanMode.shallow)) { + bool _wanted = false; + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + if (_f.baseName == (_doc_uid_base ~ _lang ~ ".ssp")) { + _wanted = true; + break; + } + } + if (!_wanted) { _f.remove; } + } + } } { podArchive_directory_tree(doc_matters, pths_pod); diff --git a/org/spine.org b/org/spine.org index 2d969e1..51926d5 100644 --- a/org/spine.org +++ b/org/spine.org @@ -289,6 +289,7 @@ scope(failure) { #+BEGIN_SRC d bool[string] opts = [ "abstraction" : false, + "abstraction-db" : false, "allow-downloads" : false, "assertions" : false, "concordance" : false, @@ -330,6 +331,7 @@ bool[string] opts = [ "manifest" : false, "hide-ocn" : false, "no-ocn" : false, + "ocda-db" : false, "ocn-off" : false, "odf" : false, "odt" : false, @@ -342,7 +344,6 @@ bool[string] opts = [ "pod2" : false, "serial" : false, "show-abstraction" : false, - "show-abstraction-db" : false, "show-config" : false, "show-curate" : false, "show-curate-authors" : false, @@ -426,6 +427,7 @@ string[string] settings = [ auto helpInfo = getopt(args, std.getopt.config.passThrough, "abstraction", "document abstraction", &opts["abstraction"], + "abstraction-db", "document abstraction (write ocda.db sqlite file)", &opts["abstraction-db"], "allow-downloads", "allow downloads (includes cgi.d from github)", &opts["allow-downloads"], "assert", "set optional assertions on", &opts["assertions"], "cgi-bin-root", "path to cgi-bin directory", &settings["cgi-bin-root"], @@ -460,6 +462,7 @@ auto helpInfo = getopt(args, "light", "default light theme", &opts["light"], "manifest", "process manifest output", &opts["manifest"], "no-ocn", "object cite numbers", &opts["no-ocn"], + "ocda-db", "document abstraction (write ocda.db sqlite file)", &opts["ocda-db"], "ocn-off", "object cite numbers", &opts["ocn-off"], "odf", "open document format text (--odt)", &opts["odf"], "odt", "open document format text", &opts["odt"], @@ -486,7 +489,6 @@ auto helpInfo = getopt(args, "serial", "serial processing", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-abstraction", "show document abstraction (write .ssp file)", &opts["show-abstraction"], - "show-abstraction-db", "show document abstraction (write .db sqlite file)", &opts["show-abstraction-db"], "show-config", "show config", &opts["show-config"], "show-curate", "show curate", &opts["show-curate"], "show-curate-authors", "show curate authors", &opts["show-curate-authors"], @@ -791,11 +793,14 @@ struct OptActions { @trusted bool latex_document_header_sty() { return (opts["latex-init"] || opts["latex-header-sty"] || opts["pdf-init"]) ? true : false; } + @trusted bool odt() { + return (opts["odf"] || opts["odt"]) ? true : false; + } @trusted bool manifest() { return opts["manifest"]; } - @trusted bool odt() { - return (opts["odf"] || opts["odt"]) ? true : false; + @trusted bool ocda_db() { + return (opts["abstraction-db"] || opts["ocda-db"]) ? true : false; } @trusted bool ocn_hidden() { return opts["hide-ocn"]; @@ -815,9 +820,6 @@ struct OptActions { @trusted bool show_abstraction() { return (opts["show-abstraction"] || pod2) ? true : false; } - @trusted bool show_abstraction_db() { - return opts["show-abstraction-db"]; - } @trusted bool show_curate() { return opts["show-curate"]; } @@ -1067,7 +1069,7 @@ struct OptActions { return ( opts["abstraction"] || show_abstraction - || show_abstraction_db + || ocda_db || concordance || source_or_pod || curate @@ -1095,7 +1097,7 @@ struct OptActions { || odt || manifest || show_abstraction - || show_abstraction_db + || ocda_db || show_make || show_metadata || show_summary @@ -1111,7 +1113,7 @@ struct OptActions { return ( opts["abstraction"] || show_abstraction - || show_abstraction_db + || ocda_db || curate || html || epub @@ -1127,7 +1129,7 @@ struct OptActions { return ( opts["abstraction"] || show_abstraction - || show_abstraction_db + || ocda_db || html || epub || odt @@ -1668,7 +1670,7 @@ if ((doc.matters.opt.action.debug_do) <> <> <> -<> +<> <> <> #+END_SRC @@ -1735,16 +1737,16 @@ if (doc.matters.opt.action.show_abstraction) { ***** show abstraction (sqlite db) --show-abstraction-db -#+NAME: spine_each_file_show_abstraction_db +#+NAME: spine_each_file_ocda_db #+BEGIN_SRC d /+ ↓ document abstraction sqlite database, built from the .ssp: the lines are emitted and read straight back, and the objects that come out are what is written, so the database cannot carry anything the .ssp does not +/ -if (doc.matters.opt.action.show_abstraction_db) { +if (doc.matters.opt.action.ocda_db) { import sisudoc.ocda.abstraction.ssp_in; - import sisudoc.outputs.io_out.create_abstraction_db; + import sisudoc.outputs.io_out.create_ocda_db; mixin spineAbstractionRead; spineAbstractionDb!()(doc, sspRoundTripAbstraction(doc)); } diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org index 4266e8c..1c93eed 100644 --- a/org/tests_for_document_abstraction_shell_scripts.org +++ b/org/tests_for_document_abstraction_shell_scripts.org @@ -87,19 +87,26 @@ test-abstraction-db-roundtrip.sh" PASSED=0 FAILED=0 RESULTS="" +LOG="${TMPDIR:-/tmp}/spine-run-tests.$$" for t in $TESTS; do printf '%s\n' "── $t" - if sh "$SCRIPT_DIR/$t" $SPINE_BIN 2>&1 | sed 's/^/ /'; then + # the status has to be the test's, not the last command of a pipeline: + # "cmd | sed" reports sed's status, which is always 0, and a harness that + # cannot fail is worse than no harness + if sh "$SCRIPT_DIR/$t" $SPINE_BIN > "$LOG" 2>&1; then + sed 's/^/ /' "$LOG" PASSED=$((PASSED + 1)) RESULTS="$RESULTS ok $t" else + sed 's/^/ /' "$LOG" FAILED=$((FAILED + 1)) RESULTS="$RESULTS FAIL $t" fi done +rm -f "$LOG" echo "" echo "summary:$RESULTS" @@ -372,7 +379,7 @@ echo "samples: $SAMPLES_DIR" echo "Generating .ssp and .ocda.db ..." rm -rf "$OUT_DIR" mkdir -p "$OUT_DIR" -$SPINE_BIN --show-abstraction --show-abstraction-db --skip-output \ +$SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 FAILURES=0 @@ -692,7 +699,7 @@ mkdir -p "$OUT_DIR" echo "spine binary: $SPINE_BIN" echo "samples: $SAMPLES_DIR" echo "Generating .ssp and .ocda.db ..." -$SPINE_BIN --show-abstraction --show-abstraction-db --skip-output \ +$SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 COUNT=0 diff --git a/src/sisudoc/ocda/abstraction/db_in.d b/src/sisudoc/ocda/abstraction/db_in.d index f223f6d..0da123d 100644 --- a/src/sisudoc/ocda/abstraction/db_in.d +++ b/src/sisudoc/ocda/abstraction/db_in.d @@ -51,7 +51,7 @@ module sisudoc.ocda.abstraction.db_in; @safe: /+ ↓ read a .ocda.db back into the document abstraction - the counterpart of sisudoc.outputs.io_out.create_abstraction_db, and the + the counterpart of sisudoc.outputs.io_out.create_ocda, and the sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, so a consumer does not have to know which of the two artefacts it was handed. diff --git a/src/sisudoc/outputs/io_out/create_abstraction_db.d b/src/sisudoc/outputs/io_out/create_abstraction_db.d deleted file mode 100644 index 9b1af4d..0000000 --- a/src/sisudoc/outputs/io_out/create_abstraction_db.d +++ /dev/null @@ -1,676 +0,0 @@ -/+ -- Name: SisuDoc Spine, Doc Reform [a part of] - - Description: documents, structuring, processing, publishing, search - - static content generator - - - Author: Ralph Amissah - [ralph.amissah@gmail.com] - - - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - - - License: AGPL 3 or later: - - Spine (SiSU), a framework for document structuring, publishing and - search - - Copyright (C) Ralph Amissah - - This program is free software: you can redistribute it and/or modify it - under the terms of the GNU AFERO General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program. If not, see [https://www.gnu.org/licenses/]. - - If you have Internet connection, the latest version of the AGPL should be - available at these locations: - [https://www.fsf.org/licensing/licenses/agpl.html] - [https://www.gnu.org/licenses/agpl.html] - - - Spine (by Doc Reform, related to SiSU) uses standard: - - docReform markup syntax - - standard SiSU markup syntax with modified headers and minor modifications - - docReform object numbering - - standard SiSU object citation numbering & system - - - Homepages: - [https://www.sisudoc.org] - [https://www.doc-reform.org] - - - Git - [https://git.sisudoc.org/] - -+/ -module sisudoc.outputs.io_out.create_abstraction_db; - -/+ ↓ write document abstraction as per-document sqlite3 database +/ -template spineAbstractionDb() { - import std.conv : to; - import std.digest : toHexString; - import std.file; - import std.path; - import std.stdio; - import std.string; - import std.array; - import d2sqlite3; - import sisudoc.outputs.io_out.paths_output; - import sisudoc.ocda.io_in.paths_source; - /+ ↓ the abstraction is passed in rather than taken from doc, because the - caller hands over one that has been through the .ssp: written as text - and read back. so the database is a function of the .ssp and cannot - carry anything the .ssp does not - +/ - void spineAbstractionDb(D,A)(D doc, A doc_abstraction) { - auto doc_matters = doc.matters; - /+ ↓ output path: beside the pod's zip and digests, not under - /abstraction/. the database carries its own images, so it is a - distributable artefact in its own right and belongs with the others: - pod// the source tree - pod/.zip the same, zipped - pod/.digests.txt what is in them - pod/.ocda.db the abstraction, one per language - +/ - auto pths_pod = spinePathsPods!()(doc_matters); - string base_pth = pths_pod.pod_dir_(); - try { - if (!exists(base_pth)) { - base_pth.mkdirRecurse; - } - } catch (Exception ex) { - } - string db_file = ((base_pth.chainPath( - doc_matters.src.doc_uid_out ~ ".ocda.db")).asNormalizedPath).array; - - /+ ↓ remove existing file to start fresh +/ - try { - if (exists(db_file)) { - remove(db_file); - } - } catch (Exception ex) { - } - - if (doc_matters.opt.action.vox_gt_1) { - writeln(" ", db_file); - } - - /+ ↓ open database and create schema +/ - auto db = Database(db_file); - /+ ↓ write once, read many: no WAL. - a WAL stamped file cannot be opened by a reader that cannot write - beside it (SQLITE_READONLY on a read-only directory, and inside a - zip or on read-only media), and this database has one writer and no - concurrency, so WAL buys nothing here. MEMORY journal keeps the bulk - load fast without leaving that mark on the file. - +/ - db.run("PRAGMA journal_mode=MEMORY"); - db.run("PRAGMA synchronous=OFF"); - - db.run(" - CREATE TABLE metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - -- objects: one row per abstraction object, in document order. - -- the fixed width arrays (ancestors, dom status, children, heading - -- ancestors text, table widths and aligns) are JSON arrays so that - -- json_extract, json_each and json_array_length can reach inside them; - -- the open ended lists (images, links, anchor tags, subtoc) are rows - -- in their own tables, not columns - CREATE TABLE objects ( - id INTEGER PRIMARY KEY, - section TEXT NOT NULL, - seq INTEGER NOT NULL, - ocn INTEGER DEFAULT 0, - is_a TEXT NOT NULL, - is_of_part TEXT, - is_of_section TEXT, - is_of_type TEXT, - heading_level INTEGER, - heading_lev_collapsed INTEGER, - identifier TEXT, - parent_ocn INTEGER DEFAULT 0, - parent_lev INTEGER DEFAULT 0, - last_descendant_ocn INTEGER DEFAULT 0, - children TEXT - CHECK(children IS NULL OR json_valid(children)), - ancestors TEXT - CHECK(ancestors IS NULL OR json_valid(ancestors)), - ancestors_collapsed TEXT - CHECK(ancestors_collapsed IS NULL OR json_valid(ancestors_collapsed)), - dom_status TEXT - CHECK(dom_status IS NULL OR json_valid(dom_status)), - dom_status_collapsed TEXT - CHECK(dom_status_collapsed IS NULL OR json_valid(dom_status_collapsed)), - dummy_heading INTEGER DEFAULT 0, - object_number_off INTEGER DEFAULT 0, - attrib TEXT, - meta_lang TEXT, - meta_syntax TEXT, - sha256 TEXT, - indent_base INTEGER DEFAULT 0, - indent_hang INTEGER DEFAULT 0, - bullet INTEGER DEFAULT 0, - lang TEXT, - has_links INTEGER DEFAULT 0, - has_notes_reg INTEGER DEFAULT 0, - has_notes_star INTEGER DEFAULT 0, - has_images INTEGER DEFAULT 0, - has_images_no_dim INTEGER DEFAULT 0, - segment TEXT, - segment_prev TEXT, - segment_next TEXT, - segment_epub TEXT, - segment_html_is TEXT, - segment_epub_is TEXT, - anchor TEXT, - heading_lev_anchor TEXT, - heading_ancestors_text TEXT - CHECK(heading_ancestors_text IS NULL OR json_valid(heading_ancestors_text)), - table_cols INTEGER, - table_widths TEXT - CHECK(table_widths IS NULL OR json_valid(table_widths)), - table_aligns TEXT - CHECK(table_aligns IS NULL OR json_valid(table_aligns)), - table_header INTEGER, - code_linenumbers INTEGER DEFAULT 0, - text TEXT, - UNIQUE(section, seq) - ); - - CREATE INDEX idx_objects_section ON objects(section); - CREATE INDEX idx_objects_ocn ON objects(ocn); - CREATE INDEX idx_objects_parent ON objects(parent_ocn); - CREATE INDEX idx_objects_is_a ON objects(is_a); - CREATE INDEX idx_objects_heading ON objects(heading_level) - WHERE heading_level IS NOT NULL; - - -- files: what this database carries with it, so that a reader needs - -- nothing beside it. role says what a file is; images are the ones the - -- output writers must have in order to produce a document. bytes are - -- stored exactly as read, so sha256 matches the digest the abstraction - -- was built with - CREATE TABLE files ( - id INTEGER PRIMARY KEY, - role TEXT NOT NULL, - name TEXT NOT NULL, - bytes INTEGER NOT NULL, - sha256 TEXT, - width INTEGER, - height INTEGER, - data BLOB, - UNIQUE(role, name) - ); - - -- the open ended per object lists, one row each, in order - CREATE TABLE object_images ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - name TEXT NOT NULL, - bytes INTEGER, - sha256 TEXT, - width INTEGER, - height INTEGER, - missing INTEGER DEFAULT 0, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_links ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - url TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_anchors ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - anchor TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - CREATE TABLE object_subtoc ( - object_id INTEGER NOT NULL REFERENCES objects(id), - seq INTEGER NOT NULL, - entry TEXT NOT NULL, - PRIMARY KEY(object_id, seq) - ); - - CREATE INDEX idx_files_role ON files(role); - CREATE INDEX idx_object_images_name ON object_images(name); - - -- full text over the object text, external content so the text is not - -- stored twice. per document: this is for exploring one abstraction, - -- the collection wide search database is a different thing - CREATE VIRTUAL TABLE objects_fts USING fts5( - text, content='objects', content_rowid='id' - ); - - -- the shapes worth asking for, named - CREATE VIEW outline AS - SELECT section, seq, ocn, heading_level, heading_lev_collapsed, - last_descendant_ocn, identifier, text - FROM objects WHERE is_a = 'heading' ORDER BY id; - CREATE VIEW citable AS - SELECT * FROM objects WHERE ocn > 0 ORDER BY ocn; - CREATE VIEW document_files AS - SELECT role, name, bytes, sha256, width, height FROM files ORDER BY role, name; - "); - - /+ ↓ populate metadata +/ - db.run("BEGIN TRANSACTION"); - - auto meta_stmt = db.prepare( - "INSERT INTO metadata (key, value) VALUES (:key, :value)" - ); - auto meta = doc_matters.conf_make_meta.meta; - - void insertMeta(string key, string value) { - if (value.length > 0) { - meta_stmt.bind(":key", key); - meta_stmt.bind(":value", value); - meta_stmt.execute(); - meta_stmt.reset(); - } - } - - /+ ↓ what this file is, so a reader can tell before it starts +/ - insertMeta("schema.name", "sisu-abstraction-db"); - insertMeta("schema.version", "1"); - insertMeta("source.filename", doc_matters.src.filename); - insertMeta("source.language", doc_matters.src.language); - - insertMeta("title.main", meta.title_main); - insertMeta("title.subtitle", meta.title_subtitle); - insertMeta("title.full", meta.title_full); - insertMeta("title.language", meta.title_language); - insertMeta("creator.author", meta.creator_author); - insertMeta("creator.author_surname", meta.creator_author_surname); - insertMeta("creator.author_surname_fn", meta.creator_author_surname_fn); - insertMeta("creator.author_email", meta.creator_author_email); - insertMeta("creator.illustrator", meta.creator_illustrator); - insertMeta("creator.translator", meta.creator_translator); - insertMeta("date.published", meta.date_published); - insertMeta("date.created", meta.date_created); - insertMeta("date.issued", meta.date_issued); - insertMeta("date.available", meta.date_available); - insertMeta("date.modified", meta.date_modified); - insertMeta("date.valid", meta.date_valid); - insertMeta("rights.copyright", meta.rights_copyright); - insertMeta("rights.license", meta.rights_license); - insertMeta("classify.topic_register", meta.classify_topic_register); - insertMeta("classify.subject", meta.classify_subject); - insertMeta("classify.keywords", meta.classify_keywords); - insertMeta("classify.loc", meta.classify_loc); - insertMeta("classify.dewey", meta.classify_dewey); - insertMeta("identifier.isbn", meta.identifier_isbn); - insertMeta("identifier.oclc", meta.identifier_oclc); - insertMeta("language.document", meta.language_document); - insertMeta("notes.abstract", meta.notes_abstract); - insertMeta("notes.description", meta.notes_description); - insertMeta("notes.summary", meta.notes_summary); - - /+ ↓ make settings +/ - auto make = doc_matters.conf_make_meta.make; - insertMeta("make.doc_type", make.doc_type); - insertMeta("make.auto_num_top_at_level", make.auto_num_top_at_level); - insertMeta("make.auto_num_top_lv", make.auto_num_top_lv.to!string); - insertMeta("make.auto_num_depth", make.auto_num_depth.to!string); - - /+ ↓ doc_has counts +/ - insertMeta("doc_has.inline_links", doc_matters.has.inline_links.to!string); - insertMeta("doc_has.inline_notes_reg", doc_matters.has.inline_notes_reg.to!string); - insertMeta("doc_has.inline_notes_star", doc_matters.has.inline_notes_star.to!string); - insertMeta("doc_has.tables", doc_matters.has.tables.to!string); - insertMeta("doc_has.codeblocks", doc_matters.has.codeblocks.to!string); - insertMeta("doc_has.images", doc_matters.has.images.to!string); - insertMeta("doc_has.poems", doc_matters.has.poems.to!string); - insertMeta("doc_has.groups", doc_matters.has.groups.to!string); - insertMeta("doc_has.blocks", doc_matters.has.blocks.to!string); - insertMeta("doc_has.quotes", doc_matters.has.quotes.to!string); - - meta_stmt.finalize(); - - /+ ↓ populate objects +/ - auto obj_stmt = db.prepare( - "INSERT INTO objects (" - ~ "section, seq, ocn, is_a, is_of_part, is_of_section, is_of_type," - ~ "heading_level, heading_lev_collapsed, identifier," - ~ "parent_ocn, parent_lev, last_descendant_ocn," - ~ "children, ancestors, ancestors_collapsed," - ~ "dom_status, dom_status_collapsed," - ~ "dummy_heading, object_number_off," - ~ "attrib, meta_lang, meta_syntax, sha256," - ~ "indent_base, indent_hang, bullet, lang," - ~ "has_links, has_notes_reg, has_notes_star, has_images, has_images_no_dim," - ~ "segment, segment_prev, segment_next, segment_epub," - ~ "segment_html_is, segment_epub_is," - ~ "anchor, heading_lev_anchor, heading_ancestors_text," - ~ "table_cols, table_widths, table_aligns, table_header," - ~ "code_linenumbers, text" - ~ ") VALUES (" - ~ ":section, :seq, :ocn, :is_a, :is_of_part, :is_of_section, :is_of_type," - ~ ":heading_level, :heading_lev_collapsed, :identifier," - ~ ":parent_ocn, :parent_lev, :last_descendant_ocn," - ~ ":children, :ancestors, :ancestors_collapsed," - ~ ":dom_status, :dom_status_collapsed," - ~ ":dummy_heading, :object_number_off," - ~ ":attrib, :meta_lang, :meta_syntax, :sha256," - ~ ":indent_base, :indent_hang, :bullet, :lang," - ~ ":has_links, :has_notes_reg, :has_notes_star, :has_images, :has_images_no_dim," - ~ ":segment, :segment_prev, :segment_next, :segment_epub," - ~ ":segment_html_is, :segment_epub_is," - ~ ":anchor, :heading_lev_anchor, :heading_ancestors_text," - ~ ":table_cols, :table_widths, :table_aligns, :table_header," - ~ ":code_linenumbers, :text" - ~ ")" - ); - - /+ ↓ the per object lists, one row each +/ - auto img_stmt = db.prepare( - "INSERT INTO object_images" - ~ " (object_id, seq, name, bytes, sha256, width, height, missing)" - ~ " VALUES (:object_id, :seq, :name, :bytes, :sha256, :width, :height, :missing)" - ); - auto lnk_stmt = db.prepare( - "INSERT INTO object_links (object_id, seq, url)" - ~ " VALUES (:object_id, :seq, :url)" - ); - auto anc_stmt = db.prepare( - "INSERT INTO object_anchors (object_id, seq, anchor)" - ~ " VALUES (:object_id, :seq, :anchor)" - ); - auto sub_stmt = db.prepare( - "INSERT INTO object_subtoc (object_id, seq, entry)" - ~ " VALUES (:object_id, :seq, :entry)" - ); - - string[] section_order = ["head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb"]; - - foreach (section; section_order) { - if (section !in doc_abstraction) continue; - auto section_objs = doc_abstraction[section]; - if (section_objs.length == 0) continue; - - foreach (seq, obj; section_objs) { - obj_stmt.bind(":section", section); - obj_stmt.bind(":seq", cast(int) seq); - obj_stmt.bind(":ocn", obj.metainfo.ocn); - obj_stmt.bind(":is_a", obj.metainfo.is_a); - - /+ ↓ nullable string fields +/ - void bindStr(string param, string val) { - import std.typecons : Nullable; - if (val.length > 0) { - obj_stmt.bind(param, val); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - /+ ↓ nullable int fields +/ - void bindInt(string param, int val, bool is_set) { - import std.typecons : Nullable; - if (is_set) { - obj_stmt.bind(param, val); - } else { - obj_stmt.bind(param, Nullable!int()); - } - } - /+ ↓ fixed width int array as a JSON array, NULL when all zero +/ - void bindJsonInts(T)(string param, T arr) { - import std.typecons : Nullable; - bool any = false; - foreach (a; arr) { if (a != 0) { any = true; break; } } - if (any) { - string[] _j; - foreach (a; arr) { _j ~= a.to!string; } - obj_stmt.bind(param, "[" ~ _j.join(",") ~ "]"); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - /+ ↓ string array as a JSON array, NULL when every slot is empty +/ - void bindJsonStrs(T)(string param, T arr) { - import std.json : JSONValue; - import std.typecons : Nullable; - bool any = false; - foreach (s; arr) { if (s.length > 0) { any = true; break; } } - if (any) { - string[] _s; - foreach (s; arr) { _s ~= s.to!string; } - obj_stmt.bind(param, JSONValue(_s).toString); - } else { - obj_stmt.bind(param, Nullable!string()); - } - } - - bindStr(":is_of_part", obj.metainfo.is_of_part); - bindStr(":is_of_section", obj.metainfo.is_of_section); - bindStr(":is_of_type", obj.metainfo.is_of_type); - - /+ ↓ heading levels, marked up and collapsed, 9 meaning unset +/ - bindInt(":heading_level", obj.metainfo.heading_lev_markup, - (obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup < 9)); - bindInt(":heading_lev_collapsed", obj.metainfo.heading_lev_collapsed, - (obj.metainfo.heading_lev_collapsed < 9)); - - bindStr(":identifier", obj.metainfo.identifier); - obj_stmt.bind(":parent_ocn", obj.metainfo.parent_ocn); - obj_stmt.bind(":parent_lev", obj.metainfo.parent_lev_markup); - obj_stmt.bind(":last_descendant_ocn", obj.metainfo.last_descendant_ocn); - - bindJsonInts(":children", obj.metainfo.children_headings); - bindJsonInts(":ancestors", obj.metainfo.markedup_ancestors); - bindJsonInts(":ancestors_collapsed", obj.metainfo.collapsed_ancestors); - bindJsonInts(":dom_status", obj.metainfo.dom_structure_markedup_tags_status); - bindJsonInts(":dom_status_collapsed", obj.metainfo.dom_structure_collapsed_tags_status); - - obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); - obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); - bindStr(":attrib", obj.metainfo.attrib); - bindStr(":meta_lang", obj.metainfo.lang); - bindStr(":meta_syntax", obj.metainfo.syntax); - - /+ ↓ object digest, hex, absent when never taken +/ - { - import std.typecons : Nullable; - bool has_sha = false; - foreach (b; obj.metainfo.sha256.text) { - if (b != 0) { has_sha = true; break; } - } - if (has_sha) { - obj_stmt.bind(":sha256", obj.metainfo.sha256.text.toHexString.to!string); - } else { - obj_stmt.bind(":sha256", Nullable!string()); - } - } - - // obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); - // obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); - obj_stmt.bind(":indent_base", obj.attrib.indent_base); - obj_stmt.bind(":indent_hang", obj.attrib.indent_hang); - obj_stmt.bind(":bullet", obj.attrib.bullet ? 1 : 0); - bindStr(":lang", obj.attrib.language); - obj_stmt.bind(":has_links", obj.has.inline_links ? 1 : 0); - obj_stmt.bind(":has_notes_reg", obj.has.inline_notes_reg ? 1 : 0); - obj_stmt.bind(":has_notes_star", obj.has.inline_notes_star ? 1 : 0); - obj_stmt.bind(":has_images", obj.has.images ? 1 : 0); - obj_stmt.bind(":has_images_no_dim", obj.has.image_without_dimensions ? 1 : 0); - bindStr(":segment", obj.tags.in_segment_html); - bindStr(":segment_prev", obj.tags.segname_prev); - bindStr(":segment_next", obj.tags.segname_next); - bindStr(":segment_epub", obj.tags.segment_anchor_tag_epub); - bindStr(":segment_html_is", obj.tags.html_segment_anchor_tag_is); - bindStr(":segment_epub_is", obj.tags.epub_segment_anchor_tag_is); - bindStr(":anchor", obj.tags.anchor_tag_html); - bindStr(":heading_lev_anchor", obj.tags.heading_lev_anchor_tag); - bindJsonStrs(":heading_ancestors_text", obj.tags.heading_ancestors_text); - - /+ ↓ table properties +/ - { - import std.typecons : Nullable; - if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { - obj_stmt.bind(":table_cols", obj.table.number_of_columns); - if (obj.table.column_widths.length > 0) { - string[] ws; - foreach (w; obj.table.column_widths) ws ~= w.to!string; - obj_stmt.bind(":table_widths", "[" ~ ws.join(",") ~ "]"); - } else { - obj_stmt.bind(":table_widths", Nullable!string()); - } - if (obj.table.column_aligns.length > 0) { - import std.json : JSONValue; - string[] _al; - foreach (a; obj.table.column_aligns) { _al ~= a.to!string; } - obj_stmt.bind(":table_aligns", JSONValue(_al).toString); - } else { - obj_stmt.bind(":table_aligns", Nullable!string()); - } - obj_stmt.bind(":table_header", obj.table.heading ? 1 : 0); - } else { - obj_stmt.bind(":table_cols", Nullable!int()); - obj_stmt.bind(":table_widths", Nullable!string()); - obj_stmt.bind(":table_aligns", Nullable!string()); - obj_stmt.bind(":table_header", Nullable!int()); - } - } - - /+ ↓ code block properties +/ - obj_stmt.bind(":code_linenumbers", - ((obj.metainfo.is_a == "code") && obj.code_block.linenumbers) ? 1 : 0); - - /+ ↓ text content +/ - bindStr(":text", obj.text); - - obj_stmt.execute(); - obj_stmt.reset(); - - /+ ↓ the per object lists, keyed on the row just written +/ - long _obj_id = db.lastInsertRowid; - { - int _n = 0; - foreach (i; obj.metainfo.sha256.images) { - img_stmt.bind(":object_id", _obj_id); - img_stmt.bind(":seq", _n); - img_stmt.bind(":name", i.fileName); - img_stmt.bind(":bytes", cast(long) i.fileSize); - img_stmt.bind(":sha256", i.fileHash_sha256.toHexString.to!string); - img_stmt.bind(":width", i.imageWidth); - img_stmt.bind(":height", i.imageHeight); - img_stmt.bind(":missing", i.fileMissing ? 1 : 0); - img_stmt.execute(); - img_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (lnk; obj.stow.link) { - if (lnk.length == 0) { continue; } - lnk_stmt.bind(":object_id", _obj_id); - lnk_stmt.bind(":seq", _n); - lnk_stmt.bind(":url", lnk); - lnk_stmt.execute(); - lnk_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (at; obj.tags.anchor_tags) { - if (at.length == 0) { continue; } - anc_stmt.bind(":object_id", _obj_id); - anc_stmt.bind(":seq", _n); - anc_stmt.bind(":anchor", at); - anc_stmt.execute(); - anc_stmt.reset(); - ++_n; - } - } - { - int _n = 0; - foreach (st; obj.tags.lev4_subtoc) { - if (st.length == 0) { continue; } - sub_stmt.bind(":object_id", _obj_id); - sub_stmt.bind(":seq", _n); - sub_stmt.bind(":entry", st); - sub_stmt.execute(); - sub_stmt.reset(); - ++_n; - } - } - } - } - - obj_stmt.finalize(); - img_stmt.finalize(); - lnk_stmt.finalize(); - anc_stmt.finalize(); - sub_stmt.finalize(); - - /+ ↓ the document's images, carried inside the file so that output can be - produced from this database alone. bytes exactly as read, and the - digest taken over them, so that it can be checked against the digest - the abstraction recorded +/ - { - import std.digest.sha : sha256Of; - auto file_stmt = db.prepare( - "INSERT OR IGNORE INTO files" - ~ " (role, name, bytes, sha256, width, height, data)" - ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" - ); - int[string] _w, _h; - foreach (section; section_order) { - if (section !in doc_abstraction) { continue; } - foreach (obj; doc_abstraction[section]) { - foreach (i; obj.metainfo.sha256.images) { - if (i.imageWidth > 0) { _w[i.fileName] = i.imageWidth; } - if (i.imageHeight > 0) { _h[i.fileName] = i.imageHeight; } - } - } - } - foreach (image; doc_matters.srcs.image_list) { - string _fn = doc_matters.src.image_dir_path ~ "/" ~ image; - ubyte[] _bytes; - try { - if (_fn.exists) { _bytes = cast(ubyte[]) _fn.read; } - } catch (Exception ex) { - _bytes = []; - } - if (_bytes.length == 0) { - if (doc_matters.opt.action.vox_gt_1) { - writeln(" image not carried into abstraction db: ", _fn); - } - continue; - } - file_stmt.bind(":role", "image"); - file_stmt.bind(":name", image); - file_stmt.bind(":bytes", cast(long) _bytes.length); - file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); - if (image in _w) { file_stmt.bind(":width", _w[image]); } - else { - import std.typecons : Nullable; - file_stmt.bind(":width", Nullable!int()); - } - if (image in _h) { file_stmt.bind(":height", _h[image]); } - else { - import std.typecons : Nullable; - file_stmt.bind(":height", Nullable!int()); - } - file_stmt.bind(":data", _bytes); - file_stmt.execute(); - file_stmt.reset(); - } - file_stmt.finalize(); - } - - /+ ↓ build the full text index from the rows just written +/ - db.run("INSERT INTO objects_fts(rowid, text)" - ~ " SELECT id, text FROM objects WHERE text IS NOT NULL"); - - db.run("COMMIT TRANSACTION"); - } -} diff --git a/src/sisudoc/outputs/io_out/create_ocda_db.d b/src/sisudoc/outputs/io_out/create_ocda_db.d new file mode 100644 index 0000000..a47db7a --- /dev/null +++ b/src/sisudoc/outputs/io_out/create_ocda_db.d @@ -0,0 +1,676 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ +module sisudoc.outputs.io_out.create_ocda_db; + +/+ ↓ write document abstraction as per-document sqlite3 database +/ +template spineAbstractionDb() { + import std.conv : to; + import std.digest : toHexString; + import std.file; + import std.path; + import std.stdio; + import std.string; + import std.array; + import d2sqlite3; + import sisudoc.outputs.io_out.paths_output; + import sisudoc.ocda.io_in.paths_source; + /+ ↓ the abstraction is passed in rather than taken from doc, because the + caller hands over one that has been through the .ssp: written as text + and read back. so the database is a function of the .ssp and cannot + carry anything the .ssp does not + +/ + void spineAbstractionDb(D,A)(D doc, A doc_abstraction) { + auto doc_matters = doc.matters; + /+ ↓ output path: beside the pod's zip and digests, not under + /abstraction/. the database carries its own images, so it is a + distributable artefact in its own right and belongs with the others: + pod// the source tree + pod/.zip the same, zipped + pod/.digests.txt what is in them + pod/.ocda.db the abstraction, one per language + +/ + auto pths_pod = spinePathsPods!()(doc_matters); + string base_pth = pths_pod.pod_dir_(); + try { + if (!exists(base_pth)) { + base_pth.mkdirRecurse; + } + } catch (Exception ex) { + } + string db_file = ((base_pth.chainPath( + doc_matters.src.doc_uid_out ~ ".ocda.db")).asNormalizedPath).array; + + /+ ↓ remove existing file to start fresh +/ + try { + if (exists(db_file)) { + remove(db_file); + } + } catch (Exception ex) { + } + + if (doc_matters.opt.action.vox_gt_1) { + writeln(" ", db_file); + } + + /+ ↓ open database and create schema +/ + auto db = Database(db_file); + /+ ↓ write once, read many: no WAL. + a WAL stamped file cannot be opened by a reader that cannot write + beside it (SQLITE_READONLY on a read-only directory, and inside a + zip or on read-only media), and this database has one writer and no + concurrency, so WAL buys nothing here. MEMORY journal keeps the bulk + load fast without leaving that mark on the file. + +/ + db.run("PRAGMA journal_mode=MEMORY"); + db.run("PRAGMA synchronous=OFF"); + + db.run(" + CREATE TABLE metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + -- objects: one row per abstraction object, in document order. + -- the fixed width arrays (ancestors, dom status, children, heading + -- ancestors text, table widths and aligns) are JSON arrays so that + -- json_extract, json_each and json_array_length can reach inside them; + -- the open ended lists (images, links, anchor tags, subtoc) are rows + -- in their own tables, not columns + CREATE TABLE objects ( + id INTEGER PRIMARY KEY, + section TEXT NOT NULL, + seq INTEGER NOT NULL, + ocn INTEGER DEFAULT 0, + is_a TEXT NOT NULL, + is_of_part TEXT, + is_of_section TEXT, + is_of_type TEXT, + heading_level INTEGER, + heading_lev_collapsed INTEGER, + identifier TEXT, + parent_ocn INTEGER DEFAULT 0, + parent_lev INTEGER DEFAULT 0, + last_descendant_ocn INTEGER DEFAULT 0, + children TEXT + CHECK(children IS NULL OR json_valid(children)), + ancestors TEXT + CHECK(ancestors IS NULL OR json_valid(ancestors)), + ancestors_collapsed TEXT + CHECK(ancestors_collapsed IS NULL OR json_valid(ancestors_collapsed)), + dom_status TEXT + CHECK(dom_status IS NULL OR json_valid(dom_status)), + dom_status_collapsed TEXT + CHECK(dom_status_collapsed IS NULL OR json_valid(dom_status_collapsed)), + dummy_heading INTEGER DEFAULT 0, + object_number_off INTEGER DEFAULT 0, + attrib TEXT, + meta_lang TEXT, + meta_syntax TEXT, + sha256 TEXT, + indent_base INTEGER DEFAULT 0, + indent_hang INTEGER DEFAULT 0, + bullet INTEGER DEFAULT 0, + lang TEXT, + has_links INTEGER DEFAULT 0, + has_notes_reg INTEGER DEFAULT 0, + has_notes_star INTEGER DEFAULT 0, + has_images INTEGER DEFAULT 0, + has_images_no_dim INTEGER DEFAULT 0, + segment TEXT, + segment_prev TEXT, + segment_next TEXT, + segment_epub TEXT, + segment_html_is TEXT, + segment_epub_is TEXT, + anchor TEXT, + heading_lev_anchor TEXT, + heading_ancestors_text TEXT + CHECK(heading_ancestors_text IS NULL OR json_valid(heading_ancestors_text)), + table_cols INTEGER, + table_widths TEXT + CHECK(table_widths IS NULL OR json_valid(table_widths)), + table_aligns TEXT + CHECK(table_aligns IS NULL OR json_valid(table_aligns)), + table_header INTEGER, + code_linenumbers INTEGER DEFAULT 0, + text TEXT, + UNIQUE(section, seq) + ); + + CREATE INDEX idx_objects_section ON objects(section); + CREATE INDEX idx_objects_ocn ON objects(ocn); + CREATE INDEX idx_objects_parent ON objects(parent_ocn); + CREATE INDEX idx_objects_is_a ON objects(is_a); + CREATE INDEX idx_objects_heading ON objects(heading_level) + WHERE heading_level IS NOT NULL; + + -- files: what this database carries with it, so that a reader needs + -- nothing beside it. role says what a file is; images are the ones the + -- output writers must have in order to produce a document. bytes are + -- stored exactly as read, so sha256 matches the digest the abstraction + -- was built with + CREATE TABLE files ( + id INTEGER PRIMARY KEY, + role TEXT NOT NULL, + name TEXT NOT NULL, + bytes INTEGER NOT NULL, + sha256 TEXT, + width INTEGER, + height INTEGER, + data BLOB, + UNIQUE(role, name) + ); + + -- the open ended per object lists, one row each, in order + CREATE TABLE object_images ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + name TEXT NOT NULL, + bytes INTEGER, + sha256 TEXT, + width INTEGER, + height INTEGER, + missing INTEGER DEFAULT 0, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_links ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + url TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_anchors ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + anchor TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + CREATE TABLE object_subtoc ( + object_id INTEGER NOT NULL REFERENCES objects(id), + seq INTEGER NOT NULL, + entry TEXT NOT NULL, + PRIMARY KEY(object_id, seq) + ); + + CREATE INDEX idx_files_role ON files(role); + CREATE INDEX idx_object_images_name ON object_images(name); + + -- full text over the object text, external content so the text is not + -- stored twice. per document: this is for exploring one abstraction, + -- the collection wide search database is a different thing + CREATE VIRTUAL TABLE objects_fts USING fts5( + text, content='objects', content_rowid='id' + ); + + -- the shapes worth asking for, named + CREATE VIEW outline AS + SELECT section, seq, ocn, heading_level, heading_lev_collapsed, + last_descendant_ocn, identifier, text + FROM objects WHERE is_a = 'heading' ORDER BY id; + CREATE VIEW citable AS + SELECT * FROM objects WHERE ocn > 0 ORDER BY ocn; + CREATE VIEW document_files AS + SELECT role, name, bytes, sha256, width, height FROM files ORDER BY role, name; + "); + + /+ ↓ populate metadata +/ + db.run("BEGIN TRANSACTION"); + + auto meta_stmt = db.prepare( + "INSERT INTO metadata (key, value) VALUES (:key, :value)" + ); + auto meta = doc_matters.conf_make_meta.meta; + + void insertMeta(string key, string value) { + if (value.length > 0) { + meta_stmt.bind(":key", key); + meta_stmt.bind(":value", value); + meta_stmt.execute(); + meta_stmt.reset(); + } + } + + /+ ↓ what this file is, so a reader can tell before it starts +/ + insertMeta("schema.name", "sisu-abstraction-db"); + insertMeta("schema.version", "1"); + insertMeta("source.filename", doc_matters.src.filename); + insertMeta("source.language", doc_matters.src.language); + + insertMeta("title.main", meta.title_main); + insertMeta("title.subtitle", meta.title_subtitle); + insertMeta("title.full", meta.title_full); + insertMeta("title.language", meta.title_language); + insertMeta("creator.author", meta.creator_author); + insertMeta("creator.author_surname", meta.creator_author_surname); + insertMeta("creator.author_surname_fn", meta.creator_author_surname_fn); + insertMeta("creator.author_email", meta.creator_author_email); + insertMeta("creator.illustrator", meta.creator_illustrator); + insertMeta("creator.translator", meta.creator_translator); + insertMeta("date.published", meta.date_published); + insertMeta("date.created", meta.date_created); + insertMeta("date.issued", meta.date_issued); + insertMeta("date.available", meta.date_available); + insertMeta("date.modified", meta.date_modified); + insertMeta("date.valid", meta.date_valid); + insertMeta("rights.copyright", meta.rights_copyright); + insertMeta("rights.license", meta.rights_license); + insertMeta("classify.topic_register", meta.classify_topic_register); + insertMeta("classify.subject", meta.classify_subject); + insertMeta("classify.keywords", meta.classify_keywords); + insertMeta("classify.loc", meta.classify_loc); + insertMeta("classify.dewey", meta.classify_dewey); + insertMeta("identifier.isbn", meta.identifier_isbn); + insertMeta("identifier.oclc", meta.identifier_oclc); + insertMeta("language.document", meta.language_document); + insertMeta("notes.abstract", meta.notes_abstract); + insertMeta("notes.description", meta.notes_description); + insertMeta("notes.summary", meta.notes_summary); + + /+ ↓ make settings +/ + auto make = doc_matters.conf_make_meta.make; + insertMeta("make.doc_type", make.doc_type); + insertMeta("make.auto_num_top_at_level", make.auto_num_top_at_level); + insertMeta("make.auto_num_top_lv", make.auto_num_top_lv.to!string); + insertMeta("make.auto_num_depth", make.auto_num_depth.to!string); + + /+ ↓ doc_has counts +/ + insertMeta("doc_has.inline_links", doc_matters.has.inline_links.to!string); + insertMeta("doc_has.inline_notes_reg", doc_matters.has.inline_notes_reg.to!string); + insertMeta("doc_has.inline_notes_star", doc_matters.has.inline_notes_star.to!string); + insertMeta("doc_has.tables", doc_matters.has.tables.to!string); + insertMeta("doc_has.codeblocks", doc_matters.has.codeblocks.to!string); + insertMeta("doc_has.images", doc_matters.has.images.to!string); + insertMeta("doc_has.poems", doc_matters.has.poems.to!string); + insertMeta("doc_has.groups", doc_matters.has.groups.to!string); + insertMeta("doc_has.blocks", doc_matters.has.blocks.to!string); + insertMeta("doc_has.quotes", doc_matters.has.quotes.to!string); + + meta_stmt.finalize(); + + /+ ↓ populate objects +/ + auto obj_stmt = db.prepare( + "INSERT INTO objects (" + ~ "section, seq, ocn, is_a, is_of_part, is_of_section, is_of_type," + ~ "heading_level, heading_lev_collapsed, identifier," + ~ "parent_ocn, parent_lev, last_descendant_ocn," + ~ "children, ancestors, ancestors_collapsed," + ~ "dom_status, dom_status_collapsed," + ~ "dummy_heading, object_number_off," + ~ "attrib, meta_lang, meta_syntax, sha256," + ~ "indent_base, indent_hang, bullet, lang," + ~ "has_links, has_notes_reg, has_notes_star, has_images, has_images_no_dim," + ~ "segment, segment_prev, segment_next, segment_epub," + ~ "segment_html_is, segment_epub_is," + ~ "anchor, heading_lev_anchor, heading_ancestors_text," + ~ "table_cols, table_widths, table_aligns, table_header," + ~ "code_linenumbers, text" + ~ ") VALUES (" + ~ ":section, :seq, :ocn, :is_a, :is_of_part, :is_of_section, :is_of_type," + ~ ":heading_level, :heading_lev_collapsed, :identifier," + ~ ":parent_ocn, :parent_lev, :last_descendant_ocn," + ~ ":children, :ancestors, :ancestors_collapsed," + ~ ":dom_status, :dom_status_collapsed," + ~ ":dummy_heading, :object_number_off," + ~ ":attrib, :meta_lang, :meta_syntax, :sha256," + ~ ":indent_base, :indent_hang, :bullet, :lang," + ~ ":has_links, :has_notes_reg, :has_notes_star, :has_images, :has_images_no_dim," + ~ ":segment, :segment_prev, :segment_next, :segment_epub," + ~ ":segment_html_is, :segment_epub_is," + ~ ":anchor, :heading_lev_anchor, :heading_ancestors_text," + ~ ":table_cols, :table_widths, :table_aligns, :table_header," + ~ ":code_linenumbers, :text" + ~ ")" + ); + + /+ ↓ the per object lists, one row each +/ + auto img_stmt = db.prepare( + "INSERT INTO object_images" + ~ " (object_id, seq, name, bytes, sha256, width, height, missing)" + ~ " VALUES (:object_id, :seq, :name, :bytes, :sha256, :width, :height, :missing)" + ); + auto lnk_stmt = db.prepare( + "INSERT INTO object_links (object_id, seq, url)" + ~ " VALUES (:object_id, :seq, :url)" + ); + auto anc_stmt = db.prepare( + "INSERT INTO object_anchors (object_id, seq, anchor)" + ~ " VALUES (:object_id, :seq, :anchor)" + ); + auto sub_stmt = db.prepare( + "INSERT INTO object_subtoc (object_id, seq, entry)" + ~ " VALUES (:object_id, :seq, :entry)" + ); + + string[] section_order = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb"]; + + foreach (section; section_order) { + if (section !in doc_abstraction) continue; + auto section_objs = doc_abstraction[section]; + if (section_objs.length == 0) continue; + + foreach (seq, obj; section_objs) { + obj_stmt.bind(":section", section); + obj_stmt.bind(":seq", cast(int) seq); + obj_stmt.bind(":ocn", obj.metainfo.ocn); + obj_stmt.bind(":is_a", obj.metainfo.is_a); + + /+ ↓ nullable string fields +/ + void bindStr(string param, string val) { + import std.typecons : Nullable; + if (val.length > 0) { + obj_stmt.bind(param, val); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + /+ ↓ nullable int fields +/ + void bindInt(string param, int val, bool is_set) { + import std.typecons : Nullable; + if (is_set) { + obj_stmt.bind(param, val); + } else { + obj_stmt.bind(param, Nullable!int()); + } + } + /+ ↓ fixed width int array as a JSON array, NULL when all zero +/ + void bindJsonInts(T)(string param, T arr) { + import std.typecons : Nullable; + bool any = false; + foreach (a; arr) { if (a != 0) { any = true; break; } } + if (any) { + string[] _j; + foreach (a; arr) { _j ~= a.to!string; } + obj_stmt.bind(param, "[" ~ _j.join(",") ~ "]"); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + /+ ↓ string array as a JSON array, NULL when every slot is empty +/ + void bindJsonStrs(T)(string param, T arr) { + import std.json : JSONValue; + import std.typecons : Nullable; + bool any = false; + foreach (s; arr) { if (s.length > 0) { any = true; break; } } + if (any) { + string[] _s; + foreach (s; arr) { _s ~= s.to!string; } + obj_stmt.bind(param, JSONValue(_s).toString); + } else { + obj_stmt.bind(param, Nullable!string()); + } + } + + bindStr(":is_of_part", obj.metainfo.is_of_part); + bindStr(":is_of_section", obj.metainfo.is_of_section); + bindStr(":is_of_type", obj.metainfo.is_of_type); + + /+ ↓ heading levels, marked up and collapsed, 9 meaning unset +/ + bindInt(":heading_level", obj.metainfo.heading_lev_markup, + (obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup < 9)); + bindInt(":heading_lev_collapsed", obj.metainfo.heading_lev_collapsed, + (obj.metainfo.heading_lev_collapsed < 9)); + + bindStr(":identifier", obj.metainfo.identifier); + obj_stmt.bind(":parent_ocn", obj.metainfo.parent_ocn); + obj_stmt.bind(":parent_lev", obj.metainfo.parent_lev_markup); + obj_stmt.bind(":last_descendant_ocn", obj.metainfo.last_descendant_ocn); + + bindJsonInts(":children", obj.metainfo.children_headings); + bindJsonInts(":ancestors", obj.metainfo.markedup_ancestors); + bindJsonInts(":ancestors_collapsed", obj.metainfo.collapsed_ancestors); + bindJsonInts(":dom_status", obj.metainfo.dom_structure_markedup_tags_status); + bindJsonInts(":dom_status_collapsed", obj.metainfo.dom_structure_collapsed_tags_status); + + obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); + obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); + bindStr(":attrib", obj.metainfo.attrib); + bindStr(":meta_lang", obj.metainfo.lang); + bindStr(":meta_syntax", obj.metainfo.syntax); + + /+ ↓ object digest, hex, absent when never taken +/ + { + import std.typecons : Nullable; + bool has_sha = false; + foreach (b; obj.metainfo.sha256.text) { + if (b != 0) { has_sha = true; break; } + } + if (has_sha) { + obj_stmt.bind(":sha256", obj.metainfo.sha256.text.toHexString.to!string); + } else { + obj_stmt.bind(":sha256", Nullable!string()); + } + } + + // obj_stmt.bind(":dummy_heading", obj.metainfo.dummy_heading ? 1 : 0); + // obj_stmt.bind(":object_number_off", obj.metainfo.object_number_off ? 1 : 0); + obj_stmt.bind(":indent_base", obj.attrib.indent_base); + obj_stmt.bind(":indent_hang", obj.attrib.indent_hang); + obj_stmt.bind(":bullet", obj.attrib.bullet ? 1 : 0); + bindStr(":lang", obj.attrib.language); + obj_stmt.bind(":has_links", obj.has.inline_links ? 1 : 0); + obj_stmt.bind(":has_notes_reg", obj.has.inline_notes_reg ? 1 : 0); + obj_stmt.bind(":has_notes_star", obj.has.inline_notes_star ? 1 : 0); + obj_stmt.bind(":has_images", obj.has.images ? 1 : 0); + obj_stmt.bind(":has_images_no_dim", obj.has.image_without_dimensions ? 1 : 0); + bindStr(":segment", obj.tags.in_segment_html); + bindStr(":segment_prev", obj.tags.segname_prev); + bindStr(":segment_next", obj.tags.segname_next); + bindStr(":segment_epub", obj.tags.segment_anchor_tag_epub); + bindStr(":segment_html_is", obj.tags.html_segment_anchor_tag_is); + bindStr(":segment_epub_is", obj.tags.epub_segment_anchor_tag_is); + bindStr(":anchor", obj.tags.anchor_tag_html); + bindStr(":heading_lev_anchor", obj.tags.heading_lev_anchor_tag); + bindJsonStrs(":heading_ancestors_text", obj.tags.heading_ancestors_text); + + /+ ↓ table properties +/ + { + import std.typecons : Nullable; + if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { + obj_stmt.bind(":table_cols", obj.table.number_of_columns); + if (obj.table.column_widths.length > 0) { + string[] ws; + foreach (w; obj.table.column_widths) ws ~= w.to!string; + obj_stmt.bind(":table_widths", "[" ~ ws.join(",") ~ "]"); + } else { + obj_stmt.bind(":table_widths", Nullable!string()); + } + if (obj.table.column_aligns.length > 0) { + import std.json : JSONValue; + string[] _al; + foreach (a; obj.table.column_aligns) { _al ~= a.to!string; } + obj_stmt.bind(":table_aligns", JSONValue(_al).toString); + } else { + obj_stmt.bind(":table_aligns", Nullable!string()); + } + obj_stmt.bind(":table_header", obj.table.heading ? 1 : 0); + } else { + obj_stmt.bind(":table_cols", Nullable!int()); + obj_stmt.bind(":table_widths", Nullable!string()); + obj_stmt.bind(":table_aligns", Nullable!string()); + obj_stmt.bind(":table_header", Nullable!int()); + } + } + + /+ ↓ code block properties +/ + obj_stmt.bind(":code_linenumbers", + ((obj.metainfo.is_a == "code") && obj.code_block.linenumbers) ? 1 : 0); + + /+ ↓ text content +/ + bindStr(":text", obj.text); + + obj_stmt.execute(); + obj_stmt.reset(); + + /+ ↓ the per object lists, keyed on the row just written +/ + long _obj_id = db.lastInsertRowid; + { + int _n = 0; + foreach (i; obj.metainfo.sha256.images) { + img_stmt.bind(":object_id", _obj_id); + img_stmt.bind(":seq", _n); + img_stmt.bind(":name", i.fileName); + img_stmt.bind(":bytes", cast(long) i.fileSize); + img_stmt.bind(":sha256", i.fileHash_sha256.toHexString.to!string); + img_stmt.bind(":width", i.imageWidth); + img_stmt.bind(":height", i.imageHeight); + img_stmt.bind(":missing", i.fileMissing ? 1 : 0); + img_stmt.execute(); + img_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (lnk; obj.stow.link) { + if (lnk.length == 0) { continue; } + lnk_stmt.bind(":object_id", _obj_id); + lnk_stmt.bind(":seq", _n); + lnk_stmt.bind(":url", lnk); + lnk_stmt.execute(); + lnk_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (at; obj.tags.anchor_tags) { + if (at.length == 0) { continue; } + anc_stmt.bind(":object_id", _obj_id); + anc_stmt.bind(":seq", _n); + anc_stmt.bind(":anchor", at); + anc_stmt.execute(); + anc_stmt.reset(); + ++_n; + } + } + { + int _n = 0; + foreach (st; obj.tags.lev4_subtoc) { + if (st.length == 0) { continue; } + sub_stmt.bind(":object_id", _obj_id); + sub_stmt.bind(":seq", _n); + sub_stmt.bind(":entry", st); + sub_stmt.execute(); + sub_stmt.reset(); + ++_n; + } + } + } + } + + obj_stmt.finalize(); + img_stmt.finalize(); + lnk_stmt.finalize(); + anc_stmt.finalize(); + sub_stmt.finalize(); + + /+ ↓ the document's images, carried inside the file so that output can be + produced from this database alone. bytes exactly as read, and the + digest taken over them, so that it can be checked against the digest + the abstraction recorded +/ + { + import std.digest.sha : sha256Of; + auto file_stmt = db.prepare( + "INSERT OR IGNORE INTO files" + ~ " (role, name, bytes, sha256, width, height, data)" + ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" + ); + int[string] _w, _h; + foreach (section; section_order) { + if (section !in doc_abstraction) { continue; } + foreach (obj; doc_abstraction[section]) { + foreach (i; obj.metainfo.sha256.images) { + if (i.imageWidth > 0) { _w[i.fileName] = i.imageWidth; } + if (i.imageHeight > 0) { _h[i.fileName] = i.imageHeight; } + } + } + } + foreach (image; doc_matters.srcs.image_list) { + string _fn = doc_matters.src.image_dir_path ~ "/" ~ image; + ubyte[] _bytes; + try { + if (_fn.exists) { _bytes = cast(ubyte[]) _fn.read; } + } catch (Exception ex) { + _bytes = []; + } + if (_bytes.length == 0) { + if (doc_matters.opt.action.vox_gt_1) { + writeln(" image not carried into abstraction db: ", _fn); + } + continue; + } + file_stmt.bind(":role", "image"); + file_stmt.bind(":name", image); + file_stmt.bind(":bytes", cast(long) _bytes.length); + file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); + if (image in _w) { file_stmt.bind(":width", _w[image]); } + else { + import std.typecons : Nullable; + file_stmt.bind(":width", Nullable!int()); + } + if (image in _h) { file_stmt.bind(":height", _h[image]); } + else { + import std.typecons : Nullable; + file_stmt.bind(":height", Nullable!int()); + } + file_stmt.bind(":data", _bytes); + file_stmt.execute(); + file_stmt.reset(); + } + file_stmt.finalize(); + } + + /+ ↓ build the full text index from the rows just written +/ + db.run("INSERT INTO objects_fts(rowid, text)" + ~ " SELECT id, text FROM objects WHERE text IS NOT NULL"); + + db.run("COMMIT TRANSACTION"); + } +} diff --git a/src/sisudoc/outputs/io_out/metadata.d b/src/sisudoc/outputs/io_out/metadata.d index f2aaa53..853fb83 100644 --- a/src/sisudoc/outputs/io_out/metadata.d +++ b/src/sisudoc/outputs/io_out/metadata.d @@ -529,7 +529,7 @@ string theme_light_1 = format(q"┃ writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); } } - if (doc_matters.opt.action.show_abstraction_db) { + if (doc_matters.opt.action.ocda_db) { try { // sha digest and size for the abstraction database, beside the pod string _db_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.doc_uid_out ~ ".ocda.db"; diff --git a/src/sisudoc/outputs/io_out/source_pod.d b/src/sisudoc/outputs/io_out/source_pod.d index 0ed26db..0b014ca 100644 --- a/src/sisudoc/outputs/io_out/source_pod.d +++ b/src/sisudoc/outputs/io_out/source_pod.d @@ -77,11 +77,15 @@ template spinePod() { regeneration, but only on the first language of a multi-language document, so that subsequent languages' files are not wiped. . - media/abstraction/ is left alone: the .ssp files there are written - by the abstraction stage, which runs before this, and for a - multi-language document they accumulate one language at a time. - That directory clears itself, in the .ssp writer, on the first - language of a run. + media/abstraction/ is left standing: the .ssp files there are + written by the abstraction stage, which runs before this, and for + a multi-language document they accumulate one language at a time. + What is pruned from it instead is any .ssp that does not belong to + a language this document has, which would otherwise linger from an + earlier run and be bundled. That is done here rather than in the + writer because pod actions run serially and the manifest's list of + languages is to hand, where the writer may be one of several + languages running at once. +/ if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0]) { string doc_pod_dir = pths_pod.base_filesystem_(doc_matters.src.filename); @@ -103,6 +107,20 @@ template spinePod() { } } } + if (exists(keep_dir) && keep_dir.isDir) { + string _doc_uid_base + = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length]; + foreach (string _f; dirEntries(keep_dir, "*.ssp", SpanMode.shallow)) { + bool _wanted = false; + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + if (_f.baseName == (_doc_uid_base ~ _lang ~ ".ssp")) { + _wanted = true; + break; + } + } + if (!_wanted) { _f.remove; } + } + } } { podArchive_directory_tree(doc_matters, pths_pod); diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d index 1c68045..568a102 100644 --- a/src/sisudoc/spine.d +++ b/src/sisudoc/spine.d @@ -115,6 +115,7 @@ string program_name = "spine"; } bool[string] opts = [ "abstraction" : false, + "abstraction-db" : false, "allow-downloads" : false, "assertions" : false, "concordance" : false, @@ -156,6 +157,7 @@ string program_name = "spine"; "manifest" : false, "hide-ocn" : false, "no-ocn" : false, + "ocda-db" : false, "ocn-off" : false, "odf" : false, "odt" : false, @@ -168,7 +170,6 @@ string program_name = "spine"; "pod2" : false, "serial" : false, "show-abstraction" : false, - "show-abstraction-db" : false, "show-config" : false, "show-curate" : false, "show-curate-authors" : false, @@ -238,6 +239,7 @@ string program_name = "spine"; auto helpInfo = getopt(args, std.getopt.config.passThrough, "abstraction", "document abstraction", &opts["abstraction"], + "abstraction-db", "document abstraction (write ocda.db sqlite file)", &opts["abstraction-db"], "allow-downloads", "allow downloads (includes cgi.d from github)", &opts["allow-downloads"], "assert", "set optional assertions on", &opts["assertions"], "cgi-bin-root", "path to cgi-bin directory", &settings["cgi-bin-root"], @@ -272,6 +274,7 @@ string program_name = "spine"; "light", "default light theme", &opts["light"], "manifest", "process manifest output", &opts["manifest"], "no-ocn", "object cite numbers", &opts["no-ocn"], + "ocda-db", "document abstraction (write ocda.db sqlite file)", &opts["ocda-db"], "ocn-off", "object cite numbers", &opts["ocn-off"], "odf", "open document format text (--odt)", &opts["odf"], "odt", "open document format text", &opts["odt"], @@ -298,7 +301,6 @@ string program_name = "spine"; "serial", "serial processing", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-abstraction", "show document abstraction (write .ssp file)", &opts["show-abstraction"], - "show-abstraction-db", "show document abstraction (write .db sqlite file)", &opts["show-abstraction-db"], "show-config", "show config", &opts["show-config"], "show-curate", "show curate", &opts["show-curate"], "show-curate-authors", "show curate authors", &opts["show-curate-authors"], @@ -590,11 +592,14 @@ string program_name = "spine"; @trusted bool latex_document_header_sty() { return (opts["latex-init"] || opts["latex-header-sty"] || opts["pdf-init"]) ? true : false; } + @trusted bool odt() { + return (opts["odf"] || opts["odt"]) ? true : false; + } @trusted bool manifest() { return opts["manifest"]; } - @trusted bool odt() { - return (opts["odf"] || opts["odt"]) ? true : false; + @trusted bool ocda_db() { + return (opts["abstraction-db"] || opts["ocda-db"]) ? true : false; } @trusted bool ocn_hidden() { return opts["hide-ocn"]; @@ -614,9 +619,6 @@ string program_name = "spine"; @trusted bool show_abstraction() { return (opts["show-abstraction"] || pod2) ? true : false; } - @trusted bool show_abstraction_db() { - return opts["show-abstraction-db"]; - } @trusted bool show_curate() { return opts["show-curate"]; } @@ -866,7 +868,7 @@ string program_name = "spine"; return ( opts["abstraction"] || show_abstraction - || show_abstraction_db + || ocda_db || concordance || source_or_pod || curate @@ -894,7 +896,7 @@ string program_name = "spine"; || odt || manifest || show_abstraction - || show_abstraction_db + || ocda_db || show_make || show_metadata || show_summary @@ -910,7 +912,7 @@ string program_name = "spine"; return ( opts["abstraction"] || show_abstraction - || show_abstraction_db + || ocda_db || curate || html || epub @@ -926,7 +928,7 @@ string program_name = "spine"; return ( opts["abstraction"] || show_abstraction - || show_abstraction_db + || ocda_db || html || epub || odt @@ -1440,9 +1442,9 @@ string program_name = "spine"; come out are what is written, so the database cannot carry anything the .ssp does not +/ - if (doc.matters.opt.action.show_abstraction_db) { + if (doc.matters.opt.action.ocda_db) { import sisudoc.ocda.abstraction.ssp_in; - import sisudoc.outputs.io_out.create_abstraction_db; + import sisudoc.outputs.io_out.create_ocda_db; mixin spineAbstractionRead; spineAbstractionDb!()(doc, sspRoundTripAbstraction(doc)); } @@ -1555,9 +1557,9 @@ string program_name = "spine"; come out are what is written, so the database cannot carry anything the .ssp does not +/ - if (doc.matters.opt.action.show_abstraction_db) { + if (doc.matters.opt.action.ocda_db) { import sisudoc.ocda.abstraction.ssp_in; - import sisudoc.outputs.io_out.create_abstraction_db; + import sisudoc.outputs.io_out.create_ocda_db; mixin spineAbstractionRead; spineAbstractionDb!()(doc, sspRoundTripAbstraction(doc)); } diff --git a/test/run-tests.sh b/test/run-tests.sh index c8869d8..85bbf03 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -64,19 +64,26 @@ test-abstraction-db-roundtrip.sh" PASSED=0 FAILED=0 RESULTS="" +LOG="${TMPDIR:-/tmp}/spine-run-tests.$$" for t in $TESTS; do printf '%s\n' "── $t" - if sh "$SCRIPT_DIR/$t" $SPINE_BIN 2>&1 | sed 's/^/ /'; then + # the status has to be the test's, not the last command of a pipeline: + # "cmd | sed" reports sed's status, which is always 0, and a harness that + # cannot fail is worse than no harness + if sh "$SCRIPT_DIR/$t" $SPINE_BIN > "$LOG" 2>&1; then + sed 's/^/ /' "$LOG" PASSED=$((PASSED + 1)) RESULTS="$RESULTS ok $t" else + sed 's/^/ /' "$LOG" FAILED=$((FAILED + 1)) RESULTS="$RESULTS FAIL $t" fi done +rm -f "$LOG" echo "" echo "summary:$RESULTS" diff --git a/test/test-abstraction-db-roundtrip.sh b/test/test-abstraction-db-roundtrip.sh old mode 100644 new mode 100755 index b3d5b0d..338f375 --- a/test/test-abstraction-db-roundtrip.sh +++ b/test/test-abstraction-db-roundtrip.sh @@ -57,7 +57,7 @@ mkdir -p "$OUT_DIR" echo "spine binary: $SPINE_BIN" echo "samples: $SAMPLES_DIR" echo "Generating .ssp and .ocda.db ..." -$SPINE_BIN --show-abstraction --show-abstraction-db --skip-output \ +$SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 COUNT=0 diff --git a/test/test-abstraction-db.sh b/test/test-abstraction-db.sh old mode 100644 new mode 100755 index 27a6e65..31e5fa5 --- a/test/test-abstraction-db.sh +++ b/test/test-abstraction-db.sh @@ -66,7 +66,7 @@ echo "samples: $SAMPLES_DIR" echo "Generating .ssp and .ocda.db ..." rm -rf "$OUT_DIR" mkdir -p "$OUT_DIR" -$SPINE_BIN --show-abstraction --show-abstraction-db --skip-output \ +$SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 FAILURES=0 -- cgit v1.2.3