From c16ada7749d842fa868b7b8b25a1ff17ffdf7b40 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Tue, 8 Sep 2026 16:09:44 -0400 Subject: output: the abstraction artefacts live with the pod pod/ holds all document source representations: pod// source tree pod//media/abstraction/.ssp abstraction, as text pod/.zip tree, zipped, .ssp included pod/.digests.txt sha256s of what is in them pod/.ocda.db abstraction, as sqlite db /abstraction/ is gone. pod//media/abstraction/.ssp preferred as having the images (found within the pod tree) which .ssp needs to reproduce a document but does not carry on its own. The .ocda.db sits carries the images as well and (like the pod.zip) can be used to reproduce a document directly. digests.txt now covers the database as well as the zip, the source and the .ssp; (as does the metadata html page). Two ordering issues addressed: - the pod builder clean-slates pod// before regenerating it, and the .ssp is now written before that runs. The clean slate now leaves media/abstraction/ alone, and the .ssp writer clears that directory itself on the first language of a run, so a .ssp for a language the document no longer is removed and cannot be bundled. - for a multi-language document the .ssp files accumulate one language at a time and are bundled on the last, which is why the directory cannot simply be emptied by whichever (language) gets there first. (assisted by Claude-Code) --- src/sisudoc/outputs/io_out/create_abstraction_db.d | 16 +++-- src/sisudoc/outputs/io_out/metadata.d | 17 ++++++ src/sisudoc/outputs/io_out/source_pod.d | 68 +++++++++++++++++----- 3 files changed, 81 insertions(+), 20 deletions(-) (limited to 'src/sisudoc/outputs') diff --git a/src/sisudoc/outputs/io_out/create_abstraction_db.d b/src/sisudoc/outputs/io_out/create_abstraction_db.d index e862aba..9b1af4d 100644 --- a/src/sisudoc/outputs/io_out/create_abstraction_db.d +++ b/src/sisudoc/outputs/io_out/create_abstraction_db.d @@ -60,6 +60,7 @@ template spineAbstractionDb() { import std.array; import d2sqlite3; import sisudoc.outputs.io_out.paths_output; + import sisudoc.ocda.io_in.paths_source; /+ ↓ the abstraction is passed in rather than taken from doc, because the caller hands over one that has been through the .ssp: written as text and read back. so the database is a function of the .ssp and cannot @@ -67,11 +68,16 @@ template spineAbstractionDb() { +/ void spineAbstractionDb(D,A)(D doc, A doc_abstraction) { auto doc_matters = doc.matters; - - /+ ↓ determine output path +/ - auto out_pth = spineOutPaths!()(doc_matters.output_path, doc_matters.src.language); - string base_dir = "abstraction"; - string base_pth = ((out_pth.output_base.chainPath(base_dir)).asNormalizedPath).array; + /+ ↓ output path: beside the pod's zip and digests, not under + /abstraction/. the database carries its own images, so it is a + distributable artefact in its own right and belongs with the others: + pod// the source tree + pod/.zip the same, zipped + pod/.digests.txt what is in them + pod/.ocda.db the abstraction, one per language + +/ + auto pths_pod = spinePathsPods!()(doc_matters); + string base_pth = pths_pod.pod_dir_(); try { if (!exists(base_pth)) { base_pth.mkdirRecurse; diff --git a/src/sisudoc/outputs/io_out/metadata.d b/src/sisudoc/outputs/io_out/metadata.d index ff9c72f..f2aaa53 100644 --- a/src/sisudoc/outputs/io_out/metadata.d +++ b/src/sisudoc/outputs/io_out/metadata.d @@ -529,6 +529,23 @@ string theme_light_1 = format(q"┃ writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); } } + if (doc_matters.opt.action.show_abstraction_db) { + try { // sha digest and size for the abstraction database, beside the pod + string _db_fn = pths_pod.pod_dir_() ~ "/" + ~ doc_matters.src.doc_uid_out ~ ".ocda.db"; + if (_db_fn.exists) { + auto data = (cast(byte[]) (_db_fn).read); + metadata_ ~= "

"; + metadata_ ~= "" ~ data.sha256Of.toHexString ~ "::" + ~ data.length.to!string ~ " - " + ~ doc_matters.src.doc_uid_out ~ ".ocda.db"; + metadata_ ~= "

"; + } + } catch (Exception ex) { + writeln("WARNING, abstraction db not read for metadata page: ", + doc_matters.src.doc_uid_out, ".ocda.db"); + } + } } if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) { metadata_ ~= "

Topics:

"; diff --git a/src/sisudoc/outputs/io_out/source_pod.d b/src/sisudoc/outputs/io_out/source_pod.d index b32abcd..0ed26db 100644 --- a/src/sisudoc/outputs/io_out/source_pod.d +++ b/src/sisudoc/outputs/io_out/source_pod.d @@ -73,13 +73,35 @@ template spinePod() { assert (doc_matters.src.filename.match(rgx_files.src_fn)); if (doc_matters.opt.action.source_or_pod) { try { - /+ ↓ clean slate: remove per-document pod directory before regeneration, - but only on the first language of a multi-language document, - so that subsequent languages' files are not wiped +/ + /+ ↓ clean slate: remove the per-document pod directory before + regeneration, but only on the first language of a multi-language + document, so that subsequent languages' files are not wiped. + . + media/abstraction/ is left alone: the .ssp files there are written + by the abstraction stage, which runs before this, and for a + multi-language document they accumulate one language at a time. + That directory clears itself, in the .ssp writer, on the first + language of a run. + +/ if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0]) { string doc_pod_dir = pths_pod.base_filesystem_(doc_matters.src.filename); + string keep_dir + = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string; if (exists(doc_pod_dir) && doc_pod_dir.isDir) { - doc_pod_dir.rmdirRecurse; + foreach (string _e; dirEntries(doc_pod_dir, SpanMode.shallow)) { + if (_e.isDir) { + if (keep_dir.startsWith(_e)) { // media/, which holds abstraction/ + foreach (string _m; dirEntries(_e, SpanMode.shallow)) { + if (_m == keep_dir) { continue; } + if (_m.isDir) { _m.rmdirRecurse; } else { _m.remove; } + } + } else { + _e.rmdirRecurse; + } + } else { + _e.remove; + } + } } } { @@ -208,24 +230,24 @@ template spinePod() { } } } - } { // bundle abstraction .ssp file (only for --pod2) + } { // the abstraction .ssp, written in place by --show-abstraction (only for --pod2) if (doc_matters.opt.action.pod2) { - if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp generated - import sisudoc.outputs.io_out.paths_output; - /+ doc_uid_out for any language follows the same pattern, differing + if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp written + /+ the .ssp is written straight into pod//media/abstraction/, + so there is nothing to copy here: it is digested and added to + the zip where it stands. + doc_uid_out for any language follows the same pattern, differing only in the trailing ".{lng}". Strip the current language to - reuse the base across all languages. +/ + reuse the base across all languages. + +/ string _doc_uid_base = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length]; foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions - auto out_pth_lng = spineOutPaths!()(doc_matters.output_path, _lang); - string abstraction_dir = ((out_pth_lng.output_base.chainPath("abstraction")).asNormalizedPath).array; string ssp_filename = _doc_uid_base ~ _lang ~ ".ssp"; - string fn_src_in = ((abstraction_dir.chainPath(ssp_filename)).asNormalizedPath).array.to!string; auto fn_src_out_pod_zip_base = pths_pod.abstraction_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ ssp_filename; - auto fn_src_out_filesystem + auto fn_src_in = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string ~ "/" ~ ssp_filename; if (exists(fn_src_in)) { @@ -235,13 +257,26 @@ template spinePod() { _digests[_lang]["ssp"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ ssp_filename ~ "\n"; } - fn_src_in.copy(fn_src_out_filesystem); zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src out NOT found (abstraction): ", fn_src_in); } } + /+ ↓ the .ocda.db sits beside the pod rather than inside it: it + carries the images itself, and putting it in the zip would + store them twice. digested here so that digests.txt covers + every artefact the pod directory holds for this document + +/ + { + string db_filename = _doc_uid_base ~ _lang ~ ".ocda.db"; + string fn_db = pths_pod.pod_dir_() ~ "/" ~ db_filename; + if (exists(fn_db)) { + ST_file_stat file_ = getFileStat(fn_db); + _digests[_lang]["ocda_db"] ~= file_.sha256hash.toHexString + ~ "::" ~ file_.fileSize.to!string ~ " - " ~ db_filename ~ "\n"; + } + } } } } @@ -539,7 +574,10 @@ template spinePod() { f.writeln(_digests[_lang]["ssi"]); } if (("ssp" in _digests[_lang]) && (_digests[_lang]["ssp"].length > 0)) { - f.writeln(_digests[_lang]["ssp"]); + f.writeln(_digests[_lang]["ssp"].strip); + } + if (("ocda_db" in _digests[_lang]) && (_digests[_lang]["ocda_db"].length > 0)) { + f.writeln(_digests[_lang]["ocda_db"].strip); } } } -- cgit v1.2.3