verify: this spine still produces this abstraction?

does this spine still produce this abstraction?

A database states an abstraction and carries the markup it was built
from, so re-parsing the one and comparing against the other says whether
the two still agree. Checked as follows:
- automatically and unskippably when a document is built from a
  database, at the moment it is parsed and before anything has been
  written from it.
- on --ocda-verify=<file>, an action of its own that materialises,
  parses every language, compares, reports and writes nothing, the exit
  status being the answer.
- on --no-verify as the escape, warning per document, for a newer spine
  reading an older artefact where the abstraction is expected to differ
  and the output is wanted anyway.

A build ends at the first failure rather than skipping the language.

A document rebuilt from a database now has its whole abstraction built.

(assisted by Claude-Code)

2 files changed, 226 insertions, 2 deletions

diff --git a/org/in_abstraction_artefacts.org b/org/in_abstraction_artefacts.org
index 71d8d65..f688c33 100644
--- a/org/in_abstraction_artefacts.org
+++ b/org/in_abstraction_artefacts.org
@@ -624,6 +624,34 @@ template spineDbLanguages() {
     return _out;
   }
 }
+/+ ↓ the .ssp digest a database recorded for each of its languages.
+     .
+     What verify holds its answer against: the database says which .ssp it was
+     built from, so re-parsing the markup it carries and emitting the .ssp again
+     either reproduces that digest or does not.
+     .
+     Its own template, for the reason spineDbLanguages is: asking a file what it
+     recorded should not instantiate the reader.
++/
+template spineDbSspDigests() {
+  @trusted string[string] spineDbSspDigests(string db_file) {
+    import std.file : exists;
+    import d2sqlite3;
+    string[string] _out;
+    if (!db_file.exists) { return _out; }
+    try {
+      auto db = Database(db_file, SQLITE_OPEN_READONLY);
+      foreach (row; db.execute(
+        "SELECT lang, ssp_digest FROM documents ORDER BY id")
+      ) {
+        _out[row["lang"].as!string] = row["ssp_digest"].as!string;
+      }
+    } catch (Exception ex) {
+      return string[string].init;
+    }
+    return _out;
+  }
+}
 template spineAbstractionDbRead() {
   import std.conv : to;
   import std.file;
diff --git a/org/spine.org b/org/spine.org
index 2ec7709..f89885b 100644
--- a/org/spine.org
+++ b/org/spine.org
@@ -73,6 +73,10 @@ string program_name = "spine";
   +/
   mixin spineOcnAlign _ocna;
   _ocna.ST_OcnProfile[] _ocn_profiles;
+  /+ ↓ what verify found, gathered as the documents go by +/
+  size_t   _verify_checked = 0;
+  size_t   _verify_failed  = 0;
+  string[] _verify_notes;
   if (_manifests.length > 1                            // _manifests[0] initialized dummy element
   && _opt_action.abstraction) {
     <<spine_initialize_output>>
@@ -82,6 +86,7 @@ string program_name = "spine";
         if (!empty(manifest.src.filename)) {
           <<spine_each_file_do_scope>>
           <<spine_each_file_do_abstraction>>
+          <<spine_each_file_do_verify_abstraction>>
           <<spine_each_file_do_show_sync>>
           <<spine_each_file_do_selected_output>>
           <<spine_each_file_do_scope_exit>>
@@ -95,6 +100,7 @@ string program_name = "spine";
         if (!empty(manifest.src.filename)) {
           <<spine_each_file_do_scope>>
           <<spine_each_file_do_abstraction>>
+          <<spine_each_file_do_verify_abstraction>>
           <<spine_each_file_do_show_no_sync>>
           <<spine_each_file_do_selected_output>>
           <<spine_each_file_do_scope_exit>>
@@ -104,6 +110,24 @@ string program_name = "spine";
       }
     }
   }
+  /+ ↓ what verify found, said once at the end.
+       A run that was only asked to verify says so plainly and its exit status
+       is the answer. A run that was building says it too, because a document
+       whose markup no longer makes its abstraction was skipped and the user has
+       to know why their output is short.
+  +/
+  if (_verify_checked > 0) {
+    foreach (_n; _verify_notes) { stderr.writeln("  ", _n); }
+    if (_opt_action.ocda_verify || _opt_action.vox_gt_1 || _verify_failed > 0) {
+      writeln("verify: ", _verify_checked - _verify_failed, " of ",
+        _verify_checked, " re-parse to the abstraction recorded with them");
+    }
+  } else if (_opt_action.ocda_verify) {
+    stderr.writeln("verify: nothing was checked. ",
+      _opt_action.ocda_verify_file.baseName,
+      " carries no markup, or none that could be read");
+    _verify_failed += 1;
+  }
   /+ ↓ the languages of a document, held against each other.
        .
        Here because it is the first point at which it can be done: a language
@@ -268,6 +292,20 @@ string program_name = "spine";
     import core.stdc.stdlib : exit;
     exit(1);
   }
+  /+ ↓ a verification that failed is a failed run, whatever else went well.
+       --no-verify has already said its piece per document and does not reach
+       here: it turns the failure into a warning and the output is produced
+       deliberately.
+  +/
+  if (_verify_failed > 0 && !(_opt_action.no_verify)) {
+    stderr.writefln(
+      "~ run FAILED ~ verify: %s of %s did not re-parse to the abstraction"
+      ~ " recorded with them",
+      _verify_failed, _verify_checked,
+    );
+    import core.stdc.stdlib : exit;
+    exit(1);
+  }
   /+ ↓ sqlite statements that failed are fatal to the run, report and exit non-zero +/
   {
     import sisudoc.outputs.io_out.sqlite_collection_db : sqliteFailureTally;
@@ -551,6 +589,7 @@ bool[string] opts = [
   "section_bookindex"           : true,
   "section_blurb"               : true,
   "backmatter"                  : true,
+  "no-verify"                   : false,
   "skip-output"                 : false,
   "strict"                      : false,
   "theme-dark"                  : false,
@@ -570,6 +609,7 @@ string[string] settings = [
   "pod-compression"             : "",
   "ssp-round-trip"              : "",
   "db-round-trip"               : "",
+  "ocda-verify"                 : "",
   "abstraction-source"          : "",
   "www-http"                    : "",
   "www-host"                    : "",
@@ -641,6 +681,8 @@ auto helpInfo = getopt(args,
   "output",                     "=/path/to/output/dir specify where to place output",             &settings["output"],
   "ssp-round-trip",             "=/path/to/file.ssp read a .ssp back and re-emit it on stdout",   &settings["ssp-round-trip"],
   "db-round-trip",              "=/path/to/file.ocda.db read it back and emit .ssp on stdout",    &settings["db-round-trip"],
+  "ocda-verify",                "=/path/to/file.ocda.db does this spine still make this abstraction from the markup it carries", &settings["ocda-verify"],
+  "no-verify",                  "do not check carried markup against the abstraction it was built with", &opts["no-verify"],
   "abstraction-source",         "=/path/to/(.sst|pod|.ssp|.ocda.db) identify it, and load it if it is an abstraction", &settings["abstraction-source"],
   "parallel",                   "parallelise document processing (opt-in; slower than serial)",   &opts["parallel"],
   "parallel-subprocesses",      "nested parallelisation",                                         &opts["parallel-subprocesses"],
@@ -1080,7 +1122,33 @@ struct OptActions {
        asked for in the same run still happens from the abstraction.
   +/
   @trusted bool needs_markup_source() {
-    return (source_or_pod || show_abstraction || ocda_db) ? true : false;
+    return (source_or_pod || show_abstraction || ocda_db || ocda_verify)
+      ? true : false;
+  }
+  /+ ↓ verify: does this spine still make this abstraction from the markup the
+       artefact carries?
+       .
+       A .ssp or a database states an abstraction and, since a .ocda.db carries
+       the markup it was built from; re-parsing the one and comparing against
+       the other is the only check that the two still agree, and it is the check
+       that says whether a document travels through time as well as through a
+       file.
+       .
+       --ocda-verify names a file and asks only that. It is an action in its own
+       right: no output is written, and the run's exit status is the answer.
+  +/
+  @trusted bool ocda_verify() {
+    return (settings["ocda-verify"].length > 0);
+  }
+  @trusted string ocda_verify_file() {
+    return settings["ocda-verify"];
+  }
+  /+ ↓ the escape, for the case it is actually wanted: a newer spine reading an
+       older artefact, where the abstraction is expected to differ and the
+       output is wanted anyway. It warns.
+  +/
+  @trusted bool no_verify() {
+    return opts["no-verify"];
   }
   @trusted bool sqlite_discrete() {
     return opts["sqlite-discrete"];
@@ -1176,7 +1244,11 @@ struct OptActions {
     return opts["backmatter"];
   }
   @trusted bool skip_output() {
-    return opts["skip-output"];
+    /+ ↓ --ocda-verify answers a question and writes nothing: the run's
+         exit status is the answer, and a check that left output behind
+         could not be run against a published tree
+    +/
+    return (opts["skip-output"] || ocda_verify) ? true : false;
   }
   @trusted bool strict() {
     return opts["strict"];
@@ -1320,6 +1392,7 @@ struct OptActions {
       opts["abstraction"]
       || show_abstraction
       || ocda_db
+      || ocda_verify
       || concordance
       || source_or_pod
       || curate
@@ -1348,6 +1421,7 @@ struct OptActions {
       || manifest
       || show_abstraction
       || ocda_db
+      || ocda_verify
       || show_make
       || show_metadata
       || show_summary
@@ -1375,11 +1449,28 @@ struct OptActions {
       || skel
     ) ? true :false;
   }
+  /+ ↓ the dom structure tags, which the .ssp records and which are only
+       computed when something needs them.
+       .
+       --ocda-verify belongs on this list and the omission cost an hour:
+       verify re-parses the carried markup and compares the .ssp it would
+       write against the one the artefact holds, and every artefact is
+       written by a run that has this on (show_abstraction and ocda_db
+       both turn it on). Without it, verify built an abstraction missing
+       the dom tags, and reported ten sound languages as ten failures.
+       .
+       Worth knowing more generally: the abstraction is not one thing
+       regardless of what was asked for. --text alone builds one without
+       these tags. That is invisible in output, since only html, epub, odt
+       and the databases use them, but anything comparing abstractions has
+       to compare like with like.
+  +/
   @trusted bool meta_processing_xml_dom() {
     return (
       opts["abstraction"]
       || show_abstraction
       || ocda_db
+      || ocda_verify
       || html
       || epub
       || odt
@@ -1507,7 +1598,17 @@ foreach (arg; args[1..$]) {
      A database without source markup still refuses, as it has to: no artefact
      written by an older spine carries any.
 +/
+/+ ↓ --ocda-verify names its file rather than taking it as an argument, so
+     that the question is unmistakable in the command line. It joins the
+     arguments here and is materialised and parsed like any other database;
+     what makes it a verification rather than a build is that nothing is
+     written and the comparison below decides the exit status.
++/
+if (_opt_action.ocda_verify) {
+  _resolved_args ~= _opt_action.ocda_verify_file;
+}
 string[] _pod_materialisations;
+string[string] _pod_came_from_db;   // pod directory -> the database
 if (_opt_action.needs_markup_source) {
   import sisudoc.ocda.abstraction.pod_from_db;
   import std.process : thisProcessID;
@@ -1525,7 +1626,18 @@ if (_opt_action.needs_markup_source) {
     auto _mat = podFromDb!()(arg, _root, _opt_action);
     if (_mat.ok) {
       _pod_materialisations ~= _root;
+      _pod_came_from_db[_mat.pod_dir] = arg;
       _args_after ~= _mat.pod_dir;
+      /+ ↓ a document rebuilt from a database gets the whole abstraction
+           built, dom structure tags and all.
+           .
+           Because that is what it will be compared against: the artefact
+           recorded its abstraction from a run that had these on, every
+           run that writes one having them on. Parsed without them, the
+           same markup makes a different .ssp, and verify would report ten
+           sound languages as ten failures. It did, for an hour.
+      +/
+      opts["abstraction"] = true;
       if (_opt_action.vox_gt_1) {
         writeln("pod from database: ", arg.baseName, " -> ", _mat.pod_dir);
       }
@@ -1983,6 +2095,90 @@ if ((doc.matters.opt.action.debug_do)
 }
 #+END_SRC
 
+*** verify _document abstraction_ [#A]
+
+#+NAME: spine_each_file_do_verify_abstraction
+#+BEGIN_SRC d
+/+ ↓ verify: the markup the database carried, re-parsed, against
+     the abstraction the database holds.
+     .
+     Here and not later because this is the last moment before
+     anything is written from it. The document has been parsed and
+     nothing has been produced yet, so a mismatch can still stop
+     the writing rather than be discovered in it.
+     .
+     Only for a document that came from a database: a pod given on
+     the command line is the source, and there is nothing to hold
+     it against.
++/
+bool _verified = true;
+if (_pod_came_from_db.length > 0) {
+  import sisudoc.ocda.abstraction.ssp_in;
+  import sisudoc.ocda.abstraction.db_in : spineDbSspDigests;
+  mixin spineAbstractionRead _vfy;
+  string _pod_key
+    = (doc.matters.pod.manifest_path.asNormalizedPath).array.to!string;
+  if (auto _db = _pod_key in _pod_came_from_db) {
+    string _made     = _vfy.sspRoundTripDocument(doc).ssp_digest;
+    auto   _recorded = spineDbSspDigests!()(*_db);
+    string _lang     = doc.matters.src.language;
+    string _was      = (_lang in _recorded) ? _recorded[_lang] : "";
+    synchronized {
+      _verify_checked += 1;
+      if (_was.length == 0) {
+        _verify_notes ~= (*_db).baseName ~ " [" ~ _lang
+          ~ "] records no abstraction digest to check against";
+        _verify_failed += 1;
+        _verified = false;
+      } else if (_was != _made) {
+        _verify_notes ~= (*_db).baseName ~ " [" ~ _lang
+          ~ "] the markup it carries no longer makes the"
+          ~ " abstraction it holds";
+        _verify_notes ~= "    recorded " ~ _was;
+        _verify_notes ~= "    made now " ~ _made;
+        _verify_failed += 1;
+        _verified = false;
+      }
+    }
+  }
+}
+if (!_verified) {
+  if (_opt_action.no_verify) {
+    stderr.writeln("WARNING: --no-verify, so this is built from",
+      " markup that no longer makes the abstraction recorded with",
+      " it: ", manifest.src.filename, " [", manifest.src.language, "]");
+  } else if (_opt_action.ocda_verify) {
+    /+ ↓ verify writes nothing, so the rest of the languages are
+         worth checking: the answer wanted is how much of the file
+         is sound, not which language failed first
+    +/
+    continue;
+  } else {
+    /+ ↓ a build stops here, and does not skip the language and
+         carry on.
+         .
+         Skipping one language of a pod leaves the pod writer
+         carrying markup for a language whose directory was never
+         made, which threw a FileException out of the run: a half
+         written pod, and an exception instead of an explanation.
+         A document whose carried markup no longer makes its
+         recorded abstraction is not one to write a pod from at
+         all, so the run ends here and says so.
+    +/
+    foreach (_n; _verify_notes) { stderr.writeln("  ", _n); }
+    stderr.writeln("~ run FAILED ~ verify: ", manifest.src.filename,
+      " [", manifest.src.language, "] is built from markup that no",
+      " longer makes the abstraction recorded with it.");
+    stderr.writeln("  Anything already written is incomplete.",
+      " --ocda-verify=<file> reports on every language, and",
+      " --no-verify builds anyway.");
+    import core.stdc.stdlib : exit;
+    stdout.flush;
+    exit(1);
+  }
+}
+#+END_SRC
+
 *** 2. _output processing_ (post abstraction processing)
 **** 0. abstraction
 ***** show document summary - abstraction _print summary_ (to screen) :abstraction:summary: