aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--org/in_source_artefact.org126
-rw-r--r--org/in_source_files.org32
-rw-r--r--org/spine.org8
-rw-r--r--src/sisudoc/ocda/abstraction/db_in.d32
-rw-r--r--src/sisudoc/ocda/abstraction/doc_from_artefact.d126
-rw-r--r--src/sisudoc/spine.d8
6 files changed, 314 insertions, 18 deletions
diff --git a/org/in_source_artefact.org b/org/in_source_artefact.org
index 0f7077d..07c865f 100644
--- a/org/in_source_artefact.org
+++ b/org/in_source_artefact.org
@@ -27,26 +27,25 @@
module sisudoc.ocda.abstraction.doc_from_artefact;
@safe:
/+ ↓ a document, from an artefact rather than from markup
-
spineAbstraction() parses markup and hands the output writers a doc:
an abstraction and a doc_matters. This does the same from a .ssp or a
.ocda.db, filling the same docMattersMake() arguments, so that what
comes out is the value the writers already take and they never learn
which of the two made it.
-
+ .
Three of the seven arguments come from the run and are the same either
way: program_info, opt_action, and the conf half of conf_make_meta,
which is site and run scoped (urls, papersize, the search database) and
has no business in a document's own artefact. The other four come from
the artefact:
-
+ .
conf_make_meta meta and make from the @meta and @make blocks, over
the site config the run assembled
doc_has docHasFromAbstraction, over the objects
doc_digest the markup digest from @source
manifest synthesised from the artefact's path and the
document's own name and language
-
+ .
What is not here, deliberately: insert_file_list, which is read only by
source_pod.d for --source and --pod2. No artefact carries the markup,
so a pod cannot be rebuilt from one, and the empty list is the truthful
@@ -57,6 +56,7 @@ template spineDocFromArtefact() {
import std.array;
import std.conv : to;
import std.path;
+ import std.process : thisProcessID;
import std.regex;
import std.stdio;
import std.string;
@@ -83,7 +83,6 @@ template spineDocFromArtefact() {
return _member;
}
/+ ↓ the document's own header, over the site config the run assembled.
-
every string field of MetaComposite is looked for under its own key,
so a field added there and emitted by the writer arrives here without
anything being added. The arr fields are split from the string they
@@ -145,14 +144,13 @@ template spineDocFromArtefact() {
return _out;
}
/+ ↓ where a document loaded from an artefact says it is.
-
an artefact names its document (% Source:) and its language (@source
language) but not where the pod holding its images sits, so that is
taken from where the artefact itself was found:
-
+ .
pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/
pod/<uid>.ocda.db -> pod/<doc>/
-
+ .
which is where --pod2 puts them both. The images beside a .ssp are
then found where html.d already looks; a .ocda.db carries its own and
does not need the directory to exist.
@@ -183,8 +181,90 @@ template spineDocFromArtefact() {
.chainPath(_ssp.source).array).to!string;
return _out;
}
+ /+ ↓ a database's images, written where the writers look for them.
+ returns the pod directory to use in place of the one beside the
+ artefact, or "" when the database carries no images and the pod
+ beside it should stand. Each blob is checked against the sha256 the
+ database recorded with it: the two were written together, so a
+ mismatch means the file is damaged, and saying so is the point of
+ having recorded the digest at all.
+ +/
+ @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) {
+ import std.digest : toHexString;
+ import std.digest.sha : sha256Of;
+ import std.file : mkdirRecurse, write, tempDir;
+ import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead;
+ mixin spineAbstractionDbRead _dbr;
+ auto _files = _dbr.dbReadFiles(_artefact, "image");
+ if (_files.length == 0) { return ""; }
+ string _root = (tempDir.chainPath("spine-ocda-"
+ ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string;
+ string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string;
+ try {
+ _img_dir.mkdirRecurse;
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not make an image directory for ", _artefact,
+ ": ", ex.msg);
+ return "";
+ }
+ foreach (_f; _files) {
+ string _got = _f.data.sha256Of.toHexString.to!string;
+ if (_f.sha256.length > 0 && _got != _f.sha256) {
+ stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName,
+ " does not match the digest recorded with it (", _f.sha256, " expected, ",
+ _got, " found); it is written out as it stands");
+ }
+ try {
+ (_img_dir.chainPath(_f.name).array).to!string.write(_f.data);
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg);
+ }
+ }
+ if (_opt_action.vox_gt_1) {
+ writeln(" ", _files.length, " image(s) from the database, in ", _img_dir);
+ }
+ return _root;
+ }
+ /+ ↓ the images a .ssp describes, against the ones beside it.
+ a .ssp carries no image bytes, so what it can do is say whether the
+ files in the pod it sits in are the ones the abstraction was built
+ from. Nothing checked that before: a changed or wrong image in the
+ directory paired silently.
+ +/
+ @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) {
+ import std.digest : toHexString;
+ import std.digest.sha : sha256Of;
+ import std.file : exists, read;
+ string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string;
+ int _checked, _missing, _wrong;
+ foreach (_section; _ssp.section_order) {
+ foreach (obj; _ssp.abstraction[_section]) {
+ foreach (_img; obj.metainfo.sha256.images) {
+ if (_img.fileName.length == 0) { continue; }
+ string _pth = (_img_dir.chainPath(_img.fileName).array).to!string;
+ if (!_pth.exists) { ++_missing; continue; }
+ ++_checked;
+ string _want = _img.fileHash_sha256.toHexString.to!string;
+ string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string;
+ if (_want != _got && _want != "00000000000000000000000000000000"
+ ~ "00000000000000000000000000000000") {
+ ++_wrong;
+ stderr.writeln("WARNING: image ", _img.fileName, " beside ",
+ _ssp.source, " is not the one the abstraction was built from");
+ }
+ }
+ }
+ }
+ if (_missing > 0) {
+ stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are",
+ " not in ", _img_dir);
+ }
+ if (_opt_action.vox_gt_1 && _checked > 0) {
+ writeln(" ", _checked, " image(s) checked against the abstraction's digests",
+ (_wrong == 0) ? ", all matching" : "");
+ }
+ }
/+ ↓ the doc the output writers take, from an artefact.
-
the reading is done here rather than by the caller, and that is not
only tidiness: main() is @system, so a template mixed in there infers
@system for everything it declares, and the output writers are @safe
@@ -203,6 +283,33 @@ template spineDocFromArtefact() {
auto _loaded = abstractionLoad(_artefact);
auto _ssp = _loaded.doc;
auto _pths = artefactPaths(_artefact, _ssp);
+ /+ ↓ the images.
+ a .ocda.db carries its own, which is what lets it travel on its
+ own, so they are taken from it and written where the output writers
+ look for images: a directory of this run's making, not the pod
+ beside the artefact, which may be somebody's published tree. A .ssp
+ only describes its images, so those are the ones in the pod it sits
+ in, and are checked against the digests it recorded.
+ +/
+ string _images_tmp;
+ if (_loaded.loaded) {
+ if (_artefact.endsWith(".ssp")) {
+ _imagesVerify(_pths.pod_dir, _ssp, _opt_action);
+ } else {
+ string _extracted = _imagesExtract(_artefact, _opt_action);
+ if (_extracted.length > 0) {
+ /+ ↓ the source path has to move with it: image_dir_path is
+ reached from the document's own file, not from the pod +/
+ _images_tmp = _extracted;
+ _pths.pod_dir = _extracted;
+ _pths.src_file_with_path = (_extracted
+ .chainPath("media").chainPath("text")
+ .chainPath(("language" in _ssp.source_info)
+ ? _ssp.source_info["language"] : "en")
+ .chainPath(_ssp.source).array).to!string;
+ }
+ }
+ }
auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path);
auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf);
auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action);
@@ -227,6 +334,7 @@ template spineDocFromArtefact() {
const auto abstraction() { return _abstraction; }
auto matters() { return _matters; }
bool loaded() { return _loaded.loaded; }
+ string images_tmp() { return _images_tmp; }
string note() { return _loaded.note; }
string source_name() { return abstractionSourceName(_loaded.source); }
}
diff --git a/org/in_source_files.org b/org/in_source_files.org
index 7d869c0..b3a5a98 100644
--- a/org/in_source_files.org
+++ b/org/in_source_files.org
@@ -1419,6 +1419,38 @@ template spineAbstractionDbRead() {
}
return doc;
}
+ /+ ↓ the files a database carries.
+
+ a .ocda.db holds its images as blobs, which is what lets it travel on
+ its own where a .ssp cannot: a .ssp describes its images by name,
+ size, pixels and sha256, but does not carry them. The role column
+ says what a file is; today only 'image' is written.
+ +/
+ struct ST_ArtefactFile {
+ string role;
+ string name;
+ ulong bytes;
+ string sha256;
+ ubyte[] data;
+ }
+ @trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") {
+ ST_ArtefactFile[] _out;
+ if (!db_file.exists) { return _out; }
+ auto db = Database(db_file, SQLITE_OPEN_READONLY);
+ foreach (row; db.execute(
+ "SELECT role, name, bytes, sha256, data FROM files WHERE role = '" ~ role
+ ~ "' ORDER BY name")
+ ) {
+ ST_ArtefactFile _f;
+ _f.role = row["role"].as!string;
+ _f.name = row["name"].as!string;
+ _f.bytes = row["bytes"].as!ulong;
+ _f.sha256 = row["sha256"].as!string;
+ _f.data = row["data"].as!(ubyte[]);
+ _out ~= _f;
+ }
+ return _out;
+ }
}
#+END_SRC
diff --git a/org/spine.org b/org/spine.org
index d3a5276..57c3273 100644
--- a/org/spine.org
+++ b/org/spine.org
@@ -144,6 +144,14 @@ string program_name = "spine";
writeln("processed artefact: ", _artefact.baseName,
" [", doc.matters.src.language, "]");
}
+ /+ ↓ the images a database carried, written out for this run only +/
+ if (doc.images_tmp.length > 0) {
+ try {
+ if (doc.images_tmp.exists) { doc.images_tmp.rmdirRecurse; }
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not remove ", doc.images_tmp, ": ", ex.msg);
+ }
+ }
}
}
if (hvst.curates.length > 0) {
diff --git a/src/sisudoc/ocda/abstraction/db_in.d b/src/sisudoc/ocda/abstraction/db_in.d
index 7f920bd..6e90820 100644
--- a/src/sisudoc/ocda/abstraction/db_in.d
+++ b/src/sisudoc/ocda/abstraction/db_in.d
@@ -305,4 +305,36 @@ template spineAbstractionDbRead() {
}
return doc;
}
+ /+ ↓ the files a database carries.
+
+ a .ocda.db holds its images as blobs, which is what lets it travel on
+ its own where a .ssp cannot: a .ssp describes its images by name,
+ size, pixels and sha256, but does not carry them. The role column
+ says what a file is; today only 'image' is written.
+ +/
+ struct ST_ArtefactFile {
+ string role;
+ string name;
+ ulong bytes;
+ string sha256;
+ ubyte[] data;
+ }
+ @trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") {
+ ST_ArtefactFile[] _out;
+ if (!db_file.exists) { return _out; }
+ auto db = Database(db_file, SQLITE_OPEN_READONLY);
+ foreach (row; db.execute(
+ "SELECT role, name, bytes, sha256, data FROM files WHERE role = '" ~ role
+ ~ "' ORDER BY name")
+ ) {
+ ST_ArtefactFile _f;
+ _f.role = row["role"].as!string;
+ _f.name = row["name"].as!string;
+ _f.bytes = row["bytes"].as!ulong;
+ _f.sha256 = row["sha256"].as!string;
+ _f.data = row["data"].as!(ubyte[]);
+ _out ~= _f;
+ }
+ return _out;
+ }
}
diff --git a/src/sisudoc/ocda/abstraction/doc_from_artefact.d b/src/sisudoc/ocda/abstraction/doc_from_artefact.d
index e45cbf0..3a32d25 100644
--- a/src/sisudoc/ocda/abstraction/doc_from_artefact.d
+++ b/src/sisudoc/ocda/abstraction/doc_from_artefact.d
@@ -50,26 +50,25 @@
module sisudoc.ocda.abstraction.doc_from_artefact;
@safe:
/+ ↓ a document, from an artefact rather than from markup
-
spineAbstraction() parses markup and hands the output writers a doc:
an abstraction and a doc_matters. This does the same from a .ssp or a
.ocda.db, filling the same docMattersMake() arguments, so that what
comes out is the value the writers already take and they never learn
which of the two made it.
-
+ .
Three of the seven arguments come from the run and are the same either
way: program_info, opt_action, and the conf half of conf_make_meta,
which is site and run scoped (urls, papersize, the search database) and
has no business in a document's own artefact. The other four come from
the artefact:
-
+ .
conf_make_meta meta and make from the @meta and @make blocks, over
the site config the run assembled
doc_has docHasFromAbstraction, over the objects
doc_digest the markup digest from @source
manifest synthesised from the artefact's path and the
document's own name and language
-
+ .
What is not here, deliberately: insert_file_list, which is read only by
source_pod.d for --source and --pod2. No artefact carries the markup,
so a pod cannot be rebuilt from one, and the empty list is the truthful
@@ -80,6 +79,7 @@ template spineDocFromArtefact() {
import std.array;
import std.conv : to;
import std.path;
+ import std.process : thisProcessID;
import std.regex;
import std.stdio;
import std.string;
@@ -106,7 +106,6 @@ template spineDocFromArtefact() {
return _member;
}
/+ ↓ the document's own header, over the site config the run assembled.
-
every string field of MetaComposite is looked for under its own key,
so a field added there and emitted by the writer arrives here without
anything being added. The arr fields are split from the string they
@@ -168,14 +167,13 @@ template spineDocFromArtefact() {
return _out;
}
/+ ↓ where a document loaded from an artefact says it is.
-
an artefact names its document (% Source:) and its language (@source
language) but not where the pod holding its images sits, so that is
taken from where the artefact itself was found:
-
+ .
pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/
pod/<uid>.ocda.db -> pod/<doc>/
-
+ .
which is where --pod2 puts them both. The images beside a .ssp are
then found where html.d already looks; a .ocda.db carries its own and
does not need the directory to exist.
@@ -206,8 +204,90 @@ template spineDocFromArtefact() {
.chainPath(_ssp.source).array).to!string;
return _out;
}
+ /+ ↓ a database's images, written where the writers look for them.
+ returns the pod directory to use in place of the one beside the
+ artefact, or "" when the database carries no images and the pod
+ beside it should stand. Each blob is checked against the sha256 the
+ database recorded with it: the two were written together, so a
+ mismatch means the file is damaged, and saying so is the point of
+ having recorded the digest at all.
+ +/
+ @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) {
+ import std.digest : toHexString;
+ import std.digest.sha : sha256Of;
+ import std.file : mkdirRecurse, write, tempDir;
+ import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead;
+ mixin spineAbstractionDbRead _dbr;
+ auto _files = _dbr.dbReadFiles(_artefact, "image");
+ if (_files.length == 0) { return ""; }
+ string _root = (tempDir.chainPath("spine-ocda-"
+ ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string;
+ string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string;
+ try {
+ _img_dir.mkdirRecurse;
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not make an image directory for ", _artefact,
+ ": ", ex.msg);
+ return "";
+ }
+ foreach (_f; _files) {
+ string _got = _f.data.sha256Of.toHexString.to!string;
+ if (_f.sha256.length > 0 && _got != _f.sha256) {
+ stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName,
+ " does not match the digest recorded with it (", _f.sha256, " expected, ",
+ _got, " found); it is written out as it stands");
+ }
+ try {
+ (_img_dir.chainPath(_f.name).array).to!string.write(_f.data);
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg);
+ }
+ }
+ if (_opt_action.vox_gt_1) {
+ writeln(" ", _files.length, " image(s) from the database, in ", _img_dir);
+ }
+ return _root;
+ }
+ /+ ↓ the images a .ssp describes, against the ones beside it.
+ a .ssp carries no image bytes, so what it can do is say whether the
+ files in the pod it sits in are the ones the abstraction was built
+ from. Nothing checked that before: a changed or wrong image in the
+ directory paired silently.
+ +/
+ @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) {
+ import std.digest : toHexString;
+ import std.digest.sha : sha256Of;
+ import std.file : exists, read;
+ string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string;
+ int _checked, _missing, _wrong;
+ foreach (_section; _ssp.section_order) {
+ foreach (obj; _ssp.abstraction[_section]) {
+ foreach (_img; obj.metainfo.sha256.images) {
+ if (_img.fileName.length == 0) { continue; }
+ string _pth = (_img_dir.chainPath(_img.fileName).array).to!string;
+ if (!_pth.exists) { ++_missing; continue; }
+ ++_checked;
+ string _want = _img.fileHash_sha256.toHexString.to!string;
+ string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string;
+ if (_want != _got && _want != "00000000000000000000000000000000"
+ ~ "00000000000000000000000000000000") {
+ ++_wrong;
+ stderr.writeln("WARNING: image ", _img.fileName, " beside ",
+ _ssp.source, " is not the one the abstraction was built from");
+ }
+ }
+ }
+ }
+ if (_missing > 0) {
+ stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are",
+ " not in ", _img_dir);
+ }
+ if (_opt_action.vox_gt_1 && _checked > 0) {
+ writeln(" ", _checked, " image(s) checked against the abstraction's digests",
+ (_wrong == 0) ? ", all matching" : "");
+ }
+ }
/+ ↓ the doc the output writers take, from an artefact.
-
the reading is done here rather than by the caller, and that is not
only tidiness: main() is @system, so a template mixed in there infers
@system for everything it declares, and the output writers are @safe
@@ -226,6 +306,33 @@ template spineDocFromArtefact() {
auto _loaded = abstractionLoad(_artefact);
auto _ssp = _loaded.doc;
auto _pths = artefactPaths(_artefact, _ssp);
+ /+ ↓ the images.
+ a .ocda.db carries its own, which is what lets it travel on its
+ own, so they are taken from it and written where the output writers
+ look for images: a directory of this run's making, not the pod
+ beside the artefact, which may be somebody's published tree. A .ssp
+ only describes its images, so those are the ones in the pod it sits
+ in, and are checked against the digests it recorded.
+ +/
+ string _images_tmp;
+ if (_loaded.loaded) {
+ if (_artefact.endsWith(".ssp")) {
+ _imagesVerify(_pths.pod_dir, _ssp, _opt_action);
+ } else {
+ string _extracted = _imagesExtract(_artefact, _opt_action);
+ if (_extracted.length > 0) {
+ /+ ↓ the source path has to move with it: image_dir_path is
+ reached from the document's own file, not from the pod +/
+ _images_tmp = _extracted;
+ _pths.pod_dir = _extracted;
+ _pths.src_file_with_path = (_extracted
+ .chainPath("media").chainPath("text")
+ .chainPath(("language" in _ssp.source_info)
+ ? _ssp.source_info["language"] : "en")
+ .chainPath(_ssp.source).array).to!string;
+ }
+ }
+ }
auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path);
auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf);
auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action);
@@ -250,6 +357,7 @@ template spineDocFromArtefact() {
const auto abstraction() { return _abstraction; }
auto matters() { return _matters; }
bool loaded() { return _loaded.loaded; }
+ string images_tmp() { return _images_tmp; }
string note() { return _loaded.note; }
string source_name() { return abstractionSourceName(_loaded.source); }
}
diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d
index 98c729c..2ce9331 100644
--- a/src/sisudoc/spine.d
+++ b/src/sisudoc/spine.d
@@ -1668,6 +1668,14 @@ string program_name = "spine";
writeln("processed artefact: ", _artefact.baseName,
" [", doc.matters.src.language, "]");
}
+ /+ ↓ the images a database carried, written out for this run only +/
+ if (doc.images_tmp.length > 0) {
+ try {
+ if (doc.images_tmp.exists) { doc.images_tmp.rmdirRecurse; }
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not remove ", doc.images_tmp, ": ", ex.msg);
+ }
+ }
}
}
if (hvst.curates.length > 0) {
generated by cgit v1.2.3 (git 2.25.1) at 2026-10-16 15:56:47 +0000