diff options
Diffstat (limited to 'src/sisudoc')
| -rw-r--r-- | src/sisudoc/ocda/abstraction/db_in.d | 32 | ||||
| -rw-r--r-- | src/sisudoc/ocda/abstraction/doc_from_artefact.d | 126 | ||||
| -rw-r--r-- | src/sisudoc/spine.d | 8 |
3 files changed, 157 insertions, 9 deletions
diff --git a/src/sisudoc/ocda/abstraction/db_in.d b/src/sisudoc/ocda/abstraction/db_in.d index 7f920bd..6e90820 100644 --- a/src/sisudoc/ocda/abstraction/db_in.d +++ b/src/sisudoc/ocda/abstraction/db_in.d @@ -305,4 +305,36 @@ template spineAbstractionDbRead() { } return doc; } + /+ ↓ the files a database carries. + + a .ocda.db holds its images as blobs, which is what lets it travel on + its own where a .ssp cannot: a .ssp describes its images by name, + size, pixels and sha256, but does not carry them. The role column + says what a file is; today only 'image' is written. + +/ + struct ST_ArtefactFile { + string role; + string name; + ulong bytes; + string sha256; + ubyte[] data; + } + @trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") { + ST_ArtefactFile[] _out; + if (!db_file.exists) { return _out; } + auto db = Database(db_file, SQLITE_OPEN_READONLY); + foreach (row; db.execute( + "SELECT role, name, bytes, sha256, data FROM files WHERE role = '" ~ role + ~ "' ORDER BY name") + ) { + ST_ArtefactFile _f; + _f.role = row["role"].as!string; + _f.name = row["name"].as!string; + _f.bytes = row["bytes"].as!ulong; + _f.sha256 = row["sha256"].as!string; + _f.data = row["data"].as!(ubyte[]); + _out ~= _f; + } + return _out; + } } diff --git a/src/sisudoc/ocda/abstraction/doc_from_artefact.d b/src/sisudoc/ocda/abstraction/doc_from_artefact.d index e45cbf0..3a32d25 100644 --- a/src/sisudoc/ocda/abstraction/doc_from_artefact.d +++ b/src/sisudoc/ocda/abstraction/doc_from_artefact.d @@ -50,26 +50,25 @@ module sisudoc.ocda.abstraction.doc_from_artefact; @safe: /+ ↓ a document, from an artefact rather than from markup - spineAbstraction() parses markup and hands the output writers a doc: an abstraction and a doc_matters. This does the same from a .ssp or a .ocda.db, filling the same docMattersMake() arguments, so that what comes out is the value the writers already take and they never learn which of the two made it. - + . Three of the seven arguments come from the run and are the same either way: program_info, opt_action, and the conf half of conf_make_meta, which is site and run scoped (urls, papersize, the search database) and has no business in a document's own artefact. The other four come from the artefact: - + . conf_make_meta meta and make from the @meta and @make blocks, over the site config the run assembled doc_has docHasFromAbstraction, over the objects doc_digest the markup digest from @source manifest synthesised from the artefact's path and the document's own name and language - + . What is not here, deliberately: insert_file_list, which is read only by source_pod.d for --source and --pod2. No artefact carries the markup, so a pod cannot be rebuilt from one, and the empty list is the truthful @@ -80,6 +79,7 @@ template spineDocFromArtefact() { import std.array; import std.conv : to; import std.path; + import std.process : thisProcessID; import std.regex; import std.stdio; import std.string; @@ -106,7 +106,6 @@ template spineDocFromArtefact() { return _member; } /+ ↓ the document's own header, over the site config the run assembled. - every string field of MetaComposite is looked for under its own key, so a field added there and emitted by the writer arrives here without anything being added. The arr fields are split from the string they @@ -168,14 +167,13 @@ template spineDocFromArtefact() { return _out; } /+ ↓ where a document loaded from an artefact says it is. - an artefact names its document (% Source:) and its language (@source language) but not where the pod holding its images sits, so that is taken from where the artefact itself was found: - + . pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/ pod/<uid>.ocda.db -> pod/<doc>/ - + . which is where --pod2 puts them both. The images beside a .ssp are then found where html.d already looks; a .ocda.db carries its own and does not need the directory to exist. @@ -206,8 +204,90 @@ template spineDocFromArtefact() { .chainPath(_ssp.source).array).to!string; return _out; } + /+ ↓ a database's images, written where the writers look for them. + returns the pod directory to use in place of the one beside the + artefact, or "" when the database carries no images and the pod + beside it should stand. Each blob is checked against the sha256 the + database recorded with it: the two were written together, so a + mismatch means the file is damaged, and saying so is the point of + having recorded the digest at all. + +/ + @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + import std.file : mkdirRecurse, write, tempDir; + import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; + mixin spineAbstractionDbRead _dbr; + auto _files = _dbr.dbReadFiles(_artefact, "image"); + if (_files.length == 0) { return ""; } + string _root = (tempDir.chainPath("spine-ocda-" + ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string; + string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string; + try { + _img_dir.mkdirRecurse; + } catch (Exception ex) { + stderr.writeln("WARNING: could not make an image directory for ", _artefact, + ": ", ex.msg); + return ""; + } + foreach (_f; _files) { + string _got = _f.data.sha256Of.toHexString.to!string; + if (_f.sha256.length > 0 && _got != _f.sha256) { + stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName, + " does not match the digest recorded with it (", _f.sha256, " expected, ", + _got, " found); it is written out as it stands"); + } + try { + (_img_dir.chainPath(_f.name).array).to!string.write(_f.data); + } catch (Exception ex) { + stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg); + } + } + if (_opt_action.vox_gt_1) { + writeln(" ", _files.length, " image(s) from the database, in ", _img_dir); + } + return _root; + } + /+ ↓ the images a .ssp describes, against the ones beside it. + a .ssp carries no image bytes, so what it can do is say whether the + files in the pod it sits in are the ones the abstraction was built + from. Nothing checked that before: a changed or wrong image in the + directory paired silently. + +/ + @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + import std.file : exists, read; + string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string; + int _checked, _missing, _wrong; + foreach (_section; _ssp.section_order) { + foreach (obj; _ssp.abstraction[_section]) { + foreach (_img; obj.metainfo.sha256.images) { + if (_img.fileName.length == 0) { continue; } + string _pth = (_img_dir.chainPath(_img.fileName).array).to!string; + if (!_pth.exists) { ++_missing; continue; } + ++_checked; + string _want = _img.fileHash_sha256.toHexString.to!string; + string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string; + if (_want != _got && _want != "00000000000000000000000000000000" + ~ "00000000000000000000000000000000") { + ++_wrong; + stderr.writeln("WARNING: image ", _img.fileName, " beside ", + _ssp.source, " is not the one the abstraction was built from"); + } + } + } + } + if (_missing > 0) { + stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are", + " not in ", _img_dir); + } + if (_opt_action.vox_gt_1 && _checked > 0) { + writeln(" ", _checked, " image(s) checked against the abstraction's digests", + (_wrong == 0) ? ", all matching" : ""); + } + } /+ ↓ the doc the output writers take, from an artefact. - the reading is done here rather than by the caller, and that is not only tidiness: main() is @system, so a template mixed in there infers @system for everything it declares, and the output writers are @safe @@ -226,6 +306,33 @@ template spineDocFromArtefact() { auto _loaded = abstractionLoad(_artefact); auto _ssp = _loaded.doc; auto _pths = artefactPaths(_artefact, _ssp); + /+ ↓ the images. + a .ocda.db carries its own, which is what lets it travel on its + own, so they are taken from it and written where the output writers + look for images: a directory of this run's making, not the pod + beside the artefact, which may be somebody's published tree. A .ssp + only describes its images, so those are the ones in the pod it sits + in, and are checked against the digests it recorded. + +/ + string _images_tmp; + if (_loaded.loaded) { + if (_artefact.endsWith(".ssp")) { + _imagesVerify(_pths.pod_dir, _ssp, _opt_action); + } else { + string _extracted = _imagesExtract(_artefact, _opt_action); + if (_extracted.length > 0) { + /+ ↓ the source path has to move with it: image_dir_path is + reached from the document's own file, not from the pod +/ + _images_tmp = _extracted; + _pths.pod_dir = _extracted; + _pths.src_file_with_path = (_extracted + .chainPath("media").chainPath("text") + .chainPath(("language" in _ssp.source_info) + ? _ssp.source_info["language"] : "en") + .chainPath(_ssp.source).array).to!string; + } + } + } auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); @@ -250,6 +357,7 @@ template spineDocFromArtefact() { const auto abstraction() { return _abstraction; } auto matters() { return _matters; } bool loaded() { return _loaded.loaded; } + string images_tmp() { return _images_tmp; } string note() { return _loaded.note; } string source_name() { return abstractionSourceName(_loaded.source); } } diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d index 98c729c..2ce9331 100644 --- a/src/sisudoc/spine.d +++ b/src/sisudoc/spine.d @@ -1668,6 +1668,14 @@ string program_name = "spine"; writeln("processed artefact: ", _artefact.baseName, " [", doc.matters.src.language, "]"); } + /+ ↓ the images a database carried, written out for this run only +/ + if (doc.images_tmp.length > 0) { + try { + if (doc.images_tmp.exists) { doc.images_tmp.rmdirRecurse; } + } catch (Exception ex) { + stderr.writeln("WARNING: could not remove ", doc.images_tmp, ": ", ex.msg); + } + } } } if (hvst.curates.length > 0) { |