path: root/src/sisudoc/ocda/abstraction/ssp.d

/+
- Name: SisuDoc Spine, Doc Reform [a part of]
  - Description: documents, structuring, processing, publishing, search
    - static content generator

  - Author: Ralph Amissah
    [ralph.amissah@gmail.com]

  - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.

  - License: AGPL 3 or later:

    Spine (SiSU), a framework for document structuring, publishing and
    search

    Copyright (C) Ralph Amissah

    This program is free software: you can redistribute it and/or modify it
    under the terms of the GNU AFERO General Public License as published by the
    Free Software Foundation, either version 3 of the License, or (at your
    option) any later version.

    This program is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
    more details.

    You should have received a copy of the GNU General Public License along with
    this program. If not, see [https://www.gnu.org/licenses/].

    If you have Internet connection, the latest version of the AGPL should be
    available at these locations:
    [https://www.fsf.org/licensing/licenses/agpl.html]
    [https://www.gnu.org/licenses/agpl.html]

  - Spine (by Doc Reform, related to SiSU) uses standard:
    - docReform markup syntax
      - standard SiSU markup syntax with modified headers and minor modifications
    - docReform object numbering
      - standard SiSU object citation numbering & system

  - Homepages:
    [https://www.sisudoc.org]
    [https://www.doc-reform.org]

  - Git
    [https://git.sisudoc.org/]

+/
module sisudoc.ocda.abstraction.ssp;
@safe:
/+ ↓ write document abstraction as human-readable .ssp text file +/
/+ ↓ escaping for values that are folded onto one line

     a .ssp property value runs to the end of its line, and
     .heading_ancestors_text joins its eight slots with "|", so a slot that
     itself contains "|", a backslash or a line break has to be escaped or
     the value cannot be read back. the pair here is the whole of the rule,
     used by the writer and by the reader.
+/
template sspEscape() {
  import std.array : replace;
  string sspEscapeValue(string _s) {
    return _s
      .replace("\\", "\\\\")
      .replace("|",  "\\|")
      .replace("\n", "\\n")
      .replace("\r", "\\r");
  }
  string[] sspUnescapeSplit(string _s) {
    string[] _out;
    string   _cur;
    bool     _esc;
    foreach (c; _s) {
      if (_esc) {
        switch (c) {
        case 'n': _cur ~= '\n'; break;
        case 'r': _cur ~= '\r'; break;
        default:  _cur ~= c;    break; // covers \\ and \|
        }
        _esc = false;
      } else if (c == '\\') {
        _esc = true;
      } else if (c == '|') {
        _out ~= _cur;
        _cur  = "";
      } else {
        _cur ~= c;
      }
    }
    _out ~= _cur;
    return _out;
  }
}
/+ ↓ one object record: the single definition of what an object looks
     like in a .ssp file. the writer emits with it, and the reader is
     checked against it by round trip, so the two cannot drift
+/
template sspObjectRecord() {
  import std.conv : to;
  import std.digest : toHexString;
  import std.stdio;
  import std.string;
  import std.array;
  import sisudoc.ocda.io_in.paths_source;
  mixin sspEscape;
  string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) {
    string[] output;
    /+ ↓ object declaration line. a heading carries its identifier here,
         on the declaration line, where it is not the ocn; every other
         object carries it as .identifier below, on the same condition
    +/
    string obj_decl = "[" ~ obj.metainfo.ocn.to!string ~ "] ";
    if (obj.metainfo.is_a == "heading") {
      string lev = obj.metainfo.marked_up_level;
      obj_decl ~= "heading :" ~ lev;
      if (obj.metainfo.identifier.length > 0
        && obj.metainfo.identifier != obj.metainfo.ocn.to!string) {
        obj_decl ~= " " ~ obj.metainfo.identifier;
      }
    } else {
      obj_decl ~= obj.metainfo.is_a;
    }
    output ~= obj_decl;
    /+ ↓ properties (only non-default values) +/
    if (obj.metainfo.is_a != "heading"
      && obj.metainfo.identifier.length > 0
      && obj.metainfo.identifier != obj.metainfo.ocn.to!string) {
      output ~= ".identifier: " ~ obj.metainfo.identifier;
    }
    if (obj.metainfo.is_of_part.length > 0) {
      output ~= ".part: " ~ obj.metainfo.is_of_part;
    }
    if (obj.metainfo.is_of_section.length > 0
      && obj.metainfo.is_of_section != section) {
      output ~= ".section: " ~ obj.metainfo.is_of_section;
    }
    /+ ↓ parent: emitted for every object except:
         - the document root, the one object that has no parent
         - toc entries, generated furniture rather than authored objects,
           whose place is given by their indent level; the toc heading
           itself does carry its parent (the document root)
         A parent ocn of 0 is not "no parent recorded", it is a parent
         heading the author has made non-substantive (ocn suppressed, or a
         dummy heading introduced to segment an output); it is structurally
         truthful and is stated explicitly rather than omitted so that the
         two cases remain distinguishable in the file
    +/
    if (!(obj.metainfo.is_a == "heading"
      && obj.metainfo.heading_lev_markup == 0)
      && obj.metainfo.is_a != "toc"
    ) {
      output ~= ".parent: " ~ obj.metainfo.parent_ocn.to!string;
    }
    if (obj.metainfo.last_descendant_ocn != 0) {
      output ~= ".last_descendant: " ~ obj.metainfo.last_descendant_ocn.to!string;
    }
    /+ ↓ child headings (from pre-computed map) +/
    if (obj.metainfo.children_headings.length > 0) {
      string[] ch;
      foreach (c; obj.metainfo.children_headings) {
        ch ~= c.to!string;
      }
      output ~= ".children: " ~ ch.join(" ");
    }
    /+ ↓ ancestors (only if non-zero) +/
    {
      bool has_anc = false;
      foreach (a; obj.metainfo.markedup_ancestors) {
        if (a != 0) { has_anc = true; break; }
      }
      if (has_anc) {
        string anc;
        foreach (i, a; obj.metainfo.markedup_ancestors) {
          if (i > 0) { anc ~= " "; }
          anc ~= a.to!string;
        }
        output ~= ".ancestors: " ~ anc;
      }
    }
    /+ ↓ collapsed ancestors (only if non-zero) +/
    {
      bool has_anc_c = false;
      foreach (a; obj.metainfo.collapsed_ancestors) {
        if (a != 0) { has_anc_c = true; break; }
      }
      if (has_anc_c) {
        string anc;
        foreach (i, a; obj.metainfo.collapsed_ancestors) {
          if (i > 0) anc ~= " ";
          anc ~= a.to!string;
        }
        output ~= ".ancestors_collapsed: " ~ anc;
      }
    }
    /+ ↓ dom structure status (only if non-zero) +/
    {
      bool has_dom = false;
      foreach (d; obj.metainfo.dom_structure_markedup_tags_status) {
        if (d != 0) { has_dom = true; break; }
      }
      if (has_dom) {
        string ds;
        foreach (i, d; obj.metainfo.dom_structure_markedup_tags_status) {
          if (i > 0) { ds ~= " "; }
          ds ~= d.to!string;
        }
        output ~= ".dom_status: " ~ ds;
      }
    }
    {
      bool has_dom_c = false;
      foreach (d; obj.metainfo.dom_structure_collapsed_tags_status) {
        if (d != 0) { has_dom_c = true; break; }
      }
      if (has_dom_c) {
        string ds;
        foreach (i, d; obj.metainfo.dom_structure_collapsed_tags_status) {
          if (i > 0) { ds ~= " "; }
          ds ~= d.to!string;
        }
        output ~= ".dom_status_collapsed: " ~ ds;
      }
    }
    if (obj.metainfo.heading_lev_collapsed < 9) {
      output ~= ".heading_lev_collapsed: " ~ obj.metainfo.heading_lev_collapsed.to!string;
    }
    if (obj.metainfo.parent_lev_markup != 0) {
      output ~= ".parent_lev: " ~ obj.metainfo.parent_lev_markup.to!string;
    }
    if (obj.metainfo.dummy_heading) {
      output ~= ".dummy: true";
    }
    if (obj.metainfo.object_number_off) {
      output ~= ".ocn_off: true";
    }
    if (obj.metainfo.is_of_type.length > 0) {
      output ~= ".is_of_type: " ~ obj.metainfo.is_of_type;
    }
    if (obj.metainfo.attrib.length > 0) {
      output ~= ".attrib: " ~ obj.metainfo.attrib;
    }
    if (obj.metainfo.lang.length > 0) {
      output ~= ".meta_lang: " ~ obj.metainfo.lang;
    }
    if (obj.metainfo.syntax.length > 0) {
      output ~= ".meta_syntax: " ~ obj.metainfo.syntax;
    }
    /+ ↓ sha256 digest +/
    {
      bool has_sha = false;
      foreach (b; obj.metainfo.sha256.text) {
        if (b != 0) { has_sha = true; break; }
      }
      if (has_sha) {
        if (_vox) {
          writeln(obj.metainfo.sha256.summary);
        }
        output ~= ".sha256: " ~ obj.metainfo.sha256.text.toHexString.to!string;
      }
    }
    /+ ↓ images, repeatable record, one per image in order of appearance:
         .image: <filename> sha256:<HEX> bytes:<n> px:<width>x<height>
         .image: <filename> missing:true
       filename first (the object's handle on the image), remaining fields
       self-describing key:value tokens so that further fields can be
       added later without invalidating existing readers. px: is the
       file's own pixel size, not the display dimensions (w##h##) which
       the object text already carries
    +/
    if (obj.metainfo.sha256.images.length > 0) {
      foreach (i; obj.metainfo.sha256.images) {
        output ~= (i.fileMissing)
        ? ".image: "          ~ i.fileName
          ~ " missing:true"
        : ".image: "          ~ i.fileName
          ~ " sha256:"        ~ i.fileHash_sha256.toHexString.to!string
          ~ " bytes:"         ~ i.fileSize.to!string
          ~ ((i.imageWidth > 0 && i.imageHeight > 0)
            ? " px:"          ~ i.imageWidth.to!string
              ~ "x"           ~ i.imageHeight.to!string
            : "");
      }
    }
    /+ ↓ text attributes +/
    if (obj.attrib.indent_base != 0 || obj.attrib.indent_hang != 0) {
      output ~= ".indent: " ~ obj.attrib.indent_base.to!string
        ~ " " ~ obj.attrib.indent_hang.to!string;
    }
    if (obj.attrib.bullet) {
      output ~= ".bullet: true";
    }
    if (obj.attrib.language.length > 0) {
      output ~= ".lang: " ~ obj.attrib.language;
    }
    /+ ↓ has flags +/
    {
      string[] has_flags;
      if (obj.has.inline_links) { has_flags ~= "links"; }
      if (obj.has.inline_notes_reg) { has_flags ~= "notes_reg"; }
      if (obj.has.inline_notes_star) { has_flags ~= "notes_star"; }
      if (obj.has.images) { has_flags ~= "images"; }
      if (obj.has.image_without_dimensions) { has_flags ~= "images_no_dim"; }
      if (has_flags.length > 0) {
        output ~= ".has: " ~ has_flags.join(" ");
      }
    }
    /+ ↓ table properties +/
    if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) {
      output ~= ".table_cols: " ~ obj.table.number_of_columns.to!string;
      if (obj.table.column_widths.length > 0) {
        string[] ws;
        foreach (w; obj.table.column_widths) ws ~= w.to!string;
        output ~= ".table_widths: " ~ ws.join(" ");
      }
      if (obj.table.column_aligns.length > 0) {
        output ~= ".table_aligns: " ~ obj.table.column_aligns.join(" ");
      }
      if (obj.table.heading) {
        output ~= ".table_header: true";
      }
    }
    /+ ↓ code block properties +/
    if (obj.metainfo.is_a == "code") {
      if (obj.code_block.linenumbers) {
        output ~= ".code_linenumbers: true";
      }
    }
    /+ ↓ stow (extracted links) +/
    if (obj.stow.link.length > 0) {
      foreach (lnk; obj.stow.link) {
        if (lnk.length > 0) {
          output ~= ".stow_link: " ~ lnk;
        }
      }
    }
    /+ ↓ tag properties +/
    if (obj.tags.in_segment_html.length > 0) {
      output ~= ".segment: " ~ obj.tags.in_segment_html;
    }
    if (obj.tags.anchor_tag_html.length > 0
      && obj.tags.anchor_tag_html != obj.tags.in_segment_html) {
      output ~= ".anchor: " ~ obj.tags.anchor_tag_html;
    }
    if (obj.tags.segname_prev.length > 0) {
      output ~= ".segment_prev: " ~ obj.tags.segname_prev;
    }
    if (obj.tags.segname_next.length > 0) {
      output ~= ".segment_next: " ~ obj.tags.segname_next;
    }
    if (obj.tags.heading_lev_anchor_tag.length > 0) {
      output ~= ".heading_lev_anchor: " ~ obj.tags.heading_lev_anchor_tag;
    }
    if (obj.tags.segment_anchor_tag_epub.length > 0) {
      output ~= ".segment_epub: " ~ obj.tags.segment_anchor_tag_epub;
    }
    /+ ↓ the segment an object belongs to, as opposed to the segment a
         heading opens. .segment (in_segment_html) is set on headings
         only, so without these two a reader cannot tell which segment a
         paragraph is in; the sqlite search writer reads the html one
    +/
    if (obj.tags.html_segment_anchor_tag_is.length > 0) {
      output ~= ".segment_html_is: " ~ obj.tags.html_segment_anchor_tag_is;
    }
    if (obj.tags.epub_segment_anchor_tag_is.length > 0) {
      output ~= ".segment_epub_is: " ~ obj.tags.epub_segment_anchor_tag_is;
    }
    /+ ↓ heading ancestors text +/
    {
      bool has_hat = false;
      foreach (h; obj.tags.heading_ancestors_text) {
        if (h.length > 0) { has_hat = true; break; }
      }
      if (has_hat) {
        string[] _hat;
        foreach (h; obj.tags.heading_ancestors_text) {
          _hat ~= sspEscapeValue(h.to!string);
        }
        output ~= ".heading_ancestors_text: " ~ _hat.join("|");
      }
    }
    /+ ↓ lev4 subtoc +/
    if (obj.tags.lev4_subtoc.length > 0) {
      foreach (st; obj.tags.lev4_subtoc) {
        if (st.length > 0) {
          output ~= ".lev4_subtoc: " ~ st;
        }
      }
    }
    /+ ↓ anchor tags +/
    if (obj.tags.anchor_tags.length > 0) {
      foreach (at; obj.tags.anchor_tags) {
        if (at.length > 0) {
          output ~= ".anchor_tag: " ~ at;
        }
      }
    }
    /+ ↓ text content +/
    if (obj.text.length > 0) {
      foreach (line; obj.text.split("\n")) {
        output ~= "| " ~ line;
      }
    }
    output ~= "";
    return output;
  }
}

/+ ↓ abstraction directories whose stale .ssp have been cleared this run, one
     entry per document. __gshared, and read and written only inside the
     synchronized block in the writer below: a document's languages share one
     abstraction directory and may be written in parallel
+/
private __gshared bool[string] _abstraction_dir_prepared;
template spineAbstractionTxt() {
  import std.conv : to;
  import std.digest : toHexString;
  import std.file;
  import std.path;
  import std.stdio;
  import std.string;
  import std.array;
  import sisudoc.ocda.io_in.paths_source;
  mixin sspObjectRecord;
  /+ ↓ the whole .ssp for a document, as lines. the file writer below emits
       these, and the abstraction db is built by reading them back, so the
       database cannot carry anything the .ssp does not
  +/
  string[] sspDocumentLines(D)(D doc) {
    auto doc_abstraction = doc.abstraction;
    auto doc_matters     = doc.matters;
    string[] output;
    /+ ↓ header comment +/
    output ~= "% SiSU Document Abstraction v0.1";
    output ~= "% Source: " ~ doc_matters.src.filename;
    output ~= "";
    /+ ↓ @meta block +/
    output ~= "@meta {";
    auto meta = doc_matters.conf_make_meta.meta;
    if (meta.title_main.length > 0) {
      output ~= "  title.main: " ~ meta.title_main;
    }
    if (meta.title_subtitle.length > 0) {
      output ~= "  title.subtitle: " ~ meta.title_subtitle;
    }
    if (meta.title_full.length > 0) {
      output ~= "  title.full: " ~ meta.title_full;
    }
    if (meta.title_language.length > 0) {
      output ~= "  title.language: " ~ meta.title_language;
    }
    if (meta.creator_author.length > 0) {
      output ~= "  creator.author: " ~ meta.creator_author;
    }
    if (meta.creator_author_surname.length > 0) {
      output ~= "  creator.author_surname: " ~ meta.creator_author_surname;
    }
    if (meta.creator_author_surname_fn.length > 0) {
      output ~= "  creator.author_surname_fn: " ~ meta.creator_author_surname_fn;
    }
    if (meta.creator_author_email.length > 0) {
      output ~= "  creator.author_email: " ~ meta.creator_author_email;
    }
    if (meta.creator_illustrator.length > 0) {
      output ~= "  creator.illustrator: " ~ meta.creator_illustrator;
    }
    if (meta.creator_translator.length > 0) {
      output ~= "  creator.translator: " ~ meta.creator_translator;
    }
    if (meta.date_published.length > 0) {
      output ~= "  date.published: " ~ meta.date_published;
    }
    if (meta.date_created.length > 0) {
      output ~= "  date.created: " ~ meta.date_created;
    }
    if (meta.date_issued.length > 0) {
      output ~= "  date.issued: " ~ meta.date_issued;
    }
    if (meta.date_available.length > 0) {
      output ~= "  date.available: " ~ meta.date_available;
    }
    if (meta.date_modified.length > 0) {
      output ~= "  date.modified: " ~ meta.date_modified;
    }
    if (meta.date_valid.length > 0) {
      output ~= "  date.valid: " ~ meta.date_valid;
    }
    if (meta.rights_copyright.length > 0) {
      output ~= "  rights.copyright: " ~ meta.rights_copyright;
    }
    if (meta.rights_license.length > 0) {
      output ~= "  rights.license: " ~ meta.rights_license;
    }
    if (meta.classify_topic_register.length > 0) {
      output ~= "  classify.topic_register: " ~ meta.classify_topic_register;
    }
    if (meta.classify_subject.length > 0) {
      output ~= "  classify.subject: " ~ meta.classify_subject;
    }
    if (meta.classify_keywords.length > 0) {
      output ~= "  classify.keywords: " ~ meta.classify_keywords;
    }
    if (meta.classify_loc.length > 0) {
      output ~= "  classify.loc: " ~ meta.classify_loc;
    }
    if (meta.classify_dewey.length > 0) {
      output ~= "  classify.dewey: " ~ meta.classify_dewey;
    }
    if (meta.identifier_isbn.length > 0) {
      output ~= "  identifier.isbn: " ~ meta.identifier_isbn;
    }
    if (meta.identifier_oclc.length > 0) {
      output ~= "  identifier.oclc: " ~ meta.identifier_oclc;
    }
    if (meta.language_document.length > 0) {
      output ~= "  language.document: " ~ meta.language_document;
    }
    if (meta.notes_abstract.length > 0) {
      output ~= "  notes.abstract: " ~ meta.notes_abstract;
    }
    if (meta.notes_description.length > 0) {
      output ~= "  notes.description: " ~ meta.notes_description;
    }
    if (meta.notes_summary.length > 0) {
      output ~= "  notes.summary: " ~ meta.notes_summary;
    } // check
    output ~= "}";
    output ~= "";
    /+ ↓ @make block +/
    output ~= "@make {";
    auto make = doc_matters.conf_make_meta.make;
    if (make.doc_type.length > 0) {
      output ~= "  doc_type: " ~ make.doc_type;
    }
    if (make.auto_num_top_at_level.length > 0) {
      output ~= "  auto_num_top_at_level: " ~ make.auto_num_top_at_level;
    } // check
    output ~= "  auto_num_top_lv: " ~ make.auto_num_top_lv.to!string;
    output ~= "  auto_num_depth: " ~ make.auto_num_depth.to!string;
    output ~= "}";
    output ~= "";
    /+ ↓ @doc_has block +/
    output ~= "@doc_has {";
    output ~= "  inline_links: " ~ doc_matters.has.inline_links.to!string;
    output ~= "  inline_notes_reg: " ~ doc_matters.has.inline_notes_reg.to!string;
    output ~= "  inline_notes_star: " ~ doc_matters.has.inline_notes_star.to!string;
    output ~= "  tables: " ~ doc_matters.has.tables.to!string;
    output ~= "  codeblocks: " ~ doc_matters.has.codeblocks.to!string;
    output ~= "  images: " ~ doc_matters.has.images.to!string;
    output ~= "  poems: " ~ doc_matters.has.poems.to!string;
    output ~= "  groups: " ~ doc_matters.has.groups.to!string;
    output ~= "  blocks: " ~ doc_matters.has.blocks.to!string;
    output ~= "  quotes: " ~ doc_matters.has.quotes.to!string;
    output ~= "}";
    output ~= "";
    /+ ↓ document sections +/
    string[] section_order = ["head", "toc", "body", "endnotes",
      "glossary", "bibliography", "bookindex", "blurb"];
    foreach (section; section_order) {
      if (section !in doc_abstraction) { continue; }
      auto section_objs = doc_abstraction[section];
      if (section_objs.length == 0) { continue; }
      output ~= "@" ~ section ~ " {";
      output ~= "";
      foreach (obj; section_objs) {
        output ~= sspObjectRecord(obj, section, doc_matters.opt.action.vox_gt_3);
      }
      output ~= "}";
      output ~= "";
    }
    return output;
  }
  @trusted void spineAbstractionTxt(D)(D doc) {
    auto doc_matters = doc.matters;
    string[] output = sspDocumentLines(doc);
    /+ ↓ write to file, into the pod's own tree:
         pod/<doc>/media/abstraction/<doc_uid_out>.ssp
         .
         where it sits beside pod/<doc>/media/image/, which is what a .ssp
         needs to be a document source: the file describes its images by
         name, size, pixels and digest, and they have to be findable. Written
         here once; --pod2 then digests and zips it where it stands rather
         than copying it in from an output directory of its own.
    +/
    auto pths_pod = spinePathsPods!()(doc_matters);
    string base_pth
      = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string;
    /+ ↓ this directory is the one part of the pod written before the pod is
         built, so the pod builder leaves it alone. It therefore clears itself
         here, or a .ssp for a language the document no longer has would linger
         and be bundled.
         .
         Every language of a document writes into this one directory, and the
         languages may be running at once, so the clearing is done once per
         directory per run, by whichever language reaches it first, with the
         lock held across it. A language that finds the directory already
         prepared passed through this lock before writing, so the clearing it
         skipped had already finished, and no .ssp written by this run can be
         removed by it. Keying the clearing on the manifest's first language
         instead makes it wait for a thread that may be scheduled last, and
         take the .ssp already written by the other languages with it.
    +/
    synchronized {
      if (base_pth !in _abstraction_dir_prepared) {
        _abstraction_dir_prepared[base_pth] = true;
        try {
          if (!exists(base_pth)) {
            base_pth.mkdirRecurse;
          } else {
            foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) {
              _f.remove;
            }
          }
        } catch (Exception ex) {
        }
      }
    }
    string out_file = ((base_pth.chainPath(
      doc_matters.src.doc_uid_out ~ ".ssp")).asNormalizedPath).array;
    if (doc_matters.opt.action.vox_gt_1) {
      writeln(" ", out_file);
    }
    auto f = File(out_file, "w");
    foreach (line; output) {
      f.writeln(line);
    }
  }
}