path: root/org/sdp.org

#+TITLE: sdp hub
#+AUTHOR: Ralph Amissah
#+EMAIL: ralph.amissah@gmail.com
#+STARTUP: indent
#+LANGUAGE: en
#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc
#+OPTIONS: author:nil email:nil creator:nil timestamp:nil
#+PROPERTY: header-args :padline no :exports code :noweb yes
#+EXPORT_SELECT_TAGS: export
#+EXPORT_EXCLUDE_TAGS: noexport
#+FILETAGS: :sdp:rel:hub:
#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n)

[[../maker.org][maker.org makefile]]  [[./][org/]]
* 0. version.txt (set version)                                      :version:

#+NAME: version_txt
#+BEGIN_SRC d  :tangle ../views/version.txt
/+ obt - org generated file +/
struct Version {
  int major;
  int minor;
  int patch;
}
enum ver = Version(0, 13, 9);
#+END_SRC

* 1. sdp (sisu document parser)                                         :sdp:

- deal with imports
- get options
  - get command line instructions
  - read config instructions
- process files as instructed by options
  - read in file
  - process file
  - output

** 0. sdp src/sdp.d                                               :template:

#+BEGIN_SRC d  :tangle ../src/sdp.d :shebang #!/usr/bin/env rdmd
/+
  sdp
+/
import
  compile_time_info,
  abstraction;
<<imports_sdp>>
<<imports_sdlang>>
<<imports_std>>
<<sdp_output_selection>>
<<mixin_sdp_version>>
<<mixin_pre_main>>
/++ A SiSU document parser writen in D. +/
void main(string[] args) {
  <<sdp_mixin>>
  <<sdp_args>>
  <<sdp_env>>
  <<sdp_conf_files>>
  foreach(fn_src; fns_src) {
    if (!empty(fn_src)) {
      <<sdp_each_file_do_scope>>
      <<sdp_abstraction>>
      <<sdp_each_file_do_debugs_checkdoc>>
      <<sdp_each_file_do_selected_output>>
      <<sdp_each_file_do_scope_exit>>
    } else {
      <<sdp_no_filename_provided>>
    }
  }
}
unittest {
  /++
  name        "sdp"
  description "A SiSU document parser writen in D."
  homepage    "http://sisudoc.org"
  +/
}
#+END_SRC

** 1. pre-loop init                                                   :init:
*** init
**** imports                                                      :import:
***** sdp                                                           :sdp:

#+NAME: imports_sdp
#+BEGIN_SRC d
/+ sdp: sisu document parser, see http://sisudoc.org +/
import
  abstraction_summary,
  ao_abstract_doc_source,
  ao_conf_make_meta,
  ao_conf_make_meta_native,
  ao_conf_make_meta_sdlang,
  ao_defaults,
  ao_doc_debugs,
  ao_read_config_files,
  ao_read_source_files,
  ao_rgx,
  output_hub;
#+END_SRC

****** notes
├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
└── views
    └── version.txt

[[./ao_abstract_doc_source.org][ao_abstract_doc_source]]
[[./ao_conf_make_meta.org][ao_conf_make_meta]]
[[./ao_defaults.org][ao_defaults]]
[[./ao_output_debugs.org][ao_output_debugs]]
[[./ao_read_source_files.org][ao_read_source_files]]
[[./compile_time_info.org][compile time info]]
[[./output.org][output]]
[[./sdp.org][sdp]]

***** sdlang                                                     :sdlang:
keep up to date, configuration in ../maker.org
check:
- http://github.com/Abscissa/SDLang-D
- https://github.com/abscissa/libInputVisitor

#+NAME: imports_sdlang
#+BEGIN_SRC d
/+ sdlang http://sdlang.org +/
import sdlang;                            // sdlang.d
#+END_SRC

****** notes

sdlang.parser,
sdlang.exceptions;

***** std                                                           :std:

#+NAME: imports_std
#+BEGIN_SRC d
/+ std +/
private import
  std.array,
  std.exception,
  std.getopt,
  std.process,
  std.stdio,
  std.regex,
  std.string,
  std.traits,
  std.typecons,
  std.utf,
  std.conv : to;
#+END_SRC

****** notes
std.conv,
std.variant,

**** mixins                                                        :mixin:
***** version.txt                                               :version:

#+NAME: mixin_sdp_version
#+BEGIN_SRC d
mixin(import("version.txt"));
#+END_SRC

***** pre main mixins
#+NAME: mixin_pre_main
#+BEGIN_SRC d
mixin CompileTimeInfo;
#+END_SRC

***** sdp "main" mixins                                             :sdp:

#+NAME: sdp_mixin
#+BEGIN_SRC d
mixin SiSUrgxInit;
mixin SiSUregisters;
mixin SiSUheaderExtractSDLang;
mixin SiSUnode;
mixin SiSUbiblio;
mixin SiSUrgxInitFlags;
mixin outputHub;
#+END_SRC

**** init                                                           :init:

#+NAME: sdp_args
#+BEGIN_SRC d
string[] fns_src;
string flag_action;
string arg_unrecognized;
enum dAM { abstraction, matters }
auto rgx = Rgx();
#+END_SRC

*** scope (run complete)                                            :scope:

#+NAME: sdp_args
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "~ run complete, ok ~ (sdp-%s.%s.%s, %s v%s, %s %s)",
      ver.major, ver.minor, ver.patch,
      __VENDOR__, __VERSION__,
      bits, os,
    );
  }
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "run failure",
    );
  }
}
#+END_SRC

*** config files and command line arguements
**** getopt args for loop                                    :args:getopt:

look into using getopt
[[http://dlang.org/phobos/std_getopt.html][getopt]]
[[http://dlang.org/library/std/getopt.html][getopt]]

#+NAME: sdp_args
#+BEGIN_SRC d
bool[string] _opt_action_bool = [
  "assertions"         : false,
  "concordance"        : false,
  "debug"              : false,
  "digest"             : false,
  "docbook"            : false,
  "epub"               : false,
  "html"               : false,
  "html_seg"           : false,
  "html_scroll"        : false,
  "manifest"           : false,
  "ocn"                : true,
  "odt"                : false,
  "pdf"                : false,
  "postgresql"         : false,
  "qrcode"             : false,
  "sisupod"            : false,
  "source"             : false,
  "sqlite"             : false,
  "text"               : false,
  "verbose"            : false,
  "xhtml"              : false,
  "xml_dom"            : false,
  "xml_sax"            : false,
  "section_toc"        : true,
  "section_body"       : true,
  "section_endnotes"   : true,
  "section_glossary"   : true,
  "section_biblio"     : true,
  "section_bookindex"  : true,
  "section_blurb"      : true,
  "backmatter"         : true,
  "skip_output"        : false,
];
auto helpInfo = getopt(args,
  std.getopt.config.passThrough,
  "assert",             "--assert set optional assertions on",                        &_opt_action_bool["assertions"],
  "concordance",        "--concordance file for document",                            &_opt_action_bool["concordance"],
  "debug",             "--debug only relevant when debug options compiled in",        &_opt_action_bool["debug"],
  "digest",             "--digest hash digest for each object",                       &_opt_action_bool["digest"],
  "docbook",            "--docbook process docbook output",                           &_opt_action_bool["docbook"],
  "epub",               "--epub process epub output",                                 &_opt_action_bool["epub"],
  "html",               "--html process html output",                                 &_opt_action_bool["html"],
  "html_seg",           "--html-seg process html output",                             &_opt_action_bool["html_seg"],
  "html_scroll",        "--html-seg process html output",                             &_opt_action_bool["html_scroll"],
  "manifest",           "--manifest process manifest output",                         &_opt_action_bool["manifest"],
  "ocn",                "--ocn object cite numbers (default)",                        &_opt_action_bool["ocn"],
  "odf",                "--odf process odf:odt output",                               &_opt_action_bool["odt"],
  "odt",                "--odt process odf:odt output",                               &_opt_action_bool["odt"],
  "pdf",                "--pdf process pdf output",                                   &_opt_action_bool["pdf"],
  "pg",                 "--pg process postgresql output",                             &_opt_action_bool["postgresql"],
  "postgresql",         "--postgresql process postgresql output",                     &_opt_action_bool["postgresql"],
  "qrcode",             "--qrcode with document metadata",                            &_opt_action_bool["qrcode"],
  "sisupod",            "--sisupod sisupod source content bundled",                   &_opt_action_bool["sisupod"],
  "source",             "--source markup source text content",                        &_opt_action_bool["source"],
  "sqlite",             "--sqlite process sqlite output",                             &_opt_action_bool["sqlite"],
  "text",               "--text process text output",                                 &_opt_action_bool["text"],
  "txt",                "--txt process text output",                                  &_opt_action_bool["text"],
  "verbose|v",          "--verbose output to terminal",                               &_opt_action_bool["verbose"],
  "xhtml",              "--xhtml process xhtml output",                               &_opt_action_bool["xhtml"],
  "xml-dom",            "--xml-dom process xml dom output",                           &_opt_action_bool["xml_dom"],
  "xml-sax",            "--xml-sax process xml sax output",                           &_opt_action_bool["xml_sax"],
  "section-toc",        "--section-toc process table of contents (default)",          &_opt_action_bool["section_toc"],
  "section-body",       "--section-body process document body (default)",             &_opt_action_bool["section_body"],
  "section-endnotes",   "--section-endnotes process document endnotes (default)",     &_opt_action_bool["section_endnotes"],
  "section-glossary",   "--section-glossary process document glossary (default)",     &_opt_action_bool["section_glossary"],
  "section-biblio",     "--section-biblio process document biblio (default)",         &_opt_action_bool["section_biblio"],
  "section-bookindex",  "--section-bookindex process document bookindex (default)",   &_opt_action_bool["section_bookindex"],
  "section-blurb",      "--section-blurb process document blurb (default)",           &_opt_action_bool["section_blurb"],
  "backmatter",         "--section-backmatter process document backmatter (default)", &_opt_action_bool["backmatter"],
  "skip_output",        "--skip-output",                                              &_opt_action_bool["skip_output"],
);
if (helpInfo.helpWanted) {
  defaultGetoptPrinter("Some information about the program.", helpInfo.options);
}
foreach(arg; args) {
  if (arg.match(rgx.flag_action)) {
    flag_action ~= " " ~ arg;   // flags not taken by getopt
  } else if (arg.match(rgx.src_pth)) {
    fns_src ~= arg;             // gather input markup source file names for processing
  } else {                      // anything remaining, unused
    arg_unrecognized ~= " " ~ arg;
  }
}
#+END_SRC

**** environment                                             :environment:

#+NAME: sdp_env
#+BEGIN_SRC d
auto env = [
  "pwd" : environment["PWD"],
  "home" : environment["HOME"],
];
#+END_SRC

**** TODO config files (load & read) (so far only SDLang)   :config:files:

#+NAME: sdp_conf_files
#+BEGIN_SRC d
auto sdl_root_configuration = ConfigHub!()("conf.sdl", env);
auto sdl_root_doc_make = ConfigHub!()("sisu_document_make", env);
auto confsdl = HeaderExtractSDL();
auto conf_settings_aa = confsdl.configSettingsSDLangToAAmake(sdl_root_configuration);
auto conf_doc_make_aa = confsdl.documentMakeSDLangToAAmake(sdl_root_doc_make);
#+END_SRC

** _2a. processing: loop each file_ [+2]                          :loop:files:
*** scope (loop)                                                    :scope:

#+NAME: sdp_each_file_do_scope
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "%s\n%s",
      "~ document complete, ok ~",
      "------------------------------------------------------------------",
    );
  }
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "~ document run failure ~ (%s  v%s)\n\t%s",
      __VENDOR__, __VERSION__,
      fn_src
    );
  }
}
enforce(
  fn_src.match(rgx.src_pth),
  "not a sisu markup filename"
);
#+END_SRC

*** 1. _document abstraction_ [#A]

- return tuple of:
  - doc_abstraction (the document)
  - doc_matters

#+NAME: sdp_abstraction
#+BEGIN_SRC d
auto t =
  SiSUabstraction!()(fn_src, _opt_action_bool, env);
static assert(!isTypeTuple!(t));
static assert(t.length==2);
auto doc_abstraction = t[dAM.abstraction];
auto doc_matters = t[dAM.matters];
#+END_SRC

*** 2. _output processing_ (post abstraction processing)
**** 0. abstraction _print summary_                    :abstraction:summary:

#+NAME: sdp_each_file_do_debugs_checkdoc
#+BEGIN_SRC d
/+ ↓ debugs +/
if (doc_matters.opt_action_bool["verbose"]) {
  SiSUabstractionSummary!()(doc_abstraction, doc_matters);
}
#+END_SRC
**** 1. _debug_ (document parts, checkdoc)                  :debug:checkdoc:
- [[./ao_output_debugs.org][ao_output_debugs]]

#+NAME: sdp_each_file_do_debugs_checkdoc
#+BEGIN_SRC d
/+ ↓ debugs +/
if ((doc_matters.opt_action_bool["debug"])
|| (doc_matters.opt_action_bool["verbose"])
) {
  SiSUdebugs!()(doc_abstraction, doc_matters);
}
#+END_SRC

**** 2. _process outputs_                                          :outputs:
- [[./output.org][output]]

#+NAME: sdp_each_file_do_selected_output
#+BEGIN_SRC d
/+ ↓ output hub +/
if (!(_opt_action_bool["skip_output"])) {
  outputHub!()(doc_abstraction, doc_matters);
}
#+END_SRC

*** scope (on loop exit)                                       :scope:exit:

#+NAME: sdp_each_file_do_scope_exit
#+BEGIN_SRC d
scope(exit) {
  debug(checkdoc) {
    writefln(
      "processed file: %s",
      fn_src
    );
  }
  destroy(fn_src);
}
#+END_SRC

** 2b. no filename provided
#+NAME: sdp_no_filename_provided
#+BEGIN_SRC d
/+ no recognized filename provided +/
writeln("no recognized filename");
break; // terminate, stop
#+END_SRC

* 2. _document abstraction functions_
** 0. abstraction template [#A]                                   :template:

#+BEGIN_SRC d  :tangle ../src/sdp/abstraction.d
template SiSUabstraction() {
  <<imports_sdp>>
  <<imports_sdlang>>
  <<imports_std>>
  <<sdp_mixin>>
  enum headBody { header, body_content, insert_filelist }
  enum makeMeta { make, meta }
  enum docAbst  { doc_abstraction, section_keys, segnames, images }
  auto rgx = Rgx();
  auto SiSUabstraction(Fn,O,E)(Fn fn_src, O opts, E env){
    <<sdp_conf_files>>
    <<sdp_each_file_do_read_and_split_sisu_markup_file_content_into_header_and_body>>
    <<sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta>>
    <<sdp_each_file_do_document_abstraction>>
    <<sdp_each_file_do_document_matters>>
    auto t = tuple(doc_abstraction, doc_matters);
    static assert(t.length==2);
    return t;
  }
}
#+END_SRC

** 1. (a) _read in raw file_ (b) split content into: _doc header & doc content_
- [[./ao_read_source_files.org][ao_read_source_files]]

- read in the source marked up document and
  - split the document into:
    - document header
    - document body
  - if a master document make a list of insert files
- return a tuple of:
  - header
  - body
  - insert file list

#+NAME: sdp_each_file_do_read_and_split_sisu_markup_file_content_into_header_and_body
#+BEGIN_SRC d
/+ ↓ read file (filename with path) +/
/+ ↓ file tuple of header and content +/
auto _header_body_inserts =
  SiSUrawMarkupContent!()(fn_src);
static assert(!isTypeTuple!(_header_body_inserts));
static assert(_header_body_inserts.length==3);
debug(header_and_body) {
  writeln(header);
  writeln(_header_body_inserts.length);
  writeln(_header_body_inserts.length[headBody.body_content][0]);
}
#+END_SRC

** 2. _document metadata_ & _make instructions_       :doc:header:metadata:make:
- [[./ao_conf_make_meta.org][ao_conf_make_meta]]

- read document header, split into:
  - metadata
  - make instructions
- read config files
  - consolidate make instructions
- return tuple of:
  - document metadata
  - make instructions (from configuration files & document header make
    instructions)

#+NAME: sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta
#+BEGIN_SRC d
/+ ↓ split header into make and meta +/
auto _make_and_meta =
  SiSUheaderExtractHub!()(_header_body_inserts[headBody.header], conf_doc_make_aa);
static assert(!isTypeTuple!(_make_and_meta));
static assert(_make_and_meta.length==2);
#+END_SRC

** 3. _document abstraction, tuple_ (pre-processing)              :processing:
- [[./ao_abstract_doc_source.org][ao_abstract_doc_source]]

- prepare the document abstraction used in downstream processing

- return tuple of:
  - document abstraction (the_document or doc_abstraction)
  - document abstraction keys
    - (head, toc, body, endnotes, glossary, bibliography, bookindex, blurb,
      tail)
    - (passed in doc_matters)
  - segnames for html epub (passed in doc_matters)
  - image list (passed in doc_matters)

#+NAME: sdp_each_file_do_document_abstraction
#+BEGIN_SRC d
/+ ↓ document abstraction: process document, return abstraction as tuple +/
auto da = SiSUdocAbstraction!()(
  (_header_body_inserts[headBody.body_content]),
  (_make_and_meta[makeMeta.make]),
  (_make_and_meta[makeMeta.meta]),
  opts
);
static assert(!isTypeTuple!(da));
static assert(da.length==4);
auto doc_abstraction = da[docAbst.doc_abstraction]; // head ~ toc ~ body ~ endnotes_seg ~ glossary ~ bibliography ~ bookindex ~blurb;
string[][string] _document_section_keys_sequenced = da[docAbst.section_keys];
string[] _doc_html_segnames = da[docAbst.segnames];
auto _images = da[docAbst.images];
#+END_SRC

** 4. _document matters_ (doc info gathered, various sources)

- prepare document_matters, miscellany about processing and the document of use
  in downstream processing

#+NAME: sdp_each_file_do_document_matters
#+BEGIN_SRC d
struct DocumentMatters {
  string[] keys_seq_seg() {
    string[] _k = _document_section_keys_sequenced["seg"];
    return _k;
  }
  string[] keys_seq_scroll() {
    string[] _k = _document_section_keys_sequenced["scroll"];
    return _k;
  }
  string[] segnames() {
    string[] _k = _doc_html_segnames;
    return _k;
  }
  auto dochead_make() {
    string[string][string] _k = _make_and_meta[makeMeta.make];
    return _k;
  }
  auto dochead_meta() {
    string[string][string] _k = _make_and_meta[makeMeta.meta];
    return _k;
  }
  auto source_filename() {
    string _k = fn_src;
    return _k;
  }
  auto language() {
    string _k;
    if (auto m = fn_src.match(rgx.language_code_and_filename)) {
      _k = m.captures[1];
    } else {
      _k = "en";
    }
    return _k;
  }
  auto file_insert_list() {
    string[] _k = _header_body_inserts[headBody.insert_filelist];
    return _k;
  }
  auto image_list() {
    auto _k = _images;
    return _k;
  }
  auto opt_action_bool() {
    bool[string] _k = opts;
    return _k;
  }
  auto environment() {
    auto _k = env;
    return _k;
  }
}
auto doc_matters = DocumentMatters();
#+END_SRC

* 3. document abstraction _summary_                               :summary:doc:
** 0. template:                                                   :template:

#+BEGIN_SRC d :tangle ../src/sdp/abstraction_summary.d
template SiSUabstractionSummary() {
  auto SiSUabstractionSummary(S,T)(
    auto return ref const S  doc_abstraction,
    auto return ref T        doc_matters,
  ) {
    <<abstraction_summary_imports>>
    mixin InternalMarkup;
    <<abstraction_summary_initialize>>
    if (doc_matters.opt_action_bool["verbose"]) {
      <<ao_abstraction_summary>>
    }
  }
}
#+END_SRC

** init
*** imports

#+name: abstraction_summary_imports
#+BEGIN_SRC d
import
  ao_defaults,
  ao_rgx;
import
  std.array,
  std.exception,
  std.stdio,
  std.regex,
  std.string,
  std.traits,
  std.typecons,
  std.uni,
  std.utf,
  std.conv : to;
#+END_SRC

*** initialize                                                     :report:

#+name: abstraction_summary_initialize
#+BEGIN_SRC d
auto markup = InlineMarkup();
#+END_SRC

** (last ocn)

#+name: ao_abstraction_summary
#+BEGIN_SRC d
string[string] check = [
  "last_obj_cite_number" : "NA [debug \"checkdoc\" not run]",
];
foreach (k; doc_matters.keys_seq_seg) {
  foreach (obj; doc_abstraction[k]) {
    if (obj.use != "empty") {
      if (!empty(obj.obj_cite_number)) {
        check["last_obj_cite_number"] = obj.obj_cite_number;
      }
    }
  }
}
#+END_SRC

** summary

#+name: ao_abstraction_summary
#+BEGIN_SRC d
auto min_repeat_number = 66;
auto char_repeat_number = (doc_matters.dochead_meta["title"]["full"].length
  + doc_matters.dochead_meta["creator"]["author"].length + 4);
char_repeat_number = (char_repeat_number > min_repeat_number)
? char_repeat_number
: min_repeat_number;
writefln(
  "%s\n\"%s\", %s\n%s\n%s\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n(%s: %s)\n%s",
  markup.repeat_character_by_number_provided("-", char_repeat_number),
  doc_matters.dochead_meta["title"]["full"],
  doc_matters.dochead_meta["creator"]["author"],
  doc_matters.source_filename,
  markup.repeat_character_by_number_provided("-", char_repeat_number),
  "length toc arr:      ",
  to!int(doc_abstraction["toc_seg"].length),
  "length doc_abstraction arr: ",
  to!int(doc_abstraction["body"].length),
  "last obj_cite_number:  ",
  to!int(check["last_obj_cite_number"]),
  "length endnotes:       ",
  (doc_abstraction["endnotes"].length > 1)
  ? (to!int(doc_abstraction["endnotes"].length))
  : 0,
  "length glossary:       ",
  (doc_abstraction["glossary"].length > 1)
  ? (to!int(doc_abstraction["glossary"].length))
  : 0,
  "length biblio:         ",
  (doc_abstraction["bibliography"].length > 1)
  ? (to!int(doc_abstraction["bibliography"].length))
  : 0,
  "length bookindex:      ",
  (doc_abstraction["bookindex_seg"].length > 1)
  ? (to!int(doc_abstraction["bookindex_seg"].length))
  : 0,
  "length blurb:          ",
  (doc_abstraction["blurb"].length > 1)
  ? (to!int(doc_abstraction["blurb"].length))
  : 0,
  __FILE__,
  __LINE__,
  markup.repeat_character_by_number_provided("-", min_repeat_number),
);
#+END_SRC

markup.repeat_character_by_number_provided("-", 10)
markup.repeat_character_by_number_provided("-", (doc_matters.dochead_meta["title"]["full"].length))
markup.repeat_character_by_number_provided("-", (doc_matters.source_filename.length))