path: root/org/sdp.org

#+TITLE: sdp hub
#+AUTHOR: Ralph Amissah
#+EMAIL: ralph.amissah@gmail.com
#+STARTUP: indent
#+LANGUAGE: en
#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc
#+OPTIONS: author:nil email:nil creator:nil timestamp:nil
#+PROPERTY: header-args :padline no :exports code :noweb yes
#+EXPORT_SELECT_TAGS: export
#+EXPORT_EXCLUDE_TAGS: noexport
#+FILETAGS: :sdp:rel:hub:
#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n)

[[../maker.org][maker.org makefile]]  [[./][org/]]

* sdp.d   sisu document parser                                       :sdp.d:
Deal with imports.
Take command line instructions and process files as instructed.

** TODO version.txt: set version                                    :version:

#+NAME: version_txt
#+BEGIN_SRC d  :tangle ../views/version.txt
/+ obt - org generated file +/
struct Version {
  int major;
  int minor;
  int patch;
}
enum ver = Version(0, 6, 1);
#+END_SRC

** pre loop init
*** imports                                                          :import:
[[./compile_time_info.org][compile time info]]
[[./ao_abstract_doc_source.org][ao_abstract_doc_source]]
[[./ao_ansi_colors.org][ao_ansi_colors]]
[[./ao_defaults.org][ao_defaults]]
[[./ao_output_debugs.org][ao_output_debugs]]
[[./ao_read_source_files.org][ao_read_source_files]]
[[./output_hub.org][output hub]]

**** sdp                                                         :import:sdp:
***** TODO src/sdp.d

├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
└── views
    └── version.txt

#+NAME: sdp_imports_use
#+BEGIN_SRC d
/+ sdp  sisu document parser http://sisudoc.org +/
import
  compile_time_info,          // sdp/compile_time_info.d
  ao_abstract_doc_source,     // sdp/ao_abstract_doc_source.d
  ao_defaults,                // sdp/ao_defaults.d
  ao_header_extract,          // sdp/ao_header_extract.d
  ao_header_extract_native,   // sdp/ao_header_extract_native.d
  ao_header_extract_sdlang,   // sdp/ao_header_extract_sdlang.d
  ao_read_config_files,       // sdp/ao_read_config_files.d
  ao_read_source_files,       // sdp/ao_read_source_files.d
  ao_output_debugs,           // sdp/ao_output_debugs.d
  ao_rgx,                     // sdp/ao_rgx.d
  ao_ansi_colors,             // sdp/ao_ansi_colors.d
  output_hub;                 // output_hub.d
  // std.conv;
#+END_SRC

**** sdlang                                                   :import:sdlang:
#+NAME: sdlang_imports_use
#+BEGIN_SRC d
/+ sdlang http://sdlang.org +/
import sdlang;                            // sdlang.d
#+END_SRC

**** std                                                         :import:std:
#+NAME: sdp_imports
#+BEGIN_SRC d
/+ sdp  sisu document parser +/
private import
  std.getopt,
  std.process,
  std.stdio,
  std.algorithm,
  std.array,
  std.container,
  std.exception,
  std.file,
  std.json,
  // std.path,
  std.range,
  std.regex,
  // std.stdio,
  std.string,
  std.traits,
  std.typecons,
  std.utf,
  // std.variant,
  std.conv : to;
#+END_SRC

**** sdp output check selection                                      :output:

#+NAME: sdp_output_selection
#+BEGIN_SRC d
struct SDPoutput {
  auto hub(S)(
    auto ref const S         contents,
    string[][string][string] bookindex_unordered_hashes,
    JSONValue[]              biblio,
    // JSONValue[string]        dochead_make_json,
    // JSONValue[string]        dochead_meta_json,
    string                   fn_src,
    bool[string]             opt_action_bool
  ) {
    mixin ScreenTxtColors;
    mixin RgxInit;
    mixin SiSUoutputHub;
    auto rgx = Rgx();
    uint return_ = 0;
    if (opt_action_bool["source"]) {
      writeln("sisupod source");
    }
    if (opt_action_bool["sisupod"]) {
      writeln("sisupod source");
    }
    if (opt_action_bool["text"]) {
      writeln("text processing");
      // auto text=SDPoutput_text();
      // text.scroll(contents, bookindex_unordered_hashes, biblio, fn_src, opt_action_bool);
      // // text.scroll(contents, bookindex_unordered_hashes, biblio, dochead_make, dochead_meta, fn_src, opt_action_bool);
    }
    if (opt_action_bool["html"]) {
      auto html=SDPoutputHTML();
      html.css_write;
      html.scroll(contents, bookindex_unordered_hashes, biblio, fn_src, opt_action_bool);
      // html.scroll(contents, bookindex_unordered_hashes, biblio, dochead_make_aa, dochead_meta_aa, fn_src, opt_action_bool);
    }
    if (opt_action_bool["epub"]) {
      writeln("epub processing");
    }
    if (opt_action_bool["pdf"]) {
      writeln("pdf processing");
    }
    if (opt_action_bool["odt"]) {
      writeln("odt processing");
    }
    if (opt_action_bool["sqlite"]) {
      writeln("sqlite processing");
    }
    if (opt_action_bool["postgresql"]) {
      writeln("pgsql processing");
    }
    return return_;
  }
}
#+END_SRC

**** version.txt                                                    :version:

#+NAME: sdp_version_mixin
#+BEGIN_SRC d
mixin(import("version.txt"));
#+END_SRC

*** mixin                                                             :mixin:

#+NAME: sdp_args
#+BEGIN_SRC d
mixin SiSUregisters;
mixin SiSUheaderExtractHub;
mixin SiSUheaderExtractSDLang;
mixin SiSUbiblio;
mixin SiSUrgxInitFlags;
mixin SiSUconfigSDLangHub;
mixin SiSUmarkupRaw;
mixin SiSUdocAbstraction;
mixin SiSUoutputDebugs;
mixin SiSUoutputHub;
mixin ScreenTxtColors;
#+END_SRC

*** init                                                               :init:

#+NAME: sdp_args
#+BEGIN_SRC d
auto raw = MarkupRaw();
auto head = HeaderDocMetadataAndMake();
auto abs = Abstraction();
auto dbg = SDPoutputDebugs();
auto output = SDPoutput();
/+
struct DocumentParts {
  string[string][] contents;
  string[string][string] meta_aa;
  string[string][string] make_aa;
  string[][string][string] bookindex_unordered_hashes;
  JSONValue[] biblio;
}
+/
string[] fns_src;
string flag_action;
string arg_unrecognized;
auto rgx = Rgx();
#+END_SRC

*** scope                                                             :scope:

#+NAME: sdp_args
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "~ run complete, ok ~ (sdp-%s.%s.%s, %s v%s, %s %s)",
      ver.major, ver.minor, ver.patch,
      __VENDOR__, __VERSION__,
      bits, os,
    );
  }
  // stderr.writeln("0");
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "%s~ run failure ~%s",
       scr_txt_color["fuchsia"], scr_txt_color["off"],
    );
  }
}
#+END_SRC

*** getopt args for loop                                        :args:getopt:

look into using getopt
[[http://dlang.org/phobos/std_getopt.html][getopt]]
[[http://dlang.org/library/std/getopt.html][getopt]]

#+NAME: sdp_args
#+BEGIN_SRC d

bool[string] opt_action_bool = [
  "assertions"      : false,
  "concordance"     : false,
  "digest"          : false,
  "docbook"         : false,
  "epub"            : false,
  "html"            : false,
  "manifest"        : false,
  "no_ocn"          : false,
  "odt"             : false,
  "pdf"             : false,
  "postgresql"      : false,
  "qrcode"          : false,
  "sisupod"         : false,
  "source"          : false,
  "sqlite"          : false,
  "text"            : false,
  "verbose"         : false,
  "xhtml"           : false,
  "xml_dom"         : false,
  "xml_sax"         : false,
];
auto helpInfo = getopt(args,
  std.getopt.config.passThrough,
  "assert",         "--assert set optional assertions on",       &opt_action_bool["assertions"],
  "concordance",    "--concordance file for document",           &opt_action_bool["concordance"],
  "digest",         "--digest hash digest for each object",      &opt_action_bool["digest"],
  "docbook",        "--docbook process docbook output",          &opt_action_bool["docbook"],
  "epub",           "--epub process epub output",                &opt_action_bool["epub"],
  "html",           "--html process html output",                &opt_action_bool["html"],
  "manifest",       "--manifest process manifest output",        &opt_action_bool["manifest"],
  "no-ocn",         "--no-ocn suppress object cite numbers",     &opt_action_bool["no_ocn"],
  "odf",            "--odf process odf:odt output",              &opt_action_bool["odt"],
  "odt",            "--odt process odf:odt output",              &opt_action_bool["odt"],
  "pdf",            "--pdf process pdf output",                  &opt_action_bool["pdf"],
  "pg",             "--pg process postgresql output",            &opt_action_bool["postgresql"],
  "postgresql",     "--postgresql process postgresql output",    &opt_action_bool["postgresql"],
  "qrcode",         "--qrcode with document metadata",           &opt_action_bool["qrcode"],
  "sisupod",        "--sisupod sisupod source content bundled",  &opt_action_bool["sisupod"],
  "source",         "--source markup source text content",       &opt_action_bool["source"],
  "sqlite",         "--sqlite process sqlite output",            &opt_action_bool["sqlite"],
  "text",           "--text process text output",                &opt_action_bool["text"],
  "txt",            "--txt process text output",                 &opt_action_bool["text"],
  "verbose|v",      "--verbose output to terminal",              &opt_action_bool["verbose"],
  "xhtml",          "--xhtml process xhtml output",              &opt_action_bool["xhtml"],
  "xml-dom",        "--xml-dom process xml dom output",          &opt_action_bool["xml_dom"],
  "xml-sax",        "--xml-sax process xml sax output",          &opt_action_bool["xml_sax"],
);
if (helpInfo.helpWanted) {
  defaultGetoptPrinter("Some information about the program.", helpInfo.options);
}
foreach(arg; args) {
  if (match(arg, rgx.flag_action)) {
    flag_action ~= " " ~ arg;   // flags not taken by getopt
  } else if (match(arg, rgx.src_pth)) {
    fns_src ~= arg;             // gather input markup source file names for processing
  } else {                      // anything remaining, unused
    arg_unrecognized ~= " " ~ arg;
  }
}
#+END_SRC

*** TODO config files (load & read) (so far only SDLang)       :config:files:

#+NAME: sdp_config_files
#+BEGIN_SRC d
auto conf = ConfigHub();
auto sdl_root_configuration = conf.configSDLang("conf.sdl");
auto sdl_root_doc_make = conf.configSDLang("sisu_document_make");
auto confsdl = HeaderExtractSDL();
auto conf_settings_aa = confsdl.configSettingsSDLangToAAmake(sdl_root_configuration);
auto conf_doc_make_aa = confsdl.documentMakeSDLangToAAmake(sdl_root_doc_make);
#+END_SRC

** each file (loop) [+2]                                         :loop:files:
*** filename provided [+1]                                     :file:process:
**** loop scope                                                       :scope:
#+NAME: sdp_each_file_do
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writeln("~ document complete, ok ~");
  }
  // stderr.writeln("0");
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "%s~ document run failure ~%s (%s  v%s)\n\t%s",
      scr_txt_color["red"], scr_txt_color["off"],
      __VENDOR__, __VERSION__,
      fn_src
    );
  }
  // stderr.writeln("1");
}
enforce(
  match(fn_src, rgx.src_pth),
  "not a sisu markup filename"
);
#+END_SRC

**** [#A] read file                                               :file:read:
[[./ao_markup_source_raw.org][ao_markup_source_raw]]

#+NAME: sdp_each_file_do
#+BEGIN_SRC d
/+ ↓ read file +/
auto header_and_content_tuple = raw.sourceContent(fn_src);
static assert(!isTypeTuple!(header_and_content_tuple));
auto header = header_and_content_tuple[0];
auto sourcefile_content = header_and_content_tuple[1];
debug(header_and_content) {
  writeln(header);
  writeln(header_and_content_tuple.length);
  writeln(sourcefile_content[0]);
}
#+END_SRC

**** [#A] read doc header: metadata & make         :doc:header:metadata:make:
#+NAME: sdp_each_file_do
#+BEGIN_SRC d
auto header_make_and_meta_tuple = head.headerContentAA(header, conf_doc_make_aa);
static assert(!isTypeTuple!(header_make_and_meta_tuple));
string[string][string] dochead_make_aa = header_make_and_meta_tuple[0];
string[string][string] dochead_meta_aa = header_make_and_meta_tuple[1];
#+END_SRC

**** [#A] processing: document abstraction, tuple                :processing:
#+NAME: sdp_each_file_do
#+BEGIN_SRC d
/+ ↓ process document, return abstraction as tuple +/
auto t = abs.abstract_doc_source(sourcefile_content, dochead_make_aa, dochead_meta_aa);
static assert(!isTypeTuple!(t));
auto doc_ao_contents = t[0]; // contents ~ endnotes ~ bookindex;
// static assert(!isIterable!(doc_ao_contents));
auto doc_ao_bookindex_unordered_hashes = t[1];
auto doc_ao_biblio = t[2];
// destroy(t);
#+END_SRC

**** TODO debug                                                       :debug:
***** [#A] debug document parts (checkdoc)                         :checkdoc:
#+NAME: sdp_each_file_do
#+BEGIN_SRC d
/+ ↓ document parts +/
debug(checkdoc) { // checkbook & dumpdoc
  dbg.abstract_doc_source_debugs(
    doc_ao_contents,
    doc_ao_bookindex_unordered_hashes,
    doc_ao_biblio,
    dochead_make_aa,
    dochead_meta_aa,
    fn_src,
    opt_action_bool
  );
}
#+END_SRC

**** TODO process outputs                                           :outputs:

#+NAME: sdp_each_file_do
#+BEGIN_SRC d
/+ ↓ output hub +/
output.hub(
  doc_ao_contents,
  doc_ao_bookindex_unordered_hashes,
  doc_ao_biblio,
  // doc_ao_make_aa,
  // doc_ao_meta_aa,
  fn_src,
  opt_action_bool
);
#+END_SRC

**** on exit                                                     :scope:exit:
#+NAME: sdp_each_file_do
#+BEGIN_SRC d
scope(exit) {
  debug(checkdoc) {
    writefln(
      "processed file: %s",
      fn_src
    );
  }
  destroy(sourcefile_content);
  destroy(t);
  destroy(doc_ao_contents);
  // destroy(doc_ao_make_aa);
  // destroy(doc_ao_meta_aa);
  // destroy(doc_ao_bookindex_unordered_hashes);
  destroy(doc_ao_biblio);
  destroy(fn_src);
}
#+END_SRC

*** no filename provided
#+NAME: sdp_no_filename_provided
#+BEGIN_SRC d
/+ no recognized filename provided +/
writeln("no recognized filename");
break;
// terminate, stop
#+END_SRC

* tangles (code structure)                                           :tangle:
** sdp                                                                :sdp.d:
*** TODO src/sdp.d

#+BEGIN_SRC d  :tangle ../src/sdp.d :shebang #!/usr/bin/env rdmd
/+
  sdp
  sdp.d
+/
<<sdp_imports_use>>
<<sdlang_imports_use>>
<<sdp_imports>>
<<sdp_output_selection>>
<<sdp_version_mixin>>
mixin CompileTimeInfo;
mixin RgxInit;
void main(string[] args) {
  <<sdp_compilation>>
  <<sdp_args>>
  <<sdp_config_files>>
  foreach(fn_src; fns_src) {
  // foreach(fn_src; fns_src) {
    if (!empty(fn_src)) {
      <<sdp_each_file_do>>
    } else {
      <<sdp_no_filename_provided>>
    }
  }
}
#+END_SRC

├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
├── views
│   └── version.txt

├── src
│   ├── sdp
│   │   ├── ao_abstract_doc_source.d
│   │   ├── ...
│   │   └── compile_time_info.d
│   └── sdp.d
├── views
│   └── version.txt

* TODO work on
** program dir structure
figure out best program dir structure for dub and compilers, issue with rdmd

** sisu file structure

|---------------------+------------------------------------------+------------------------+--------|
| header              | sisu /header markup/                       | markup                 |        |
| - metadata          |                                          |                        |        |
| - make instructions |                                          |                        |        |
|---------------------+------------------------------------------+------------------------+--------|
| table of contents   | markup of headings                       | (regular content)      | output |
|---------------------+------------------------------------------+------------------------+--------|
| substantive content | sisu /content markup/                      | markup                 | output |
|                     | headings (providing document structure), | (regular content)      |        |
|                     | paragraphs, blocks                       |                        |        |
|                     | blocks (code, poem, group, table)        |                        |        |
|---------------------+------------------------------------------+------------------------+--------|
| endnotes            | markup within substantive content        | markup                 | output |
|                     | (extracted from sisu /content markup/)     | (from regular content) |        |
|---------------------+------------------------------------------+------------------------+--------|
| glossary            | identify special section                 | markup                 | output |
|                     | regular /content markup/                   |                        |        |
|---------------------+------------------------------------------+------------------------+--------|
| bibliography        | identify section,                        | markup (special)       | output |
|                     | special /bibliography markup/              |                        |        |
|---------------------+------------------------------------------+------------------------+--------|
| book index          | extracted from markup attached to        | markup                 | output |
|                     | related substantive content objects      |                        |        |
|                     | (special tags in sisu /content markup/)    | (from regular content) |        |
|---------------------+------------------------------------------+------------------------+--------|
| metadata            |                                          | (from regular header)  | output |
|---------------------+------------------------------------------+------------------------+--------|

** config                                                            :config:
using sdlang in sdp
*** sdp config and header? file format? sdl ? yml ? json ?       :sdl:sdlang:

[[https://sdlang.org/][SDL: Simple Declarative Language]]  [[http://sdl4r.rubyforge.org/syntaxhighlighter_brush.html][highlighter]]
https://github.com/Abscissa/SDLang-D
https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md

**** build/ compile

The recommended way to use SDLang-D is via DUB. Just add a dependency to
sdlang-d in your project's dub.json or dub.sdl file as shown here. Then simply
build your project with DUB as usual. dub dependency
http://code.dlang.org/packages/sdlang-d


Alternatively, you can git clone both SDLang-D and the latest version of
libInputVisitor,

#+BEGIN_SRC d :tangle no
git clone https://github.com/Abscissa/SDLang-D
git clone https://github.com/abscissa/libInputVisitor
#+END_SRC

 and when running the compiler include:

#+BEGIN_SRC d :tangle no
 -I{path to SDLang-D}/src -I{path to libInputVisitor}/src
#+END_SRC

**** Importing

To use SDL, first import the module sdlang:

#+BEGIN_SRC d :tangle no
import sdlang;
#+END_SRC

If you're not using DUB, then you must also include the path the SDLang-D sources when you compile:

#+BEGIN_SRC d :tangle no
rdmd --build-only -I{path to sdlang}/src -I{path to libInputVisitor}/src {other flags} yourProgram.d
#+END_SRC

**** misc
http://forum.dlang.org/thread/hphtqkkmrfnlcipnxzai@forum.dlang.org
http://forum.dlang.org/thread/gnfctbuhiemidetngrzi@forum.dlang.org?page=23#post-rlxlfveyyzgewhkxhhta:40forum.dlang.org

*** other links
http://semitwist.com/sdlang-d-docs/v0.9.3/sdlang.html  http://semitwist.com/sdlang-d-docs/

** read markup files
**** regular .sst
relatively straight forward
**** master .ssm
master files have been able to read in inser files .ssi and regular files .sst
***** reading in .ssi files is straightforward
***** reading in .sst files is more problematic
.sst files have their own root (structure)
either
- the root needs to be disabled - not used
or
- the root tree needs to be demoted, which is only possible if markup from
  heading D is not reached then A - C could be demoted to B - D
- the other issue there is that it is common not to write out heading level A
  text but to rely on the metadata for title and author, the issue there is that
  at present the header for .sst files that are imported is just lopped off and
  thrown away. At least the title and author information for each imported .sst
  file would have to read and available for use in its header A that is demoted
  to B

** processing files, currently using utf8
** src dir structure & files
#+BEGIN_SRC txt :tangle no
tree  /home/ralph/sisu_www/current/src/democratizing_innovation.eric_von_hippel.sst

/home/ralph/sisu_www/current/src/
democratizing_innovation.eric_von_hippel.sst
└── sisupod
    ├── doc
    │   ├── en
    │   │   └── democratizing_innovation.eric_von_hippel.sst
    │   └── _sisu
    │       └── sisu_document_make  // [interesting as part of larger conf.sdl]
    └── image
        ├── di_evh_f10-1.png
        ├── di_evh_f11-1.png
        ├── di_evh_f11-2.png
        ├── di_evh_f1-1.png
        ├── di_evh_f5-1.png
        └── di_evh.png

#+END_SRC

** check
#+BEGIN_SRC text
./.sisu ./_sisu ~/.sisu /etc/.sisu
#+END_SRC

#+BEGIN_SRC d
// string[string] envVars = environment.toAA();
// writeln(envVars);
/+
writefln(
  "pwd: %s; home: %s",
  environment["PWD"],
  environment["HOME"]
);
+/
#+END_SRC