path: root/org/sdp.org

#+TITLE: sdp hub
#+AUTHOR: Ralph Amissah
#+EMAIL: ralph.amissah@gmail.com
#+STARTUP: indent
#+LANGUAGE: en
#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc
#+OPTIONS: author:nil email:nil creator:nil timestamp:nil
#+PROPERTY: header-args :padline no :exports code :noweb yes
#+EXPORT_SELECT_TAGS: export
#+EXPORT_EXCLUDE_TAGS: noexport
#+FILETAGS: :sdp:rel:hub:
#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n)

[[../maker.org][maker.org makefile]]  [[./][org/]]
* 0. Code Skeleton / Outline / Structure (tangles)                   :tangle:
** TODO version.txt: set version (sisu document parser)            :version:

#+NAME: version_txt
#+BEGIN_SRC d  :tangle ../views/version.txt
/+ obt - org generated file +/
struct Version {
  int major;
  int minor;
  int patch;
}
enum ver = Version(0, 11, 0);
#+END_SRC

** sdp src/sdp.d                                                       :sdp:

#+BEGIN_SRC d  :tangle ../src/sdp.d :shebang #!/usr/bin/env rdmd
/+
  sdp
+/
<<imports_sdp>>
<<imports_sdlang>>
<<imports_std>>
<<sdp_output_selection>>
<<mixin_sdp_version>>
<<mixin_pre_main>>
/++ A SiSU document parser writen in D. +/
void main(string[] args) {
  <<sdp_mixin>>
  <<sdp_args>>
  <<sdp_conf_files>>
  foreach(fn_src; fns_src) {
    if (!empty(fn_src)) {
      <<sdp_each_file_do_scope>>
      <<sdp_each_file_do_read_sisu_markup_file>>
      <<sdp_each_file_do_split_sisu_markup_file_content_into_header_and_body>>
      <<sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta>>
      <<sdp_each_file_do_document_abstraction>>
      <<sdp_each_file_do_debugs_checkdoc>>
      <<sdp_each_file_do_selected_output>>
      <<sdp_each_file_do_scope_exit>>
    } else {
      <<sdp_no_filename_provided>>
    }
  }
}
unittest {
  /++
	name        "sdp"
	description "A SiSU document parser writen in D."
	homepage    "http://sisudoc.org"
  +/
}
#+END_SRC

* 1. sdp.d (sisu document parser)                                       :sdp:

- deal with imports
- get options
  - get command line instructions
  - read config instructions
- process files as instructed by options
  - read in file
  - proess file
  - output

** 0. pre-loop init                                                   :init:
*** init
**** imports                                                      :import:
***** sdp                                                           :sdp:

#+NAME: imports_sdp
#+BEGIN_SRC d
/+ sdp: sisu document parser, see http://sisudoc.org +/
import
  compile_time_info,
  ao_abstract_doc_source,
  ao_conf_make_meta,
  ao_conf_make_meta_native,
  ao_conf_make_meta_sdlang,
  ao_defaults,
  ao_output_debugs,
  ao_read_config_files,
  ao_read_source_files,
  ao_rgx,
  output_hub,
  output_epub,
  output_html,
  output_xhtmls,
  source_sisupod;
#+END_SRC

****** notes
├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
└── views
    └── version.txt

[[./ao_abstract_doc_source.org][ao_abstract_doc_source]]
[[./ao_conf_make_meta.org][ao_conf_make_meta]]
[[./ao_defaults.org][ao_defaults]]
[[./ao_output_debugs.org][ao_output_debugs]]
[[./ao_read_source_files.org][ao_read_source_files]]
[[./compile_time_info.org][compile time info]]
[[./output.org][output]]
[[./sdp.org][sdp]]

***** sdlang                                                     :sdlang:
keep up to date, configuration in ../maker.org
check:
- http://github.com/Abscissa/SDLang-D
- https://github.com/abscissa/libInputVisitor

#+NAME: imports_sdlang
#+BEGIN_SRC d
/+ sdlang http://sdlang.org +/
import sdlang;                            // sdlang.d
#+END_SRC

****** notes

sdlang.parser,
sdlang.exceptions;

***** std                                                           :std:

#+NAME: imports_std
#+BEGIN_SRC d
/+ std +/
private import
  std.algorithm,
  std.array,
  std.container,
  std.exception,
  std.getopt,
  std.json,
  std.process,
  std.stdio,
  std.file,
  std.path,
  std.range,
  std.regex,
  std.string,
  std.traits,
  std.typecons,
  std.utf,
  std.conv : to;
#+END_SRC

****** notes
std.conv,
std.variant,

**** mixins                                                        :mixin:
***** version.txt                                               :version:

#+NAME: mixin_sdp_version
#+BEGIN_SRC d
mixin(import("version.txt"));
#+END_SRC

***** pre main mixins
#+NAME: mixin_pre_main
#+BEGIN_SRC d
mixin CompileTimeInfo;
mixin RgxInit;
#+END_SRC

***** sdp "main" mixins                                             :sdp:

#+NAME: sdp_mixin
#+BEGIN_SRC d
mixin SiSUregisters;
mixin SiSUheaderExtractHub;
mixin SiSUheaderExtractSDLang;
mixin SiSUnode;
mixin SiSUbiblio;
mixin SiSUrgxInitFlags;
mixin SiSUconfigSDLangHub;
mixin SiSUmarkupRaw;
mixin SiSUdocAbstraction;
mixin SiSUoutputDebugs;
mixin SiSUoutputHub;
#+END_SRC

**** init                                                           :init:

#+NAME: sdp_args
#+BEGIN_SRC d
auto raw = MarkupRaw();
auto head = HeaderDocMetadataAndMake();
auto abs = Abstraction();
auto dbg = SDPoutputDebugs();
auto output = SDPoutput();
/+
struct DocumentParts {
  string[string][] contents;
  string[string][string] meta_aa;
  string[string][string] make_aa;
  string[][string][string] bookindex_unordered_hashes;
  JSONValue[] biblio;
}
+/
string[] fns_src;
string flag_action;
string arg_unrecognized;
auto rgx = Rgx();
#+END_SRC

*** scope (run complete)                                            :scope:

#+NAME: sdp_args
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "~ run complete, ok ~ (sdp-%s.%s.%s, %s v%s, %s %s)",
      ver.major, ver.minor, ver.patch,
      __VENDOR__, __VERSION__,
      bits, os,
    );
  }
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "run failure",
    );
  }
}
#+END_SRC

*** config files and command line arguements
**** getopt args for loop                                    :args:getopt:

look into using getopt
[[http://dlang.org/phobos/std_getopt.html][getopt]]
[[http://dlang.org/library/std/getopt.html][getopt]]

#+NAME: sdp_args
#+BEGIN_SRC d
bool[string] _opt_action_bool = [
  "assertions"         : false,
  "concordance"        : false,
  "digest"             : false,
  "docbook"            : false,
  "epub"               : false,
  "html"               : false,
  "html_seg"           : false,
  "html_scroll"        : false,
  "manifest"           : false,
  "ocn"                : true,
  "odt"                : false,
  "pdf"                : false,
  "postgresql"         : false,
  "qrcode"             : false,
  "sisupod"            : false,
  "source"             : false,
  "sqlite"             : false,
  "text"               : false,
  "verbose"            : false,
  "xhtml"              : false,
  "xml_dom"            : false,
  "xml_sax"            : false,
  "section_toc"        : true,
  "section_body"       : true,
  "section_endnotes"   : true,
  "section_glossary"   : true,
  "section_biblio"     : true,
  "section_bookindex"  : true,
  "section_blurb"      : true,
  "backmatter"         : true,
  "skip_output"        : false,
];
auto helpInfo = getopt(args,
  std.getopt.config.passThrough,
  "assert",             "--assert set optional assertions on",                        &_opt_action_bool["assertions"],
  "concordance",        "--concordance file for document",                            &_opt_action_bool["concordance"],
  "digest",             "--digest hash digest for each object",                       &_opt_action_bool["digest"],
  "docbook",            "--docbook process docbook output",                           &_opt_action_bool["docbook"],
  "epub",               "--epub process epub output",                                 &_opt_action_bool["epub"],
  "html",               "--html process html output",                                 &_opt_action_bool["html"],
  "html_seg",           "--html-seg process html output",                             &_opt_action_bool["html_seg"],
  "html_scroll",        "--html-seg process html output",                             &_opt_action_bool["html_scroll"],
  "manifest",           "--manifest process manifest output",                         &_opt_action_bool["manifest"],
  "ocn",                "--ocn object cite numbers (default)",                        &_opt_action_bool["ocn"],
  "odf",                "--odf process odf:odt output",                               &_opt_action_bool["odt"],
  "odt",                "--odt process odf:odt output",                               &_opt_action_bool["odt"],
  "pdf",                "--pdf process pdf output",                                   &_opt_action_bool["pdf"],
  "pg",                 "--pg process postgresql output",                             &_opt_action_bool["postgresql"],
  "postgresql",         "--postgresql process postgresql output",                     &_opt_action_bool["postgresql"],
  "qrcode",             "--qrcode with document metadata",                            &_opt_action_bool["qrcode"],
  "sisupod",            "--sisupod sisupod source content bundled",                   &_opt_action_bool["sisupod"],
  "source",             "--source markup source text content",                        &_opt_action_bool["source"],
  "sqlite",             "--sqlite process sqlite output",                             &_opt_action_bool["sqlite"],
  "text",               "--text process text output",                                 &_opt_action_bool["text"],
  "txt",                "--txt process text output",                                  &_opt_action_bool["text"],
  "verbose|v",          "--verbose output to terminal",                               &_opt_action_bool["verbose"],
  "xhtml",              "--xhtml process xhtml output",                               &_opt_action_bool["xhtml"],
  "xml-dom",            "--xml-dom process xml dom output",                           &_opt_action_bool["xml_dom"],
  "xml-sax",            "--xml-sax process xml sax output",                           &_opt_action_bool["xml_sax"],
  "section-toc",        "--section-toc process table of contents (default)",          &_opt_action_bool["section_toc"],
  "section-body",       "--section-body process document body (default)",             &_opt_action_bool["section_body"],
  "section-endnotes",   "--section-endnotes process document endnotes (default)",     &_opt_action_bool["section_endnotes"],
  "section-glossary",   "--section-glossary process document glossary (default)",     &_opt_action_bool["section_glossary"],
  "section-biblio",     "--section-biblio process document biblio (default)",         &_opt_action_bool["section_biblio"],
  "section-bookindex",  "--section-bookindex process document bookindex (default)",   &_opt_action_bool["section_bookindex"],
  "section-blurb",      "--section-blurb process document blurb (default)",           &_opt_action_bool["section_blurb"],
  "backmatter",         "--section-backmatter process document backmatter (default)", &_opt_action_bool["backmatter"],
  "skip_output",        "--skip-output",                                              &_opt_action_bool["skip_output"],
);
if (helpInfo.helpWanted) {
  defaultGetoptPrinter("Some information about the program.", helpInfo.options);
}
foreach(arg; args) {
  if (match(arg, rgx.flag_action)) {
    flag_action ~= " " ~ arg;   // flags not taken by getopt
  } else if (match(arg, rgx.src_pth)) {
    fns_src ~= arg;             // gather input markup source file names for processing
  } else {                      // anything remaining, unused
    arg_unrecognized ~= " " ~ arg;
  }
}
#+END_SRC

**** TODO config files (load & read) (so far only SDLang)   :config:files:

#+NAME: sdp_conf_files
#+BEGIN_SRC d
auto conf = ConfigHub();
auto sdl_root_configuration = conf.configSDLang("conf.sdl");
auto sdl_root_doc_make = conf.configSDLang("sisu_document_make");
auto confsdl = HeaderExtractSDL();
auto conf_settings_aa = confsdl.configSettingsSDLangToAAmake(sdl_root_configuration);
auto conf_doc_make_aa = confsdl.documentMakeSDLangToAAmake(sdl_root_doc_make);
#+END_SRC

** _1a. processing: loop each file_ [+2]                          :loop:files:
*** scope (loop)                                                    :scope:

#+NAME: sdp_each_file_do_scope
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "%s\n%s",
      "~ document complete, ok ~",
      "-------------------------------",
    );
  }
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "~ document run failure ~ (%s  v%s)\n\t%s",
      __VENDOR__, __VERSION__,
      fn_src
    );
  }
}
enforce(
  match(fn_src, rgx.src_pth),
  "not a sisu markup filename"
);
#+END_SRC

*** 0. prepare document, _document abstraction_
**** 0. _read in raw file_ (using filename with path)            :file:read:
- [[./ao_read_source_files.org][ao_read_source_files]]

#+NAME: sdp_each_file_do_read_sisu_markup_file
#+BEGIN_SRC d
/+ ↓ read file +/
auto read_in_file_string = raw.sourceContent(fn_src);
#+END_SRC

**** 1. split raw content file into: _doc header & doc content_
- [[./ao_read_source_files.org][ao_read_source_files]]

#+NAME: sdp_each_file_do_split_sisu_markup_file_content_into_header_and_body
#+BEGIN_SRC d
/+ ↓ file tuple of header and content +/
auto header_and_body_tuple = raw.sourceContentSplitIntoHeaderAndBody(read_in_file_string, fn_src);
auto header = header_and_body_tuple[0];
auto content_body = header_and_body_tuple[1];
auto _file_insert_list = header_and_body_tuple[2];
debug(header_and_body) {
  writeln(header);
  writeln(header_and_body_tuple.length);
  writeln(content_body[0]);
}
#+END_SRC

**** 2. split doc header into: _metadata & make_  :doc:header:metadata:make:
- [[./ao_conf_make_meta.org][ao_conf_make_meta]]

#+NAME: sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta
#+BEGIN_SRC d
/+ ↓ split header into make and meta +/
auto header_make_and_meta_tuple = head.headerContentAA(header, conf_doc_make_aa);
static assert(!isTypeTuple!(header_make_and_meta_tuple));
string[string][string] _dochead_make = header_make_and_meta_tuple[0];
string[string][string] _dochead_meta = header_make_and_meta_tuple[1];
#+END_SRC

**** 3. processing: _document abstraction, tuple_               :processing:
- [[./ao_abstract_doc_source.org][ao_abstract_doc_source]]

#+NAME: sdp_each_file_do_document_abstraction
#+BEGIN_SRC d
/+ ↓ document abstraction: process document, return abstraction as tuple +/
auto t = abs.abstract_doc_source(content_body, _dochead_make, _dochead_meta, _opt_action_bool);
static assert(!isTypeTuple!(t));
auto doc_abstraction = t[0]; // head ~ toc ~ contents ~ endnotes_seg ~ glossary ~ bibliography ~ bookindex ~blurb;
string[][string] document_section_keys_sequenced = t[1];
string[] doc_html_segnames = t[2];
#+END_SRC

**** 4. _document matters_ (compiled from various sources)

#+NAME: sdp_each_file_do_document_abstraction
#+BEGIN_SRC d
struct DocumentMatters {
  string[] keys_seq_seg() {
    string[] _k = document_section_keys_sequenced["seg"];
    return _k;
  }
  string[] keys_seq_scroll() {
    string[] _k = document_section_keys_sequenced["scroll"];
    return _k;
  }
  string[] segnames() {
    string[] _k = doc_html_segnames;
    return _k;
  }
  auto dochead_make() {
    string[string][string] _k = _dochead_make;
    return _k;
  }
  auto dochead_meta() {
    string[string][string] _k = _dochead_meta;
    return _k;
  }
  auto source_filename() {
    string _k = fn_src;
    return _k;
  }
  auto file_insert_list() {
    string[] _k = _file_insert_list;
    return _k;
  }
  auto opt_action_bool() {
    bool[string] _k = _opt_action_bool;
    return _k;
  }
}
auto doc_matters = DocumentMatters();
#+END_SRC

*** 1. _output processing_ (post abstraction processing)
**** 0. _debug_ (document parts, checkdoc)                  :debug:checkdoc:
- [[./ao_output_debugs.org][ao_output_debugs]]

#+NAME: sdp_each_file_do_debugs_checkdoc
#+BEGIN_SRC d
/+ ↓ debugs +/
debug(checkdoc) {
  dbg.abstract_doc_source_debugs(
    doc_abstraction,
    doc_matters,
  );
}
#+END_SRC

**** 1. _process outputs_                                          :outputs:
- [[./output.org][output]]

#+NAME: sdp_each_file_do_selected_output
#+BEGIN_SRC d
/+ ↓ output hub +/
if (!(_opt_action_bool["skip_output"])) {
  output.hub(
    doc_abstraction,
    doc_matters,
  );
}
#+END_SRC

*** scope (on loop exit)                                       :scope:exit:

#+NAME: sdp_each_file_do_scope_exit
#+BEGIN_SRC d
scope(exit) {
  debug(checkdoc) {
    writefln(
      "processed file: %s",
      fn_src
    );
  }
  destroy(content_body);
  destroy(t);
  destroy(doc_abstraction);
  destroy(doc_html_segnames);
  destroy(fn_src);
}
#+END_SRC

** 1b. no filename provided
#+NAME: sdp_no_filename_provided
#+BEGIN_SRC d
/+ no recognized filename provided +/
writeln("no recognized filename");
break;
#+END_SRC

* Notes
** directory structure
*** program file arrangement

#+BEGIN_SRC shell
├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
├── views
│   └── version.txt

├── src
│   ├── sdp
│   │   ├── ao_abstract_doc_source.d
│   │   ├── ...
│   │   └── compile_time_info.d
│   └── sdp.d
├── views
│   └── version.txt
#+END_SRC

*** markup files

#+BEGIN_SRC shell
#+END_SRC

*** source file bundles

#+BEGIN_SRC shell
dir structure
/tmp/_sisu_processing_/ralph/en/sisupod
  ├── doc
  │   └── en
  └── image
#+END_SRC

*** src dir structure & files
#+BEGIN_SRC txt :tangle no
tree  /home/ralph/sisu_www/current/src/democratizing_innovation.eric_von_hippel.sst

/home/ralph/sisu_www/current/src/
democratizing_innovation.eric_von_hippel.sst
└── sisupod
    ├── doc
    │   ├── en
    │   │   └── democratizing_innovation.eric_von_hippel.sst
    │   └── _sisu
    │       └── sisu_document_make  // [interesting as part of larger conf.sdl]
    └── image
        ├── di_evh_f10-1.png
        ├── di_evh_f11-1.png
        ├── di_evh_f11-2.png
        ├── di_evh_f1-1.png
        ├── di_evh_f5-1.png
        └── di_evh.png

#+END_SRC


** document abstraction
*** terminology / glossary

- header
  - document header, containing document specific
    - (i) metadata information or
    - (ii) make instructions

- (document) structure
  - relationship between headings and sub-headings, and the objects they
    contain.
  - Document structure is extracted from
    - heading levels, which are either:
      - explicitly marked up, or;
      - determined from a make regex provided in the document header.
    Use of document structure allow for the meaningful representation of
    documents in alternative ways and the use of ocn permits easy reference
    across different output formats.

- heading
  - document heading, each heading is marked indicating its level (in relation
    to other headings), and this is used as basis for determininge document
    structure. There are 8 levels, which are can be distinguesed as being one of
    three types: (i) 1 title level (marked up A or numeric 0); (ii) 3 optional
    document division levels, above text separating headings (marked up B - D,
    or numeric 1 to 3); (iii) 4 text headings (marked up 1 - 4, or numeric 4
    to 7)

- levels == heading levels
  - document heading level, see heading and structure

- marked up headings / mark up level

- collapsed headings / collapsed levels

- numeric levels

- dummy heading
  - a markup level 1 / dummy level 4 that does not exist in the original text
    that is manually inserted to maintain the documents structure rule that text
    follows a heading of markup level 1 (rather than A to D) (numeric level 4
    rather than 0 to 3)

- relatives? relations
  - ancestors
    - headings preceding current level under which it occurs
  - decendants
    - sub-headings contained beneath current level
    - range of objects contained by a heading (ocn ranges for each heading in document body)
  see ancestors and decendants

- document ...

- ancestors
  - heading levels above the current heading level which it logically falls
    under and to which it belongs (headings preceding current level under which
    it occurs)

- decendants
  - decendant headings are sub-headings beneath the current heading level,
    heading levels below the current heading level which are derived from it and
    belong to it (sub-headings contained beneath current level); decendant
    objects are the range of objects contained by a heading (ocn ranges for each
    heading in document body)

- (document) sections
  - a document can be divided into 3 parts: front; body and; back. Front matter
    includes the table of contents (which is generated from headings) and any
    parts of the document that are presented before the document body (this
    might include a copyright notice for example). The document body, the
    substantive part of the document, all its substantive objects, including:
    headings, paragraphs, tables, verse etc. This is followed by optional
    backmatter: endnotes, generated from inline markup; glossary, from section
    using a subset of regular markup, with an indication that section is to be
    treated as glossary. Note two things glossary might do that it does not,
    there is: no automatic (sorting) alphabetisation of listing; no creation of
    term anchor tags (perhaps it should); bibliography, created from a specially
    marked up section, with indication that section is to be treated as
    bibliography; bookindex generated from dedicated markup appended to objects
    providing index terms and the relevant range; blurb made up of ordinary
    markup, with indication that section is to be treated as blurb

|-----------------------+------------------------------------------+------------------------+--------|
| _header_                |                                          |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - metadata            | sisu /header markup/                       | markup                 |        |
| - make instructions   |                                          |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| _front matter_          |                                          |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - table of contents   | markup of headings                       | (regular content)      | output |
|-----------------------+------------------------------------------+------------------------+--------|
| _body_                  |                                          |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - substantive content | sisu /content markup/                      | markup                 | output |
|                       | headings (providing document structure), | (regular content)      |        |
|                       | paragraphs, blocks                       |                        |        |
|                       | blocks (code, poem, group, table)        |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| _backmatter_            |                                          |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - endnotes            | markup within substantive content        | markup                 | output |
|                       | (extracted from sisu /content markup/)     | (from regular content) |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - glossary            | identify special section                 | markup                 | output |
|                       | regular /content markup/                   |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - bibliography        | identify section,                        | markup (special)       | output |
|                       | special /bibliography markup/              |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - book index          | extracted from markup attached to        | markup                 | output |
|                       | related substantive content objects      |                        |        |
|                       | (special tags in sisu /content markup/)    | (from regular content) |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - blurb               |                                          |                        |        |
|-----------------------+------------------------------------------+------------------------+--------|
| - metadata            |                                          | (from regular header)  | output |
|-----------------------+------------------------------------------+------------------------+--------|

- segment, segmented text
  - certain forms of output are conveniently segmented, e.g. epub and segmented
    html. The document is broken into chunks indicated by markup level 1 heading
    (numeric level 4 headings) as the significant level at which the document
    should be segmented, and including all decendant objects of that level. For
    a longer text/book this will usually the chapter level. (this is significant
    in e.g. for epub and segmented html, which are broken by segment, usually
    chosen to be chapter)

- scroll
  - the document as a "scroll", e.g. as a single text file, or continuous html
    document

- object
  - a unit of text. Objects include:
    - headings;
    - paragraphs;
    - code blocks;
    - grouped text;
    - verse of poems;
    - tables.
  - Each substantive object is given an object number, that should make it citable.

- ocn (object citation number / citation number)
  - numbers assigned sequentially to each substantive object of a document.
    - An ocn has the characteristic of remaining identical across output
      formats.
    - Translations should be prepared so number remains identical across objects
      in different languages

- citation number (see ocn / object citation number)

- document abstraction (== internal representation) intermediate step,
  - preprocessing of document, into abstraction / representation that is used by
    all downstream processing, i.e. for all output formats. This allows
    normalisation, reducing alternative markup options to common
    representations, e.g. code blocks (open and close), tables, ways of
    instructing that text be bold, shortuct way of providing and endnote
    reference to a link

- (document) internal representation (== document abstraction)
  - see document abstraction

- node representation

- attribute (object attributes)
  - when the document is abstracted attributes associated with an object, for
    example for a:
    - paragraph,
      - indent (hang ... check & add),
      - bulleted,
    - code block,
      - the language syntax,
      - whether the block is numbered

- inline markup
  - when the document is abstracted, markup that remains embedded in the text,
    such as its
    - font face
      - bold,
      - italic,
      -  emphasis,
      -  underscore,
      -  strike,
      - superscript,
      -  subscript,
    - links,
    - endnotes

- sequential all objects backkeeping number?

*** configuration (flies and command line)
**** alternative config sources

- ./_sisu/conf.sdl
- ./_sisu/sisu_document_make
- markup document header, make section
- command line options

**** conf.sdl
(site wide make instruction contained in same file as site configuration file)

#+BEGIN_SRC shell
/*
  name "SiSU sdl config"
  author email="ralph@amissah.com"
  description "site or directory wide environment defaults set here using sdlang see http://sdlang.org system environment info / resource configuration file, for sisu"
  license "GPL v3 or later"
    this file should be configured and live in
      /etc/sisu     #per environment settings, overridden by:
      ~/.sisu       #per user settings, overridden by:
     ./_sisu/config #per local directory settings
*/

/* presentation/web directory, main path and subdirectories (most subdirectories are created automatically based on markup directory name) */
// webserv \
//   url_root      'http://www.sisudoc.org' // without dir stub, e.g. this dir would map to http://www.sisudoc.org/samples
//   path          '~/sisu_www'             // /srv/www #either (i) / [full path from root] or (ii) ~/ [home] or (iii) ./ [pwd] or (iv) will be made from home
//   #path         './tested'              // either (i) / [full path from root] or (ii) ~/ [home] or (iii) ./ [pwd] or (iv) will be made from home
//   #images       'sisu/image'
//   #man          'man'
//   #cgi          '/usr/local/lib/sisu-cgi'
// show_output_on  'filesystem'  // for -v and -u url information, alternatives: 'filesystem','webserver','remote_webserver','local:8111','localhost','localhost:8080','webrick','path'
//// show_output_on 'filesystem_url'

// webserv_cgi \
//   host:         localhost
//   base_path:    ~
//   port:         '8081'
//   user:         ~
//   file_links:   www.sisudoc.org

/* processing directories, main path and subdirectories */
// processing \
//  path=         "~" \
//   dir=         "_sisu_processing~" \
// //  metaverse=    'metaverse' \
// //  tune=         'tune' \
// //  latex=        'tex' \
// //  texinfo=      'texinfo' \
// //  concord_max=  400000

/* output_dir_structure_by: language (language_and_filetype); filetype; or filename (original v1 & v2) */
output_dir_structure_by "language"

/* flag - set (non-default) processing flag shortcuts -1, -2 etc. (here adding color and verbosity as default)
   making color default -c is toggle, and will now toggle color off */
flag \
  color= true \
  act0=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --xhtml --xml-sax --xml-dom --sqlite --manifest --verbose" \
  act1=  "--digest --text --html --manifest" \
  act2=  "--digest --text --html --epub --pdf --manifest" \
  act3=  "--digest --qrcode --text --html --epub --concordance --pdf --manifest" \
  act4=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --manifest" \
  act5=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --sqlite --manifest" \
  act6=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --xhtml --xml-sax --xml-dom --sqlite --manifest" \
  act7=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --xhtml --xml-sax --xml-dom --sqlite --source --sisupod --manifest" \
  act8=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --xhtml --xml-sax --xml-dom --pg --update --manifest" \
  act9=  "--digest --qrcode --text --html --epub --concordance --pdf --odf --docbook --xhtml --xml-sax --xml-dom --pg --update --source --sisupod --manifest"

/* papersize, (LaTeX/pdf) available values: A4, US_letter, book_b5, book_a5, US_legal */
default \
  papersize=         "a4,letter" // 'a4,letter,b5,a5,legal'

// texpdf_font       'Liberation Sans'
// texpdf_font_sans  'Liberation Sans'
// texpdf_font_serif 'Liberation Serif'
// texpdf_font_mono  'Liberation Mono' #'Inconsolata'
// text_wrap:        78
// emphasis:         'bold' #make *{emphasis}* 'bold', 'italics' or 'underscore', default if not configured is 'bold'
// language:         'fr'
// language:         'en'
// digest:           'sha' #sha is sha256, default is md5

/* settings used by ssh scp */
// remote \
//   user         'ralph' \
//   host         'www.sisudoc.org' \
//   path         '/srv/www' // '/srv/sisudoc/web' // '.' #no trailing slash eg 'sisu/www'

/* webrick information */

/* sql database info, postgresql */
// db
//   engine
//     default    'postgresql'
// # share_source:  true
//   postgresql \
//     port       '5432' \
//     user       'ralph' \\ '[provide username]'
//     #host       'sisudoc.org'

/* search */
// search \
//  sisu
//    flag              false
//    action            "http://www.sisudoc.org/cgi-bin/sisu_lng_pgsql.cgi"
//   #action            "http://www.sisudoc.org/cgi-bin/sisu_pgsql.cgi"
//   #action            "http://search.sisudoc.org"
//     db               "current"
//     title             'SiSU search form (sample)'
// html
//   minitoc             true
// manifest
//   minitoc             true
make \
  breaks="break=1" \
  home_button_text="{SiSU}http://sisudoc.org; {sources / git}http://git.sisudoc.org/gitweb/"
//  footer="{SiSU}http://sisudoc.org; {sources / git}http://git.sisudoc.org/gitweb/"
//  home_button_image="{sisu.png }http://sisudoc.org"

link "{SiSU}http://sisudoc.org"
link "{sources / git}http://git.sisudoc.org/gitweb/"
#+END_SRC

**** sisu_document_make
(site wide make instruction)

sample
#+BEGIN_SRC shell
make \
  breaks="break=1" \
  home_button_text="{SiSU}http://sisudoc.org; {sources / git}http://git.sisudoc.org/gitweb/"
//  footer="{SiSU}http://sisudoc.org; {sources / git}http://git.sisudoc.org/gitweb/" \
//  home_button_text="{sisu.png }http://sisudoc.org" \

links \
  link="{SiSU}http://sisudoc.org" \
  link="{sources / git}http://git.sisudoc.org/gitweb/"
#+END_SRC

**** markup document header, make section
(document specific make instructions)

sample

#+BEGIN_SRC shell
% SiSU 8.0

title "The Wealth of Networks" \
  sub="How Social Production Transforms Markets and Freedom" \
  language="US"

creator \
  author="Benkler, Yochai"

date \
  published="2006-04-03" \
  created="2006-04-03" \
  issued="2006-04-03" \
  available="2006-04-03" \
  modified="2006-04-03" \
  valid="2006-04-03"

rights \
  copyright="Copyright (C) 2006 Yochai Benkler." \
  license="All rights reserved. Subject to the exception immediately following, this book may not be reproduced, in whole or in part, including illustrations, in any form (beyond that copying permitted by Sections 107 and 108 of the U.S. Copyright Law and except by reviewers for the public press), without written permission from the publishers. http://creativecommons.org/licenses/by-nc-sa/2.5/ The author has made an online version of the book available under a Creative Commons Noncommercial Sharealike license; it can be accessed through the author's website at http://www.benkler.org."

classify \
  topic_register="SiSU markup sample:book:discourse;networks;Internet;intellectual property:patents|copyright;intellectual property:copyright:creative commons;economics;society;copyright;patents;book:subject:information society|information networks|economics|public policy|society|copyright|patents"

make \
  breaks="new=:B; break=1" \
  home_button_image="{won_benkler.png }http://cyber.law.harvard.edu/wealth_of_networks/Main_Page" \
  footer="{The Wealth of Networks}http://cyber.law.harvard.edu/wealth_of_networks/Main_Page; {Yochai Benkler}http://http://www.doctorow.com"
#+END_SRC

**** hierarchy

#+BEGIN_SRC text
./.sisu ./_sisu ~/.sisu /etc/.sisu
#+END_SRC

*** links
**** internal
**** configure
site url base, needed e.g. for pdf documents that do not use relative links

*** substitutions
**** configure
substitutions to occur within a document

*** source
**** sisupod

*** abstract document structure

** TODO work on
*** sisu document structure

- header

- table of contents
  - scroll            #[ocn]
  - seg               ../[fn]/#[ocn]
- substantive contents
  contents
  footnotes
  - seg               #[ocn]
  internal links
  - scroll            #[ocn]
  - seg               ../[fn]/#[ocn]
- endnotes
  - scroll
  - seg               ../[fn]/#[ocn]
- glossary
- bibliography
- book index
  - scroll            #[ocn]
  - seg               ../[fn]/#[ocn]
- metadata

*** read markup files
**** regular .sst
relatively straight forward
**** master .ssm
master files have been able to read in inser files .ssi and regular files .sst
***** reading in .ssi files is straightforward
***** reading in .sst files is more problematic
.sst files have their own root (structure)
either
- the root needs to be disabled - not used
or
- the root tree needs to be demoted, which is only possible if markup from
  heading D is not reached then A - C could be demoted to B - D
- the other issue there is that it is common not to write out heading level A
  text but to rely on the metadata for title and author, the issue there is that
  at present the header for .sst files that are imported is just lopped off and
  thrown away. At least the title and author information for each imported .sst
  file would have to read and available for use in its header A that is demoted
  to B