diff options
| author | Ralph Amissah <ralph@amissah.com> | 2018-04-23 12:07:22 -0400 | 
|---|---|---|
| committer | Ralph Amissah <ralph@amissah.com> | 2019-04-10 15:14:14 -0400 | 
| commit | db92f9af3293f8009543448a100b08b7a2f2ffbe (patch) | |
| tree | 026d884db374b4ab6c575b2b56edc2526de4adc4 | |
| parent | sqlite3 some stuff & rearrange files, dlang, org (diff) | |
0.26.1 single sqlite file, premature?
| -rw-r--r-- | org/default_paths.org | 54 | ||||
| -rw-r--r-- | org/default_regex.org | 2 | ||||
| -rw-r--r-- | org/output_hub.org | 34 | ||||
| -rw-r--r-- | org/output_sqlite.org | 280 | ||||
| -rw-r--r-- | org/output_sqlite_discrete.org | 1359 | ||||
| -rw-r--r-- | org/sdp.org | 2 | ||||
| -rw-r--r-- | src/sdp/meta/rgx.d | 2 | ||||
| -rw-r--r-- | src/sdp/output/hub.d | 35 | ||||
| -rw-r--r-- | src/sdp/output/paths_source.d | 54 | ||||
| -rw-r--r-- | src/sdp/output/rgx.d | 2 | ||||
| -rw-r--r-- | src/sdp/output/sqlite.d | 1930 | ||||
| -rw-r--r-- | src/sdp/output/sqlite_discrete.d | 1023 | ||||
| -rw-r--r-- | views/version.txt | 2 | 
13 files changed, 1331 insertions, 3448 deletions
| diff --git a/org/default_paths.org b/org/default_paths.org index af96bea..308475a 100644 --- a/org/default_paths.org +++ b/org/default_paths.org @@ -204,6 +204,7 @@ template PathMatters() {          return Pod_();        }        auto src() { +        string _sep = "␣";          struct SRC_ {            auto is_pod() {              auto _src_is_pod = (_manifest.pod_manifest_path.length > 0) ? true : false; @@ -212,19 +213,70 @@ template PathMatters() {            auto path_and_fn() {              return _fns;            } +          auto pod_name() { /+ work on +/ +            auto _pod_name = (is_pod) +            ? _manifest.pod_manifest_path +            : ""; +            return _pod_name; +          }            auto filename() {              auto _fn = (path_and_fn).baseName;              return _fn;            } +          auto filename_base() { +            auto _fn = filename.stripExtension; +            return _fn; +          } +          auto filename_extension() { +            auto _ext = filename.match(rgx.src_pth_sst_or_ssm).captures["extension"]; +            return _ext; +          }            auto lng() {              string _k; -            if (auto m = (path_and_fn).match(rgx.language_code_and_filename)) { +            if (auto m = path_and_fn.match(rgx.language_code_and_filename)) {                _k = m.captures[1];              } else {                _k = "en";              }              return _k;            } +          auto docname_composite_unique_per_src_doc() { +          /+ +            z pod name if any + src filename + lng code +             filename ~ "." ~  lng +             * unique per src doc +             used by +             - sqlite discrete index (multilingual, each language of a document) +          +/ +            string _fn; +            if (pod_name.baseName == filename_base) { +              _fn = filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; +            } else if (!(pod_name.empty)) { +              _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; +            } else { +              _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; +            } +            return _fn; +          } +          auto docname_composite_unique_per_src_pod() { +          /+ +            z pod name if any + src filename (without lng code) +             filename ~ _sep ~ lng +             * unique per src pod +             used by +             - sisupod (multilingual collection) +             - sqlite discrete index (multilingual collection) +          +/ +            string _fn; +            if (pod_name.baseName == filename_base) { +              _fn = filename_base ~ _sep ~ filename_extension; +            } else if (!(pod_name.empty)) { +              _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension; +            } else { +              _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension; +            } +            return _fn; +          }            auto language() {              return lng();            } diff --git a/org/default_regex.org b/org/default_regex.org index c848109..e824577 100644 --- a/org/default_regex.org +++ b/org/default_regex.org @@ -420,7 +420,7 @@ static special_markup_chars                           = ctRegex!(`[【】〖〗  #+name: prgmkup_rgx  #+BEGIN_SRC d -static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`); +static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);  static src_pth_pod_sst_or_ssm                         = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);  static src_pth_contents                               = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/sisupod[.]manifest$`);  static src_pth_pod_root                               = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); diff --git a/org/output_hub.org b/org/output_hub.org index 6d62477..3129b72 100644 --- a/org/output_hub.org +++ b/org/output_hub.org @@ -29,6 +29,35 @@ template outputHub() {    void outputHub(D,I)(D doc_abstraction, I doc_matters) {      mixin SiSUoutputRgxInit;      static auto rgx = Rgx(); +    debug (substitutions) { +      enum Substitute { match, markup, html, } +      writeln(__LINE__, ":", __FILE__, ": DEBUG substitutions:"); +      writeln("Doc Title:            ", doc_matters.conf_make_meta.meta.title_full); +      if (doc_matters.conf_make_meta.make.bold) { +        writeln("regex to match:       ", doc_matters.conf_make_meta.make.bold[Substitute.match]); +        writeln("substitution to make: ", doc_matters.conf_make_meta.make.bold[Substitute.html]); + +        auto _w = "1. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" +          .replaceAll( +            regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), +            doc_matters.conf_make_meta.make.bold[Substitute.html] +          ); +        writeln(_w); + +        writeln("2a. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" +          .replaceAll( +            regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), +            doc_matters.conf_make_meta.make.bold[Substitute.html] +          )); + +        auto _v = regex(doc_matters.conf_make_meta.make.bold[Substitute.match]); +        writeln("2b. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" +          .replaceAll( +            _v, +            doc_matters.conf_make_meta.make.bold[Substitute.html] +          )); +      } +    }      <<output_options>>    }  } @@ -50,7 +79,6 @@ import sdp.output,    sdp.output.epub3,    sdp.output.html,    sdp.output.sqlite, -  sdp.output.sqlite_discrete,    sdp.output.xmls,    sdp.output.source_sisupod,    sdp.output.create_zip_file, @@ -202,7 +230,7 @@ if (doc_matters.opt.action.sqlite_discrete) {    if ((doc_matters.opt.action.verbose)) {      writeln("sqlite processing... ");    } -  SQLiteDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); +  SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);  }  #+END_SRC @@ -214,7 +242,7 @@ if (doc_matters.opt.action.sqlite_update) {    if ((doc_matters.opt.action.verbose)) {      writeln("sqlite processing... ");    } -  SQLiteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); +  SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters);  }  #+END_SRC diff --git a/org/output_sqlite.org b/org/output_sqlite.org index 7fb843d..566775b 100644 --- a/org/output_sqlite.org +++ b/org/output_sqlite.org @@ -17,10 +17,67 @@  * sql  ** 0. module, templates +*** hub +**** collection +  #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d  module sdp.output.sqlite;  <<output_imports>> +import d2sqlite3; +import std.typecons : Nullable; +mixin SiSUoutputRgxInit; +static auto rgx = Rgx();  long _metadata_tid_lastrowid; +template SQLiteHubBuildTablesAndPopulate() { +  void SQLiteHubBuildTablesAndPopulate(D,I)( +    auto ref const D    doc_abstraction, +    auto ref       I    doc_matters, +  ) { +    auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); +    template SQLiteDbStatementComposite() { +      void SQLiteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        <<sqlite_db_statement_composite_collection>> +      } +    } +    SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); +  } +} +#+END_SRC + +**** discrete + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLiteHubDiscreteBuildTablesAndPopulate() { +  void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)( +    auto ref const D    doc_abstraction, +    auto ref       I    doc_matters, +  ) { +    auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); +    template SQLiteDiscreteDbStatementComposite() { +      void SQLiteDiscreteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        <<sqlite_db_statement_composite_discrete>> +      } +    } +    SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); +  } +} +#+END_SRC + +*** db run + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d  template SQLiteDbRun() {    void SQLiteDbRun(Db,St,O)(      auto ref       Db   db, @@ -37,7 +94,9 @@ template SQLiteDbRun() {          db_statement ~          "\nCOMMIT;\n"        ); -      if (!(opt_action.sqlite_create)) { +      if (!(opt_action.sqlite_discrete) +        && !(opt_action.sqlite_create) +      ) {          _metadata_tid_lastrowid = db.lastInsertRowid();          writeln("last row id: ", _metadata_tid_lastrowid);        } @@ -54,83 +113,70 @@ template SQLiteDbRun() {      }    }  } -template SQLiteBuildTablesAndPopulate() { -  void SQLiteBuildTablesAndPopulate(D,I)( -    auto ref const D    doc_abstraction, +#+END_SRC + +*** munge + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLinsertDelimiter() { +  auto SQLinsertDelimiter(string _txt) { +    _txt = _txt +      .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +    return _txt; +  } +} +template SQLiteFormatAndLoadObject() { +  auto SQLiteFormatAndLoadObject(I)(      auto ref       I    doc_matters,    ) { -    import d2sqlite3; -    import std.typecons : Nullable;      mixin SiSUoutputRgxInit; -    static auto rgx = Rgx(); -    auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); -    pth_sqlite.base.mkdirRecurse; -    auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); -    template SQLiteDbStatementComposite() { -      void SQLiteDbStatementComposite(Db,D,I)( -        auto ref       Db   db, -        auto ref const D    doc_abstraction, -        auto ref       I    doc_matters, -      ) { -        <<sqlite_db_statement_composite>> -      } -    } -    template SQLinsertDelimiter() { -      auto SQLinsertDelimiter(string _txt) { -        _txt = _txt -          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); -        return _txt; -      } +    struct sqlite_format_and_load_objects { +      <<sanitize_text_for_search>> +      <<sanitize_and_munge_inline_html>> +      <<html_objects>> +      <<sqlite_load_object>> +      <<hub_format_and_sqlite_load_objects>>      } -    template SQLiteFormatAndLoadObject() { -      auto SQLiteFormatAndLoadObject(I)( -        auto ref       I    doc_matters, -      ) { -        mixin SiSUoutputRgxInit; -        struct sqlite_format_and_load_objects { -          <<sanitize_text_for_search>> -          <<sanitize_and_munge_inline_html>> -          <<html_objects>> -          <<sqlite_load_object>> -          <<hub_format_and_sqlite_load_objects>> -        } -        return sqlite_format_and_load_objects(); -      } -    } -    template SQLiteTablesReCreate() { -      string SQLiteTablesReCreate()() { -        string _sql_instruct; -        _sql_instruct = format(q"¶ -          <<sqlite_statement_drop_existing_index>> -          <<sqlite_statement_drop_existing_tables>> -          <<sqlite_statement_create_table_metadata_and_src_txt>> -          <<sqlite_statement_create_table_objects>> -          <<sqlite_statement_create_index>> -        ¶",); -        return _sql_instruct; -      } -    } -    template SQLiteInsertMetadata() { -      string SQLiteInsertMetadata(I)( -        auto ref       I    doc_matters, -      ) { -        <<sqlite_formatted_insertions_doc_matters_metadata>> -        return _insert_metadata; -      } -    } -    template SQLiteInsertDocObjectsLoop() { -      string SQLiteInsertDocObjectsLoop(P,I)( -        auto ref       P    doc_parts, -        auto ref       I    _metadata_tid, -      ) { -        string insertDocObjectsRow(O)(O obj) { -          <<sqlite_formatted_insertions_doc_objects>> -          return _insert_doc_objects_row; -        } -        <<sqlite_objects_loop>> -      } +    return sqlite_format_and_load_objects(); +  } +} +#+END_SRC + +*** sqlite instructions + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLiteTablesReCreate() { +  string SQLiteTablesReCreate()() { +    string _sql_instruct; +    _sql_instruct = format(q"¶ +      <<sqlite_statement_drop_existing_index>> +      <<sqlite_statement_drop_existing_tables>> +      <<sqlite_statement_create_table_metadata_and_src_txt>> +      <<sqlite_statement_create_table_objects>> +      <<sqlite_statement_create_index>> +    ¶",); +    return _sql_instruct; +  } +} +template SQLiteInsertMetadata() { +  string SQLiteInsertMetadata(I)( +    auto ref       I    doc_matters, +  ) { +    <<sqlite_formatted_insertions_doc_matters_metadata>> +    return _insert_metadata; +  } +} +template SQLiteInsertDocObjectsLoop() { +  string SQLiteInsertDocObjectsLoop(D,I,X)( +    auto ref const D    doc_abstraction, +    auto ref       I    doc_matters, +    auto ref       X    _metadata_tid, +  ) { +    string insertDocObjectsRow(O)(O obj) { +      <<sqlite_formatted_insertions_doc_objects>> +      return _insert_doc_objects_row;      } -    SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); +    <<sqlite_objects_loop>>    }  }  template SQLiteTablesCreate() { @@ -169,30 +215,45 @@ template SQLiteTablesDrop() {  #+END_SRC  ** 1. [#A] sqlite_db_statement +*** collection -#+name: sqlite_db_statement_composite +#+name: sqlite_db_statement_composite_collection  #+BEGIN_SRC d  { +  string _db_statement;    { -    string _db_statement; -    { -      if ((doc_matters.opt.action.sqlite_create)) { -        auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); -        pth_sqlite.base.mkdirRecurse; -        _db_statement ~= SQLiteTablesReCreate!()(); -        SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); -      } -      if ((doc_matters.opt.action.sqlite_update)) { // TODO -        _db_statement ~= SQLiteInsertMetadata!()(doc_matters); -        SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); -        /+ get tid (lastrowid or max) for use in doc_objects table +/ -        _metadata_tid_lastrowid = db.lastInsertRowid(); -        _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX -        SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); -      } +    if ((doc_matters.opt.action.sqlite_create)) { +      auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); +      pth_sqlite.base.mkdirRecurse; +      _db_statement ~= SQLiteTablesReCreate!()(); +      SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); +    } +    if ((doc_matters.opt.action.sqlite_update)) { // TODO +      _db_statement ~= SQLiteInsertMetadata!()(doc_matters); +      SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); +      /+ get tid (lastrowid or max) for use in doc_objects table +/ +      _metadata_tid_lastrowid = db.lastInsertRowid(); +      _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX +      SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects");      } -    db.close;    } +  db.close; +} +#+END_SRC + +*** discrete + +#+name: sqlite_db_statement_composite_discrete +#+BEGIN_SRC d +{ +  string _db_statement; +  { +    _db_statement ~= SQLiteTablesReCreate!()(); +    _db_statement ~= SQLiteInsertMetadata!()(doc_matters); +    _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX +    SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); +  } +  db.close;  }  #+END_SRC @@ -869,7 +930,7 @@ auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);  string[string] obj_txt;  string doc_text;  string[] _insert_doc_objects; -foreach (part; doc_parts) { +foreach (part; doc_matters.xml.keys_seq.sql) {    foreach (obj; doc_abstraction[part]) {      switch (obj.of_part) {      case "frontmatter":              assert(part == "head", part); @@ -1023,7 +1084,6 @@ return _insert_doc_objects.join.to!(char[]).toUTF8;  ** 5. SQL statements  *** create tables -  **** DROP INDEX IF EXISTS  #+name: sqlite_statement_drop_existing_index @@ -1061,7 +1121,8 @@ DROP TABLE IF EXISTS urls;  #+BEGIN_SRC sql  CREATE TABLE metadata_and_text (    tid                              INTEGER           PRIMARY KEY, -/* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +  src_composite_id_per_txt         VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +  src_composite_id_per_pod         VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */    title                            VARCHAR(800)  NOT NULL,    title_main                       VARCHAR(400)  NOT NULL,    title_sub                        VARCHAR(400)      NULL, @@ -1208,9 +1269,8 @@ CREATE INDEX idx_language ON metadata_and_text(language_document_char);  CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);  #+END_SRC -**** TODO local site link & info - -*** sql insert statement formatted doc objects +*** inserts +**** INSERT doc objects  lid unique, increment by 1 per object, not ocn @@ -1219,14 +1279,15 @@ either:  - increment by adding 1 for each document,  - make hash of document filename or url and use? -**** sql statement: dlang format +***** sql statement: dlang format +  #+name: sqlite_formatted_insertions_doc_objects  #+BEGIN_SRC d  string _insert_doc_objects_row;  _insert_doc_objects_row = format(q"¶  #+END_SRC -**** INSERT INTO +***** INSERT INTO  #+name: sqlite_formatted_insertions_doc_objects  #+BEGIN_SRC sql @@ -1241,7 +1302,7 @@ _insert_doc_objects_row = format(q"¶    )  #+END_SRC -**** VALUES +***** VALUES  #+name: sqlite_formatted_insertions_doc_objects  #+BEGIN_SRC sql @@ -1256,7 +1317,7 @@ _insert_doc_objects_row = format(q"¶    );  #+END_SRC -**** sql statement: dlang values for formatting +***** dlang values for formatting  #+name: sqlite_formatted_insertions_doc_objects  #+BEGIN_SRC d @@ -1271,20 +1332,23 @@ _insert_doc_objects_row = format(q"¶  );  #+END_SRC -*** sqlite insert statement formatted doc matters metadata -**** sql statement: dlang format +**** INSERT doc matters metadata +***** sql statement: dlang format +  #+name: sqlite_formatted_insertions_doc_matters_metadata  #+BEGIN_SRC d  string _insert_metadata;  _insert_metadata = format(q"¶  #+END_SRC -**** INSERT INTO +***** INSERT INTO  #+name: sqlite_formatted_insertions_doc_matters_metadata  #+BEGIN_SRC sql    INSERT INTO metadata_and_text (      src_filename, +    src_composite_id_per_txt, +    src_composite_id_per_pod,      title,      title_main,      title_sub, @@ -1331,7 +1395,7 @@ _insert_metadata = format(q"¶    )  #+END_SRC -**** VALUES +***** VALUES  #+name: sqlite_formatted_insertions_doc_matters_metadata  #+BEGIN_SRC sql @@ -1379,16 +1443,20 @@ _insert_metadata = format(q"¶      '%s',      '%s',      '%s', +    '%s', +    '%s',      '%s'    );  #+END_SRC -**** sql statement: values for formatting +***** dlang values for formatting  #+name: sqlite_formatted_insertions_doc_matters_metadata  #+BEGIN_SRC d  ¶",    SQLinsertDelimiter!()(doc_matters.src.filename), +  SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), +  SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod),    SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full),    SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main),    SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), diff --git a/org/output_sqlite_discrete.org b/org/output_sqlite_discrete.org deleted file mode 100644 index c2f273c..0000000 --- a/org/output_sqlite_discrete.org +++ /dev/null @@ -1,1359 +0,0 @@ -#+TITLE:       sdp output sqlite discrete -#+AUTHOR:      Ralph Amissah -#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+KEYWORDS -#+LANGUAGE:    en -#+STARTUP:     indent content -#+OPTIONS:     H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t -#+OPTIONS:     TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc -#+OPTIONS:     author:nil email:nil creator:nil timestamp:nil -#+PROPERTY:    header-args :padline no :exports code :noweb yes -#+EXPORT_SELECT_TAGS:  export -#+EXPORT_EXCLUDE_TAGS: noexport -#+FILETAGS:            :sdp:niu:output: -#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n) - -* sql -** 0. module, templates - -#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite_discrete.d -module sdp.output.sqlite_discrete; -<<output_imports>> -template SQLiteDiscreteDbRun() { -  void SQLiteDiscreteDbRun(Db,St,O)( -    auto ref       Db   db, -    auto ref       St   db_statement, -    auto ref       O    opt_action, -               string   note, -  ) { -    debug(sql_statement) { -      writeln(db_statement); -    } -    try { -      db.run( -        "\nBEGIN;\n" ~ -        db_statement ~ -        "\nCOMMIT;\n" -      ); -    } catch (ErrnoException ex) { -      writeln("ERROR SQLite : ", ex); -    } catch (Exception ex) { -      writeln("ERROR SQLite : ", ex); -    } -    if (opt_action.debug_do) { -      writeln(note); -      if (opt_action.verbose) { -        writeln(db_statement); -      } -    } -  } -} -template SQLiteDiscreteBuildTablesAndPopulate() { -  void SQLiteDiscreteBuildTablesAndPopulate(D,I)( -    auto ref const D    doc_abstraction, -    auto ref       I    doc_matters, -  ) { -    import d2sqlite3; -    import std.typecons : Nullable; -    mixin SiSUoutputRgxInit; -    static auto rgx = Rgx(); -    auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); -    pth_sqlite.base.mkdirRecurse; -    auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); -    template SQLiteDiscreteDbStatementComposite() { -      void SQLiteDiscreteDbStatementComposite(Db,D,I)( -        auto ref       Db   db, -        auto ref const D    doc_abstraction, -        auto ref       I    doc_matters, -      ) { -        <<sqlite_db_statement_composite>> -      } -    } -    template SQLinsertDelimiter() { -      auto SQLinsertDelimiter(string _txt) { -        _txt = _txt -          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); -        return _txt; -      } -    } -    template SQLiteFormatAndLoadObject() { -      auto SQLiteFormatAndLoadObject(I)( -        auto ref       I    doc_matters, -      ) { -        mixin SiSUoutputRgxInit; -        struct sqlite_format_and_load_objects { -          <<sanitize_text_for_search>> -          <<sanitize_and_munge_inline_html>> -          <<html_objects>> -          <<sqlite_load_object>> -          <<hub_format_and_sqlite_load_objects>> -        } -        return sqlite_format_and_load_objects(); -      } -    } -    template SQLiteDiscreteTablesReCreate() { -      string SQLiteDiscreteTablesReCreate()() { -        string _sql_instruct; -        _sql_instruct = format(q"¶ -          <<sqlite_statement_drop_existing_index>> -          <<sqlite_statement_drop_existing_tables>> -          <<sqlite_statement_create_table_metadata_and_src_txt>> -          <<sqlite_statement_create_table_objects>> -          <<sqlite_statement_create_index>> -        ¶",); -        return _sql_instruct; -      } -    } -    template SQLiteDiscreteInsertMetadata() { -      string SQLiteDiscreteInsertMetadata(I)( -        auto ref       I    doc_matters, -      ) { -        <<sqlite_formatted_insertions_doc_matters_metadata>> -        return _insert_metadata; -      } -    } -    template SQLiteDiscreteInsertDocObjectsLoop() { -      string SQLiteDiscreteInsertDocObjectsLoop(P,I)( -        auto ref       P    doc_parts, -        auto ref       I    tid, -      ) { -        string insertDocObjectsRow(O)(O obj) { -          <<sqlite_formatted_insertions_doc_objects>> -          return _insert_doc_objects_row; -        } -        <<sqlite_objects_loop>> -      } -    } -    SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); -  } -} -#+END_SRC - -** 1. sqlite_db_statement [#A] - -#+name: sqlite_db_statement_composite -#+BEGIN_SRC d -{ -  string _db_statement; -  { -    _db_statement ~= SQLiteDiscreteTablesReCreate!()(); -    _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters); -    _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX -    SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); -  } -  db.close; -} -#+END_SRC - -** 2. imports - -#+name: output_imports -#+BEGIN_SRC d -import sdp.output; -import -  std.file, -  std.conv : to; -#+END_SRC - -** 3. format and load template -*** 1. prepare objects (munge, sanitize, markup) -**** 1. _text_ generic munge (sanitize text for search) - -- [3/4] (search text, applies to all but code blocks) -  - [ ] remove whitespace, paragraph on single line (formatting kept so far) -  - [X] remove font face attributes -  - [X] move embedded endnotes -    - [X] place after text object -    - [X] remove embedded endnote numbers (rely on html output to represent) -  - [X] urls -    - [X] clean url markers, leave plain link text -    - [X] place urls after text object and its endnotes - -#+name: sanitize_text_for_search -#+BEGIN_SRC d -auto generic_munge_sanitize_text_for_search( -  string _txt, -) { -  string _notes; -  string _urls; -  if (_txt.matchFirst(rgx.inline_notes_al_gen)) { -    foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { -      _notes ~= "\n" ~ m["text"]; -    } -    _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); -  } -  if (_txt.matchFirst(rgx.inline_link)) { -    foreach (m; _txt.matchAll(rgx.inline_link)) { -      if (m["link"].match(rgx.url)) { -        _urls ~= "\n" ~ m["link"]; -      } -    } -    _txt = _txt.replaceAll(rgx.inline_link_clean, ""); -  } -  if (_notes.length > 0) { -    _txt ~= _notes; -  } -  if (_urls.length > 0) { -    _txt ~= _urls; -  } -  if (doc_matters.opt.action.debug_do) { -    writeln(_txt, "\n"); -  } -  debug(sql_text_clean) { -    writeln(_txt); -  } -  return _txt; -} -#+END_SRC - -**** 2. _html_ - -light html objects -- place endnotes after text object -- light inline html markup - -***** munge -****** general munge (special characters, inline markup, move notes) - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -auto munge_html(O)( -  auto return ref const O    obj, -) { -  string _html_special_characters(string _txt){ -    _txt = _txt -      .replaceAll(rgx.xhtml_ampersand,    "&") -      .replaceAll(rgx.xhtml_quotation,    """) -      .replaceAll(rgx.xhtml_less_than,    "<") -      .replaceAll(rgx.xhtml_greater_than, ">") -      .replaceAll(rgx.nbsp_char,          " ") -      .replaceAll(rgx.xhtml_line_break,   "<br />"); -    return _txt; -  } -  string _html_font_face(string _txt){ -    _txt = _txt -      .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") -      .replaceAll(rgx.inline_bold,        "<b>$1</b>") -      .replaceAll(rgx.inline_underscore,  "<u>$1</u>") -      .replaceAll(rgx.inline_italics,     "<i>$1</i>") -      .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") -      .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") -      .replaceAll(rgx.inline_strike,      "<del>$1</del>") -      .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") -      .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") -      .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); -    return _txt; -  } -  string _notes; -  string _urls; -  string _txt = _html_font_face(_html_special_characters(obj.text)); -  if (_txt.matchFirst(rgx.inline_notes_al_gen)) { -    foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { -      _notes ~= "\n" ~ m["text"]; -    } -    _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> "); -  } -  if (_txt.matchFirst(rgx.inline_link)) { -    foreach (m; _txt.matchAll(rgx.inline_link)) { -    } -    _txt = _txt.replaceAll(rgx.inline_link_clean, ""); -  } -  if (_notes.length > 0) { -    _txt ~= _notes; -  } -  if (doc_matters.opt.action.debug_do) { -    writeln(_txt, "\n"); -  } -  return _txt; -} -#+END_SRC - -****** special characters - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -string html_special_characters(string _txt){ -  _txt = _txt -    .replaceAll(rgx.xhtml_ampersand,    "&") -    .replaceAll(rgx.xhtml_quotation,    """) -    .replaceAll(rgx.xhtml_less_than,    "<") -    .replaceAll(rgx.xhtml_greater_than, ">") -    .replaceAll(rgx.nbsp_char,          " ") -    .replaceAll(rgx.xhtml_line_break,   "<br />"); -  return _txt; -} -#+END_SRC - -****** special characters for code - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -string html_special_characters_code(string _txt){ -  _txt = _txt -    .replaceAll(rgx.xhtml_ampersand,    "&") -    .replaceAll(rgx.xhtml_quotation,    """) -    .replaceAll(rgx.xhtml_less_than,    "<") -    .replaceAll(rgx.xhtml_greater_than, ">") -    .replaceAll(rgx.nbsp_char,          " "); -  return _txt; -} -#+END_SRC - -****** font_face - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -string html_font_face(string _txt){ -  _txt = _txt -    .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") -    .replaceAll(rgx.inline_bold,        "<b>$1</b>") -    .replaceAll(rgx.inline_underscore,  "<u>$1</u>") -    .replaceAll(rgx.inline_italics,     "<i>$1</i>") -    .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") -    .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") -    .replaceAll(rgx.inline_strike,      "<del>$1</del>") -    .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") -    .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") -    .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); -  return _txt; -} -#+END_SRC - -***** objects -****** heading - -#+name: html_objects -#+BEGIN_SRC d -auto html_heading(O)( -  auto return ref const O    obj, -) { -  string _txt = munge_html(obj); -  string o = format(q"¶<p class="%s"><b> -      %s -    </b></p>¶", -      obj.is_a, -      _txt, -    ); -  return o; -} -#+END_SRC - -******* +fancy+ - -##+name: prepare_objects_html -#+BEGIN_SRC d -auto html_heading(O)( -  auto return ref const O    obj, -) { -  string o; -  string _txt = munge_html(obj); -  o = format(q"¶<h%s class="%s"> -      %s -    </h%s>¶", -      obj.heading_lev_markup, -      obj.is_a, -      _txt, -      obj.heading_lev_markup, -    ); -  return o; -} -#+END_SRC - -****** para - -#+name: html_objects -#+BEGIN_SRC d -auto html_para(O)( -  auto return ref const O    obj, -) { -  string _txt = munge_html(obj); -  _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; -  string o = format(q"¶<p class="%s" indent="h%si%s"> -    %s -  </p>¶", -      obj.is_a, -      obj.indent_hang, -      obj.indent_base, -      _txt -    ); -  return o; -} -#+END_SRC - -****** quote - -#+name: html_objects -#+BEGIN_SRC d -auto html_quote(O)( -  auto return ref const O    obj, -) { -  string _txt = munge_html(obj); -  string o = format(q"¶<p class="%s"> -    %s -  </p>¶", -      obj.is_a, -      _txt -    ); -  return o; -} -#+END_SRC - -****** group - -#+name: html_objects -#+BEGIN_SRC d -auto html_group(O)( -  auto return ref const O    obj, -) { -  string _txt = munge_html(obj); -  string o = format(q"¶<p class="%s"> -    %s -  </p>¶", -      obj.is_a, -      _txt -    ); -  return o; -} -#+END_SRC - -****** block - -#+name: html_objects -#+BEGIN_SRC d -auto html_block(O)( -  auto return ref const O    obj, -) { -  string _txt = munge_html(obj); -  string o = format(q"¶ -  <p class="%s">%s</p>¶", -      obj.is_a, -      _txt.stripRight -    ); -  return o; -} -#+END_SRC - -****** verse - -#+name: html_objects -#+BEGIN_SRC d -auto html_verse(O)( -  auto return ref const O    obj, -) { -  string _txt = munge_html(obj); -  string o = format(q"¶<p class="%s">%s</p>¶", -      obj.is_a, -      _txt -    ); -  return o; -} -#+END_SRC - -****** code - -#+name: html_objects -#+BEGIN_SRC d -auto html_code(O)( -  auto return ref const O    obj, -) { -  string _txt = html_special_characters_code(obj.text); -  string o = format(q"¶<p class="%s">%s</p>¶", -      obj.is_a, -      _txt -    ); -  return o; -} -#+END_SRC - -****** table - -#+name: html_objects -#+BEGIN_SRC d -auto html_table(O)( -  auto return ref const O     obj, -) { -  auto _tablarize(O)( -    auto return ref const O    obj, -    string                     _txt, -  ) { -    string[] _table_rows = _txt.split(rgx.table_delimiter_row); -    string[] _table_cols; -    string _table; -    string _tablenote; -    foreach(row_idx, row; _table_rows) { -      _table_cols = row.split(rgx.table_delimiter_col); -        _table ~= "<tr>"; -        foreach(col_idx, cell; _table_cols) { -          if ((_table_cols.length == 1) -          && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) -            _tablenote ~= cell; -          } else { -            string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; -            string _align = ("style=\"text-align:" -            ~ ((obj.table_column_aligns[col_idx] == "l") -            ? "left\"" : "right\"")); -            _table ~= "<" -              ~ _col_is -              ~ " width=\"" -              ~ obj.table_column_widths[col_idx].to!string -              ~ "%\" " -              ~ _align -              ~ ">"; -            _table ~= cell; -            _table ~= "</" -              ~ _col_is -              ~ ">"; -          } -        } -        _table ~= "</tr>"; -      } -    auto t = tuple( -      _table, -      _tablenote, -    ); -    return t; -  } -  string _txt = munge_html(obj); -  auto t = _tablarize(obj, _txt); -  _txt = t[0]; -  string _note = t[1]; -  string o = format(q"¶<p class="%s"> -    <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center"> -      %s -    </table> -    %s -  </p>¶", -    obj.is_a, -    _txt, -    _note -  ); -  return o; -} -#+END_SRC - -*** 2. _sqlite_ (loop preparation & pre-loop action) -**** prepare insert statements and do pre-loop inserts -***** 1. _metadata & src_ text -***** 2. _doc objects_ (used with doc_objects in document loop) -******  TODO (within loop not here - insert doc objects - -work out - -*** 3. hub (sqlite_format_and_load_objects) -**** sql related - -#+name: sqlite_load_object -#+BEGIN_SRC d -auto sqlite_load_string(O,Dm)( -  auto return ref const O     obj, -  auto return ref       Dm    doc_matters, -) { -  string o; -  return o; -} -#+END_SRC - -#+name: sqlite_load_object -#+BEGIN_SRC d -auto postgresql_load_string(O,Dm)( -  auto return ref const O     obj, -  auto return ref       Dm    doc_matters, -) { -  string o; -  return o; -} -#+END_SRC - -#+name: sqlite_load_object -#+BEGIN_SRC d -string sqlite_statement(O)( -  auto return ref const O    obj, -  string                     _txt, -  string                     _html, -) { -  void _sql_exe(O)( -    string                   _sql, -  ) { -    writeln(_html); -    writeln(_sql); -  } -  string _sql; -  return _sql; -} -#+END_SRC - -**** heading - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto heading(O)( -  auto return ref const O      obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_heading(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -**** para - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto para(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_para(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -**** quote - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto quote(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_quote(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC -**** group - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto group(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_group(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -**** block - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto block(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_block(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -**** verse - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto verse(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_verse(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -**** code - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto code(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_code(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -**** table - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto table(O)( -  auto return ref const O     obj, -) { -  string[string] obj_txt = [ -    "text": generic_munge_sanitize_text_for_search(obj.text), -    "html": html_table(obj) -  ]; -  if (doc_matters.opt.action.debug_do) { -    debug(sql_txt) { -      writeln(obj_txt["text"]); -    } -    debug(sql_html) { -      writeln(obj_txt["html"]); -    } -  } else { -    // load sql -  } -  return obj_txt; -} -#+END_SRC - -** 4. loop, identify, load - loop template - -#+name: sqlite_objects_loop -#+BEGIN_SRC d -auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); -string[string] obj_txt; -string doc_text; -string[] _insert_doc_objects; -foreach (part; doc_parts) { -  foreach (obj; doc_abstraction[part]) { -    switch (obj.of_part) { -    case "frontmatter":              assert(part == "head", part); -      switch (obj.is_of) { -      case "para": -        switch (obj.is_a) { -        case "heading": -          obj_txt = format_and_sqlite_load.heading(obj); -          break; -        default: -          if ((doc_matters.opt.action.debug_do)) { -            writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -          } -          break; -        } -        break; -      default: -        if ((doc_matters.opt.action.debug_do)) { -          writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -        } -        break; -      } -      break; -    case "body": //                    assert(part == "body", part); // TODO broken -      switch (obj.is_of) { -      case "para": -        switch (obj.is_a) { -        case "heading": -          debug (asserts) { // TODO consider and fix or remove -            if (part != "body") { -              writeln(__LINE__, ": ", obj.text); -            } -          } -          obj_txt = format_and_sqlite_load.heading(obj); -          break; -        case "para": -          obj_txt = format_and_sqlite_load.para(obj); -          break; -        default: -          if ((doc_matters.opt.action.debug_do)) { -            writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -          } -          break; -        } -        break; -      case "block": -        switch (obj.is_a) { -        case "quote": -          obj_txt = format_and_sqlite_load.quote(obj); -          break; -        case "group": -          obj_txt = format_and_sqlite_load.group(obj); -          break; -        case "block": -          obj_txt = format_and_sqlite_load.block(obj); -          break; -        case "poem":                        // double check on keeping both poem & verse -          break; -        case "verse": -          obj_txt = format_and_sqlite_load.verse(obj); -          break; -        case "code": -          obj_txt = format_and_sqlite_load.code(obj); -          break; -        case "table": -          obj_txt = format_and_sqlite_load.table(obj); -          break; -        default: -          if ((doc_matters.opt.action.debug_do)) { -            writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -          } -          break; -        } -        break; -      default: -        if ((doc_matters.opt.action.debug_do)) { -          writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -        } -        break; -      } -      break; -    case "backmatter": -      assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); -      switch (obj.is_of) { -      case "para": -        switch (obj.is_a) { -        case "heading": -          obj_txt = format_and_sqlite_load.heading(obj); -          break; -        case "endnote":              assert(part == "endnotes", part); -          obj_txt = format_and_sqlite_load.para(obj); -          break; -        case "glossary":             assert(part == "glossary", part); -          obj_txt = format_and_sqlite_load.para(obj); -          break; -        case "bibliography":         assert(part == "bibliography", part); -          obj_txt = format_and_sqlite_load.para(obj); -          break; -        case "bookindex":            assert(part == "bookindex_seg", part); -          obj_txt = format_and_sqlite_load.para(obj); -          break; -        case "blurb":                assert(part == "blurb", part); -          obj_txt = format_and_sqlite_load.para(obj); -          break; -        default: -          if ((doc_matters.opt.action.debug_do)) { -            writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -          } -          break; -        } -        break; -      default: -        if ((doc_matters.opt.action.debug_do)) { -          writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -        } -        break; -      } -      break; -    case "comment": -      break; -    default: -      if ((doc_matters.opt.action.debug_do)) { -        writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from -        writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -        writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from -      } -      break; -    } -    if (obj.is_a == "heading") { -      if ((doc_matters.opt.action.verbose)) { -        writeln( -          "markup: ", obj.heading_lev_markup, -          "> ", obj.dom_markedup, -          "; collapsed: ", obj.heading_lev_collapsed, -          "> ", obj.dom_collapsed, -          "; ocn: ", obj.ocn, -          " node: ", obj.node, -          "; parent: ", obj.parent_lev_markup, -          "; ocn: ", obj.parent_ocn, -          "; ", -        ); -      } -    } -    if (!(obj.is_a == "comment")) { -      _insert_doc_objects ~= insertDocObjectsRow(obj); -    } -  } // loop closes -} -return _insert_doc_objects.join.to!(char[]).toUTF8; -#+END_SRC - -** 5. SQL statements -*** create tables - -**** DROP INDEX IF EXISTS - -#+name: sqlite_statement_drop_existing_index -#+BEGIN_SRC sql -DROP INDEX IF EXISTS ocn; -DROP INDEX IF EXISTS digest_clean; -DROP INDEX IF EXISTS digest_all; -DROP INDEX IF EXISTS clean; -DROP INDEX IF EXISTS lev0; -DROP INDEX IF EXISTS lev1; -DROP INDEX IF EXISTS lev2; -DROP INDEX IF EXISTS lev3; -DROP INDEX IF EXISTS lev4; -DROP INDEX IF EXISTS lev5; -DROP INDEX IF EXISTS lev6; -DROP INDEX IF EXISTS title; -DROP INDEX IF EXISTS creator_author; -DROP INDEX IF EXISTS src_filename; -DROP INDEX IF EXISTS language_document_char; -DROP INDEX IF EXISTS classify_topic_register; -#+END_SRC - -**** DROP TABLE IF EXISTS - -#+name: sqlite_statement_drop_existing_tables -#+BEGIN_SRC sql -DROP TABLE IF EXISTS metadata_and_text; -DROP TABLE IF EXISTS doc_objects; -DROP TABLE IF EXISTS urls; -#+END_SRC - -**** CREATE TABLE metadata_and_text - -#+name: sqlite_statement_create_table_metadata_and_src_txt -#+BEGIN_SRC sql -CREATE TABLE metadata_and_text ( -  tid                              INTEGER           PRIMARY KEY, -/* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ -  title                            VARCHAR(800)  NOT NULL, -  title_main                       VARCHAR(400)  NOT NULL, -  title_sub                        VARCHAR(400)      NULL, -  title_short                      VARCHAR(400)      NULL, -  title_edition                    VARCHAR(10)       NULL, -  title_note                       VARCHAR(2500)     NULL, -  title_language                   VARCHAR(100)      NULL, -  title_language_char              VARCHAR(6)        NULL, -  creator_author                   VARCHAR(600)  NOT NULL, -  creator_author_email             VARCHAR(100)      NULL, -  creator_author_hon               VARCHAR(100)      NULL, -  creator_author_nationality       VARCHAR(100)      NULL, -  creator_editor                   VARCHAR(600)      NULL, -  creator_contributor              VARCHAR(600)      NULL, -  creator_illustrator              VARCHAR(600)      NULL, -  creator_photographer             VARCHAR(600)      NULL, -  creator_translator               VARCHAR(600)      NULL, -  creator_prepared_by              VARCHAR(600)      NULL, -  creator_digitized_by             VARCHAR(600)      NULL, -  creator_audio                    VARCHAR(600)      NULL, -  creator_video                    VARCHAR(600)      NULL, -  language_document                VARCHAR(100)      NULL, -  language_document_char           VARCHAR(6)    NOT NULL, -  language_original                VARCHAR(100)      NULL, -  language_original_char           VARCHAR(6)        NULL, -  date_added_to_site               VARCHAR(10)       NULL, -  date_available                   VARCHAR(10)       NULL, -  date_created                     VARCHAR(10)       NULL, -  date_issued                      VARCHAR(10)       NULL, -  date_modified                    VARCHAR(10)       NULL, -  date_published                   VARCHAR(10)       NULL, -  date_valid                       VARCHAR(10)       NULL, -  date_translated                  VARCHAR(10)       NULL, -  date_original_publication        VARCHAR(10)       NULL, -  date_generated                   VARCHAR(10)       NULL, -  publisher                        VARCHAR(600)      NULL, -  original_publisher               VARCHAR(600)      NULL, -  original_language                VARCHAR(100)      NULL, -  original_language_char           VARCHAR(6)        NULL, -  original_source                  VARCHAR(600)      NULL, -  original_institution             VARCHAR(600)      NULL, -  original_nationality             VARCHAR(100)      NULL, -  original_title                   VARCHAR(800)      NULL, -  rights_copyright                 VARCHAR(2500)     NULL, -  rights_copyright_audio           VARCHAR(2500)     NULL, -  rights_copyright_cover           VARCHAR(2500)     NULL, -  rights_copyright_illustrations   VARCHAR(2500)     NULL, -  rights_copyright_photographs     VARCHAR(2500)     NULL, -  rights_copyright_text            VARCHAR(2500)     NULL, -  rights_copyright_translation     VARCHAR(2500)     NULL, -  rights_copyright_video           VARCHAR(2500)     NULL, -  rights_license                   VARCHAR(2500)     NULL, -  identifier_oclc                  VARCHAR(30)       NULL, -  identifier_isbn                  VARCHAR(16)       NULL, -  classify_topic_register          VARCHAR(2500)     NULL, -  classify_subject                 VARCHAR(600)      NULL, -  classify_loc                     VARCHAR(30)       NULL, -  classify_dewey                   VARCHAR(30)       NULL, -  classify_keywords                VARCHAR(600)      NULL, -  notes_abstract                   TEXT              NULL, -  notes_description                TEXT              NULL, -  notes_comment                    TEXT              NULL, -  notes_coverage                   VARCHAR(200)      NULL, -  notes_relation                   VARCHAR(200)      NULL, -  notes_history                    VARCHAR(600)      NULL, -  notes_type                       VARCHAR(600)      NULL, -  notes_format                     VARCHAR(600)      NULL, -  notes_prefix                     TEXT              NULL, -  notes_prefix_a                   TEXT              NULL, -  notes_prefix_b                   TEXT              NULL, -  notes_suffix                     TEXT              NULL, -  src_filename                     VARCHAR(256)  NOT NULL, -  src_fingerprint                  VARCHAR(256)      NULL, -  src_filesize                     VARCHAR(10)       NULL, -  src_wordcount                    VARCHAR(10)       NULL, -  pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ -  pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ -  pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ -  src_text                         TEXT              NULL, -  fulltext                         TEXT              NULL, -  links                            TEXT              NULL -); -#+END_SRC - -**** CREATE TABLE doc_objects - -#+name: sqlite_statement_create_table_objects -#+BEGIN_SRC sql -CREATE TABLE doc_objects ( -  lid                              INTEGER PRIMARY KEY, -  metadata_tid                     INTEGER REFERENCES metadata_and_text, -  ocn                              SMALLINT, -  ocnd                             VARCHAR(6), -  ocns                             VARCHAR(6), -  clean                            TEXT NULL, -  body                             TEXT NULL, -  book_idx                         TEXT NULL, -  seg                              VARCHAR(256) NULL, -  lev_an                           VARCHAR(1), -  lev                              SMALLINT NULL, -  lev0                             SMALLINT, -  lev1                             SMALLINT, -  lev2                             SMALLINT, -  lev3                             SMALLINT, -  lev4                             SMALLINT, -  lev5                             SMALLINT, -  lev6                             SMALLINT, -  lev7                             SMALLINT, -  en_a                             SMALLINT NULL, -  en_z                             SMALLINT NULL, -  en_a_asterisk                    SMALLINT NULL, -  en_z_asterisk                    SMALLINT NULL, -  en_a_plus                        SMALLINT NULL, -  en_z_plus                        SMALLINT NULL, -  t_of                             VARCHAR(16), -  t_is                             VARCHAR(16), -  node                             VARCHAR(16) NULL, -  parent                           VARCHAR(16) NULL, -  digest_clean                     CHAR(256), -  digest_all                       CHAR(256), -  types                            CHAR(1) NULL -); -#+END_SRC - -**** CREATE INDEX - -#+name: sqlite_statement_create_index -#+BEGIN_SRC sql -CREATE INDEX idx_ocn ON doc_objects(ocn); -CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); -CREATE INDEX idx_digest_all ON doc_objects(digest_all); -CREATE INDEX idx_clean ON doc_objects(clean); -CREATE INDEX idx_lev0 ON doc_objects(lev0); -CREATE INDEX idx_lev1 ON doc_objects(lev1); -CREATE INDEX idx_lev2 ON doc_objects(lev2); -CREATE INDEX idx_lev3 ON doc_objects(lev3); -CREATE INDEX idx_lev4 ON doc_objects(lev4); -CREATE INDEX idx_lev5 ON doc_objects(lev5); -CREATE INDEX idx_lev6 ON doc_objects(lev6); -CREATE INDEX idx_title ON metadata_and_text(title); -CREATE INDEX idx_author ON metadata_and_text(creator_author); -CREATE INDEX idx_filename ON metadata_and_text(src_filename); -CREATE INDEX idx_language ON metadata_and_text(language_document_char); -CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); -#+END_SRC - -**** TODO local site link & info - -*** sql insert statement formatted doc objects - -lid unique, increment by 1 per object, not ocn - -metadata tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -**** sql statement: dlang format -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC d -string _insert_doc_objects_row; -_insert_doc_objects_row = format(q"¶ -#+END_SRC - -**** INSERT INTO - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC sql -  INSERT INTO doc_objects ( -    metadata_tid, -    ocn, -    clean, -    body, -    lev, -    t_of, -    t_is -  ) -#+END_SRC - -**** VALUES - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC sql -  VALUES ( -    %d, -    %s, -    '%s', -    '%s', -    %s, -    '%s', -    '%s' -  ); -#+END_SRC - -**** sql statement: dlang values for formatting - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC d -¶", -  1, -  obj.ocn, -  SQLinsertDelimiter!()(obj_txt["text"]), -  SQLinsertDelimiter!()(obj_txt["html"]), -  obj.heading_lev_markup, -  obj.is_of, -  obj.is_a, -); -#+END_SRC - -*** sqlite insert statement formatted doc matters metadata -**** sql statement: dlang format -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC d -string _insert_metadata; -_insert_metadata = format(q"¶ -#+END_SRC - -**** INSERT INTO - -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC sql -  INSERT INTO metadata_and_text ( -    src_filename, -    title, -    title_main, -    title_sub, -    title_short, -    title_edition, -    title_language, -    classify_dewey, -    classify_keywords, -    classify_loc, -    classify_subject, -    classify_topic_register, -    creator_author, -    creator_author_email, -    creator_illustrator, -    creator_translator, -    date_added_to_site, -    date_available, -    date_created, -    date_issued, -    date_modified, -    date_published, -    date_valid, -    identifier_isbn, -    identifier_oclc, -    language_document, -    language_document_char, -    notes_abstract, -    notes_description, -    original_publisher, -    original_language, -    original_language_char, -    original_source, -    original_title, -    publisher, -    rights_copyright, -    rights_copyright_audio, -    rights_copyright_cover, -    rights_copyright_illustrations, -    rights_copyright_photographs, -    rights_copyright_text, -    rights_copyright_translation, -    rights_copyright_video, -    rights_license -  ) -#+END_SRC - -**** VALUES - -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC sql -  VALUES ( -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s', -    '%s' -  ); -#+END_SRC - -**** sql statement: values for formatting - -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC d -¶", -  SQLinsertDelimiter!()(doc_matters.src.filename), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), -  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) -); -#+END_SRC - -* __END__ diff --git a/org/sdp.org b/org/sdp.org index 6b9052b..a3ddf40 100644 --- a/org/sdp.org +++ b/org/sdp.org @@ -26,7 +26,7 @@ struct Version {    int minor;    int patch;  } -enum ver = Version(0, 26, 0); +enum ver = Version(0, 26, 1);  #+END_SRC  ** compilation restrictions (supported compilers) diff --git a/src/sdp/meta/rgx.d b/src/sdp/meta/rgx.d index 0f60847..6006122 100644 --- a/src/sdp/meta/rgx.d +++ b/src/sdp/meta/rgx.d @@ -199,7 +199,7 @@ static template SiSUrgxInit() {      static nbsp_and_space                                 = ctRegex!(` [ ]`, "mg");      static nbsp_char_and_space                            = ctRegex!(`░[ ]`, "mg");      static special_markup_chars                           = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg"); -    static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`); +    static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);      static src_pth_pod_sst_or_ssm                         = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);      static src_pth_contents                               = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/sisupod[.]manifest$`);      static src_pth_pod_root                               = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d index 373092b..d395269 100644 --- a/src/sdp/output/hub.d +++ b/src/sdp/output/hub.d @@ -8,7 +8,6 @@ template outputHub() {      sdp.output.epub3,      sdp.output.html,      sdp.output.sqlite, -    sdp.output.sqlite_discrete,      sdp.output.xmls,      sdp.output.source_sisupod,      sdp.output.create_zip_file, @@ -16,6 +15,35 @@ template outputHub() {    void outputHub(D,I)(D doc_abstraction, I doc_matters) {      mixin SiSUoutputRgxInit;      static auto rgx = Rgx(); +    debug (substitutions) { +      enum Substitute { match, markup, html, } +      writeln(__LINE__, ":", __FILE__, ": DEBUG substitutions:"); +      writeln("Doc Title:            ", doc_matters.conf_make_meta.meta.title_full); +      if (doc_matters.conf_make_meta.make.bold) { +        writeln("regex to match:       ", doc_matters.conf_make_meta.make.bold[Substitute.match]); +        writeln("substitution to make: ", doc_matters.conf_make_meta.make.bold[Substitute.html]); + +        auto _w = "1. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" +          .replaceAll( +            regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), +            doc_matters.conf_make_meta.make.bold[Substitute.html] +          ); +        writeln(_w); + +        writeln("2a. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" +          .replaceAll( +            regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), +            doc_matters.conf_make_meta.make.bold[Substitute.html] +          )); + +        auto _v = regex(doc_matters.conf_make_meta.make.bold[Substitute.match]); +        writeln("2b. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" +          .replaceAll( +            _v, +            doc_matters.conf_make_meta.make.bold[Substitute.html] +          )); +      } +    }      if (doc_matters.opt.action.verbose) {        writeln(doc_matters.xml.keys_seq.seg);      } @@ -109,13 +137,13 @@ template outputHub() {        if ((doc_matters.opt.action.verbose)) {          writeln("sqlite processing... ");        } -      SQLiteDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); +      SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);      }      if (doc_matters.opt.action.sqlite_update) {        if ((doc_matters.opt.action.verbose)) {          writeln("sqlite processing... ");        } -      SQLiteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); +      SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters);      }      if (doc_matters.opt.action.postgresql) {        /+ mixin outputPostgreSQL; +/ @@ -128,7 +156,6 @@ template outputHubOp() {      sdp.output.epub3,      sdp.output.html,      sdp.output.sqlite, -    sdp.output.sqlite_discrete,      sdp.output.xmls,      sdp.output.source_sisupod,      sdp.output.create_zip_file, diff --git a/src/sdp/output/paths_source.d b/src/sdp/output/paths_source.d index c420d7e..d04c9f6 100644 --- a/src/sdp/output/paths_source.d +++ b/src/sdp/output/paths_source.d @@ -151,6 +151,7 @@ template PathMatters() {          return Pod_();        }        auto src() { +        string _sep = "␣";          struct SRC_ {            auto is_pod() {              auto _src_is_pod = (_manifest.pod_manifest_path.length > 0) ? true : false; @@ -159,19 +160,70 @@ template PathMatters() {            auto path_and_fn() {              return _fns;            } +          auto pod_name() { /+ work on +/ +            auto _pod_name = (is_pod) +            ? _manifest.pod_manifest_path +            : ""; +            return _pod_name; +          }            auto filename() {              auto _fn = (path_and_fn).baseName;              return _fn;            } +          auto filename_base() { +            auto _fn = filename.stripExtension; +            return _fn; +          } +          auto filename_extension() { +            auto _ext = filename.match(rgx.src_pth_sst_or_ssm).captures["extension"]; +            return _ext; +          }            auto lng() {              string _k; -            if (auto m = (path_and_fn).match(rgx.language_code_and_filename)) { +            if (auto m = path_and_fn.match(rgx.language_code_and_filename)) {                _k = m.captures[1];              } else {                _k = "en";              }              return _k;            } +          auto docname_composite_unique_per_src_doc() { +          /+ +            z pod name if any + src filename + lng code +             filename ~ "." ~  lng +             * unique per src doc +             used by +             - sqlite discrete index (multilingual, each language of a document) +          +/ +            string _fn; +            if (pod_name.baseName == filename_base) { +              _fn = filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; +            } else if (!(pod_name.empty)) { +              _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; +            } else { +              _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; +            } +            return _fn; +          } +          auto docname_composite_unique_per_src_pod() { +          /+ +            z pod name if any + src filename (without lng code) +             filename ~ _sep ~ lng +             * unique per src pod +             used by +             - sisupod (multilingual collection) +             - sqlite discrete index (multilingual collection) +          +/ +            string _fn; +            if (pod_name.baseName == filename_base) { +              _fn = filename_base ~ _sep ~ filename_extension; +            } else if (!(pod_name.empty)) { +              _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension; +            } else { +              _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension; +            } +            return _fn; +          }            auto language() {              return lng();            } diff --git a/src/sdp/output/rgx.d b/src/sdp/output/rgx.d index d559491..4bfa72a 100644 --- a/src/sdp/output/rgx.d +++ b/src/sdp/output/rgx.d @@ -16,7 +16,7 @@ static template SiSUoutputRgxInit() {      static nbsp_and_space                                 = ctRegex!(` [ ]`, "mg");      static nbsp_char_and_space                            = ctRegex!(`░[ ]`, "mg");      static special_markup_chars                           = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg"); -    static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`); +    static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);      static src_pth_pod_sst_or_ssm                         = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);      static src_pth_contents                               = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/sisupod[.]manifest$`);      static src_pth_pod_root                               = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d index 2409e3b..fc1843f 100644 --- a/src/sdp/output/sqlite.d +++ b/src/sdp/output/sqlite.d @@ -3,7 +3,79 @@ import sdp.output;  import    std.file,    std.conv : to; +import d2sqlite3; +import std.typecons : Nullable; +mixin SiSUoutputRgxInit; +static auto rgx = Rgx();  long _metadata_tid_lastrowid; +template SQLiteHubBuildTablesAndPopulate() { +  void SQLiteHubBuildTablesAndPopulate(D,I)( +    auto ref const D    doc_abstraction, +    auto ref       I    doc_matters, +  ) { +    auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); +    template SQLiteDbStatementComposite() { +      void SQLiteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        { +          string _db_statement; +          { +            if ((doc_matters.opt.action.sqlite_create)) { +              auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); +              pth_sqlite.base.mkdirRecurse; +              _db_statement ~= SQLiteTablesReCreate!()(); +              SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); +            } +            if ((doc_matters.opt.action.sqlite_update)) { // TODO +              _db_statement ~= SQLiteInsertMetadata!()(doc_matters); +              SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); +              /+ get tid (lastrowid or max) for use in doc_objects table +/ +              _metadata_tid_lastrowid = db.lastInsertRowid(); +              _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX +              SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); +            } +          } +          db.close; +        } +      } +    } +    SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); +  } +} +template SQLiteHubDiscreteBuildTablesAndPopulate() { +  void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)( +    auto ref const D    doc_abstraction, +    auto ref       I    doc_matters, +  ) { +    auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); +    template SQLiteDiscreteDbStatementComposite() { +      void SQLiteDiscreteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        { +          string _db_statement; +          { +            _db_statement ~= SQLiteTablesReCreate!()(); +            _db_statement ~= SQLiteInsertMetadata!()(doc_matters); +            _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX +            SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); +          } +          db.close; +        } +      } +    } +    SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); +  } +}  template SQLiteDbRun() {    void SQLiteDbRun(Db,St,O)(      auto ref       Db   db, @@ -20,7 +92,9 @@ template SQLiteDbRun() {          db_statement ~          "\nCOMMIT;\n"        ); -      if (!(opt_action.sqlite_create)) { +      if (!(opt_action.sqlite_discrete) +        && !(opt_action.sqlite_create) +      ) {          _metadata_tid_lastrowid = db.lastInsertRowid();          writeln("last row id: ", _metadata_tid_lastrowid);        } @@ -37,1005 +111,968 @@ template SQLiteDbRun() {      }    }  } -template SQLiteBuildTablesAndPopulate() { -  void SQLiteBuildTablesAndPopulate(D,I)( -    auto ref const D    doc_abstraction, +template SQLinsertDelimiter() { +  auto SQLinsertDelimiter(string _txt) { +    _txt = _txt +      .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +    return _txt; +  } +} +template SQLiteFormatAndLoadObject() { +  auto SQLiteFormatAndLoadObject(I)(      auto ref       I    doc_matters,    ) { -    import d2sqlite3; -    import std.typecons : Nullable;      mixin SiSUoutputRgxInit; -    static auto rgx = Rgx(); -    auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); -    pth_sqlite.base.mkdirRecurse; -    auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); -    template SQLiteDbStatementComposite() { -      void SQLiteDbStatementComposite(Db,D,I)( -        auto ref       Db   db, -        auto ref const D    doc_abstraction, -        auto ref       I    doc_matters, +    struct sqlite_format_and_load_objects { +      auto generic_munge_sanitize_text_for_search( +        string _txt,        ) { -        { -          { -            string _db_statement; -            { -              if ((doc_matters.opt.action.sqlite_create)) { -                auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); -                pth_sqlite.base.mkdirRecurse; -                _db_statement ~= SQLiteTablesReCreate!()(); -                SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); -              } -              if ((doc_matters.opt.action.sqlite_update)) { // TODO -                _db_statement ~= SQLiteInsertMetadata!()(doc_matters); -                SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); -                /+ get tid (lastrowid or max) for use in doc_objects table +/ -                _metadata_tid_lastrowid = db.lastInsertRowid(); -                _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX -                SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); -              } +        string _notes; +        string _urls; +        if (_txt.matchFirst(rgx.inline_notes_al_gen)) { +          foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { +            _notes ~= "\n" ~ m["text"]; +          } +          _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); +        } +        if (_txt.matchFirst(rgx.inline_link)) { +          foreach (m; _txt.matchAll(rgx.inline_link)) { +            if (m["link"].match(rgx.url)) { +              _urls ~= "\n" ~ m["link"];              } -            db.close;            } +          _txt = _txt.replaceAll(rgx.inline_link_clean, ""); +        } +        if (_notes.length > 0) { +          _txt ~= _notes; +        } +        if (_urls.length > 0) { +          _txt ~= _urls;          } +        if (doc_matters.opt.action.debug_do) { +          writeln(_txt, "\n"); +        } +        debug(sql_text_clean) { +          writeln(_txt); +        } +        return _txt;        } -    } -    template SQLinsertDelimiter() { -      auto SQLinsertDelimiter(string _txt) { +      auto munge_html(O)( +        auto return ref const O    obj, +      ) { +        string _html_special_characters(string _txt){ +          _txt = _txt +            .replaceAll(rgx.xhtml_ampersand,    "&") +            .replaceAll(rgx.xhtml_quotation,    """) +            .replaceAll(rgx.xhtml_less_than,    "<") +            .replaceAll(rgx.xhtml_greater_than, ">") +            .replaceAll(rgx.nbsp_char,          " ") +            .replaceAll(rgx.xhtml_line_break,   "<br />"); +          return _txt; +        } +        string _html_font_face(string _txt){ +          _txt = _txt +            .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") +            .replaceAll(rgx.inline_bold,        "<b>$1</b>") +            .replaceAll(rgx.inline_underscore,  "<u>$1</u>") +            .replaceAll(rgx.inline_italics,     "<i>$1</i>") +            .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") +            .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") +            .replaceAll(rgx.inline_strike,      "<del>$1</del>") +            .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") +            .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") +            .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); +          return _txt; +        } +        string _notes; +        string _urls; +        string _txt = _html_font_face(_html_special_characters(obj.text)); +        if (_txt.matchFirst(rgx.inline_notes_al_gen)) { +          foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { +            _notes ~= "\n" ~ m["text"]; +          } +          _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> "); +        } +        if (_txt.matchFirst(rgx.inline_link)) { +          foreach (m; _txt.matchAll(rgx.inline_link)) { +          } +          _txt = _txt.replaceAll(rgx.inline_link_clean, ""); +        } +        if (_notes.length > 0) { +          _txt ~= _notes; +        } +        if (doc_matters.opt.action.debug_do) { +          writeln(_txt, "\n"); +        } +        return _txt; +      } +      string html_special_characters(string _txt){          _txt = _txt -          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +          .replaceAll(rgx.xhtml_ampersand,    "&") +          .replaceAll(rgx.xhtml_quotation,    """) +          .replaceAll(rgx.xhtml_less_than,    "<") +          .replaceAll(rgx.xhtml_greater_than, ">") +          .replaceAll(rgx.nbsp_char,          " ") +          .replaceAll(rgx.xhtml_line_break,   "<br />");          return _txt;        } -    } -    template SQLiteFormatAndLoadObject() { -      auto SQLiteFormatAndLoadObject(I)( -        auto ref       I    doc_matters, +      string html_special_characters_code(string _txt){ +        _txt = _txt +          .replaceAll(rgx.xhtml_ampersand,    "&") +          .replaceAll(rgx.xhtml_quotation,    """) +          .replaceAll(rgx.xhtml_less_than,    "<") +          .replaceAll(rgx.xhtml_greater_than, ">") +          .replaceAll(rgx.nbsp_char,          " "); +        return _txt; +      } +      string html_font_face(string _txt){ +        _txt = _txt +          .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") +          .replaceAll(rgx.inline_bold,        "<b>$1</b>") +          .replaceAll(rgx.inline_underscore,  "<u>$1</u>") +          .replaceAll(rgx.inline_italics,     "<i>$1</i>") +          .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") +          .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") +          .replaceAll(rgx.inline_strike,      "<del>$1</del>") +          .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") +          .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") +          .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); +        return _txt; +      } +      auto html_heading(O)( +        auto return ref const O    obj,        ) { -        mixin SiSUoutputRgxInit; -        struct sqlite_format_and_load_objects { -          auto generic_munge_sanitize_text_for_search( -            string _txt, -          ) { -            string _notes; -            string _urls; -            if (_txt.matchFirst(rgx.inline_notes_al_gen)) { -              foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { -                _notes ~= "\n" ~ m["text"]; -              } -              _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); -            } -            if (_txt.matchFirst(rgx.inline_link)) { -              foreach (m; _txt.matchAll(rgx.inline_link)) { -                if (m["link"].match(rgx.url)) { -                  _urls ~= "\n" ~ m["link"]; +        string _txt = munge_html(obj); +        string o = format(q"¶<p class="%s"><b> +            %s +          </b></p>¶", +            obj.is_a, +            _txt, +          ); +        return o; +      } +      auto html_para(O)( +        auto return ref const O    obj, +      ) { +        string _txt = munge_html(obj); +        _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; +        string o = format(q"¶<p class="%s" indent="h%si%s"> +          %s +        </p>¶", +            obj.is_a, +            obj.indent_hang, +            obj.indent_base, +            _txt +          ); +        return o; +      } +      auto html_quote(O)( +        auto return ref const O    obj, +      ) { +        string _txt = munge_html(obj); +        string o = format(q"¶<p class="%s"> +          %s +        </p>¶", +            obj.is_a, +            _txt +          ); +        return o; +      } +      auto html_group(O)( +        auto return ref const O    obj, +      ) { +        string _txt = munge_html(obj); +        string o = format(q"¶<p class="%s"> +          %s +        </p>¶", +            obj.is_a, +            _txt +          ); +        return o; +      } +      auto html_block(O)( +        auto return ref const O    obj, +      ) { +        string _txt = munge_html(obj); +        string o = format(q"¶ +        <p class="%s">%s</p>¶", +            obj.is_a, +            _txt.stripRight +          ); +        return o; +      } +      auto html_verse(O)( +        auto return ref const O    obj, +      ) { +        string _txt = munge_html(obj); +        string o = format(q"¶<p class="%s">%s</p>¶", +            obj.is_a, +            _txt +          ); +        return o; +      } +      auto html_code(O)( +        auto return ref const O    obj, +      ) { +        string _txt = html_special_characters_code(obj.text); +        string o = format(q"¶<p class="%s">%s</p>¶", +            obj.is_a, +            _txt +          ); +        return o; +      } +      auto html_table(O)( +        auto return ref const O     obj, +      ) { +        auto _tablarize(O)( +          auto return ref const O    obj, +          string                     _txt, +        ) { +          string[] _table_rows = _txt.split(rgx.table_delimiter_row); +          string[] _table_cols; +          string _table; +          string _tablenote; +          foreach(row_idx, row; _table_rows) { +            _table_cols = row.split(rgx.table_delimiter_col); +              _table ~= "<tr>"; +              foreach(col_idx, cell; _table_cols) { +                if ((_table_cols.length == 1) +                && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) +                  _tablenote ~= cell; +                } else { +                  string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; +                  string _align = ("style=\"text-align:" +                  ~ ((obj.table_column_aligns[col_idx] == "l") +                  ? "left\"" : "right\"")); +                  _table ~= "<" +                    ~ _col_is +                    ~ " width=\"" +                    ~ obj.table_column_widths[col_idx].to!string +                    ~ "%\" " +                    ~ _align +                    ~ ">"; +                  _table ~= cell; +                  _table ~= "</" +                    ~ _col_is +                    ~ ">";                  }                } -              _txt = _txt.replaceAll(rgx.inline_link_clean, ""); -            } -            if (_notes.length > 0) { -              _txt ~= _notes; -            } -            if (_urls.length > 0) { -              _txt ~= _urls; -            } -            if (doc_matters.opt.action.debug_do) { -              writeln(_txt, "\n"); -            } -            debug(sql_text_clean) { -              writeln(_txt); -            } -            return _txt; -          } -          auto munge_html(O)( -            auto return ref const O    obj, -          ) { -            string _html_special_characters(string _txt){ -              _txt = _txt -                .replaceAll(rgx.xhtml_ampersand,    "&") -                .replaceAll(rgx.xhtml_quotation,    """) -                .replaceAll(rgx.xhtml_less_than,    "<") -                .replaceAll(rgx.xhtml_greater_than, ">") -                .replaceAll(rgx.nbsp_char,          " ") -                .replaceAll(rgx.xhtml_line_break,   "<br />"); -              return _txt; -            } -            string _html_font_face(string _txt){ -              _txt = _txt -                .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") -                .replaceAll(rgx.inline_bold,        "<b>$1</b>") -                .replaceAll(rgx.inline_underscore,  "<u>$1</u>") -                .replaceAll(rgx.inline_italics,     "<i>$1</i>") -                .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") -                .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") -                .replaceAll(rgx.inline_strike,      "<del>$1</del>") -                .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") -                .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") -                .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); -              return _txt; +              _table ~= "</tr>";              } -            string _notes; -            string _urls; -            string _txt = _html_font_face(_html_special_characters(obj.text)); -            if (_txt.matchFirst(rgx.inline_notes_al_gen)) { -              foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { -                _notes ~= "\n" ~ m["text"]; -              } -              _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> "); -            } -            if (_txt.matchFirst(rgx.inline_link)) { -              foreach (m; _txt.matchAll(rgx.inline_link)) { -              } -              _txt = _txt.replaceAll(rgx.inline_link_clean, ""); -            } -            if (_notes.length > 0) { -              _txt ~= _notes; -            } -            if (doc_matters.opt.action.debug_do) { -              writeln(_txt, "\n"); -            } -            return _txt; -          } -          string html_special_characters(string _txt){ -            _txt = _txt -              .replaceAll(rgx.xhtml_ampersand,    "&") -              .replaceAll(rgx.xhtml_quotation,    """) -              .replaceAll(rgx.xhtml_less_than,    "<") -              .replaceAll(rgx.xhtml_greater_than, ">") -              .replaceAll(rgx.nbsp_char,          " ") -              .replaceAll(rgx.xhtml_line_break,   "<br />"); -            return _txt; -          } -          string html_special_characters_code(string _txt){ -            _txt = _txt -              .replaceAll(rgx.xhtml_ampersand,    "&") -              .replaceAll(rgx.xhtml_quotation,    """) -              .replaceAll(rgx.xhtml_less_than,    "<") -              .replaceAll(rgx.xhtml_greater_than, ">") -              .replaceAll(rgx.nbsp_char,          " "); -            return _txt; -          } -          string html_font_face(string _txt){ -            _txt = _txt -              .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") -              .replaceAll(rgx.inline_bold,        "<b>$1</b>") -              .replaceAll(rgx.inline_underscore,  "<u>$1</u>") -              .replaceAll(rgx.inline_italics,     "<i>$1</i>") -              .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") -              .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") -              .replaceAll(rgx.inline_strike,      "<del>$1</del>") -              .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") -              .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") -              .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); -            return _txt; -          } -          auto html_heading(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s"><b> -                %s -              </b></p>¶", -                obj.is_a, -                _txt, -              ); -            return o; -          } -          auto html_para(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; -            string o = format(q"¶<p class="%s" indent="h%si%s"> -              %s -            </p>¶", -                obj.is_a, -                obj.indent_hang, -                obj.indent_base, -                _txt -              ); -            return o; -          } -          auto html_quote(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s"> -              %s -            </p>¶", -                obj.is_a, -                _txt -              ); -            return o; -          } -          auto html_group(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s"> -              %s -            </p>¶", -                obj.is_a, -                _txt -              ); -            return o; +          auto t = tuple( +            _table, +            _tablenote, +          ); +          return t; +        } +        string _txt = munge_html(obj); +        auto t = _tablarize(obj, _txt); +        _txt = t[0]; +        string _note = t[1]; +        string o = format(q"¶<p class="%s"> +          <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center"> +            %s +          </table> +          %s +        </p>¶", +          obj.is_a, +          _txt, +          _note +        ); +        return o; +      } +      auto sqlite_load_string(O,Dm)( +        auto return ref const O     obj, +        auto return ref       Dm    doc_matters, +      ) { +        string o; +        return o; +      } +      auto postgresql_load_string(O,Dm)( +        auto return ref const O     obj, +        auto return ref       Dm    doc_matters, +      ) { +        string o; +        return o; +      } +      string sqlite_statement(O)( +        auto return ref const O    obj, +        string                     _txt, +        string                     _html, +      ) { +        void _sql_exe(O)( +          string                   _sql, +        ) { +          writeln(_html); +          writeln(_sql); +        } +        string _sql; +        return _sql; +      } +      auto heading(O)( +        auto return ref const O      obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_heading(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto html_block(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶ -            <p class="%s">%s</p>¶", -                obj.is_a, -                _txt.stripRight -              ); -            return o; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto html_verse(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s">%s</p>¶", -                obj.is_a, -                _txt -              ); -            return o; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto para(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_para(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto html_code(O)( -            auto return ref const O    obj, -          ) { -            string _txt = html_special_characters_code(obj.text); -            string o = format(q"¶<p class="%s">%s</p>¶", -                obj.is_a, -                _txt -              ); -            return o; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto html_table(O)( -            auto return ref const O     obj, -          ) { -            auto _tablarize(O)( -              auto return ref const O    obj, -              string                     _txt, -            ) { -              string[] _table_rows = _txt.split(rgx.table_delimiter_row); -              string[] _table_cols; -              string _table; -              string _tablenote; -              foreach(row_idx, row; _table_rows) { -                _table_cols = row.split(rgx.table_delimiter_col); -                  _table ~= "<tr>"; -                  foreach(col_idx, cell; _table_cols) { -                    if ((_table_cols.length == 1) -                    && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) -                      _tablenote ~= cell; -                    } else { -                      string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; -                      string _align = ("style=\"text-align:" -                      ~ ((obj.table_column_aligns[col_idx] == "l") -                      ? "left\"" : "right\"")); -                      _table ~= "<" -                        ~ _col_is -                        ~ " width=\"" -                        ~ obj.table_column_widths[col_idx].to!string -                        ~ "%\" " -                        ~ _align -                        ~ ">"; -                      _table ~= cell; -                      _table ~= "</" -                        ~ _col_is -                        ~ ">"; -                    } -                  } -                  _table ~= "</tr>"; -                } -              auto t = tuple( -                _table, -                _tablenote, -              ); -              return t; -            } -            string _txt = munge_html(obj); -            auto t = _tablarize(obj, _txt); -            _txt = t[0]; -            string _note = t[1]; -            string o = format(q"¶<p class="%s"> -              <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center"> -                %s -              </table> -              %s -            </p>¶", -              obj.is_a, -              _txt, -              _note -            ); -            return o; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto quote(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_quote(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto sqlite_load_string(O,Dm)( -            auto return ref const O     obj, -            auto return ref       Dm    doc_matters, -          ) { -            string o; -            return o; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto postgresql_load_string(O,Dm)( -            auto return ref const O     obj, -            auto return ref       Dm    doc_matters, -          ) { -            string o; -            return o; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto group(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_group(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          string sqlite_statement(O)( -            auto return ref const O    obj, -            string                     _txt, -            string                     _html, -          ) { -            void _sql_exe(O)( -              string                   _sql, -            ) { -              writeln(_html); -              writeln(_sql); -            } -            string _sql; -            return _sql; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto heading(O)( -            auto return ref const O      obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_heading(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto block(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_block(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto para(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_para(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto quote(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_quote(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto verse(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_verse(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto group(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_group(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto block(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_block(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto code(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_code(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto verse(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_verse(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } -          auto code(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_code(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +        } else { +          // load sql +        } +        return obj_txt; +      } +      auto table(O)( +        auto return ref const O     obj, +      ) { +        string[string] obj_txt = [ +          "text": generic_munge_sanitize_text_for_search(obj.text), +          "html": html_table(obj) +        ]; +        if (doc_matters.opt.action.debug_do) { +          debug(sql_txt) { +            writeln(obj_txt["text"]);            } -          auto table(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_table(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; +          debug(sql_html) { +            writeln(obj_txt["html"]);            } +        } else { +          // load sql          } -        return sqlite_format_and_load_objects(); +        return obj_txt;        }      } -    template SQLiteTablesReCreate() { -      string SQLiteTablesReCreate()() { -        string _sql_instruct; -        _sql_instruct = format(q"¶ -          DROP INDEX IF EXISTS ocn; -          DROP INDEX IF EXISTS digest_clean; -          DROP INDEX IF EXISTS digest_all; -          DROP INDEX IF EXISTS clean; -          DROP INDEX IF EXISTS lev0; -          DROP INDEX IF EXISTS lev1; -          DROP INDEX IF EXISTS lev2; -          DROP INDEX IF EXISTS lev3; -          DROP INDEX IF EXISTS lev4; -          DROP INDEX IF EXISTS lev5; -          DROP INDEX IF EXISTS lev6; -          DROP INDEX IF EXISTS title; -          DROP INDEX IF EXISTS creator_author; -          DROP INDEX IF EXISTS src_filename; -          DROP INDEX IF EXISTS language_document_char; -          DROP INDEX IF EXISTS classify_topic_register; -          DROP TABLE IF EXISTS metadata_and_text; -          DROP TABLE IF EXISTS doc_objects; -          DROP TABLE IF EXISTS urls; -          CREATE TABLE metadata_and_text ( -            tid                              INTEGER           PRIMARY KEY, -          /* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ -            title                            VARCHAR(800)  NOT NULL, -            title_main                       VARCHAR(400)  NOT NULL, -            title_sub                        VARCHAR(400)      NULL, -            title_short                      VARCHAR(400)      NULL, -            title_edition                    VARCHAR(10)       NULL, -            title_note                       VARCHAR(2500)     NULL, -            title_language                   VARCHAR(100)      NULL, -            title_language_char              VARCHAR(6)        NULL, -            creator_author                   VARCHAR(600)  NOT NULL, -            creator_author_email             VARCHAR(100)      NULL, -            creator_author_hon               VARCHAR(100)      NULL, -            creator_author_nationality       VARCHAR(100)      NULL, -            creator_editor                   VARCHAR(600)      NULL, -            creator_contributor              VARCHAR(600)      NULL, -            creator_illustrator              VARCHAR(600)      NULL, -            creator_photographer             VARCHAR(600)      NULL, -            creator_translator               VARCHAR(600)      NULL, -            creator_prepared_by              VARCHAR(600)      NULL, -            creator_digitized_by             VARCHAR(600)      NULL, -            creator_audio                    VARCHAR(600)      NULL, -            creator_video                    VARCHAR(600)      NULL, -            language_document                VARCHAR(100)      NULL, -            language_document_char           VARCHAR(6)    NOT NULL, -            language_original                VARCHAR(100)      NULL, -            language_original_char           VARCHAR(6)        NULL, -            date_added_to_site               VARCHAR(10)       NULL, -            date_available                   VARCHAR(10)       NULL, -            date_created                     VARCHAR(10)       NULL, -            date_issued                      VARCHAR(10)       NULL, -            date_modified                    VARCHAR(10)       NULL, -            date_published                   VARCHAR(10)       NULL, -            date_valid                       VARCHAR(10)       NULL, -            date_translated                  VARCHAR(10)       NULL, -            date_original_publication        VARCHAR(10)       NULL, -            date_generated                   VARCHAR(10)       NULL, -            publisher                        VARCHAR(600)      NULL, -            original_publisher               VARCHAR(600)      NULL, -            original_language                VARCHAR(100)      NULL, -            original_language_char           VARCHAR(6)        NULL, -            original_source                  VARCHAR(600)      NULL, -            original_institution             VARCHAR(600)      NULL, -            original_nationality             VARCHAR(100)      NULL, -            original_title                   VARCHAR(800)      NULL, -            rights_copyright                 VARCHAR(2500)     NULL, -            rights_copyright_audio           VARCHAR(2500)     NULL, -            rights_copyright_cover           VARCHAR(2500)     NULL, -            rights_copyright_illustrations   VARCHAR(2500)     NULL, -            rights_copyright_photographs     VARCHAR(2500)     NULL, -            rights_copyright_text            VARCHAR(2500)     NULL, -            rights_copyright_translation     VARCHAR(2500)     NULL, -            rights_copyright_video           VARCHAR(2500)     NULL, -            rights_license                   VARCHAR(2500)     NULL, -            identifier_oclc                  VARCHAR(30)       NULL, -            identifier_isbn                  VARCHAR(16)       NULL, -            classify_topic_register          VARCHAR(2500)     NULL, -            classify_subject                 VARCHAR(600)      NULL, -            classify_loc                     VARCHAR(30)       NULL, -            classify_dewey                   VARCHAR(30)       NULL, -            classify_keywords                VARCHAR(600)      NULL, -            notes_abstract                   TEXT              NULL, -            notes_description                TEXT              NULL, -            notes_comment                    TEXT              NULL, -            notes_coverage                   VARCHAR(200)      NULL, -            notes_relation                   VARCHAR(200)      NULL, -            notes_history                    VARCHAR(600)      NULL, -            notes_type                       VARCHAR(600)      NULL, -            notes_format                     VARCHAR(600)      NULL, -            notes_prefix                     TEXT              NULL, -            notes_prefix_a                   TEXT              NULL, -            notes_prefix_b                   TEXT              NULL, -            notes_suffix                     TEXT              NULL, -            src_filename                     VARCHAR(256)  NOT NULL, -            src_fingerprint                  VARCHAR(256)      NULL, -            src_filesize                     VARCHAR(10)       NULL, -            src_wordcount                    VARCHAR(10)       NULL, -            pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ -            pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ -            pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ -            src_text                         TEXT              NULL, -            fulltext                         TEXT              NULL, -            links                            TEXT              NULL -          ); -          CREATE TABLE doc_objects ( -            lid                              INTEGER PRIMARY KEY, -            metadata_tid                     INTEGER REFERENCES metadata_and_text, -            ocn                              SMALLINT, -            ocnd                             VARCHAR(6), -            ocns                             VARCHAR(6), -            clean                            TEXT NULL, -            body                             TEXT NULL, -            book_idx                         TEXT NULL, -            seg                              VARCHAR(256) NULL, -            lev_an                           VARCHAR(1), -            lev                              SMALLINT NULL, -            lev0                             SMALLINT, -            lev1                             SMALLINT, -            lev2                             SMALLINT, -            lev3                             SMALLINT, -            lev4                             SMALLINT, -            lev5                             SMALLINT, -            lev6                             SMALLINT, -            lev7                             SMALLINT, -            en_a                             SMALLINT NULL, -            en_z                             SMALLINT NULL, -            en_a_asterisk                    SMALLINT NULL, -            en_z_asterisk                    SMALLINT NULL, -            en_a_plus                        SMALLINT NULL, -            en_z_plus                        SMALLINT NULL, -            t_of                             VARCHAR(16), -            t_is                             VARCHAR(16), -            node                             VARCHAR(16) NULL, -            parent                           VARCHAR(16) NULL, -            digest_clean                     CHAR(256), -            digest_all                       CHAR(256), -            types                            CHAR(1) NULL -          ); -          CREATE INDEX idx_ocn ON doc_objects(ocn); -          CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); -          CREATE INDEX idx_digest_all ON doc_objects(digest_all); -          CREATE INDEX idx_clean ON doc_objects(clean); -          CREATE INDEX idx_lev0 ON doc_objects(lev0); -          CREATE INDEX idx_lev1 ON doc_objects(lev1); -          CREATE INDEX idx_lev2 ON doc_objects(lev2); -          CREATE INDEX idx_lev3 ON doc_objects(lev3); -          CREATE INDEX idx_lev4 ON doc_objects(lev4); -          CREATE INDEX idx_lev5 ON doc_objects(lev5); -          CREATE INDEX idx_lev6 ON doc_objects(lev6); -          CREATE INDEX idx_title ON metadata_and_text(title); -          CREATE INDEX idx_author ON metadata_and_text(creator_author); -          CREATE INDEX idx_filename ON metadata_and_text(src_filename); -          CREATE INDEX idx_language ON metadata_and_text(language_document_char); -          CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); -        ¶",); -        return _sql_instruct; -      } -    } -    template SQLiteInsertMetadata() { -      string SQLiteInsertMetadata(I)( -        auto ref       I    doc_matters, -      ) { -        string _insert_metadata; -        _insert_metadata = format(q"¶ -          INSERT INTO metadata_and_text ( -            src_filename, -            title, -            title_main, -            title_sub, -            title_short, -            title_edition, -            title_language, -            classify_dewey, -            classify_keywords, -            classify_loc, -            classify_subject, -            classify_topic_register, -            creator_author, -            creator_author_email, -            creator_illustrator, -            creator_translator, -            date_added_to_site, -            date_available, -            date_created, -            date_issued, -            date_modified, -            date_published, -            date_valid, -            identifier_isbn, -            identifier_oclc, -            language_document, -            language_document_char, -            notes_abstract, -            notes_description, -            original_publisher, -            original_language, -            original_language_char, -            original_source, -            original_title, -            publisher, -            rights_copyright, -            rights_copyright_audio, -            rights_copyright_cover, -            rights_copyright_illustrations, -            rights_copyright_photographs, -            rights_copyright_text, -            rights_copyright_translation, -            rights_copyright_video, -            rights_license -          ) -          VALUES ( -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s' -          ); -        ¶", -          SQLinsertDelimiter!()(doc_matters.src.filename), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) +    return sqlite_format_and_load_objects(); +  } +} +template SQLiteTablesReCreate() { +  string SQLiteTablesReCreate()() { +    string _sql_instruct; +    _sql_instruct = format(q"¶ +      DROP INDEX IF EXISTS ocn; +      DROP INDEX IF EXISTS digest_clean; +      DROP INDEX IF EXISTS digest_all; +      DROP INDEX IF EXISTS clean; +      DROP INDEX IF EXISTS lev0; +      DROP INDEX IF EXISTS lev1; +      DROP INDEX IF EXISTS lev2; +      DROP INDEX IF EXISTS lev3; +      DROP INDEX IF EXISTS lev4; +      DROP INDEX IF EXISTS lev5; +      DROP INDEX IF EXISTS lev6; +      DROP INDEX IF EXISTS title; +      DROP INDEX IF EXISTS creator_author; +      DROP INDEX IF EXISTS src_filename; +      DROP INDEX IF EXISTS language_document_char; +      DROP INDEX IF EXISTS classify_topic_register; +      DROP TABLE IF EXISTS metadata_and_text; +      DROP TABLE IF EXISTS doc_objects; +      DROP TABLE IF EXISTS urls; +      CREATE TABLE metadata_and_text ( +        tid                              INTEGER           PRIMARY KEY, +        src_composite_id_per_txt         VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +        src_composite_id_per_pod         VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +        title                            VARCHAR(800)  NOT NULL, +        title_main                       VARCHAR(400)  NOT NULL, +        title_sub                        VARCHAR(400)      NULL, +        title_short                      VARCHAR(400)      NULL, +        title_edition                    VARCHAR(10)       NULL, +        title_note                       VARCHAR(2500)     NULL, +        title_language                   VARCHAR(100)      NULL, +        title_language_char              VARCHAR(6)        NULL, +        creator_author                   VARCHAR(600)  NOT NULL, +        creator_author_email             VARCHAR(100)      NULL, +        creator_author_hon               VARCHAR(100)      NULL, +        creator_author_nationality       VARCHAR(100)      NULL, +        creator_editor                   VARCHAR(600)      NULL, +        creator_contributor              VARCHAR(600)      NULL, +        creator_illustrator              VARCHAR(600)      NULL, +        creator_photographer             VARCHAR(600)      NULL, +        creator_translator               VARCHAR(600)      NULL, +        creator_prepared_by              VARCHAR(600)      NULL, +        creator_digitized_by             VARCHAR(600)      NULL, +        creator_audio                    VARCHAR(600)      NULL, +        creator_video                    VARCHAR(600)      NULL, +        language_document                VARCHAR(100)      NULL, +        language_document_char           VARCHAR(6)    NOT NULL, +        language_original                VARCHAR(100)      NULL, +        language_original_char           VARCHAR(6)        NULL, +        date_added_to_site               VARCHAR(10)       NULL, +        date_available                   VARCHAR(10)       NULL, +        date_created                     VARCHAR(10)       NULL, +        date_issued                      VARCHAR(10)       NULL, +        date_modified                    VARCHAR(10)       NULL, +        date_published                   VARCHAR(10)       NULL, +        date_valid                       VARCHAR(10)       NULL, +        date_translated                  VARCHAR(10)       NULL, +        date_original_publication        VARCHAR(10)       NULL, +        date_generated                   VARCHAR(10)       NULL, +        publisher                        VARCHAR(600)      NULL, +        original_publisher               VARCHAR(600)      NULL, +        original_language                VARCHAR(100)      NULL, +        original_language_char           VARCHAR(6)        NULL, +        original_source                  VARCHAR(600)      NULL, +        original_institution             VARCHAR(600)      NULL, +        original_nationality             VARCHAR(100)      NULL, +        original_title                   VARCHAR(800)      NULL, +        rights_copyright                 VARCHAR(2500)     NULL, +        rights_copyright_audio           VARCHAR(2500)     NULL, +        rights_copyright_cover           VARCHAR(2500)     NULL, +        rights_copyright_illustrations   VARCHAR(2500)     NULL, +        rights_copyright_photographs     VARCHAR(2500)     NULL, +        rights_copyright_text            VARCHAR(2500)     NULL, +        rights_copyright_translation     VARCHAR(2500)     NULL, +        rights_copyright_video           VARCHAR(2500)     NULL, +        rights_license                   VARCHAR(2500)     NULL, +        identifier_oclc                  VARCHAR(30)       NULL, +        identifier_isbn                  VARCHAR(16)       NULL, +        classify_topic_register          VARCHAR(2500)     NULL, +        classify_subject                 VARCHAR(600)      NULL, +        classify_loc                     VARCHAR(30)       NULL, +        classify_dewey                   VARCHAR(30)       NULL, +        classify_keywords                VARCHAR(600)      NULL, +        notes_abstract                   TEXT              NULL, +        notes_description                TEXT              NULL, +        notes_comment                    TEXT              NULL, +        notes_coverage                   VARCHAR(200)      NULL, +        notes_relation                   VARCHAR(200)      NULL, +        notes_history                    VARCHAR(600)      NULL, +        notes_type                       VARCHAR(600)      NULL, +        notes_format                     VARCHAR(600)      NULL, +        notes_prefix                     TEXT              NULL, +        notes_prefix_a                   TEXT              NULL, +        notes_prefix_b                   TEXT              NULL, +        notes_suffix                     TEXT              NULL, +        src_filename                     VARCHAR(256)  NOT NULL, +        src_fingerprint                  VARCHAR(256)      NULL, +        src_filesize                     VARCHAR(10)       NULL, +        src_wordcount                    VARCHAR(10)       NULL, +        pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +        pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +        pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ +        src_text                         TEXT              NULL, +        fulltext                         TEXT              NULL, +        links                            TEXT              NULL +      ); +      CREATE TABLE doc_objects ( +        lid                              INTEGER PRIMARY KEY, +        metadata_tid                     INTEGER REFERENCES metadata_and_text, +        ocn                              SMALLINT, +        ocnd                             VARCHAR(6), +        ocns                             VARCHAR(6), +        clean                            TEXT NULL, +        body                             TEXT NULL, +        book_idx                         TEXT NULL, +        seg                              VARCHAR(256) NULL, +        lev_an                           VARCHAR(1), +        lev                              SMALLINT NULL, +        lev0                             SMALLINT, +        lev1                             SMALLINT, +        lev2                             SMALLINT, +        lev3                             SMALLINT, +        lev4                             SMALLINT, +        lev5                             SMALLINT, +        lev6                             SMALLINT, +        lev7                             SMALLINT, +        en_a                             SMALLINT NULL, +        en_z                             SMALLINT NULL, +        en_a_asterisk                    SMALLINT NULL, +        en_z_asterisk                    SMALLINT NULL, +        en_a_plus                        SMALLINT NULL, +        en_z_plus                        SMALLINT NULL, +        t_of                             VARCHAR(16), +        t_is                             VARCHAR(16), +        node                             VARCHAR(16) NULL, +        parent                           VARCHAR(16) NULL, +        digest_clean                     CHAR(256), +        digest_all                       CHAR(256), +        types                            CHAR(1) NULL +      ); +      CREATE INDEX idx_ocn ON doc_objects(ocn); +      CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); +      CREATE INDEX idx_digest_all ON doc_objects(digest_all); +      CREATE INDEX idx_clean ON doc_objects(clean); +      CREATE INDEX idx_lev0 ON doc_objects(lev0); +      CREATE INDEX idx_lev1 ON doc_objects(lev1); +      CREATE INDEX idx_lev2 ON doc_objects(lev2); +      CREATE INDEX idx_lev3 ON doc_objects(lev3); +      CREATE INDEX idx_lev4 ON doc_objects(lev4); +      CREATE INDEX idx_lev5 ON doc_objects(lev5); +      CREATE INDEX idx_lev6 ON doc_objects(lev6); +      CREATE INDEX idx_title ON metadata_and_text(title); +      CREATE INDEX idx_author ON metadata_and_text(creator_author); +      CREATE INDEX idx_filename ON metadata_and_text(src_filename); +      CREATE INDEX idx_language ON metadata_and_text(language_document_char); +      CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); +    ¶",); +    return _sql_instruct; +  } +} +template SQLiteInsertMetadata() { +  string SQLiteInsertMetadata(I)( +    auto ref       I    doc_matters, +  ) { +    string _insert_metadata; +    _insert_metadata = format(q"¶ +      INSERT INTO metadata_and_text ( +        src_filename, +        src_composite_id_per_txt, +        src_composite_id_per_pod, +        title, +        title_main, +        title_sub, +        title_short, +        title_edition, +        title_language, +        classify_dewey, +        classify_keywords, +        classify_loc, +        classify_subject, +        classify_topic_register, +        creator_author, +        creator_author_email, +        creator_illustrator, +        creator_translator, +        date_added_to_site, +        date_available, +        date_created, +        date_issued, +        date_modified, +        date_published, +        date_valid, +        identifier_isbn, +        identifier_oclc, +        language_document, +        language_document_char, +        notes_abstract, +        notes_description, +        original_publisher, +        original_language, +        original_language_char, +        original_source, +        original_title, +        publisher, +        rights_copyright, +        rights_copyright_audio, +        rights_copyright_cover, +        rights_copyright_illustrations, +        rights_copyright_photographs, +        rights_copyright_text, +        rights_copyright_translation, +        rights_copyright_video, +        rights_license +      ) +      VALUES ( +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s', +        '%s' +      ); +    ¶", +      SQLinsertDelimiter!()(doc_matters.src.filename), +      SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), +      SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), +      SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) +    ); +    return _insert_metadata; +  } +} +template SQLiteInsertDocObjectsLoop() { +  string SQLiteInsertDocObjectsLoop(D,I,X)( +    auto ref const D    doc_abstraction, +    auto ref       I    doc_matters, +    auto ref       X    _metadata_tid, +  ) { +    string insertDocObjectsRow(O)(O obj) { +      string _insert_doc_objects_row; +      _insert_doc_objects_row = format(q"¶ +        INSERT INTO doc_objects ( +          metadata_tid, +          ocn, +          clean, +          body, +          lev, +          t_of, +          t_is +        ) +        VALUES ( +          %d, +          %s, +          '%s', +          '%s', +          %s, +          '%s', +          '%s'          ); -        return _insert_metadata; -      } +      ¶", +        _metadata_tid, +        obj.ocn, +        SQLinsertDelimiter!()(obj_txt["text"]), +        SQLinsertDelimiter!()(obj_txt["html"]), +        obj.heading_lev_markup, +        obj.is_of, +        obj.is_a, +      ); +      return _insert_doc_objects_row;      } -    template SQLiteInsertDocObjectsLoop() { -      string SQLiteInsertDocObjectsLoop(P,I)( -        auto ref       P    doc_parts, -        auto ref       I    _metadata_tid, -      ) { -        string insertDocObjectsRow(O)(O obj) { -          string _insert_doc_objects_row; -          _insert_doc_objects_row = format(q"¶ -            INSERT INTO doc_objects ( -              metadata_tid, -              ocn, -              clean, -              body, -              lev, -              t_of, -              t_is -            ) -            VALUES ( -              %d, -              %s, -              '%s', -              '%s', -              %s, -              '%s', -              '%s' -            ); -          ¶", -            _metadata_tid, -            obj.ocn, -            SQLinsertDelimiter!()(obj_txt["text"]), -            SQLinsertDelimiter!()(obj_txt["html"]), -            obj.heading_lev_markup, -            obj.is_of, -            obj.is_a, -          ); -          return _insert_doc_objects_row; -        } -        auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); -        string[string] obj_txt; -        string doc_text; -        string[] _insert_doc_objects; -        foreach (part; doc_parts) { -          foreach (obj; doc_abstraction[part]) { -            switch (obj.of_part) { -            case "frontmatter":              assert(part == "head", part); -              switch (obj.is_of) { -              case "para": -                switch (obj.is_a) { -                case "heading": -                  obj_txt = format_and_sqlite_load.heading(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              default: -                if ((doc_matters.opt.action.debug_do)) { -                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -                } -                break; +    auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); +    string[string] obj_txt; +    string doc_text; +    string[] _insert_doc_objects; +    foreach (part; doc_matters.xml.keys_seq.sql) { +      foreach (obj; doc_abstraction[part]) { +        switch (obj.of_part) { +        case "frontmatter":              assert(part == "head", part); +          switch (obj.is_of) { +          case "para": +            switch (obj.is_a) { +            case "heading": +              obj_txt = format_and_sqlite_load.heading(obj); +              break; +            default: +              if ((doc_matters.opt.action.debug_do)) { +                writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);                }                break; -            case "body": //                    assert(part == "body", part); // TODO broken -              switch (obj.is_of) { -              case "para": -                switch (obj.is_a) { -                case "heading": -                  debug (asserts) { // TODO consider and fix or remove -                    if (part != "body") { -                      writeln(__LINE__, ": ", obj.text); -                    } -                  } -                  obj_txt = format_and_sqlite_load.heading(obj); -                  break; -                case "para": -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              case "block": -                switch (obj.is_a) { -                case "quote": -                  obj_txt = format_and_sqlite_load.quote(obj); -                  break; -                case "group": -                  obj_txt = format_and_sqlite_load.group(obj); -                  break; -                case "block": -                  obj_txt = format_and_sqlite_load.block(obj); -                  break; -                case "poem":                        // double check on keeping both poem & verse -                  break; -                case "verse": -                  obj_txt = format_and_sqlite_load.verse(obj); -                  break; -                case "code": -                  obj_txt = format_and_sqlite_load.code(obj); -                  break; -                case "table": -                  obj_txt = format_and_sqlite_load.table(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              default: -                if ((doc_matters.opt.action.debug_do)) { -                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); +            } +            break; +          default: +            if ((doc_matters.opt.action.debug_do)) { +              writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); +            } +            break; +          } +          break; +        case "body": //                    assert(part == "body", part); // TODO broken +          switch (obj.is_of) { +          case "para": +            switch (obj.is_a) { +            case "heading": +              debug (asserts) { // TODO consider and fix or remove +                if (part != "body") { +                  writeln(__LINE__, ": ", obj.text);                  } -                break;                } +              obj_txt = format_and_sqlite_load.heading(obj);                break; -            case "backmatter": -              assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); -              switch (obj.is_of) { -              case "para": -                switch (obj.is_a) { -                case "heading": -                  obj_txt = format_and_sqlite_load.heading(obj); -                  break; -                case "endnote":              assert(part == "endnotes", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "glossary":             assert(part == "glossary", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "bibliography":         assert(part == "bibliography", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "bookindex":            assert(part == "bookindex_seg", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "blurb":                assert(part == "blurb", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              default: -                if ((doc_matters.opt.action.debug_do)) { -                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -                } -                break; +            case "para": +              obj_txt = format_and_sqlite_load.para(obj); +              break; +            default: +              if ((doc_matters.opt.action.debug_do)) { +                writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);                }                break; -            case "comment": +            } +            break; +          case "block": +            switch (obj.is_a) { +            case "quote": +              obj_txt = format_and_sqlite_load.quote(obj); +              break; +            case "group": +              obj_txt = format_and_sqlite_load.group(obj); +              break; +            case "block": +              obj_txt = format_and_sqlite_load.block(obj); +              break; +            case "poem":                        // double check on keeping both poem & verse +              break; +            case "verse": +              obj_txt = format_and_sqlite_load.verse(obj); +              break; +            case "code": +              obj_txt = format_and_sqlite_load.code(obj); +              break; +            case "table": +              obj_txt = format_and_sqlite_load.table(obj);                break;              default:                if ((doc_matters.opt.action.debug_do)) { -                writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from                }                break;              } -            if (obj.is_a == "heading") { -              if ((doc_matters.opt.action.verbose)) { -                writeln( -                  "markup: ", obj.heading_lev_markup, -                  "> ", obj.dom_markedup, -                  "; collapsed: ", obj.heading_lev_collapsed, -                  "> ", obj.dom_collapsed, -                  "; ocn: ", obj.ocn, -                  " node: ", obj.node, -                  "; parent: ", obj.parent_lev_markup, -                  "; ocn: ", obj.parent_ocn, -                  "; ", -                ); +            break; +          default: +            if ((doc_matters.opt.action.debug_do)) { +              writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); +            } +            break; +          } +          break; +        case "backmatter": +          assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); +          switch (obj.is_of) { +          case "para": +            switch (obj.is_a) { +            case "heading": +              obj_txt = format_and_sqlite_load.heading(obj); +              break; +            case "endnote":              assert(part == "endnotes", part); +              obj_txt = format_and_sqlite_load.para(obj); +              break; +            case "glossary":             assert(part == "glossary", part); +              obj_txt = format_and_sqlite_load.para(obj); +              break; +            case "bibliography":         assert(part == "bibliography", part); +              obj_txt = format_and_sqlite_load.para(obj); +              break; +            case "bookindex":            assert(part == "bookindex_seg", part); +              obj_txt = format_and_sqlite_load.para(obj); +              break; +            case "blurb":                assert(part == "blurb", part); +              obj_txt = format_and_sqlite_load.para(obj); +              break; +            default: +              if ((doc_matters.opt.action.debug_do)) { +                writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);                } +              break;              } -            if (!(obj.is_a == "comment")) { -              _insert_doc_objects ~= insertDocObjectsRow(obj); +            break; +          default: +            if ((doc_matters.opt.action.debug_do)) { +              writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);              } -          } // loop closes +            break; +          } +          break; +        case "comment": +          break; +        default: +          if ((doc_matters.opt.action.debug_do)) { +            writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from +            writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); +            writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from +          } +          break;          } -        return _insert_doc_objects.join.to!(char[]).toUTF8; -      } +        if (obj.is_a == "heading") { +          if ((doc_matters.opt.action.verbose)) { +            writeln( +              "markup: ", obj.heading_lev_markup, +              "> ", obj.dom_markedup, +              "; collapsed: ", obj.heading_lev_collapsed, +              "> ", obj.dom_collapsed, +              "; ocn: ", obj.ocn, +              " node: ", obj.node, +              "; parent: ", obj.parent_lev_markup, +              "; ocn: ", obj.parent_ocn, +              "; ", +            ); +          } +        } +        if (!(obj.is_a == "comment")) { +          _insert_doc_objects ~= insertDocObjectsRow(obj); +        } +      } // loop closes      } -    SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); +    return _insert_doc_objects.join.to!(char[]).toUTF8;    }  }  template SQLiteTablesCreate() { @@ -1066,7 +1103,8 @@ template SQLiteTablesCreate() {            DROP TABLE IF EXISTS urls;            CREATE TABLE metadata_and_text (              tid                              INTEGER           PRIMARY KEY, -          /* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +            src_composite_id_per_txt         VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +            src_composite_id_per_pod         VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */              title                            VARCHAR(800)  NOT NULL,              title_main                       VARCHAR(400)  NOT NULL,              title_sub                        VARCHAR(400)      NULL, diff --git a/src/sdp/output/sqlite_discrete.d b/src/sdp/output/sqlite_discrete.d deleted file mode 100644 index 40e0b15..0000000 --- a/src/sdp/output/sqlite_discrete.d +++ /dev/null @@ -1,1023 +0,0 @@ -module sdp.output.sqlite_discrete; -import sdp.output; -import -  std.file, -  std.conv : to; -template SQLiteDiscreteDbRun() { -  void SQLiteDiscreteDbRun(Db,St,O)( -    auto ref       Db   db, -    auto ref       St   db_statement, -    auto ref       O    opt_action, -               string   note, -  ) { -    debug(sql_statement) { -      writeln(db_statement); -    } -    try { -      db.run( -        "\nBEGIN;\n" ~ -        db_statement ~ -        "\nCOMMIT;\n" -      ); -    } catch (ErrnoException ex) { -      writeln("ERROR SQLite : ", ex); -    } catch (Exception ex) { -      writeln("ERROR SQLite : ", ex); -    } -    if (opt_action.debug_do) { -      writeln(note); -      if (opt_action.verbose) { -        writeln(db_statement); -      } -    } -  } -} -template SQLiteDiscreteBuildTablesAndPopulate() { -  void SQLiteDiscreteBuildTablesAndPopulate(D,I)( -    auto ref const D    doc_abstraction, -    auto ref       I    doc_matters, -  ) { -    import d2sqlite3; -    import std.typecons : Nullable; -    mixin SiSUoutputRgxInit; -    static auto rgx = Rgx(); -    auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); -    pth_sqlite.base.mkdirRecurse; -    auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); -    template SQLiteDiscreteDbStatementComposite() { -      void SQLiteDiscreteDbStatementComposite(Db,D,I)( -        auto ref       Db   db, -        auto ref const D    doc_abstraction, -        auto ref       I    doc_matters, -      ) { -        { -          string _db_statement; -          { -            _db_statement ~= SQLiteDiscreteTablesReCreate!()(); -            _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters); -            _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX -            SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); -          } -          db.close; -        } -      } -    } -    template SQLinsertDelimiter() { -      auto SQLinsertDelimiter(string _txt) { -        _txt = _txt -          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); -        return _txt; -      } -    } -    template SQLiteFormatAndLoadObject() { -      auto SQLiteFormatAndLoadObject(I)( -        auto ref       I    doc_matters, -      ) { -        mixin SiSUoutputRgxInit; -        struct sqlite_format_and_load_objects { -          auto generic_munge_sanitize_text_for_search( -            string _txt, -          ) { -            string _notes; -            string _urls; -            if (_txt.matchFirst(rgx.inline_notes_al_gen)) { -              foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { -                _notes ~= "\n" ~ m["text"]; -              } -              _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); -            } -            if (_txt.matchFirst(rgx.inline_link)) { -              foreach (m; _txt.matchAll(rgx.inline_link)) { -                if (m["link"].match(rgx.url)) { -                  _urls ~= "\n" ~ m["link"]; -                } -              } -              _txt = _txt.replaceAll(rgx.inline_link_clean, ""); -            } -            if (_notes.length > 0) { -              _txt ~= _notes; -            } -            if (_urls.length > 0) { -              _txt ~= _urls; -            } -            if (doc_matters.opt.action.debug_do) { -              writeln(_txt, "\n"); -            } -            debug(sql_text_clean) { -              writeln(_txt); -            } -            return _txt; -          } -          auto munge_html(O)( -            auto return ref const O    obj, -          ) { -            string _html_special_characters(string _txt){ -              _txt = _txt -                .replaceAll(rgx.xhtml_ampersand,    "&") -                .replaceAll(rgx.xhtml_quotation,    """) -                .replaceAll(rgx.xhtml_less_than,    "<") -                .replaceAll(rgx.xhtml_greater_than, ">") -                .replaceAll(rgx.nbsp_char,          " ") -                .replaceAll(rgx.xhtml_line_break,   "<br />"); -              return _txt; -            } -            string _html_font_face(string _txt){ -              _txt = _txt -                .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") -                .replaceAll(rgx.inline_bold,        "<b>$1</b>") -                .replaceAll(rgx.inline_underscore,  "<u>$1</u>") -                .replaceAll(rgx.inline_italics,     "<i>$1</i>") -                .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") -                .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") -                .replaceAll(rgx.inline_strike,      "<del>$1</del>") -                .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") -                .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") -                .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); -              return _txt; -            } -            string _notes; -            string _urls; -            string _txt = _html_font_face(_html_special_characters(obj.text)); -            if (_txt.matchFirst(rgx.inline_notes_al_gen)) { -              foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { -                _notes ~= "\n" ~ m["text"]; -              } -              _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> "); -            } -            if (_txt.matchFirst(rgx.inline_link)) { -              foreach (m; _txt.matchAll(rgx.inline_link)) { -              } -              _txt = _txt.replaceAll(rgx.inline_link_clean, ""); -            } -            if (_notes.length > 0) { -              _txt ~= _notes; -            } -            if (doc_matters.opt.action.debug_do) { -              writeln(_txt, "\n"); -            } -            return _txt; -          } -          string html_special_characters(string _txt){ -            _txt = _txt -              .replaceAll(rgx.xhtml_ampersand,    "&") -              .replaceAll(rgx.xhtml_quotation,    """) -              .replaceAll(rgx.xhtml_less_than,    "<") -              .replaceAll(rgx.xhtml_greater_than, ">") -              .replaceAll(rgx.nbsp_char,          " ") -              .replaceAll(rgx.xhtml_line_break,   "<br />"); -            return _txt; -          } -          string html_special_characters_code(string _txt){ -            _txt = _txt -              .replaceAll(rgx.xhtml_ampersand,    "&") -              .replaceAll(rgx.xhtml_quotation,    """) -              .replaceAll(rgx.xhtml_less_than,    "<") -              .replaceAll(rgx.xhtml_greater_than, ">") -              .replaceAll(rgx.nbsp_char,          " "); -            return _txt; -          } -          string html_font_face(string _txt){ -            _txt = _txt -              .replaceAll(rgx.inline_emphasis,    "<em>$1</em>") -              .replaceAll(rgx.inline_bold,        "<b>$1</b>") -              .replaceAll(rgx.inline_underscore,  "<u>$1</u>") -              .replaceAll(rgx.inline_italics,     "<i>$1</i>") -              .replaceAll(rgx.inline_superscript, "<sup>$1</sup>") -              .replaceAll(rgx.inline_subscript,   "<sub>$1</sub>") -              .replaceAll(rgx.inline_strike,      "<del>$1</del>") -              .replaceAll(rgx.inline_insert,      "<ins>$1</ins>") -              .replaceAll(rgx.inline_mono,        "<tt>$1</tt>") -              .replaceAll(rgx.inline_cite,        "<cite>$1</cite>"); -            return _txt; -          } -          auto html_heading(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s"><b> -                %s -              </b></p>¶", -                obj.is_a, -                _txt, -              ); -            return o; -          } -          auto html_para(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; -            string o = format(q"¶<p class="%s" indent="h%si%s"> -              %s -            </p>¶", -                obj.is_a, -                obj.indent_hang, -                obj.indent_base, -                _txt -              ); -            return o; -          } -          auto html_quote(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s"> -              %s -            </p>¶", -                obj.is_a, -                _txt -              ); -            return o; -          } -          auto html_group(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s"> -              %s -            </p>¶", -                obj.is_a, -                _txt -              ); -            return o; -          } -          auto html_block(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶ -            <p class="%s">%s</p>¶", -                obj.is_a, -                _txt.stripRight -              ); -            return o; -          } -          auto html_verse(O)( -            auto return ref const O    obj, -          ) { -            string _txt = munge_html(obj); -            string o = format(q"¶<p class="%s">%s</p>¶", -                obj.is_a, -                _txt -              ); -            return o; -          } -          auto html_code(O)( -            auto return ref const O    obj, -          ) { -            string _txt = html_special_characters_code(obj.text); -            string o = format(q"¶<p class="%s">%s</p>¶", -                obj.is_a, -                _txt -              ); -            return o; -          } -          auto html_table(O)( -            auto return ref const O     obj, -          ) { -            auto _tablarize(O)( -              auto return ref const O    obj, -              string                     _txt, -            ) { -              string[] _table_rows = _txt.split(rgx.table_delimiter_row); -              string[] _table_cols; -              string _table; -              string _tablenote; -              foreach(row_idx, row; _table_rows) { -                _table_cols = row.split(rgx.table_delimiter_col); -                  _table ~= "<tr>"; -                  foreach(col_idx, cell; _table_cols) { -                    if ((_table_cols.length == 1) -                    && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) -                      _tablenote ~= cell; -                    } else { -                      string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; -                      string _align = ("style=\"text-align:" -                      ~ ((obj.table_column_aligns[col_idx] == "l") -                      ? "left\"" : "right\"")); -                      _table ~= "<" -                        ~ _col_is -                        ~ " width=\"" -                        ~ obj.table_column_widths[col_idx].to!string -                        ~ "%\" " -                        ~ _align -                        ~ ">"; -                      _table ~= cell; -                      _table ~= "</" -                        ~ _col_is -                        ~ ">"; -                    } -                  } -                  _table ~= "</tr>"; -                } -              auto t = tuple( -                _table, -                _tablenote, -              ); -              return t; -            } -            string _txt = munge_html(obj); -            auto t = _tablarize(obj, _txt); -            _txt = t[0]; -            string _note = t[1]; -            string o = format(q"¶<p class="%s"> -              <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center"> -                %s -              </table> -              %s -            </p>¶", -              obj.is_a, -              _txt, -              _note -            ); -            return o; -          } -          auto sqlite_load_string(O,Dm)( -            auto return ref const O     obj, -            auto return ref       Dm    doc_matters, -          ) { -            string o; -            return o; -          } -          auto postgresql_load_string(O,Dm)( -            auto return ref const O     obj, -            auto return ref       Dm    doc_matters, -          ) { -            string o; -            return o; -          } -          string sqlite_statement(O)( -            auto return ref const O    obj, -            string                     _txt, -            string                     _html, -          ) { -            void _sql_exe(O)( -              string                   _sql, -            ) { -              writeln(_html); -              writeln(_sql); -            } -            string _sql; -            return _sql; -          } -          auto heading(O)( -            auto return ref const O      obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_heading(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto para(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_para(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto quote(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_quote(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto group(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_group(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto block(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_block(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto verse(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_verse(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto code(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_code(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -          auto table(O)( -            auto return ref const O     obj, -          ) { -            string[string] obj_txt = [ -              "text": generic_munge_sanitize_text_for_search(obj.text), -              "html": html_table(obj) -            ]; -            if (doc_matters.opt.action.debug_do) { -              debug(sql_txt) { -                writeln(obj_txt["text"]); -              } -              debug(sql_html) { -                writeln(obj_txt["html"]); -              } -            } else { -              // load sql -            } -            return obj_txt; -          } -        } -        return sqlite_format_and_load_objects(); -      } -    } -    template SQLiteDiscreteTablesReCreate() { -      string SQLiteDiscreteTablesReCreate()() { -        string _sql_instruct; -        _sql_instruct = format(q"¶ -          DROP INDEX IF EXISTS ocn; -          DROP INDEX IF EXISTS digest_clean; -          DROP INDEX IF EXISTS digest_all; -          DROP INDEX IF EXISTS clean; -          DROP INDEX IF EXISTS lev0; -          DROP INDEX IF EXISTS lev1; -          DROP INDEX IF EXISTS lev2; -          DROP INDEX IF EXISTS lev3; -          DROP INDEX IF EXISTS lev4; -          DROP INDEX IF EXISTS lev5; -          DROP INDEX IF EXISTS lev6; -          DROP INDEX IF EXISTS title; -          DROP INDEX IF EXISTS creator_author; -          DROP INDEX IF EXISTS src_filename; -          DROP INDEX IF EXISTS language_document_char; -          DROP INDEX IF EXISTS classify_topic_register; -          DROP TABLE IF EXISTS metadata_and_text; -          DROP TABLE IF EXISTS doc_objects; -          DROP TABLE IF EXISTS urls; -          CREATE TABLE metadata_and_text ( -            tid                              INTEGER           PRIMARY KEY, -          /* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ -            title                            VARCHAR(800)  NOT NULL, -            title_main                       VARCHAR(400)  NOT NULL, -            title_sub                        VARCHAR(400)      NULL, -            title_short                      VARCHAR(400)      NULL, -            title_edition                    VARCHAR(10)       NULL, -            title_note                       VARCHAR(2500)     NULL, -            title_language                   VARCHAR(100)      NULL, -            title_language_char              VARCHAR(6)        NULL, -            creator_author                   VARCHAR(600)  NOT NULL, -            creator_author_email             VARCHAR(100)      NULL, -            creator_author_hon               VARCHAR(100)      NULL, -            creator_author_nationality       VARCHAR(100)      NULL, -            creator_editor                   VARCHAR(600)      NULL, -            creator_contributor              VARCHAR(600)      NULL, -            creator_illustrator              VARCHAR(600)      NULL, -            creator_photographer             VARCHAR(600)      NULL, -            creator_translator               VARCHAR(600)      NULL, -            creator_prepared_by              VARCHAR(600)      NULL, -            creator_digitized_by             VARCHAR(600)      NULL, -            creator_audio                    VARCHAR(600)      NULL, -            creator_video                    VARCHAR(600)      NULL, -            language_document                VARCHAR(100)      NULL, -            language_document_char           VARCHAR(6)    NOT NULL, -            language_original                VARCHAR(100)      NULL, -            language_original_char           VARCHAR(6)        NULL, -            date_added_to_site               VARCHAR(10)       NULL, -            date_available                   VARCHAR(10)       NULL, -            date_created                     VARCHAR(10)       NULL, -            date_issued                      VARCHAR(10)       NULL, -            date_modified                    VARCHAR(10)       NULL, -            date_published                   VARCHAR(10)       NULL, -            date_valid                       VARCHAR(10)       NULL, -            date_translated                  VARCHAR(10)       NULL, -            date_original_publication        VARCHAR(10)       NULL, -            date_generated                   VARCHAR(10)       NULL, -            publisher                        VARCHAR(600)      NULL, -            original_publisher               VARCHAR(600)      NULL, -            original_language                VARCHAR(100)      NULL, -            original_language_char           VARCHAR(6)        NULL, -            original_source                  VARCHAR(600)      NULL, -            original_institution             VARCHAR(600)      NULL, -            original_nationality             VARCHAR(100)      NULL, -            original_title                   VARCHAR(800)      NULL, -            rights_copyright                 VARCHAR(2500)     NULL, -            rights_copyright_audio           VARCHAR(2500)     NULL, -            rights_copyright_cover           VARCHAR(2500)     NULL, -            rights_copyright_illustrations   VARCHAR(2500)     NULL, -            rights_copyright_photographs     VARCHAR(2500)     NULL, -            rights_copyright_text            VARCHAR(2500)     NULL, -            rights_copyright_translation     VARCHAR(2500)     NULL, -            rights_copyright_video           VARCHAR(2500)     NULL, -            rights_license                   VARCHAR(2500)     NULL, -            identifier_oclc                  VARCHAR(30)       NULL, -            identifier_isbn                  VARCHAR(16)       NULL, -            classify_topic_register          VARCHAR(2500)     NULL, -            classify_subject                 VARCHAR(600)      NULL, -            classify_loc                     VARCHAR(30)       NULL, -            classify_dewey                   VARCHAR(30)       NULL, -            classify_keywords                VARCHAR(600)      NULL, -            notes_abstract                   TEXT              NULL, -            notes_description                TEXT              NULL, -            notes_comment                    TEXT              NULL, -            notes_coverage                   VARCHAR(200)      NULL, -            notes_relation                   VARCHAR(200)      NULL, -            notes_history                    VARCHAR(600)      NULL, -            notes_type                       VARCHAR(600)      NULL, -            notes_format                     VARCHAR(600)      NULL, -            notes_prefix                     TEXT              NULL, -            notes_prefix_a                   TEXT              NULL, -            notes_prefix_b                   TEXT              NULL, -            notes_suffix                     TEXT              NULL, -            src_filename                     VARCHAR(256)  NOT NULL, -            src_fingerprint                  VARCHAR(256)      NULL, -            src_filesize                     VARCHAR(10)       NULL, -            src_wordcount                    VARCHAR(10)       NULL, -            pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ -            pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ -            pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ -            src_text                         TEXT              NULL, -            fulltext                         TEXT              NULL, -            links                            TEXT              NULL -          ); -          CREATE TABLE doc_objects ( -            lid                              INTEGER PRIMARY KEY, -            metadata_tid                     INTEGER REFERENCES metadata_and_text, -            ocn                              SMALLINT, -            ocnd                             VARCHAR(6), -            ocns                             VARCHAR(6), -            clean                            TEXT NULL, -            body                             TEXT NULL, -            book_idx                         TEXT NULL, -            seg                              VARCHAR(256) NULL, -            lev_an                           VARCHAR(1), -            lev                              SMALLINT NULL, -            lev0                             SMALLINT, -            lev1                             SMALLINT, -            lev2                             SMALLINT, -            lev3                             SMALLINT, -            lev4                             SMALLINT, -            lev5                             SMALLINT, -            lev6                             SMALLINT, -            lev7                             SMALLINT, -            en_a                             SMALLINT NULL, -            en_z                             SMALLINT NULL, -            en_a_asterisk                    SMALLINT NULL, -            en_z_asterisk                    SMALLINT NULL, -            en_a_plus                        SMALLINT NULL, -            en_z_plus                        SMALLINT NULL, -            t_of                             VARCHAR(16), -            t_is                             VARCHAR(16), -            node                             VARCHAR(16) NULL, -            parent                           VARCHAR(16) NULL, -            digest_clean                     CHAR(256), -            digest_all                       CHAR(256), -            types                            CHAR(1) NULL -          ); -          CREATE INDEX idx_ocn ON doc_objects(ocn); -          CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); -          CREATE INDEX idx_digest_all ON doc_objects(digest_all); -          CREATE INDEX idx_clean ON doc_objects(clean); -          CREATE INDEX idx_lev0 ON doc_objects(lev0); -          CREATE INDEX idx_lev1 ON doc_objects(lev1); -          CREATE INDEX idx_lev2 ON doc_objects(lev2); -          CREATE INDEX idx_lev3 ON doc_objects(lev3); -          CREATE INDEX idx_lev4 ON doc_objects(lev4); -          CREATE INDEX idx_lev5 ON doc_objects(lev5); -          CREATE INDEX idx_lev6 ON doc_objects(lev6); -          CREATE INDEX idx_title ON metadata_and_text(title); -          CREATE INDEX idx_author ON metadata_and_text(creator_author); -          CREATE INDEX idx_filename ON metadata_and_text(src_filename); -          CREATE INDEX idx_language ON metadata_and_text(language_document_char); -          CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); -        ¶",); -        return _sql_instruct; -      } -    } -    template SQLiteDiscreteInsertMetadata() { -      string SQLiteDiscreteInsertMetadata(I)( -        auto ref       I    doc_matters, -      ) { -        string _insert_metadata; -        _insert_metadata = format(q"¶ -          INSERT INTO metadata_and_text ( -            src_filename, -            title, -            title_main, -            title_sub, -            title_short, -            title_edition, -            title_language, -            classify_dewey, -            classify_keywords, -            classify_loc, -            classify_subject, -            classify_topic_register, -            creator_author, -            creator_author_email, -            creator_illustrator, -            creator_translator, -            date_added_to_site, -            date_available, -            date_created, -            date_issued, -            date_modified, -            date_published, -            date_valid, -            identifier_isbn, -            identifier_oclc, -            language_document, -            language_document_char, -            notes_abstract, -            notes_description, -            original_publisher, -            original_language, -            original_language_char, -            original_source, -            original_title, -            publisher, -            rights_copyright, -            rights_copyright_audio, -            rights_copyright_cover, -            rights_copyright_illustrations, -            rights_copyright_photographs, -            rights_copyright_text, -            rights_copyright_translation, -            rights_copyright_video, -            rights_license -          ) -          VALUES ( -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s', -            '%s' -          ); -        ¶", -          SQLinsertDelimiter!()(doc_matters.src.filename), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), -          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) -        ); -        return _insert_metadata; -      } -    } -    template SQLiteDiscreteInsertDocObjectsLoop() { -      string SQLiteDiscreteInsertDocObjectsLoop(P,I)( -        auto ref       P    doc_parts, -        auto ref       I    tid, -      ) { -        string insertDocObjectsRow(O)(O obj) { -          string _insert_doc_objects_row; -          _insert_doc_objects_row = format(q"¶ -            INSERT INTO doc_objects ( -              metadata_tid, -              ocn, -              clean, -              body, -              lev, -              t_of, -              t_is -            ) -            VALUES ( -              %d, -              %s, -              '%s', -              '%s', -              %s, -              '%s', -              '%s' -            ); -          ¶", -            1, -            obj.ocn, -            SQLinsertDelimiter!()(obj_txt["text"]), -            SQLinsertDelimiter!()(obj_txt["html"]), -            obj.heading_lev_markup, -            obj.is_of, -            obj.is_a, -          ); -          return _insert_doc_objects_row; -        } -        auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); -        string[string] obj_txt; -        string doc_text; -        string[] _insert_doc_objects; -        foreach (part; doc_parts) { -          foreach (obj; doc_abstraction[part]) { -            switch (obj.of_part) { -            case "frontmatter":              assert(part == "head", part); -              switch (obj.is_of) { -              case "para": -                switch (obj.is_a) { -                case "heading": -                  obj_txt = format_and_sqlite_load.heading(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              default: -                if ((doc_matters.opt.action.debug_do)) { -                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -                } -                break; -              } -              break; -            case "body": //                    assert(part == "body", part); // TODO broken -              switch (obj.is_of) { -              case "para": -                switch (obj.is_a) { -                case "heading": -                  debug (asserts) { // TODO consider and fix or remove -                    if (part != "body") { -                      writeln(__LINE__, ": ", obj.text); -                    } -                  } -                  obj_txt = format_and_sqlite_load.heading(obj); -                  break; -                case "para": -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              case "block": -                switch (obj.is_a) { -                case "quote": -                  obj_txt = format_and_sqlite_load.quote(obj); -                  break; -                case "group": -                  obj_txt = format_and_sqlite_load.group(obj); -                  break; -                case "block": -                  obj_txt = format_and_sqlite_load.block(obj); -                  break; -                case "poem":                        // double check on keeping both poem & verse -                  break; -                case "verse": -                  obj_txt = format_and_sqlite_load.verse(obj); -                  break; -                case "code": -                  obj_txt = format_and_sqlite_load.code(obj); -                  break; -                case "table": -                  obj_txt = format_and_sqlite_load.table(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              default: -                if ((doc_matters.opt.action.debug_do)) { -                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -                } -                break; -              } -              break; -            case "backmatter": -              assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); -              switch (obj.is_of) { -              case "para": -                switch (obj.is_a) { -                case "heading": -                  obj_txt = format_and_sqlite_load.heading(obj); -                  break; -                case "endnote":              assert(part == "endnotes", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "glossary":             assert(part == "glossary", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "bibliography":         assert(part == "bibliography", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "bookindex":            assert(part == "bookindex_seg", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                case "blurb":                assert(part == "blurb", part); -                  obj_txt = format_and_sqlite_load.para(obj); -                  break; -                default: -                  if ((doc_matters.opt.action.debug_do)) { -                    writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                  } -                  break; -                } -                break; -              default: -                if ((doc_matters.opt.action.debug_do)) { -                  writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); -                } -                break; -              } -              break; -            case "comment": -              break; -            default: -              if ((doc_matters.opt.action.debug_do)) { -                writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from -                writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); -                writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from -              } -              break; -            } -            if (obj.is_a == "heading") { -              if ((doc_matters.opt.action.verbose)) { -                writeln( -                  "markup: ", obj.heading_lev_markup, -                  "> ", obj.dom_markedup, -                  "; collapsed: ", obj.heading_lev_collapsed, -                  "> ", obj.dom_collapsed, -                  "; ocn: ", obj.ocn, -                  " node: ", obj.node, -                  "; parent: ", obj.parent_lev_markup, -                  "; ocn: ", obj.parent_ocn, -                  "; ", -                ); -              } -            } -            if (!(obj.is_a == "comment")) { -              _insert_doc_objects ~= insertDocObjectsRow(obj); -            } -          } // loop closes -        } -        return _insert_doc_objects.join.to!(char[]).toUTF8; -      } -    } -    SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); -  } -} diff --git a/views/version.txt b/views/version.txt index b3b647a..e143575 100644 --- a/views/version.txt +++ b/views/version.txt @@ -4,7 +4,7 @@ struct Version {    int minor;    int patch;  } -enum ver = Version(0, 26, 0); +enum ver = Version(0, 26, 1);  version (Posix) {    version (DigitalMars) {    } else version (LDC) { | 
