diff options
| author | Ralph Amissah <ralph@amissah.com> | 2018-04-14 16:39:52 -0400 | 
|---|---|---|
| committer | Ralph Amissah <ralph@amissah.com> | 2019-04-10 15:14:14 -0400 | 
| commit | 479e71e90146b1067120faf489af1b13a5a0cf3d (patch) | |
| tree | 5e725994ae345dd89687fae2fe90979eb3cf6f8d | |
| parent | 0.26.0 sqlite single statement insertion of objects (diff) | |
sqlite3 some stuff & rearrange files, dlang, org
| -rw-r--r-- | dub.sdl | 2 | ||||
| -rw-r--r-- | dub.selections.json | 2 | ||||
| -rw-r--r-- | maker.org | 2 | ||||
| -rw-r--r-- | org/default_paths.org | 42 | ||||
| -rw-r--r-- | org/meta_read_source_files.org | 10 | ||||
| -rw-r--r-- | org/output_hub.org | 4 | ||||
| -rw-r--r-- | org/output_sisupod.org | 26 | ||||
| -rw-r--r-- | org/output_sqlite.org | 1209 | ||||
| -rw-r--r-- | org/output_sqlite_discrete.org | 1093 | ||||
| -rw-r--r-- | org/sdp.org | 10 | ||||
| -rw-r--r-- | src/sdp/meta/read_config_files.d | 6 | ||||
| -rw-r--r-- | src/sdp/output/hub.d | 4 | ||||
| -rw-r--r-- | src/sdp/output/paths_output.d | 42 | ||||
| -rw-r--r-- | src/sdp/output/source_sisupod.d | 26 | ||||
| -rw-r--r-- | src/sdp/output/sqlite.d | 806 | ||||
| -rw-r--r-- | src/sdp/output/sqlite_discrete.d | 491 | ||||
| -rwxr-xr-x | src/sdp/sdp.d | 4 | 
17 files changed, 1922 insertions, 1857 deletions
| @@ -10,7 +10,7 @@ stringImportPaths "./views"  buildRequirements "allowWarnings"  dependency        "toml"      version="~>0.4.0-rc.2" # https://code.dlang.org/packages/toml       https://github.com/toml-lang/toml/wiki  dependency        "toml:json" version="~>0.4.0-rc.2" -dependency        "d2sqlite3" version="~>0.16.0"     # https://code.dlang.org/packages/d2sqlite3  http://biozic.github.io/d2sqlite3/d2sqlite3.html +dependency        "d2sqlite3" version="~>0.16.1"     # https://code.dlang.org/packages/d2sqlite3  http://biozic.github.io/d2sqlite3/d2sqlite3.html  subconfiguration  "d2sqlite3" "all-included"  dependency        "archive"   version="~>0.6.0"      # http://code.dlang.org/packages/archive     https://github.com/rcythr/archive  configuration "reggae" { diff --git a/dub.selections.json b/dub.selections.json index dda658d..61a1ad5 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -2,7 +2,7 @@  	"fileVersion": 1,  	"versions": {  		"archive": "0.6.0", -		"d2sqlite3": "0.16.0", +		"d2sqlite3": "0.16.1",  		"libinputvisitor": "1.2.2",  		"sdlang-d": "0.10.2",  		"taggedalgebraic": "0.10.9", @@ -815,7 +815,7 @@ stringImportPaths "./views"  buildRequirements "allowWarnings"  dependency        "toml"      version="~>0.4.0-rc.2" # https://code.dlang.org/packages/toml       https://github.com/toml-lang/toml/wiki  dependency        "toml:json" version="~>0.4.0-rc.2" -dependency        "d2sqlite3" version="~>0.16.0"     # https://code.dlang.org/packages/d2sqlite3  http://biozic.github.io/d2sqlite3/d2sqlite3.html +dependency        "d2sqlite3" version="~>0.16.1"     # https://code.dlang.org/packages/d2sqlite3  http://biozic.github.io/d2sqlite3/d2sqlite3.html  subconfiguration  "d2sqlite3" "all-included"  dependency        "archive"   version="~>0.6.0"      # http://code.dlang.org/packages/archive     https://github.com/rcythr/archive  #+END_SRC diff --git a/org/default_paths.org b/org/default_paths.org index cc78c46..af96bea 100644 --- a/org/default_paths.org +++ b/org/default_paths.org @@ -880,6 +880,27 @@ template SiSUoutPaths() {      return _PathsStruct();    }  } +template SiSUoutPathSQLite() { +  auto SiSUoutPathSQLite(Po)( +    Po  output_pth_root, +  ) { +    struct _PathsStruct { +      string output_root() { +        string out_root_; +        if (output_pth_root.length > 0) { +          out_root_ = output_pth_root; +        } else { +          out_root_ = "sisugen"; +        } +        return out_root_; +      } +      string output_base() { +        return asNormalizedPath(output_root).array; +      } +    } +    return _PathsStruct(); +  } +}  #+END_SRC  ** set @@ -891,7 +912,6 @@ template SiSUoutPathsFnPd() {    auto SiSUoutPathsFnPd(Fn,Pn)(      Fn  fn_src_pth,      Pn  pod_name -    // Pn  pod_name = "",    ) {      struct _PathsStruct {        string base_filename() { @@ -1092,14 +1112,16 @@ template SiSUpathsSQLiteDiscrete() {      Po  output_pth_root,      Lng lng,    ) { -    auto out_pth = SiSUoutPaths!()(output_pth_root, lng); -    string base_dir = "sqlite";      struct _PathsStruct {        string base_filename(string fn_src) {          return fn_src.baseName.stripExtension;        }        string base() { -        return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array; +        auto out_pth = SiSUoutPaths!()(output_pth_root, lng); +        string base_dir = "sqlite"; +        return asNormalizedPath((out_pth.output_base).chainPath(base_dir)).array; +        // return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array; // single file for each file, multiple languages +        // return asNormalizedPath((out_pth.output_base).chainPath(base_dir)).array; // separate files for each language        }        string seg(string fn_src) {          return asNormalizedPath(base.chainPath(base_filename(fn_src))).array; @@ -1120,23 +1142,19 @@ template SiSUpathsSQLiteDiscrete() {  template SiSUpathsSQLite() {    mixin SiSUrgxInit;    static auto rgx = Rgx(); -  auto SiSUpathsSQLite(Po,Lng)( +  auto SiSUpathsSQLite(Po)(      Po  output_pth_root, -    Lng lng,    ) { -    auto out_pth = SiSUoutPaths!()(output_pth_root, lng); -    string base_dir = "sqlite";      struct _PathsStruct {        string base_filename(string fn_src) {          return fn_src.baseName.stripExtension;        }        string base() { +        auto out_pth = SiSUoutPathSQLite!()(output_pth_root); // decide whether to have separate files for each language +        string base_dir = "sqlite";          return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array;        } -      string seg(string fn_src) { -        return asNormalizedPath(base.chainPath(base_filename(fn_src))).array; -      } -      string sqlite_file(string fn_src) { +      string sqlite_file(string fn_src = "sdp-niu") {          return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array;        }      } diff --git a/org/meta_read_source_files.org b/org/meta_read_source_files.org index ab73040..b726631 100644 --- a/org/meta_read_source_files.org +++ b/org/meta_read_source_files.org @@ -70,10 +70,8 @@ static template readConfigSite() {              config_file_str = conf_file.readText;              break;            } -        } -        catch (ErrnoException ex) { -        } -        catch (FileException ex) { +        } catch (ErrnoException ex) { +        } catch (FileException ex) {          }        }        if (config_file_str.length > 0) { break; } @@ -385,7 +383,7 @@ static template SiSUrawMarkupContent() {  #+END_SRC  ** get markup source, read file                              :source:markup: -*** read file, source string                                  :string: +*** read file, source string [#A]                                  :string:  #+name: meta_markup_source_raw_read_file_source_string  #+BEGIN_SRC d @@ -417,7 +415,7 @@ final private string readInMarkupSource(in char[] fn_src) {  }  #+END_SRC -*** document header & content, array.length == 2               :array: +*** document header & content, array.length == 2 [#A]               :array:  here you split document header and body, an array.length == 2  split is on first match of level A~ (which is required) diff --git a/org/output_hub.org b/org/output_hub.org index 0221652..6d62477 100644 --- a/org/output_hub.org +++ b/org/output_hub.org @@ -34,7 +34,7 @@ template outputHub() {  }  template outputHubOp() {    <<output_imports>> -  void outputHubOp(O)(O opt_action) { +  void outputHubOp(E,O)(E env, O opt_action) {      mixin SiSUoutputRgxInit;      static auto rgx = Rgx();      <<output_options_op>> @@ -227,7 +227,7 @@ if ((opt_action.sqlite_create)) {    if ((opt_action.verbose)) {      writeln("sqlite create table... ");    } -  SQLiteTablesCreate!()(); +  SQLiteTablesCreate!()(env, opt_action);  }  if ((opt_action.sqlite_drop)) {    if ((opt_action.verbose)) { diff --git a/org/output_sisupod.org b/org/output_sisupod.org index 5b1f6ab..a53f9c7 100644 --- a/org/output_sisupod.org +++ b/org/output_sisupod.org @@ -132,7 +132,7 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;        ~ "/" ~ image;      if (exists(fn_src_in)) {        debug(io) { -        writeln("WARNING (io debug) src out found: ", fn_src_in); +        writeln("(io debug) src out found: ", fn_src_in);        }        if (doc_matters.opt.action.source) {          fn_src_in.copy(fn_src_out_filesystem); @@ -163,7 +163,7 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;      ~ "/" ~ "sisu_document_make";    if (exists(fn_src_in)) {      debug(io) { -      writeln("WARNING (io debug) src out found: ", fn_src_in); +      writeln("(io debug) src out found: ", fn_src_in);      }      if (doc_matters.opt.action.source) {        fn_src_in.copy(fn_src_out_filesystem); @@ -194,7 +194,7 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;    string[] filelist_src_zpod_arr;    if (exists(fn_src_in)) {      debug(io) { -      writeln("WARNING (io debug) src out found: ", fn_src_in); +      writeln("(io debug) src in found: ", fn_src_in);      }      filelist_src_out_sisupod_arr ~= fn_src_out_sisupod_zip_base;      filelist_src_zpod_arr ~= fn_src_out_inside_pod; @@ -217,7 +217,7 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;    } else {      if (doc_matters.opt.action.verbose      || doc_matters.opt.action.debug_do) { -      writeln("WARNING (io) src out NOT found (markup source): ", fn_src_in); +      writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);      }    }  } { /+ bundle insert files +/ @@ -248,7 +248,7 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;          ).filesystem_open_zpod.to!string;        if (exists(fn_src_in)) {          debug(io) { -          writeln("WARNING (io debug) src out found: ", fn_src_in); +          writeln("(io debug) src out found: ", fn_src_in);          }          if (doc_matters.opt.action.source) {            fn_src_in.copy(fn_src_out_filesystem); @@ -272,9 +272,14 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;    }  } {    auto fn_src_in = doc_matters.src.filename; -  if (exists(fn_src_in)) { -    if (doc_matters.opt.action.sisupod) { +  if (doc_matters.opt.action.sisupod) { +    if (exists(doc_matters.src.file_with_absolute_path)) {        createZipFile!()(fn_sisupod, zip.build()); +    } else { +      writeln("WARNING check missing source file(s): ", doc_matters.opt.action.sisupod); +    } +    if (!(exists(fn_sisupod))) { +      writeln("WARNING failed to create sisupod zip archive: ", fn_sisupod);      }    }  } @@ -286,8 +291,11 @@ auto fn_sisupod = pths_sisupod.sisupod_filename(doc_matters.src.filename).zpod;  #+BEGIN_SRC d  if (exists(fn_sisupod)) {    try { -    auto data = (cast(byte[]) (fn_sisupod).read); -    writefln("%-(%02x%) %s", data.sha256Of, fn_sisupod); +    if (doc_matters.opt.action.verbose) { +      auto data = (cast(byte[]) (fn_sisupod).read); +      writeln(doc_matters.src.filename, " >> "); +      writefln("%-(%02x%) %s", data.sha256Of, fn_sisupod); +    }      debug(sisupod) {        try {          auto zipped = new ZipArchive((fn_sisupod).read); diff --git a/org/output_sqlite.org b/org/output_sqlite.org index 0e7f125..7fb843d 100644 --- a/org/output_sqlite.org +++ b/org/output_sqlite.org @@ -20,18 +20,71 @@  #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d  module sdp.output.sqlite;  <<output_imports>> +long _metadata_tid_lastrowid; +template SQLiteDbRun() { +  void SQLiteDbRun(Db,St,O)( +    auto ref       Db   db, +    auto ref       St   db_statement, +    auto ref       O    opt_action, +               string   note, +  ) { +    debug(sql_statement) { +      writeln(db_statement); +    } +    try { +      db.run( +        "\nBEGIN;\n" ~ +        db_statement ~ +        "\nCOMMIT;\n" +      ); +      if (!(opt_action.sqlite_create)) { +        _metadata_tid_lastrowid = db.lastInsertRowid(); +        writeln("last row id: ", _metadata_tid_lastrowid); +      } +    } catch (ErrnoException ex) { +      writeln("ERROR SQLite : ", ex); +    } catch (Exception ex) { +      writeln("ERROR SQLite : ", ex); +    } +    if (opt_action.debug_do) { +      writeln(note); +      if (opt_action.verbose) { +        writeln(db_statement); +      } +    } +  } +}  template SQLiteBuildTablesAndPopulate() {    void SQLiteBuildTablesAndPopulate(D,I)(      auto ref const D    doc_abstraction, -    auto ref I          doc_matters, +    auto ref       I    doc_matters,    ) {      import d2sqlite3;      import std.typecons : Nullable;      mixin SiSUoutputRgxInit;      static auto rgx = Rgx(); +    auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); +    template SQLiteDbStatementComposite() { +      void SQLiteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        <<sqlite_db_statement_composite>> +      } +    } +    template SQLinsertDelimiter() { +      auto SQLinsertDelimiter(string _txt) { +        _txt = _txt +          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +        return _txt; +      } +    }      template SQLiteFormatAndLoadObject() {        auto SQLiteFormatAndLoadObject(I)( -        auto ref              I    doc_matters, +        auto ref       I    doc_matters,        ) {          mixin SiSUoutputRgxInit;          struct sqlite_format_and_load_objects { @@ -44,34 +97,68 @@ template SQLiteBuildTablesAndPopulate() {          return sqlite_format_and_load_objects();        }      } -    template SQLiteInstruct() { -      Statement SQLiteInstruct(I)( -        auto ref              I    doc_matters, +    template SQLiteTablesReCreate() { +      string SQLiteTablesReCreate()() { +        string _sql_instruct; +        _sql_instruct = format(q"¶ +          <<sqlite_statement_drop_existing_index>> +          <<sqlite_statement_drop_existing_tables>> +          <<sqlite_statement_create_table_metadata_and_src_txt>> +          <<sqlite_statement_create_table_objects>> +          <<sqlite_statement_create_index>> +        ¶",); +        return _sql_instruct; +      } +    } +    template SQLiteInsertMetadata() { +      string SQLiteInsertMetadata(I)( +        auto ref       I    doc_matters,        ) { -        <<sqlite_db_initialize>> -          <<sqlite_table_initialize>> -          <<sqlite_create_table_metadata_and_src_txt>> -          /* */<<sqlite_create_table_urls>> -          <<sqlite_create_table_objects>> -        <<sqlite_close_initialize>> -        <<sqlite_insert_metadata_and_src_text>> -        //<<sqlite_insert_urls>> -        <<sqlite_insert_doc_objects>> +        <<sqlite_formatted_insertions_doc_matters_metadata>> +        return _insert_metadata;        }      } -    template SQLiteObjectsLoop() { -      void SQLiteObjectsLoop(P)( -        auto ref P          doc_parts, +    template SQLiteInsertDocObjectsLoop() { +      string SQLiteInsertDocObjectsLoop(P,I)( +        auto ref       P    doc_parts, +        auto ref       I    _metadata_tid,        ) { +        string insertDocObjectsRow(O)(O obj) { +          <<sqlite_formatted_insertions_doc_objects>> +          return _insert_doc_objects_row; +        }          <<sqlite_objects_loop>>        }      } -    SQLiteObjectsLoop!()(doc_matters.xml.keys_seq.sql); +    SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);    }  }  template SQLiteTablesCreate() { -  void SQLiteTablesCreate()() { -    writeln("table create"); +  void SQLiteTablesCreate(E,O)(E env, O opt_action) { +    import d2sqlite3; +    template SQLiteTablesReCreate() { +      string SQLiteTablesReCreate()() { +        string _sql_instruct; +        _sql_instruct = format(q"¶ +          <<sqlite_statement_drop_existing_index>> +          <<sqlite_statement_drop_existing_tables>> +          <<sqlite_statement_create_table_metadata_and_src_txt>> +          <<sqlite_statement_create_table_objects>> +          <<sqlite_statement_create_index>> +        ¶",); +        return _sql_instruct; +      } +    } +    if (opt_action.sqlite_create) { +      string _db_statement; +      auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE +      pth_sqlite.base.mkdirRecurse; +      auto db = Database(pth_sqlite.sqlite_file); // ISSUE +      { +        _db_statement ~= SQLiteTablesReCreate!()(); +      } +      SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE"); +    }    }  }  template SQLiteTablesDrop() { @@ -81,7 +168,35 @@ template SQLiteTablesDrop() {  }  #+END_SRC -** 1. imports +** 1. [#A] sqlite_db_statement + +#+name: sqlite_db_statement_composite +#+BEGIN_SRC d +{ +  { +    string _db_statement; +    { +      if ((doc_matters.opt.action.sqlite_create)) { +        auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); +        pth_sqlite.base.mkdirRecurse; +        _db_statement ~= SQLiteTablesReCreate!()(); +        SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); +      } +      if ((doc_matters.opt.action.sqlite_update)) { // TODO +        _db_statement ~= SQLiteInsertMetadata!()(doc_matters); +        SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); +        /+ get tid (lastrowid or max) for use in doc_objects table +/ +        _metadata_tid_lastrowid = db.lastInsertRowid(); +        _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX +        SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); +      } +    } +    db.close; +  } +} +#+END_SRC + +** 2. imports  #+name: output_imports  #+BEGIN_SRC d @@ -91,7 +206,7 @@ import    std.conv : to;  #+END_SRC -** 2. format and load template +** 3. format and load template  *** 1. prepare objects (munge, sanitize, markup)  **** 1. _text_ generic munge (sanitize text for search) @@ -112,7 +227,6 @@ auto generic_munge_sanitize_text_for_search(  ) {    string _notes;    string _urls; -  _txt = _txt.replaceAll(rgx.inline_fontface_clean, "");    if (_txt.matchFirst(rgx.inline_notes_al_gen)) {      foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {        _notes ~= "\n" ~ m["text"]; @@ -429,9 +543,17 @@ auto html_table(O)(              string _align = ("style=\"text-align:"              ~ ((obj.table_column_aligns[col_idx] == "l")              ? "left\"" : "right\"")); -            _table ~= "<" ~ _col_is ~ " width=\"" ~ obj.table_column_widths[col_idx].to!string ~ "%\" " ~ _align ~ ">"; +            _table ~= "<" +              ~ _col_is +              ~ " width=\"" +              ~ obj.table_column_widths[col_idx].to!string +              ~ "%\" " +              ~ _align +              ~ ">";              _table ~= cell; -            _table ~= "</" ~ _col_is ~ ">"; +            _table ~= "</" +              ~ _col_is +              ~ ">";            }          }          _table ~= "</tr>"; @@ -461,176 +583,17 @@ auto html_table(O)(  #+END_SRC  *** 2. _sqlite_ (loop preparation & pre-loop action) -***** initialize: prepare statements drop & create db & tables anew -****** d sqlite statements (initialize database) { - -#+name: sqlite_db_initialize -#+BEGIN_SRC d -auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path, doc_matters.src.language); -pth_sqlite.base.mkdirRecurse; -auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); -// auto db = Database(":memory:"); // open database in memory -if (doc_matters.opt.action.sqlite_create) { -  if ((doc_matters.opt.action.verbose)) { writeln("sqlite create tables... "); } -  db.run(" -#+END_SRC - -*******  0. drop table if exists - -#+name: sqlite_table_initialize -#+BEGIN_SRC sql -DROP TABLE IF EXISTS metadata_and_text; -DROP TABLE IF EXISTS doc_objects; -DROP TABLE IF EXISTS urls; -#+END_SRC - -*******  1. create tables -********  metadata & src text - -#+name: sqlite_create_table_metadata_and_src_txt -#+BEGIN_SRC sql -CREATE TABLE metadata_and_text ( -  tid                              BIGINT            PRIMARY KEY, -  title                            VARCHAR(800)  NOT NULL, -  title_main                       VARCHAR(400)      NULL, /*NOT*/ -  title_sub                        VARCHAR(400)      NULL, -  title_short                      VARCHAR(400)      NULL, -  title_edition                    VARCHAR(10)       NULL, -  title_note                       VARCHAR(2500)     NULL, -  title_language                   VARCHAR(100)      NULL, -  title_language_char              VARCHAR(6)        NULL, -  creator_author                   VARCHAR(600)      NULL, -  creator_author_email             VARCHAR(100)      NULL, -  creator_author_hon               VARCHAR(100)      NULL, -  creator_author_nationality       VARCHAR(100)      NULL, -  creator_editor                   VARCHAR(600)      NULL, -  creator_contributor              VARCHAR(600)      NULL, -  creator_illustrator              VARCHAR(600)      NULL, -  creator_photographer             VARCHAR(600)      NULL, -  creator_translator               VARCHAR(600)      NULL, -  creator_prepared_by              VARCHAR(600)      NULL, -  creator_digitized_by             VARCHAR(600)      NULL, -  creator_audio                    VARCHAR(600)      NULL, -  creator_video                    VARCHAR(600)      NULL, -  language_document                VARCHAR(100)      NULL, -  language_document_char           VARCHAR(6)        NULL, /*NOT*/ -  language_original                VARCHAR(100)      NULL, -  language_original_char           VARCHAR(6)        NULL, -  date_added_to_site               VARCHAR(10)       NULL, -  date_available                   VARCHAR(10)       NULL, -  date_created                     VARCHAR(10)       NULL, -  date_issued                      VARCHAR(10)       NULL, -  date_modified                    VARCHAR(10)       NULL, -  date_published                   VARCHAR(10)       NULL, -  date_valid                       VARCHAR(10)       NULL, -  date_translated                  VARCHAR(10)       NULL, -  date_original_publication        VARCHAR(10)       NULL, -  date_generated                   VARCHAR(10)       NULL, -  publisher                        VARCHAR(600)      NULL, -  original_publisher               VARCHAR(600)      NULL, -  original_language                VARCHAR(100)      NULL, -  original_language_char           VARCHAR(6)        NULL, -  original_source                  VARCHAR(600)      NULL, -  original_institution             VARCHAR(600)      NULL, -  original_nationality             VARCHAR(100)      NULL, -  original_title                   VARCHAR(800)      NULL, -  rights_copyright                 VARCHAR(2500)     NULL, -  rights_copyright_audio           VARCHAR(2500)     NULL, -  rights_copyright_cover           VARCHAR(2500)     NULL, -  rights_copyright_illustrations   VARCHAR(2500)     NULL, -  rights_copyright_photographs     VARCHAR(2500)     NULL, -  rights_copyright_text            VARCHAR(2500)     NULL, -  rights_copyright_translation     VARCHAR(2500)     NULL, -  rights_copyright_video           VARCHAR(2500)     NULL, -  rights_license                   VARCHAR(2500)     NULL, -  identifier_oclc                  VARCHAR(30)       NULL, -  identifier_isbn                  VARCHAR(16)       NULL, -  classify_topic_register          VARCHAR(2500)     NULL, -  classify_subject                 VARCHAR(600)      NULL, -  classify_loc                     VARCHAR(30)       NULL, -  classify_dewey                   VARCHAR(30)       NULL, -  classify_keywords                VARCHAR(600)      NULL, -  notes_abstract                   TEXT              NULL, -  notes_description                TEXT              NULL, -  notes_comment                    TEXT              NULL, -  notes_coverage                   VARCHAR(200)      NULL, -  notes_relation                   VARCHAR(200)      NULL, -  notes_history                    VARCHAR(600)      NULL, -  notes_type                       VARCHAR(600)      NULL, -  notes_format                     VARCHAR(600)      NULL, -  notes_prefix                     TEXT              NULL, -  notes_prefix_a                   TEXT              NULL, -  notes_prefix_b                   TEXT              NULL, -  notes_suffix                     TEXT              NULL, -  src_filename                     VARCHAR(256)      NULL, -  src_fingerprint                  VARCHAR(256)      NULL, -  src_filesize                     VARCHAR(10)       NULL, -  src_wordcount                    VARCHAR(10)       NULL, -  src_text                         TEXT              NULL, -  fulltext                         TEXT              NULL, -  links                            TEXT              NULL -); -#+END_SRC - -********  doc_objects - -#+name: sqlite_create_table_objects -#+BEGIN_SRC sql -CREATE TABLE doc_objects ( -  lid                              BIGINT PRIMARY KEY, -  metadata_tid                     BIGINT REFERENCES metadata_and_text, -  ocn                              SMALLINT, -  ocnd                             VARCHAR(6), -  ocns                             VARCHAR(6), -  clean                            TEXT NULL, -  body                             TEXT NULL, -  book_idx                         TEXT NULL, -  seg                              VARCHAR(256) NULL, -  lev_an                           VARCHAR(1), -  lev                              SMALLINT NULL, -  lev0                             SMALLINT, -  lev1                             SMALLINT, -  lev2                             SMALLINT, -  lev3                             SMALLINT, -  lev4                             SMALLINT, -  lev5                             SMALLINT, -  lev6                             SMALLINT, -  lev7                             SMALLINT, -  en_a                             SMALLINT NULL, -  en_z                             SMALLINT NULL, -  en_a_asterisk                    SMALLINT NULL, -  en_z_asterisk                    SMALLINT NULL, -  en_a_plus                        SMALLINT NULL, -  en_z_plus                        SMALLINT NULL, -  t_of                             VARCHAR(16), -  t_is                             VARCHAR(16), -  node                             VARCHAR(16) NULL, -  parent                           VARCHAR(16) NULL, -  digest_clean                     CHAR(256), -  digest_all                       CHAR(256), -  types                            CHAR(1) NULL -) -#+END_SRC - -****** d } - -#+name: sqlite_close_initialize -#+BEGIN_SRC d -  "); -} -#+END_SRC - -***** prepare insert statements and do pre-loop inserts -****** 1. _metadata & src_ text -******* TODO sql remove selected -******** d { sql statement +**** prepare insert statements and do pre-loop inserts +***** 1. _metadata & src_ text +****** TODO sql remove selected +******* d { sql statement  ##+name: sqlite_remove_selected  #+BEGIN_SRC d  Statement insert_metadata = db.prepare("  #+END_SRC -********* remove selected +******** remove selected  identify and remove, identify by  - filename & language code @@ -644,413 +607,20 @@ identify and remove, identify by     AND metadata_and_text.language_document_char = '#{doc_matters.conf_make_meta.meta.language_document_char}'  #+END_SRC -******** d } +******* d }  ##+name: sqlite_remove_selected  #+BEGIN_SRC d  ");  #+END_SRC -******* sql statement -******** d { sql statement - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -Statement insert_metadata = db.prepare(" -#+END_SRC - -tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -********* sql insert into - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC sql -  INSERT INTO metadata_and_text ( -    tid, -    title, -    title_main, -    title_sub, -    title_short, -    title_edition, -    title_note, -    title_language, -    title_language_char, -    creator_author, -    creator_author_email, -    creator_author_hon, -    creator_author_nationality, -    creator_editor, -    creator_contributor, -    creator_illustrator, -    creator_photographer, -    creator_translator, -    creator_prepared_by, -    creator_digitized_by, -    creator_audio, -    creator_video, -    language_document, -    language_document_char, -    language_original, -    language_original_char, -    date_added_to_site, -    date_available, -    date_created, -    date_issued, -    date_modified, -    date_published, -    date_valid, -    date_translated, -    date_original_publication, -    date_generated, -    publisher, -    original_publisher, -    original_language, -    original_language_char, -    original_source, -    original_institution, -    original_nationality, -    original_title, -    rights_copyright, -    rights_copyright_audio, -    rights_copyright_cover, -    rights_copyright_illustrations, -    rights_copyright_photographs, -    rights_copyright_text, -    rights_copyright_translation, -    rights_copyright_video, -    rights_license, -    identifier_oclc, -    identifier_isbn, -    classify_topic_register, -    classify_subject, -    classify_loc, -    classify_dewey, -    classify_keywords, -    notes_abstract, -    notes_description, -    notes_comment, -    notes_coverage, -    notes_relation, -    notes_history, -    notes_type, -    notes_format, -    notes_prefix, -    notes_prefix_a, -    notes_prefix_b, -    notes_suffix, -    src_filename, -    src_fingerprint, -    src_filesize, -    src_wordcount, -    src_text, -    fulltext, -    links -  ) -#+END_SRC - -********* sql values - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC sql -  VALUES ( -    :tid, -    :title, -    :title_main, -    :title_sub, -    :title_short, -    :title_edition, -    :title_note, -    :title_language, -    :title_language_char, -    :creator_author, -    :creator_author_email, -    :creator_author_hon, -    :creator_author_nationality, -    :creator_editor, -    :creator_contributor, -    :creator_illustrator, -    :creator_photographer, -    :creator_translator, -    :creator_prepared_by, -    :creator_digitized_by, -    :creator_audio, -    :creator_video, -    :language_document, -    :language_document_char, -    :language_original, -    :language_original_char, -    :date_added_to_site, -    :date_available, -    :date_created, -    :date_issued, -    :date_modified, -    :date_published, -    :date_valid, -    :date_translated, -    :date_original_publication, -    :date_generated, -    :publisher, -    :original_publisher, -    :original_language, -    :original_language_char, -    :original_source, -    :original_institution, -    :original_nationality, -    :original_title, -    :rights_copyright, -    :rights_copyright_audio, -    :rights_copyright_cover, -    :rights_copyright_illustrations, -    :rights_copyright_photographs, -    :rights_copyright_text, -    :rights_copyright_translation, -    :rights_copyright_video, -    :rights_license, -    :identifier_oclc, -    :identifier_isbn, -    :classify_topic_register, -    :classify_subject, -    :classify_loc, -    :classify_dewey, -    :classify_keywords, -    :notes_abstract, -    :notes_description, -    :notes_comment, -    :notes_coverage, -    :notes_relation, -    :notes_history, -    :notes_type, -    :notes_format, -    :notes_prefix, -    :notes_prefix_a, -    :notes_prefix_b, -    :notes_suffix, -    :src_filename, -    :src_fingerprint, -    :src_filesize, -    :src_wordcount, -    :src_text, -    :fulltext, -    :links -  ) -#+END_SRC - -******** d } - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -"); -#+END_SRC - -*******  TODO insert document_metadata - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -// insert_metadata.bind(":tid",                               ); // unique identifier you could try use sha of title author & language? or auto increment?? -insert_metadata.bind(":title",                             doc_matters.conf_make_meta.meta.title_full); -insert_metadata.bind(":title_main",                        doc_matters.conf_make_meta.meta.title_main); -insert_metadata.bind(":title_sub",                         doc_matters.conf_make_meta.meta.title_subtitle); -insert_metadata.bind(":title_short",                       doc_matters.conf_make_meta.meta.title_short); // -insert_metadata.bind(":title_edition",                     doc_matters.conf_make_meta.meta.title_edition); -insert_metadata.bind(":title_language",                    doc_matters.conf_make_meta.meta.title_language); -// insert_metadata.bind(":title_language_char",            doc_matters.conf_make_meta.meta.title_language_char); -insert_metadata.bind(":classify_dewey",                    doc_matters.conf_make_meta.meta.classify_dewey); -insert_metadata.bind(":classify_keywords",                 doc_matters.conf_make_meta.meta.classify_keywords); -insert_metadata.bind(":classify_loc",                      doc_matters.conf_make_meta.meta.classify_loc); -insert_metadata.bind(":classify_subject",                  doc_matters.conf_make_meta.meta.classify_subject); -insert_metadata.bind(":classify_topic_register",           doc_matters.conf_make_meta.meta.classify_topic_register); -insert_metadata.bind(":creator_author",                    doc_matters.conf_make_meta.meta.creator_author); -insert_metadata.bind(":creator_author_email",              doc_matters.conf_make_meta.meta.creator_author_email); -// insert_metadata.bind(":creator_editor",                 doc_matters.conf_make_meta.meta.creator_editor); -// insert_metadata.bind(":creator_contributor",            doc_matters.conf_make_meta.meta.creator_contributor); -insert_metadata.bind(":creator_illustrator",               doc_matters.conf_make_meta.meta.creator_illustrator); -// insert_metadata.bind(":creator_photographer",           doc_matters.conf_make_meta.meta.creator_photographer); -insert_metadata.bind(":creator_translator",                doc_matters.conf_make_meta.meta.creator_translator); -// insert_metadata.bind(":creator_audio",                  doc_matters.conf_make_meta.meta.creator_audio); -// insert_metadata.bind(":creator_video",                  doc_matters.conf_make_meta.meta.creator_video); -insert_metadata.bind(":date_added_to_site",                doc_matters.conf_make_meta.meta.date_added_to_site); -insert_metadata.bind(":date_available",                    doc_matters.conf_make_meta.meta.date_available); -insert_metadata.bind(":date_created",                      doc_matters.conf_make_meta.meta.date_created); -insert_metadata.bind(":date_issued",                       doc_matters.conf_make_meta.meta.date_issued); -insert_metadata.bind(":date_modified",                     doc_matters.conf_make_meta.meta.date_modified); -insert_metadata.bind(":date_published",                    doc_matters.conf_make_meta.meta.date_published); -insert_metadata.bind(":date_valid",                        doc_matters.conf_make_meta.meta.date_valid); -// insert_metadata.bind(":date_translated",                doc_matters.conf_make_meta.meta.date_translated); -// insert_metadata.bind(":date_original_publication",      doc_matters.conf_make_meta.meta.date_original_publication); -// insert_metadata.bind(":date_generated",                 doc_matters.conf_make_meta.meta.date_generated); -insert_metadata.bind(":identifier_isbn",                   doc_matters.conf_make_meta.meta.identifier_isbn); -insert_metadata.bind(":identifier_oclc",                   doc_matters.conf_make_meta.meta.identifier_oclc); -insert_metadata.bind(":language_document",                 doc_matters.conf_make_meta.meta.language_document); -insert_metadata.bind(":language_document_char",            doc_matters.conf_make_meta.meta.language_document_char); -// insert_metadata.bind(":language_original",              doc_matters.conf_make_meta.meta.language_original); -// insert_metadata.bind(":language_original_char",         doc_matters.conf_make_meta.meta.language_original_char); -insert_metadata.bind(":notes_abstract",                    doc_matters.conf_make_meta.meta.notes_abstract); -insert_metadata.bind(":notes_description",                 doc_matters.conf_make_meta.meta.notes_description); -insert_metadata.bind(":original_publisher",                doc_matters.conf_make_meta.meta.original_publisher); -insert_metadata.bind(":original_language",                 doc_matters.conf_make_meta.meta.original_language); -insert_metadata.bind(":original_language_char",            doc_matters.conf_make_meta.meta.original_language_char); -insert_metadata.bind(":original_source",                   doc_matters.conf_make_meta.meta.original_source); -insert_metadata.bind(":original_title",                    doc_matters.conf_make_meta.meta.original_title); -insert_metadata.bind(":publisher",                         doc_matters.conf_make_meta.meta.publisher); -// insert_metadata.bind(":rights",                         doc_matters.conf_make_meta.meta.rights); -insert_metadata.bind(":rights_copyright",                  doc_matters.conf_make_meta.meta.rights_copyright); -insert_metadata.bind(":rights_copyright_audio",            doc_matters.conf_make_meta.meta.rights_copyright_audio); -insert_metadata.bind(":rights_copyright_cover",            doc_matters.conf_make_meta.meta.rights_copyright_cover); -insert_metadata.bind(":rights_copyright_illustrations",    doc_matters.conf_make_meta.meta.rights_copyright_illustrations); -insert_metadata.bind(":rights_copyright_photographs",      doc_matters.conf_make_meta.meta.rights_copyright_photographs); -// insert_metadata.bind(":rights_copyright_preparation",   doc_matters.conf_make_meta.meta.rights_preparation); -insert_metadata.bind(":rights_copyright_text",      doc_matters.conf_make_meta.meta.rights_copyright_text); -insert_metadata.bind(":rights_copyright_translation",      doc_matters.conf_make_meta.meta.rights_copyright_translation); -insert_metadata.bind(":rights_copyright_video",            doc_matters.conf_make_meta.meta.rights_copyright_video); -// insert_metadata.bind(":rights_copyright_digitization",  doc_matters.conf_make_meta.meta.rights_digitization); -// insert_metadata.bind(":rights_copyright_audio",         doc_matters.conf_make_meta.meta.rights_audio); -// insert_metadata.bind(":rights_copyright_video",         doc_matters.conf_make_meta.meta.rights_video); -insert_metadata.bind(":rights_license",                    doc_matters.conf_make_meta.meta.rights_license); -// insert_metadata.bind(":src_filename",                   doc_matters.conf_make_meta.meta.src_filename); -// insert_metadata.bind(":src_fingerprint",                doc_matters.conf_make_meta.meta.src_fingerprint); -// insert_metadata.bind(":src_filesize",                   doc_matters.conf_make_meta.meta.src_filesize); -// insert_metadata.bind(":src_wordcount",                  doc_matters.conf_make_meta.meta.src_wordcount); -// insert_metadata.bind(":src_text",                       doc_matters.conf_make_meta.meta.src_text); -// insert_metadata.bind(":fulltext",                       doc_matters.conf_make_meta.meta.fulltext); -// insert_metadata.bind(":links",                          doc_matters.conf_make_meta.meta.links); -insert_metadata.execute(); insert_metadata.reset(); -/+ watch +/ -writeln(" ", pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); -if ((doc_matters.opt.action.verbose)) { -  writeln("sql statement executed"); -} -assert(db.totalChanges == 1); -#+END_SRC - -###+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -insert_metadata.inject( -  1, -  doc_matters.dochead_meta["title"]["full"], -  "", -  "", -  "", -  "", -  "", -  "", -  "", -  doc_matters.dochead_meta["creator"]["author"] -); -#+END_SRC - -****** 2. _doc objects_ (used with doc_objects in document loop) -******* prepare sql statement -******** d { sql statement - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC d -Statement insert_doc_objects = db.prepare(" -#+END_SRC - -********* sql insert into - -lid unique, increment by 1 per object, not ocn - -metadata tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC sql -  INSERT INTO doc_objects ( -    lid, -    metadata_tid, -    ocn, -    ocnd, -    ocns, -    clean, -    body, -    book_idx, -    seg, -    lev_an, -    lev, -    lev0, -    lev1, -    lev2, -    lev3, -    lev4, -    lev5, -    lev6, -    lev7, -    en_a, -    en_z, -    en_a_asterisk, -    en_z_asterisk, -    en_a_plus, -    en_z_plus, -    t_of, -    t_is, -    node, -    parent, -    digest_clean, -    digest_all, -    types -  ) -#+END_SRC - -********* sql values - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC sql -  VALUES ( -    :lid, -    :metadata_tid, -    :ocn, -    :ocnd, -    :ocns, -    :clean, -    :body, -    :book_idx, -    :seg, -    :lev_an, -    :lev, -    :lev0, -    :lev1, -    :lev2, -    :lev3, -    :lev4, -    :lev5, -    :lev6, -    :lev7, -    :en_a, -    :en_z, -    :en_a_asterisk, -    :en_z_asterisk, -    :en_a_plus, -    :en_z_plus, -    :t_of, -    :t_is, -    :node, -    :parent, -    :digest_clean, -    :digest_all, -    :types -  ) -#+END_SRC - -******** d } - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC d -"); -return insert_doc_objects; -#+END_SRC - -*******  TODO (within loop not here - insert doc objects +***** 2. _doc objects_ (used with doc_objects in document loop) +******  TODO (within loop not here - insert doc objects  work out  *** 3. hub (sqlite_format_and_load_objects) -***** sql related +**** sql related  #+name: sqlite_load_object  #+BEGIN_SRC d @@ -1092,7 +662,7 @@ string sqlite_statement(O)(  }  #+END_SRC -***** heading +**** heading  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1117,7 +687,7 @@ auto heading(O)(  }  #+END_SRC -***** para +**** para  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1142,7 +712,7 @@ auto para(O)(  }  #+END_SRC -***** quote +**** quote  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1166,7 +736,7 @@ auto quote(O)(    return obj_txt;  }  #+END_SRC -***** group +**** group  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1191,7 +761,7 @@ auto group(O)(  }  #+END_SRC -***** block +**** block  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1216,7 +786,7 @@ auto block(O)(  }  #+END_SRC -***** verse +**** verse  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1241,7 +811,7 @@ auto verse(O)(  }  #+END_SRC -***** code +**** code  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1266,7 +836,7 @@ auto code(O)(  }  #+END_SRC -***** table +**** table  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1291,14 +861,14 @@ auto table(O)(  }  #+END_SRC -** 3. loop, identify, load - loop template +** 4. loop, identify, load - loop template  #+name: sqlite_objects_loop  #+BEGIN_SRC d -Statement insert_doc_objects = SQLiteInstruct!()(doc_matters);  auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);  string[string] obj_txt;  string doc_text; +string[] _insert_doc_objects;  foreach (part; doc_parts) {    foreach (obj; doc_abstraction[part]) {      switch (obj.of_part) { @@ -1443,35 +1013,426 @@ foreach (part; doc_parts) {          );        }      } -    insert_doc_objects.bind(":t_of",        obj.is_of); -    insert_doc_objects.bind(":t_is",        obj.is_a); -    insert_doc_objects.bind(":ocn",         obj.ocn); -    insert_doc_objects.bind(":clean",       obj_txt["text"]); // consider whether book index info should be made available within clear text for search -    insert_doc_objects.bind(":body",        obj_txt["html"]); -    // insert_doc_objects.bind(":book_idx",    ""); // not needed, but, consider whether should be made available within object for clear text search -    insert_doc_objects.bind(":lev",         obj.heading_lev_markup); -    // // insert_doc_objects.bind(":dom_markedup",    ""); // should make lev sequence below obsolete -    // // insert_doc_objects.bind(":dom_collapsed", ""); // should add info -    // insert_doc_objects.bind(":lev0",        ""); -    // insert_doc_objects.bind(":lev1",        ""); -    // insert_doc_objects.bind(":lev2",        ""); -    // insert_doc_objects.bind(":lev3",        ""); -    // insert_doc_objects.bind(":lev4",        ""); -    // insert_doc_objects.bind(":lev5",        ""); -    // insert_doc_objects.bind(":lev6",        ""); -    // insert_doc_objects.bind(":lev7",        ""); -    // insert_doc_objects.bind(":node",        ""); -    // insert_doc_objects.bind(":type",        ""); -    // insert_doc_objects.bind(":parent_ocn",                ""); -    // insert_doc_objects.bind(":ancestors",                 ""); -    // insert_doc_objects.bind(":heading_lev_markup",        ""); -    // insert_doc_objects.bind(":heading_lev_collapsed",     ""); -    // insert_doc_objects.bind(":parent_lev_markup",        ""); -    // insert_doc_objects.bind(":heading_ancestors",         ""); -    // insert_doc_objects.bind(":node",                      ""); -    insert_doc_objects.execute(); insert_doc_objects.reset(); -  } +    if (!(obj.is_a == "comment")) { +      _insert_doc_objects ~= insertDocObjectsRow(obj); +    } +  } // loop closes  } +return _insert_doc_objects.join.to!(char[]).toUTF8; +#+END_SRC + +** 5. SQL statements +*** create tables + +**** DROP INDEX IF EXISTS + +#+name: sqlite_statement_drop_existing_index +#+BEGIN_SRC sql +DROP INDEX IF EXISTS ocn; +DROP INDEX IF EXISTS digest_clean; +DROP INDEX IF EXISTS digest_all; +DROP INDEX IF EXISTS clean; +DROP INDEX IF EXISTS lev0; +DROP INDEX IF EXISTS lev1; +DROP INDEX IF EXISTS lev2; +DROP INDEX IF EXISTS lev3; +DROP INDEX IF EXISTS lev4; +DROP INDEX IF EXISTS lev5; +DROP INDEX IF EXISTS lev6; +DROP INDEX IF EXISTS title; +DROP INDEX IF EXISTS creator_author; +DROP INDEX IF EXISTS src_filename; +DROP INDEX IF EXISTS language_document_char; +DROP INDEX IF EXISTS classify_topic_register; +#+END_SRC + +**** DROP TABLE IF EXISTS + +#+name: sqlite_statement_drop_existing_tables +#+BEGIN_SRC sql +DROP TABLE IF EXISTS metadata_and_text; +DROP TABLE IF EXISTS doc_objects; +DROP TABLE IF EXISTS urls; +#+END_SRC + +**** CREATE TABLE metadata_and_text + +#+name: sqlite_statement_create_table_metadata_and_src_txt +#+BEGIN_SRC sql +CREATE TABLE metadata_and_text ( +  tid                              INTEGER           PRIMARY KEY, +/* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +  title                            VARCHAR(800)  NOT NULL, +  title_main                       VARCHAR(400)  NOT NULL, +  title_sub                        VARCHAR(400)      NULL, +  title_short                      VARCHAR(400)      NULL, +  title_edition                    VARCHAR(10)       NULL, +  title_note                       VARCHAR(2500)     NULL, +  title_language                   VARCHAR(100)      NULL, +  title_language_char              VARCHAR(6)        NULL, +  creator_author                   VARCHAR(600)  NOT NULL, +  creator_author_email             VARCHAR(100)      NULL, +  creator_author_hon               VARCHAR(100)      NULL, +  creator_author_nationality       VARCHAR(100)      NULL, +  creator_editor                   VARCHAR(600)      NULL, +  creator_contributor              VARCHAR(600)      NULL, +  creator_illustrator              VARCHAR(600)      NULL, +  creator_photographer             VARCHAR(600)      NULL, +  creator_translator               VARCHAR(600)      NULL, +  creator_prepared_by              VARCHAR(600)      NULL, +  creator_digitized_by             VARCHAR(600)      NULL, +  creator_audio                    VARCHAR(600)      NULL, +  creator_video                    VARCHAR(600)      NULL, +  language_document                VARCHAR(100)      NULL, +  language_document_char           VARCHAR(6)    NOT NULL, +  language_original                VARCHAR(100)      NULL, +  language_original_char           VARCHAR(6)        NULL, +  date_added_to_site               VARCHAR(10)       NULL, +  date_available                   VARCHAR(10)       NULL, +  date_created                     VARCHAR(10)       NULL, +  date_issued                      VARCHAR(10)       NULL, +  date_modified                    VARCHAR(10)       NULL, +  date_published                   VARCHAR(10)       NULL, +  date_valid                       VARCHAR(10)       NULL, +  date_translated                  VARCHAR(10)       NULL, +  date_original_publication        VARCHAR(10)       NULL, +  date_generated                   VARCHAR(10)       NULL, +  publisher                        VARCHAR(600)      NULL, +  original_publisher               VARCHAR(600)      NULL, +  original_language                VARCHAR(100)      NULL, +  original_language_char           VARCHAR(6)        NULL, +  original_source                  VARCHAR(600)      NULL, +  original_institution             VARCHAR(600)      NULL, +  original_nationality             VARCHAR(100)      NULL, +  original_title                   VARCHAR(800)      NULL, +  rights_copyright                 VARCHAR(2500)     NULL, +  rights_copyright_audio           VARCHAR(2500)     NULL, +  rights_copyright_cover           VARCHAR(2500)     NULL, +  rights_copyright_illustrations   VARCHAR(2500)     NULL, +  rights_copyright_photographs     VARCHAR(2500)     NULL, +  rights_copyright_text            VARCHAR(2500)     NULL, +  rights_copyright_translation     VARCHAR(2500)     NULL, +  rights_copyright_video           VARCHAR(2500)     NULL, +  rights_license                   VARCHAR(2500)     NULL, +  identifier_oclc                  VARCHAR(30)       NULL, +  identifier_isbn                  VARCHAR(16)       NULL, +  classify_topic_register          VARCHAR(2500)     NULL, +  classify_subject                 VARCHAR(600)      NULL, +  classify_loc                     VARCHAR(30)       NULL, +  classify_dewey                   VARCHAR(30)       NULL, +  classify_keywords                VARCHAR(600)      NULL, +  notes_abstract                   TEXT              NULL, +  notes_description                TEXT              NULL, +  notes_comment                    TEXT              NULL, +  notes_coverage                   VARCHAR(200)      NULL, +  notes_relation                   VARCHAR(200)      NULL, +  notes_history                    VARCHAR(600)      NULL, +  notes_type                       VARCHAR(600)      NULL, +  notes_format                     VARCHAR(600)      NULL, +  notes_prefix                     TEXT              NULL, +  notes_prefix_a                   TEXT              NULL, +  notes_prefix_b                   TEXT              NULL, +  notes_suffix                     TEXT              NULL, +  src_filename                     VARCHAR(256)  NOT NULL, +  src_fingerprint                  VARCHAR(256)      NULL, +  src_filesize                     VARCHAR(10)       NULL, +  src_wordcount                    VARCHAR(10)       NULL, +  pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +  pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +  pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ +  src_text                         TEXT              NULL, +  fulltext                         TEXT              NULL, +  links                            TEXT              NULL +); +#+END_SRC + +**** CREATE TABLE doc_objects + +#+name: sqlite_statement_create_table_objects +#+BEGIN_SRC sql +CREATE TABLE doc_objects ( +  lid                              INTEGER PRIMARY KEY, +  metadata_tid                     INTEGER REFERENCES metadata_and_text, +  ocn                              SMALLINT, +  ocnd                             VARCHAR(6), +  ocns                             VARCHAR(6), +  clean                            TEXT NULL, +  body                             TEXT NULL, +  book_idx                         TEXT NULL, +  seg                              VARCHAR(256) NULL, +  lev_an                           VARCHAR(1), +  lev                              SMALLINT NULL, +  lev0                             SMALLINT, +  lev1                             SMALLINT, +  lev2                             SMALLINT, +  lev3                             SMALLINT, +  lev4                             SMALLINT, +  lev5                             SMALLINT, +  lev6                             SMALLINT, +  lev7                             SMALLINT, +  en_a                             SMALLINT NULL, +  en_z                             SMALLINT NULL, +  en_a_asterisk                    SMALLINT NULL, +  en_z_asterisk                    SMALLINT NULL, +  en_a_plus                        SMALLINT NULL, +  en_z_plus                        SMALLINT NULL, +  t_of                             VARCHAR(16), +  t_is                             VARCHAR(16), +  node                             VARCHAR(16) NULL, +  parent                           VARCHAR(16) NULL, +  digest_clean                     CHAR(256), +  digest_all                       CHAR(256), +  types                            CHAR(1) NULL +); +#+END_SRC + +**** CREATE INDEX + +#+name: sqlite_statement_create_index +#+BEGIN_SRC sql +CREATE INDEX idx_ocn ON doc_objects(ocn); +CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); +CREATE INDEX idx_digest_all ON doc_objects(digest_all); +CREATE INDEX idx_clean ON doc_objects(clean); +CREATE INDEX idx_lev0 ON doc_objects(lev0); +CREATE INDEX idx_lev1 ON doc_objects(lev1); +CREATE INDEX idx_lev2 ON doc_objects(lev2); +CREATE INDEX idx_lev3 ON doc_objects(lev3); +CREATE INDEX idx_lev4 ON doc_objects(lev4); +CREATE INDEX idx_lev5 ON doc_objects(lev5); +CREATE INDEX idx_lev6 ON doc_objects(lev6); +CREATE INDEX idx_title ON metadata_and_text(title); +CREATE INDEX idx_author ON metadata_and_text(creator_author); +CREATE INDEX idx_filename ON metadata_and_text(src_filename); +CREATE INDEX idx_language ON metadata_and_text(language_document_char); +CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); +#+END_SRC + +**** TODO local site link & info + +*** sql insert statement formatted doc objects + +lid unique, increment by 1 per object, not ocn + +metadata tid document number unique +either: +- increment by adding 1 for each document, +- make hash of document filename or url and use? + +**** sql statement: dlang format +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC d +string _insert_doc_objects_row; +_insert_doc_objects_row = format(q"¶ +#+END_SRC + +**** INSERT INTO + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC sql +  INSERT INTO doc_objects ( +    metadata_tid, +    ocn, +    clean, +    body, +    lev, +    t_of, +    t_is +  ) +#+END_SRC + +**** VALUES + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC sql +  VALUES ( +    %d, +    %s, +    '%s', +    '%s', +    %s, +    '%s', +    '%s' +  ); +#+END_SRC + +**** sql statement: dlang values for formatting + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC d +¶", +  _metadata_tid, +  obj.ocn, +  SQLinsertDelimiter!()(obj_txt["text"]), +  SQLinsertDelimiter!()(obj_txt["html"]), +  obj.heading_lev_markup, +  obj.is_of, +  obj.is_a, +); +#+END_SRC + +*** sqlite insert statement formatted doc matters metadata +**** sql statement: dlang format +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC d +string _insert_metadata; +_insert_metadata = format(q"¶ +#+END_SRC + +**** INSERT INTO + +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC sql +  INSERT INTO metadata_and_text ( +    src_filename, +    title, +    title_main, +    title_sub, +    title_short, +    title_edition, +    title_language, +    classify_dewey, +    classify_keywords, +    classify_loc, +    classify_subject, +    classify_topic_register, +    creator_author, +    creator_author_email, +    creator_illustrator, +    creator_translator, +    date_added_to_site, +    date_available, +    date_created, +    date_issued, +    date_modified, +    date_published, +    date_valid, +    identifier_isbn, +    identifier_oclc, +    language_document, +    language_document_char, +    notes_abstract, +    notes_description, +    original_publisher, +    original_language, +    original_language_char, +    original_source, +    original_title, +    publisher, +    rights_copyright, +    rights_copyright_audio, +    rights_copyright_cover, +    rights_copyright_illustrations, +    rights_copyright_photographs, +    rights_copyright_text, +    rights_copyright_translation, +    rights_copyright_video, +    rights_license +  ) +#+END_SRC + +**** VALUES + +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC sql +  VALUES ( +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s' +  ); +#+END_SRC + +**** sql statement: values for formatting + +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC d +¶", +  SQLinsertDelimiter!()(doc_matters.src.filename), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) +);  #+END_SRC  * __END__ diff --git a/org/output_sqlite_discrete.org b/org/output_sqlite_discrete.org index fced21d..c2f273c 100644 --- a/org/output_sqlite_discrete.org +++ b/org/output_sqlite_discrete.org @@ -20,18 +20,66 @@  #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite_discrete.d  module sdp.output.sqlite_discrete;  <<output_imports>> +template SQLiteDiscreteDbRun() { +  void SQLiteDiscreteDbRun(Db,St,O)( +    auto ref       Db   db, +    auto ref       St   db_statement, +    auto ref       O    opt_action, +               string   note, +  ) { +    debug(sql_statement) { +      writeln(db_statement); +    } +    try { +      db.run( +        "\nBEGIN;\n" ~ +        db_statement ~ +        "\nCOMMIT;\n" +      ); +    } catch (ErrnoException ex) { +      writeln("ERROR SQLite : ", ex); +    } catch (Exception ex) { +      writeln("ERROR SQLite : ", ex); +    } +    if (opt_action.debug_do) { +      writeln(note); +      if (opt_action.verbose) { +        writeln(db_statement); +      } +    } +  } +}  template SQLiteDiscreteBuildTablesAndPopulate() {    void SQLiteDiscreteBuildTablesAndPopulate(D,I)(      auto ref const D    doc_abstraction, -    auto ref I          doc_matters, +    auto ref       I    doc_matters,    ) {      import d2sqlite3;      import std.typecons : Nullable;      mixin SiSUoutputRgxInit;      static auto rgx = Rgx(); +    auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); +    template SQLiteDiscreteDbStatementComposite() { +      void SQLiteDiscreteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        <<sqlite_db_statement_composite>> +      } +    } +    template SQLinsertDelimiter() { +      auto SQLinsertDelimiter(string _txt) { +        _txt = _txt +          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +        return _txt; +      } +    }      template SQLiteFormatAndLoadObject() {        auto SQLiteFormatAndLoadObject(I)( -        auto ref              I    doc_matters, +        auto ref       I    doc_matters,        ) {          mixin SiSUoutputRgxInit;          struct sqlite_format_and_load_objects { @@ -44,45 +92,61 @@ template SQLiteDiscreteBuildTablesAndPopulate() {          return sqlite_format_and_load_objects();        }      } -    template SQLiteInstruct() { -      void SQLiteInstruct(Db,I)( -                              Db   db, -        auto ref              I    doc_matters, +    template SQLiteDiscreteTablesReCreate() { +      string SQLiteDiscreteTablesReCreate()() { +        string _sql_instruct; +        _sql_instruct = format(q"¶ +          <<sqlite_statement_drop_existing_index>> +          <<sqlite_statement_drop_existing_tables>> +          <<sqlite_statement_create_table_metadata_and_src_txt>> +          <<sqlite_statement_create_table_objects>> +          <<sqlite_statement_create_index>> +        ¶",); +        return _sql_instruct; +      } +    } +    template SQLiteDiscreteInsertMetadata() { +      string SQLiteDiscreteInsertMetadata(I)( +        auto ref       I    doc_matters,        ) { -        <<sqlite_db_initialize>> -          <<sqlite_table_initialize>> -          <<sqlite_create_table_metadata_and_src_txt>> -          /* */<<sqlite_create_table_urls>> -          <<sqlite_create_table_objects>> -        <<sqlite_close_initialize>> -        <<sqlite_insert_metadata_and_src_text>> -        //<<sqlite_insert_urls>> +        <<sqlite_formatted_insertions_doc_matters_metadata>> +        return _insert_metadata;        }      } -    template SQLiteObjectsLoop() { -      void SQLiteObjectsLoop(P)( -        auto ref P          doc_parts, +    template SQLiteDiscreteInsertDocObjectsLoop() { +      string SQLiteDiscreteInsertDocObjectsLoop(P,I)( +        auto ref       P    doc_parts, +        auto ref       I    tid,        ) { -        <<sqlite_insert_doc_objects>> +        string insertDocObjectsRow(O)(O obj) { +          <<sqlite_formatted_insertions_doc_objects>> +          return _insert_doc_objects_row; +        }          <<sqlite_objects_loop>>        }      } -    SQLiteObjectsLoop!()(doc_matters.xml.keys_seq.sql); +    SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);    }  } -template SQLiteDiscreteTablesCreate() { -  void SQLiteDiscreteTablesCreate()() { -    writeln("table create"); -  } -} -template SQLiteDiscreteTablesDrop() { -  void SQLiteDiscreteTablesDrop()() { -    writeln("table drop"); +#+END_SRC + +** 1. sqlite_db_statement [#A] + +#+name: sqlite_db_statement_composite +#+BEGIN_SRC d +{ +  string _db_statement; +  { +    _db_statement ~= SQLiteDiscreteTablesReCreate!()(); +    _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters); +    _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX +    SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects");    } +  db.close;  }  #+END_SRC -** 1. imports +** 2. imports  #+name: output_imports  #+BEGIN_SRC d @@ -92,7 +156,7 @@ import    std.conv : to;  #+END_SRC -** 2. format and load template +** 3. format and load template  *** 1. prepare objects (munge, sanitize, markup)  **** 1. _text_ generic munge (sanitize text for search) @@ -469,532 +533,15 @@ auto html_table(O)(  #+END_SRC  *** 2. _sqlite_ (loop preparation & pre-loop action) -***** initialize: prepare statements drop & create db & tables anew -****** d sqlite statements (initialize database) { - -#+name: sqlite_db_initialize -#+BEGIN_SRC d -db.run(" -#+END_SRC - -*******  0. drop table if exists - -#+name: sqlite_table_initialize -#+BEGIN_SRC sql -DROP TABLE IF EXISTS metadata_and_text; -DROP TABLE IF EXISTS doc_objects; -DROP TABLE IF EXISTS urls; -BEGIN; -#+END_SRC - -*******  1. create tables -********  metadata & src text - -#+name: sqlite_create_table_metadata_and_src_txt -#+BEGIN_SRC sql -CREATE TABLE metadata_and_text ( -  tid                              BIGINT            PRIMARY KEY, -  title                            VARCHAR(800)  NOT NULL, -  title_main                       VARCHAR(400)      NULL, /*NOT*/ -  title_sub                        VARCHAR(400)      NULL, -  title_short                      VARCHAR(400)      NULL, -  title_edition                    VARCHAR(10)       NULL, -  title_note                       VARCHAR(2500)     NULL, -  title_language                   VARCHAR(100)      NULL, -  title_language_char              VARCHAR(6)        NULL, -  creator_author                   VARCHAR(600)      NULL, -  creator_author_email             VARCHAR(100)      NULL, -  creator_author_hon               VARCHAR(100)      NULL, -  creator_author_nationality       VARCHAR(100)      NULL, -  creator_editor                   VARCHAR(600)      NULL, -  creator_contributor              VARCHAR(600)      NULL, -  creator_illustrator              VARCHAR(600)      NULL, -  creator_photographer             VARCHAR(600)      NULL, -  creator_translator               VARCHAR(600)      NULL, -  creator_prepared_by              VARCHAR(600)      NULL, -  creator_digitized_by             VARCHAR(600)      NULL, -  creator_audio                    VARCHAR(600)      NULL, -  creator_video                    VARCHAR(600)      NULL, -  language_document                VARCHAR(100)      NULL, -  language_document_char           VARCHAR(6)        NULL, /*NOT*/ -  language_original                VARCHAR(100)      NULL, -  language_original_char           VARCHAR(6)        NULL, -  date_added_to_site               VARCHAR(10)       NULL, -  date_available                   VARCHAR(10)       NULL, -  date_created                     VARCHAR(10)       NULL, -  date_issued                      VARCHAR(10)       NULL, -  date_modified                    VARCHAR(10)       NULL, -  date_published                   VARCHAR(10)       NULL, -  date_valid                       VARCHAR(10)       NULL, -  date_translated                  VARCHAR(10)       NULL, -  date_original_publication        VARCHAR(10)       NULL, -  date_generated                   VARCHAR(10)       NULL, -  publisher                        VARCHAR(600)      NULL, -  original_publisher               VARCHAR(600)      NULL, -  original_language                VARCHAR(100)      NULL, -  original_language_char           VARCHAR(6)        NULL, -  original_source                  VARCHAR(600)      NULL, -  original_institution             VARCHAR(600)      NULL, -  original_nationality             VARCHAR(100)      NULL, -  original_title                   VARCHAR(800)      NULL, -  rights_copyright                 VARCHAR(2500)     NULL, -  rights_copyright_audio           VARCHAR(2500)     NULL, -  rights_copyright_cover           VARCHAR(2500)     NULL, -  rights_copyright_illustrations   VARCHAR(2500)     NULL, -  rights_copyright_photographs     VARCHAR(2500)     NULL, -  rights_copyright_text            VARCHAR(2500)     NULL, -  rights_copyright_translation     VARCHAR(2500)     NULL, -  rights_copyright_video           VARCHAR(2500)     NULL, -  rights_license                   VARCHAR(2500)     NULL, -  identifier_oclc                  VARCHAR(30)       NULL, -  identifier_isbn                  VARCHAR(16)       NULL, -  classify_topic_register          VARCHAR(2500)     NULL, -  classify_subject                 VARCHAR(600)      NULL, -  classify_loc                     VARCHAR(30)       NULL, -  classify_dewey                   VARCHAR(30)       NULL, -  classify_keywords                VARCHAR(600)      NULL, -  notes_abstract                   TEXT              NULL, -  notes_description                TEXT              NULL, -  notes_comment                    TEXT              NULL, -  notes_coverage                   VARCHAR(200)      NULL, -  notes_relation                   VARCHAR(200)      NULL, -  notes_history                    VARCHAR(600)      NULL, -  notes_type                       VARCHAR(600)      NULL, -  notes_format                     VARCHAR(600)      NULL, -  notes_prefix                     TEXT              NULL, -  notes_prefix_a                   TEXT              NULL, -  notes_prefix_b                   TEXT              NULL, -  notes_suffix                     TEXT              NULL, -  src_filename                     VARCHAR(256)      NULL, -  src_fingerprint                  VARCHAR(256)      NULL, -  src_filesize                     VARCHAR(10)       NULL, -  src_wordcount                    VARCHAR(10)       NULL, -  src_text                         TEXT              NULL, -  fulltext                         TEXT              NULL, -  links                            TEXT              NULL -); -#+END_SRC - -********  doc_objects - -#+name: sqlite_create_table_objects -#+BEGIN_SRC sql -CREATE TABLE doc_objects ( -  lid                              BIGINT PRIMARY KEY, -  metadata_tid                     BIGINT REFERENCES metadata_and_text, -  ocn                              SMALLINT, -  ocnd                             VARCHAR(6), -  ocns                             VARCHAR(6), -  clean                            TEXT NULL, -  body                             TEXT NULL, -  book_idx                         TEXT NULL, -  seg                              VARCHAR(256) NULL, -  lev_an                           VARCHAR(1), -  lev                              SMALLINT NULL, -  lev0                             SMALLINT, -  lev1                             SMALLINT, -  lev2                             SMALLINT, -  lev3                             SMALLINT, -  lev4                             SMALLINT, -  lev5                             SMALLINT, -  lev6                             SMALLINT, -  lev7                             SMALLINT, -  en_a                             SMALLINT NULL, -  en_z                             SMALLINT NULL, -  en_a_asterisk                    SMALLINT NULL, -  en_z_asterisk                    SMALLINT NULL, -  en_a_plus                        SMALLINT NULL, -  en_z_plus                        SMALLINT NULL, -  t_of                             VARCHAR(16), -  t_is                             VARCHAR(16), -  node                             VARCHAR(16) NULL, -  parent                           VARCHAR(16) NULL, -  digest_clean                     CHAR(256), -  digest_all                       CHAR(256), -  types                            CHAR(1) NULL -); -#+END_SRC - -****** d } - -#+name: sqlite_close_initialize -#+BEGIN_SRC d -  COMMIT -"); -#+END_SRC - -***** prepare insert statements and do pre-loop inserts -****** 1. _metadata & src_ text -******* sql statement -******** d { sql statement - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -Statement insert_metadata = db.prepare(" -#+END_SRC - -tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -********* sql insert into - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC sql -  INSERT INTO metadata_and_text ( -    tid, -    title, -    title_main, -    title_sub, -    title_short, -    title_edition, -    title_note, -    title_language, -    title_language_char, -    creator_author, -    creator_author_email, -    creator_author_hon, -    creator_author_nationality, -    creator_editor, -    creator_contributor, -    creator_illustrator, -    creator_photographer, -    creator_translator, -    creator_prepared_by, -    creator_digitized_by, -    creator_audio, -    creator_video, -    language_document, -    language_document_char, -    language_original, -    language_original_char, -    date_added_to_site, -    date_available, -    date_created, -    date_issued, -    date_modified, -    date_published, -    date_valid, -    date_translated, -    date_original_publication, -    date_generated, -    publisher, -    original_publisher, -    original_language, -    original_language_char, -    original_source, -    original_institution, -    original_nationality, -    original_title, -    rights_copyright, -    rights_copyright_audio, -    rights_copyright_cover, -    rights_copyright_illustrations, -    rights_copyright_photographs, -    rights_copyright_text, -    rights_copyright_translation, -    rights_copyright_video, -    rights_license, -    identifier_oclc, -    identifier_isbn, -    classify_topic_register, -    classify_subject, -    classify_loc, -    classify_dewey, -    classify_keywords, -    notes_abstract, -    notes_description, -    notes_comment, -    notes_coverage, -    notes_relation, -    notes_history, -    notes_type, -    notes_format, -    notes_prefix, -    notes_prefix_a, -    notes_prefix_b, -    notes_suffix, -    src_filename, -    src_fingerprint, -    src_filesize, -    src_wordcount, -    src_text, -    fulltext, -    links -  ) -#+END_SRC - -********* sql values - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC sql -  VALUES ( -    :tid, -    :title, -    :title_main, -    :title_sub, -    :title_short, -    :title_edition, -    :title_note, -    :title_language, -    :title_language_char, -    :creator_author, -    :creator_author_email, -    :creator_author_hon, -    :creator_author_nationality, -    :creator_editor, -    :creator_contributor, -    :creator_illustrator, -    :creator_photographer, -    :creator_translator, -    :creator_prepared_by, -    :creator_digitized_by, -    :creator_audio, -    :creator_video, -    :language_document, -    :language_document_char, -    :language_original, -    :language_original_char, -    :date_added_to_site, -    :date_available, -    :date_created, -    :date_issued, -    :date_modified, -    :date_published, -    :date_valid, -    :date_translated, -    :date_original_publication, -    :date_generated, -    :publisher, -    :original_publisher, -    :original_language, -    :original_language_char, -    :original_source, -    :original_institution, -    :original_nationality, -    :original_title, -    :rights_copyright, -    :rights_copyright_audio, -    :rights_copyright_cover, -    :rights_copyright_illustrations, -    :rights_copyright_photographs, -    :rights_copyright_text, -    :rights_copyright_translation, -    :rights_copyright_video, -    :rights_license, -    :identifier_oclc, -    :identifier_isbn, -    :classify_topic_register, -    :classify_subject, -    :classify_loc, -    :classify_dewey, -    :classify_keywords, -    :notes_abstract, -    :notes_description, -    :notes_comment, -    :notes_coverage, -    :notes_relation, -    :notes_history, -    :notes_type, -    :notes_format, -    :notes_prefix, -    :notes_prefix_a, -    :notes_prefix_b, -    :notes_suffix, -    :src_filename, -    :src_fingerprint, -    :src_filesize, -    :src_wordcount, -    :src_text, -    :fulltext, -    :links -  ) -#+END_SRC - -******** d } - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -"); -#+END_SRC - -*******  TODO insert document_metadata - -#+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -// insert_metadata.bind(":tid",                               ); // unique identifier you could try use sha of title author & language? or auto increment?? -insert_metadata.bind(":title",                             doc_matters.conf_make_meta.meta.title_full); -insert_metadata.bind(":title_main",                        doc_matters.conf_make_meta.meta.title_main); -insert_metadata.bind(":title_sub",                         doc_matters.conf_make_meta.meta.title_subtitle); -insert_metadata.bind(":title_short",                       doc_matters.conf_make_meta.meta.title_short); // -insert_metadata.bind(":title_edition",                     doc_matters.conf_make_meta.meta.title_edition); -insert_metadata.bind(":title_language",                    doc_matters.conf_make_meta.meta.title_language); -// insert_metadata.bind(":title_language_char",            doc_matters.conf_make_meta.meta.title_language_char); -insert_metadata.bind(":classify_dewey",                    doc_matters.conf_make_meta.meta.classify_dewey); -insert_metadata.bind(":classify_keywords",                 doc_matters.conf_make_meta.meta.classify_keywords); -insert_metadata.bind(":classify_loc",                      doc_matters.conf_make_meta.meta.classify_loc); -insert_metadata.bind(":classify_subject",                  doc_matters.conf_make_meta.meta.classify_subject); -insert_metadata.bind(":classify_topic_register",           doc_matters.conf_make_meta.meta.classify_topic_register); -insert_metadata.bind(":creator_author",                    doc_matters.conf_make_meta.meta.creator_author); -insert_metadata.bind(":creator_author_email",              doc_matters.conf_make_meta.meta.creator_author_email); -// insert_metadata.bind(":creator_editor",                 doc_matters.conf_make_meta.meta.creator_editor); -// insert_metadata.bind(":creator_contributor",            doc_matters.conf_make_meta.meta.creator_contributor); -insert_metadata.bind(":creator_illustrator",               doc_matters.conf_make_meta.meta.creator_illustrator); -// insert_metadata.bind(":creator_photographer",           doc_matters.conf_make_meta.meta.creator_photographer); -insert_metadata.bind(":creator_translator",                doc_matters.conf_make_meta.meta.creator_translator); -// insert_metadata.bind(":creator_audio",                  doc_matters.conf_make_meta.meta.creator_audio); -// insert_metadata.bind(":creator_video",                  doc_matters.conf_make_meta.meta.creator_video); -insert_metadata.bind(":date_added_to_site",                doc_matters.conf_make_meta.meta.date_added_to_site); -insert_metadata.bind(":date_available",                    doc_matters.conf_make_meta.meta.date_available); -insert_metadata.bind(":date_created",                      doc_matters.conf_make_meta.meta.date_created); -insert_metadata.bind(":date_issued",                       doc_matters.conf_make_meta.meta.date_issued); -insert_metadata.bind(":date_modified",                     doc_matters.conf_make_meta.meta.date_modified); -insert_metadata.bind(":date_published",                    doc_matters.conf_make_meta.meta.date_published); -insert_metadata.bind(":date_valid",                        doc_matters.conf_make_meta.meta.date_valid); -// insert_metadata.bind(":date_translated",                doc_matters.conf_make_meta.meta.date_translated); -// insert_metadata.bind(":date_original_publication",      doc_matters.conf_make_meta.meta.date_original_publication); -// insert_metadata.bind(":date_generated",                 doc_matters.conf_make_meta.meta.date_generated); -insert_metadata.bind(":identifier_isbn",                   doc_matters.conf_make_meta.meta.identifier_isbn); -insert_metadata.bind(":identifier_oclc",                   doc_matters.conf_make_meta.meta.identifier_oclc); -insert_metadata.bind(":language_document",                 doc_matters.conf_make_meta.meta.language_document); -insert_metadata.bind(":language_document_char",            doc_matters.conf_make_meta.meta.language_document_char); -// insert_metadata.bind(":language_original",              doc_matters.conf_make_meta.meta.language_original); -// insert_metadata.bind(":language_original_char",         doc_matters.conf_make_meta.meta.language_original_char); -insert_metadata.bind(":notes_abstract",                    doc_matters.conf_make_meta.meta.notes_abstract); -insert_metadata.bind(":notes_description",                 doc_matters.conf_make_meta.meta.notes_description); -insert_metadata.bind(":original_publisher",                doc_matters.conf_make_meta.meta.original_publisher); -insert_metadata.bind(":original_language",                 doc_matters.conf_make_meta.meta.original_language); -insert_metadata.bind(":original_language_char",            doc_matters.conf_make_meta.meta.original_language_char); -insert_metadata.bind(":original_source",                   doc_matters.conf_make_meta.meta.original_source); -insert_metadata.bind(":original_title",                    doc_matters.conf_make_meta.meta.original_title); -insert_metadata.bind(":publisher",                         doc_matters.conf_make_meta.meta.publisher); -// insert_metadata.bind(":rights",                         doc_matters.conf_make_meta.meta.rights); -insert_metadata.bind(":rights_copyright",                  doc_matters.conf_make_meta.meta.rights_copyright); -insert_metadata.bind(":rights_copyright_audio",            doc_matters.conf_make_meta.meta.rights_copyright_audio); -insert_metadata.bind(":rights_copyright_cover",            doc_matters.conf_make_meta.meta.rights_copyright_cover); -insert_metadata.bind(":rights_copyright_illustrations",    doc_matters.conf_make_meta.meta.rights_copyright_illustrations); -insert_metadata.bind(":rights_copyright_photographs",      doc_matters.conf_make_meta.meta.rights_copyright_photographs); -// insert_metadata.bind(":rights_copyright_preparation",   doc_matters.conf_make_meta.meta.rights_preparation); -insert_metadata.bind(":rights_copyright_text",      doc_matters.conf_make_meta.meta.rights_copyright_text); -insert_metadata.bind(":rights_copyright_translation",      doc_matters.conf_make_meta.meta.rights_copyright_translation); -insert_metadata.bind(":rights_copyright_video",            doc_matters.conf_make_meta.meta.rights_copyright_video); -// insert_metadata.bind(":rights_copyright_digitization",  doc_matters.conf_make_meta.meta.rights_digitization); -// insert_metadata.bind(":rights_copyright_audio",         doc_matters.conf_make_meta.meta.rights_audio); -// insert_metadata.bind(":rights_copyright_video",         doc_matters.conf_make_meta.meta.rights_video); -insert_metadata.bind(":rights_license",                    doc_matters.conf_make_meta.meta.rights_license); -// insert_metadata.bind(":src_filename",                   doc_matters.conf_make_meta.meta.src_filename); -// insert_metadata.bind(":src_fingerprint",                doc_matters.conf_make_meta.meta.src_fingerprint); -// insert_metadata.bind(":src_filesize",                   doc_matters.conf_make_meta.meta.src_filesize); -// insert_metadata.bind(":src_wordcount",                  doc_matters.conf_make_meta.meta.src_wordcount); -// insert_metadata.bind(":src_text",                       doc_matters.conf_make_meta.meta.src_text); -// insert_metadata.bind(":fulltext",                       doc_matters.conf_make_meta.meta.fulltext); -// insert_metadata.bind(":links",                          doc_matters.conf_make_meta.meta.links); -insert_metadata.execute(); insert_metadata.reset(); -/+ watch +/ -if ((doc_matters.opt.action.verbose)) { -  writeln("sql statement executed"); -} -assert(db.totalChanges == 1); -#+END_SRC - -###+name: sqlite_insert_metadata_and_src_text -#+BEGIN_SRC d -insert_metadata.inject( -  1, -  doc_matters.dochead_meta["title"]["full"], -  "", -  "", -  "", -  "", -  "", -  "", -  "", -  doc_matters.dochead_meta["creator"]["author"] -); -#+END_SRC - -****** 2. _doc objects_ (used with doc_objects in document loop) -******* prepare sql statement, sql utf8 -******** d { sql statement - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC d -string insertDocObjectsRow(O)(O obj) { -  auto sql_insert_delimiter(string _txt) { -    _txt = _txt -      .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); -    return _txt; -  } -  string _insert_doc_objects_row; - _insert_doc_objects_row = format(q"¶ -#+END_SRC - -********* sql statement: formatted - -lid unique, increment by 1 per object, not ocn - -metadata tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -********** INSERT INTO - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC sql -    INSERT INTO doc_objects ( -      ocn, -      clean, -      body, -      lev, -      t_of, -      t_is -    ) -#+END_SRC - -********** VALUES - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC sql -    VALUES ( -      %s, -      '%s', -      '%s', -      %s, -      '%s', -      '%s' -    ); -#+END_SRC - -********* sql statement: values for formatting - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC sql -  ¶", -    obj.ocn, -    sql_insert_delimiter(obj_txt["text"]), -    sql_insert_delimiter(obj_txt["html"]), -    obj.heading_lev_markup, -    obj.is_of, -    obj.is_a, -  ); -#+END_SRC - -******** d } - -#+name: sqlite_insert_doc_objects -#+BEGIN_SRC d -  return _insert_doc_objects_row; -} -#+END_SRC - -*******  TODO (within loop not here - insert doc objects +**** prepare insert statements and do pre-loop inserts +***** 1. _metadata & src_ text +***** 2. _doc objects_ (used with doc_objects in document loop) +******  TODO (within loop not here - insert doc objects  work out  *** 3. hub (sqlite_format_and_load_objects) -***** sql related +**** sql related  #+name: sqlite_load_object  #+BEGIN_SRC d @@ -1036,7 +583,7 @@ string sqlite_statement(O)(  }  #+END_SRC -***** heading +**** heading  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1061,7 +608,7 @@ auto heading(O)(  }  #+END_SRC -***** para +**** para  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1086,7 +633,7 @@ auto para(O)(  }  #+END_SRC -***** quote +**** quote  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1110,7 +657,7 @@ auto quote(O)(    return obj_txt;  }  #+END_SRC -***** group +**** group  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1135,7 +682,7 @@ auto group(O)(  }  #+END_SRC -***** block +**** block  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1160,7 +707,7 @@ auto block(O)(  }  #+END_SRC -***** verse +**** verse  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1185,7 +732,7 @@ auto verse(O)(  }  #+END_SRC -***** code +**** code  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1210,7 +757,7 @@ auto code(O)(  }  #+END_SRC -***** table +**** table  #+name: hub_format_and_sqlite_load_objects  #+BEGIN_SRC d @@ -1235,19 +782,14 @@ auto table(O)(  }  #+END_SRC -** 3. loop, identify, load - loop template +** 4. loop, identify, load - loop template  #+name: sqlite_objects_loop  #+BEGIN_SRC d -auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); -pth_sqlite.base.mkdirRecurse; -auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); -SQLiteInstruct!()(db, doc_matters); // consider best location, need to feed individual objects for sqlite table: doc_objects, possibly a separate template?  auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);  string[string] obj_txt;  string doc_text;  string[] _insert_doc_objects; -_insert_doc_objects ~= "BEGIN;\n";  foreach (part; doc_parts) {    foreach (obj; doc_abstraction[part]) {      switch (obj.of_part) { @@ -1397,12 +939,421 @@ foreach (part; doc_parts) {      }    } // loop closes  } -_insert_doc_objects ~= "COMMIT"; -debug(sql_statement) { -  writeln("#+BEGIN_SRC sql\n", _insert_doc_objects.join, "\n#+END_SRC"); -} -std.utf.validate(_insert_doc_objects.join); // TODO -db.run(_insert_doc_objects.join.to!(char[]).toUTF8); +return _insert_doc_objects.join.to!(char[]).toUTF8; +#+END_SRC + +** 5. SQL statements +*** create tables + +**** DROP INDEX IF EXISTS + +#+name: sqlite_statement_drop_existing_index +#+BEGIN_SRC sql +DROP INDEX IF EXISTS ocn; +DROP INDEX IF EXISTS digest_clean; +DROP INDEX IF EXISTS digest_all; +DROP INDEX IF EXISTS clean; +DROP INDEX IF EXISTS lev0; +DROP INDEX IF EXISTS lev1; +DROP INDEX IF EXISTS lev2; +DROP INDEX IF EXISTS lev3; +DROP INDEX IF EXISTS lev4; +DROP INDEX IF EXISTS lev5; +DROP INDEX IF EXISTS lev6; +DROP INDEX IF EXISTS title; +DROP INDEX IF EXISTS creator_author; +DROP INDEX IF EXISTS src_filename; +DROP INDEX IF EXISTS language_document_char; +DROP INDEX IF EXISTS classify_topic_register; +#+END_SRC + +**** DROP TABLE IF EXISTS + +#+name: sqlite_statement_drop_existing_tables +#+BEGIN_SRC sql +DROP TABLE IF EXISTS metadata_and_text; +DROP TABLE IF EXISTS doc_objects; +DROP TABLE IF EXISTS urls; +#+END_SRC + +**** CREATE TABLE metadata_and_text + +#+name: sqlite_statement_create_table_metadata_and_src_txt +#+BEGIN_SRC sql +CREATE TABLE metadata_and_text ( +  tid                              INTEGER           PRIMARY KEY, +/* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +  title                            VARCHAR(800)  NOT NULL, +  title_main                       VARCHAR(400)  NOT NULL, +  title_sub                        VARCHAR(400)      NULL, +  title_short                      VARCHAR(400)      NULL, +  title_edition                    VARCHAR(10)       NULL, +  title_note                       VARCHAR(2500)     NULL, +  title_language                   VARCHAR(100)      NULL, +  title_language_char              VARCHAR(6)        NULL, +  creator_author                   VARCHAR(600)  NOT NULL, +  creator_author_email             VARCHAR(100)      NULL, +  creator_author_hon               VARCHAR(100)      NULL, +  creator_author_nationality       VARCHAR(100)      NULL, +  creator_editor                   VARCHAR(600)      NULL, +  creator_contributor              VARCHAR(600)      NULL, +  creator_illustrator              VARCHAR(600)      NULL, +  creator_photographer             VARCHAR(600)      NULL, +  creator_translator               VARCHAR(600)      NULL, +  creator_prepared_by              VARCHAR(600)      NULL, +  creator_digitized_by             VARCHAR(600)      NULL, +  creator_audio                    VARCHAR(600)      NULL, +  creator_video                    VARCHAR(600)      NULL, +  language_document                VARCHAR(100)      NULL, +  language_document_char           VARCHAR(6)    NOT NULL, +  language_original                VARCHAR(100)      NULL, +  language_original_char           VARCHAR(6)        NULL, +  date_added_to_site               VARCHAR(10)       NULL, +  date_available                   VARCHAR(10)       NULL, +  date_created                     VARCHAR(10)       NULL, +  date_issued                      VARCHAR(10)       NULL, +  date_modified                    VARCHAR(10)       NULL, +  date_published                   VARCHAR(10)       NULL, +  date_valid                       VARCHAR(10)       NULL, +  date_translated                  VARCHAR(10)       NULL, +  date_original_publication        VARCHAR(10)       NULL, +  date_generated                   VARCHAR(10)       NULL, +  publisher                        VARCHAR(600)      NULL, +  original_publisher               VARCHAR(600)      NULL, +  original_language                VARCHAR(100)      NULL, +  original_language_char           VARCHAR(6)        NULL, +  original_source                  VARCHAR(600)      NULL, +  original_institution             VARCHAR(600)      NULL, +  original_nationality             VARCHAR(100)      NULL, +  original_title                   VARCHAR(800)      NULL, +  rights_copyright                 VARCHAR(2500)     NULL, +  rights_copyright_audio           VARCHAR(2500)     NULL, +  rights_copyright_cover           VARCHAR(2500)     NULL, +  rights_copyright_illustrations   VARCHAR(2500)     NULL, +  rights_copyright_photographs     VARCHAR(2500)     NULL, +  rights_copyright_text            VARCHAR(2500)     NULL, +  rights_copyright_translation     VARCHAR(2500)     NULL, +  rights_copyright_video           VARCHAR(2500)     NULL, +  rights_license                   VARCHAR(2500)     NULL, +  identifier_oclc                  VARCHAR(30)       NULL, +  identifier_isbn                  VARCHAR(16)       NULL, +  classify_topic_register          VARCHAR(2500)     NULL, +  classify_subject                 VARCHAR(600)      NULL, +  classify_loc                     VARCHAR(30)       NULL, +  classify_dewey                   VARCHAR(30)       NULL, +  classify_keywords                VARCHAR(600)      NULL, +  notes_abstract                   TEXT              NULL, +  notes_description                TEXT              NULL, +  notes_comment                    TEXT              NULL, +  notes_coverage                   VARCHAR(200)      NULL, +  notes_relation                   VARCHAR(200)      NULL, +  notes_history                    VARCHAR(600)      NULL, +  notes_type                       VARCHAR(600)      NULL, +  notes_format                     VARCHAR(600)      NULL, +  notes_prefix                     TEXT              NULL, +  notes_prefix_a                   TEXT              NULL, +  notes_prefix_b                   TEXT              NULL, +  notes_suffix                     TEXT              NULL, +  src_filename                     VARCHAR(256)  NOT NULL, +  src_fingerprint                  VARCHAR(256)      NULL, +  src_filesize                     VARCHAR(10)       NULL, +  src_wordcount                    VARCHAR(10)       NULL, +  pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +  pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +  pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ +  src_text                         TEXT              NULL, +  fulltext                         TEXT              NULL, +  links                            TEXT              NULL +); +#+END_SRC + +**** CREATE TABLE doc_objects + +#+name: sqlite_statement_create_table_objects +#+BEGIN_SRC sql +CREATE TABLE doc_objects ( +  lid                              INTEGER PRIMARY KEY, +  metadata_tid                     INTEGER REFERENCES metadata_and_text, +  ocn                              SMALLINT, +  ocnd                             VARCHAR(6), +  ocns                             VARCHAR(6), +  clean                            TEXT NULL, +  body                             TEXT NULL, +  book_idx                         TEXT NULL, +  seg                              VARCHAR(256) NULL, +  lev_an                           VARCHAR(1), +  lev                              SMALLINT NULL, +  lev0                             SMALLINT, +  lev1                             SMALLINT, +  lev2                             SMALLINT, +  lev3                             SMALLINT, +  lev4                             SMALLINT, +  lev5                             SMALLINT, +  lev6                             SMALLINT, +  lev7                             SMALLINT, +  en_a                             SMALLINT NULL, +  en_z                             SMALLINT NULL, +  en_a_asterisk                    SMALLINT NULL, +  en_z_asterisk                    SMALLINT NULL, +  en_a_plus                        SMALLINT NULL, +  en_z_plus                        SMALLINT NULL, +  t_of                             VARCHAR(16), +  t_is                             VARCHAR(16), +  node                             VARCHAR(16) NULL, +  parent                           VARCHAR(16) NULL, +  digest_clean                     CHAR(256), +  digest_all                       CHAR(256), +  types                            CHAR(1) NULL +); +#+END_SRC + +**** CREATE INDEX + +#+name: sqlite_statement_create_index +#+BEGIN_SRC sql +CREATE INDEX idx_ocn ON doc_objects(ocn); +CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); +CREATE INDEX idx_digest_all ON doc_objects(digest_all); +CREATE INDEX idx_clean ON doc_objects(clean); +CREATE INDEX idx_lev0 ON doc_objects(lev0); +CREATE INDEX idx_lev1 ON doc_objects(lev1); +CREATE INDEX idx_lev2 ON doc_objects(lev2); +CREATE INDEX idx_lev3 ON doc_objects(lev3); +CREATE INDEX idx_lev4 ON doc_objects(lev4); +CREATE INDEX idx_lev5 ON doc_objects(lev5); +CREATE INDEX idx_lev6 ON doc_objects(lev6); +CREATE INDEX idx_title ON metadata_and_text(title); +CREATE INDEX idx_author ON metadata_and_text(creator_author); +CREATE INDEX idx_filename ON metadata_and_text(src_filename); +CREATE INDEX idx_language ON metadata_and_text(language_document_char); +CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); +#+END_SRC + +**** TODO local site link & info + +*** sql insert statement formatted doc objects + +lid unique, increment by 1 per object, not ocn + +metadata tid document number unique +either: +- increment by adding 1 for each document, +- make hash of document filename or url and use? + +**** sql statement: dlang format +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC d +string _insert_doc_objects_row; +_insert_doc_objects_row = format(q"¶ +#+END_SRC + +**** INSERT INTO + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC sql +  INSERT INTO doc_objects ( +    metadata_tid, +    ocn, +    clean, +    body, +    lev, +    t_of, +    t_is +  ) +#+END_SRC + +**** VALUES + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC sql +  VALUES ( +    %d, +    %s, +    '%s', +    '%s', +    %s, +    '%s', +    '%s' +  ); +#+END_SRC + +**** sql statement: dlang values for formatting + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC d +¶", +  1, +  obj.ocn, +  SQLinsertDelimiter!()(obj_txt["text"]), +  SQLinsertDelimiter!()(obj_txt["html"]), +  obj.heading_lev_markup, +  obj.is_of, +  obj.is_a, +); +#+END_SRC + +*** sqlite insert statement formatted doc matters metadata +**** sql statement: dlang format +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC d +string _insert_metadata; +_insert_metadata = format(q"¶ +#+END_SRC + +**** INSERT INTO + +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC sql +  INSERT INTO metadata_and_text ( +    src_filename, +    title, +    title_main, +    title_sub, +    title_short, +    title_edition, +    title_language, +    classify_dewey, +    classify_keywords, +    classify_loc, +    classify_subject, +    classify_topic_register, +    creator_author, +    creator_author_email, +    creator_illustrator, +    creator_translator, +    date_added_to_site, +    date_available, +    date_created, +    date_issued, +    date_modified, +    date_published, +    date_valid, +    identifier_isbn, +    identifier_oclc, +    language_document, +    language_document_char, +    notes_abstract, +    notes_description, +    original_publisher, +    original_language, +    original_language_char, +    original_source, +    original_title, +    publisher, +    rights_copyright, +    rights_copyright_audio, +    rights_copyright_cover, +    rights_copyright_illustrations, +    rights_copyright_photographs, +    rights_copyright_text, +    rights_copyright_translation, +    rights_copyright_video, +    rights_license +  ) +#+END_SRC + +**** VALUES + +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC sql +  VALUES ( +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s', +    '%s' +  ); +#+END_SRC + +**** sql statement: values for formatting + +#+name: sqlite_formatted_insertions_doc_matters_metadata +#+BEGIN_SRC d +¶", +  SQLinsertDelimiter!()(doc_matters.src.filename), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), +  SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) +);  #+END_SRC  * __END__ diff --git a/org/sdp.org b/org/sdp.org index 4057d96..6b9052b 100644 --- a/org/sdp.org +++ b/org/sdp.org @@ -276,7 +276,7 @@ auto helpInfo = getopt(args,    std.getopt.config.passThrough,    "assert",             "--assert set optional assertions on",                        &opts["assertions"],    "concordance",        "--concordance file for document",                            &opts["concordance"], -  "debug",              "--debug only relevant when debug options compiled in",       &opts["debug"], +  "debug",              "--debug",                                                    &opts["debug"],    "digest",             "--digest hash digest for each object",                       &opts["digest"],    "docbook",            "--docbook process docbook output",                           &opts["docbook"],    "epub",               "--epub process epub output",                                 &opts["epub"], @@ -541,7 +541,7 @@ if (!(_opt_action.skip_output)) {    debug(steps) {      writeln("step0 commence → (without processing files)");    } -  outputHubOp!()(_opt_action); +  outputHubOp!()(_env, _opt_action);    debug(steps) {      writeln("- step0 complete");    } @@ -578,7 +578,7 @@ enforce(  );  #+END_SRC -*** 1. _document abstraction_                                                      >> +*** 1. _document abstraction_ [#A]                                                >>  - return tuple of:    - doc_abstraction (the document) @@ -770,7 +770,7 @@ debug(steps) {  }  #+END_SRC -** 3. _document abstraction, tuple_ (output-pre-processing)                       >> +** 3. _document abstraction, tuple_ (output-pre-processing) [#A]                   >>  - [[./meta_abstraction.org][meta_abstraction]]  - prepare the document abstraction used in downstream processing @@ -807,7 +807,7 @@ debug(steps) {  }  #+END_SRC -** 4. _document matters_ (doc info gathered, various sources)                     >> +** 4. _document matters_ (doc info gathered, various sources) [#A]                 >>  - prepare document_matters, miscellany about processing and the document of use    in downstream processing diff --git a/src/sdp/meta/read_config_files.d b/src/sdp/meta/read_config_files.d index 56a2bf8..22d285f 100644 --- a/src/sdp/meta/read_config_files.d +++ b/src/sdp/meta/read_config_files.d @@ -125,10 +125,8 @@ static template readConfigSite() {              config_file_str = conf_file.readText;              break;            } -        } -        catch (ErrnoException ex) { -        } -        catch (FileException ex) { +        } catch (ErrnoException ex) { +        } catch (FileException ex) {          }        }        if (config_file_str.length > 0) { break; } diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d index 830c6d0..373092b 100644 --- a/src/sdp/output/hub.d +++ b/src/sdp/output/hub.d @@ -133,14 +133,14 @@ template outputHubOp() {      sdp.output.source_sisupod,      sdp.output.create_zip_file,      sdp.output.paths_output; -  void outputHubOp(O)(O opt_action) { +  void outputHubOp(E,O)(E env, O opt_action) {      mixin SiSUoutputRgxInit;      static auto rgx = Rgx();      if ((opt_action.sqlite_create)) {        if ((opt_action.verbose)) {          writeln("sqlite create table... ");        } -      SQLiteTablesCreate!()(); +      SQLiteTablesCreate!()(env, opt_action);      }      if ((opt_action.sqlite_drop)) {        if ((opt_action.verbose)) { diff --git a/src/sdp/output/paths_output.d b/src/sdp/output/paths_output.d index d58b990..cd4817b 100644 --- a/src/sdp/output/paths_output.d +++ b/src/sdp/output/paths_output.d @@ -29,12 +29,32 @@ template SiSUoutPaths() {      return _PathsStruct();    }  } +template SiSUoutPathSQLite() { +  auto SiSUoutPathSQLite(Po)( +    Po  output_pth_root, +  ) { +    struct _PathsStruct { +      string output_root() { +        string out_root_; +        if (output_pth_root.length > 0) { +          out_root_ = output_pth_root; +        } else { +          out_root_ = "sisugen"; +        } +        return out_root_; +      } +      string output_base() { +        return asNormalizedPath(output_root).array; +      } +    } +    return _PathsStruct(); +  } +}  template SiSUoutPathsFnPd() {    /+ TODO stuff to work out here +/    auto SiSUoutPathsFnPd(Fn,Pn)(      Fn  fn_src_pth,      Pn  pod_name -    // Pn  pod_name = "",    ) {      struct _PathsStruct {        string base_filename() { @@ -216,14 +236,16 @@ template SiSUpathsSQLiteDiscrete() {      Po  output_pth_root,      Lng lng,    ) { -    auto out_pth = SiSUoutPaths!()(output_pth_root, lng); -    string base_dir = "sqlite";      struct _PathsStruct {        string base_filename(string fn_src) {          return fn_src.baseName.stripExtension;        }        string base() { -        return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array; +        auto out_pth = SiSUoutPaths!()(output_pth_root, lng); +        string base_dir = "sqlite"; +        return asNormalizedPath((out_pth.output_base).chainPath(base_dir)).array; +        // return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array; // single file for each file, multiple languages +        // return asNormalizedPath((out_pth.output_base).chainPath(base_dir)).array; // separate files for each language        }        string seg(string fn_src) {          return asNormalizedPath(base.chainPath(base_filename(fn_src))).array; @@ -238,23 +260,19 @@ template SiSUpathsSQLiteDiscrete() {  template SiSUpathsSQLite() {    mixin SiSUrgxInit;    static auto rgx = Rgx(); -  auto SiSUpathsSQLite(Po,Lng)( +  auto SiSUpathsSQLite(Po)(      Po  output_pth_root, -    Lng lng,    ) { -    auto out_pth = SiSUoutPaths!()(output_pth_root, lng); -    string base_dir = "sqlite";      struct _PathsStruct {        string base_filename(string fn_src) {          return fn_src.baseName.stripExtension;        }        string base() { +        auto out_pth = SiSUoutPathSQLite!()(output_pth_root); // decide whether to have separate files for each language +        string base_dir = "sqlite";          return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array;        } -      string seg(string fn_src) { -        return asNormalizedPath(base.chainPath(base_filename(fn_src))).array; -      } -      string sqlite_file(string fn_src) { +      string sqlite_file(string fn_src = "sdp-niu") {          return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array;        }      } diff --git a/src/sdp/output/source_sisupod.d b/src/sdp/output/source_sisupod.d index 6c50cf2..50b37b0 100644 --- a/src/sdp/output/source_sisupod.d +++ b/src/sdp/output/source_sisupod.d @@ -77,7 +77,7 @@ template SiSUpod() {              ~ "/" ~ image;            if (exists(fn_src_in)) {              debug(io) { -              writeln("WARNING (io debug) src out found: ", fn_src_in); +              writeln("(io debug) src out found: ", fn_src_in);              }              if (doc_matters.opt.action.source) {                fn_src_in.copy(fn_src_out_filesystem); @@ -108,7 +108,7 @@ template SiSUpod() {            ~ "/" ~ "sisu_document_make";          if (exists(fn_src_in)) {            debug(io) { -            writeln("WARNING (io debug) src out found: ", fn_src_in); +            writeln("(io debug) src out found: ", fn_src_in);            }            if (doc_matters.opt.action.source) {              fn_src_in.copy(fn_src_out_filesystem); @@ -139,7 +139,7 @@ template SiSUpod() {          string[] filelist_src_zpod_arr;          if (exists(fn_src_in)) {            debug(io) { -            writeln("WARNING (io debug) src out found: ", fn_src_in); +            writeln("(io debug) src in found: ", fn_src_in);            }            filelist_src_out_sisupod_arr ~= fn_src_out_sisupod_zip_base;            filelist_src_zpod_arr ~= fn_src_out_inside_pod; @@ -162,7 +162,7 @@ template SiSUpod() {          } else {            if (doc_matters.opt.action.verbose            || doc_matters.opt.action.debug_do) { -            writeln("WARNING (io) src out NOT found (markup source): ", fn_src_in); +            writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);            }          }        } { /+ bundle insert files +/ @@ -193,7 +193,7 @@ template SiSUpod() {                ).filesystem_open_zpod.to!string;              if (exists(fn_src_in)) {                debug(io) { -                writeln("WARNING (io debug) src out found: ", fn_src_in); +                writeln("(io debug) src out found: ", fn_src_in);                }                if (doc_matters.opt.action.source) {                  fn_src_in.copy(fn_src_out_filesystem); @@ -217,16 +217,24 @@ template SiSUpod() {          }        } {          auto fn_src_in = doc_matters.src.filename; -        if (exists(fn_src_in)) { -          if (doc_matters.opt.action.sisupod) { +        if (doc_matters.opt.action.sisupod) { +          if (exists(doc_matters.src.file_with_absolute_path)) {              createZipFile!()(fn_sisupod, zip.build()); +          } else { +            writeln("WARNING check missing source file(s): ", doc_matters.opt.action.sisupod); +          } +          if (!(exists(fn_sisupod))) { +            writeln("WARNING failed to create sisupod zip archive: ", fn_sisupod);            }          }        }        if (exists(fn_sisupod)) {          try { -          auto data = (cast(byte[]) (fn_sisupod).read); -          writefln("%-(%02x%) %s", data.sha256Of, fn_sisupod); +          if (doc_matters.opt.action.verbose) { +            auto data = (cast(byte[]) (fn_sisupod).read); +            writeln(doc_matters.src.filename, " >> "); +            writefln("%-(%02x%) %s", data.sha256Of, fn_sisupod); +          }            debug(sisupod) {              try {                auto zipped = new ZipArchive((fn_sisupod).read); diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d index 480c317..2409e3b 100644 --- a/src/sdp/output/sqlite.d +++ b/src/sdp/output/sqlite.d @@ -3,18 +3,92 @@ import sdp.output;  import    std.file,    std.conv : to; +long _metadata_tid_lastrowid; +template SQLiteDbRun() { +  void SQLiteDbRun(Db,St,O)( +    auto ref       Db   db, +    auto ref       St   db_statement, +    auto ref       O    opt_action, +               string   note, +  ) { +    debug(sql_statement) { +      writeln(db_statement); +    } +    try { +      db.run( +        "\nBEGIN;\n" ~ +        db_statement ~ +        "\nCOMMIT;\n" +      ); +      if (!(opt_action.sqlite_create)) { +        _metadata_tid_lastrowid = db.lastInsertRowid(); +        writeln("last row id: ", _metadata_tid_lastrowid); +      } +    } catch (ErrnoException ex) { +      writeln("ERROR SQLite : ", ex); +    } catch (Exception ex) { +      writeln("ERROR SQLite : ", ex); +    } +    if (opt_action.debug_do) { +      writeln(note); +      if (opt_action.verbose) { +        writeln(db_statement); +      } +    } +  } +}  template SQLiteBuildTablesAndPopulate() {    void SQLiteBuildTablesAndPopulate(D,I)(      auto ref const D    doc_abstraction, -    auto ref I          doc_matters, +    auto ref       I    doc_matters,    ) {      import d2sqlite3;      import std.typecons : Nullable;      mixin SiSUoutputRgxInit;      static auto rgx = Rgx(); +    auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); +    template SQLiteDbStatementComposite() { +      void SQLiteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        { +          { +            string _db_statement; +            { +              if ((doc_matters.opt.action.sqlite_create)) { +                auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); +                pth_sqlite.base.mkdirRecurse; +                _db_statement ~= SQLiteTablesReCreate!()(); +                SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); +              } +              if ((doc_matters.opt.action.sqlite_update)) { // TODO +                _db_statement ~= SQLiteInsertMetadata!()(doc_matters); +                SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); +                /+ get tid (lastrowid or max) for use in doc_objects table +/ +                _metadata_tid_lastrowid = db.lastInsertRowid(); +                _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX +                SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); +              } +            } +            db.close; +          } +        } +      } +    } +    template SQLinsertDelimiter() { +      auto SQLinsertDelimiter(string _txt) { +        _txt = _txt +          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +        return _txt; +      } +    }      template SQLiteFormatAndLoadObject() {        auto SQLiteFormatAndLoadObject(I)( -        auto ref              I    doc_matters, +        auto ref       I    doc_matters,        ) {          mixin SiSUoutputRgxInit;          struct sqlite_format_and_load_objects { @@ -23,7 +97,6 @@ template SQLiteBuildTablesAndPopulate() {            ) {              string _notes;              string _urls; -            _txt = _txt.replaceAll(rgx.inline_fontface_clean, "");              if (_txt.matchFirst(rgx.inline_notes_al_gen)) {                foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {                  _notes ~= "\n" ~ m["text"]; @@ -239,9 +312,17 @@ template SQLiteBuildTablesAndPopulate() {                        string _align = ("style=\"text-align:"                        ~ ((obj.table_column_aligns[col_idx] == "l")                        ? "left\"" : "right\"")); -                      _table ~= "<" ~ _col_is ~ " width=\"" ~ obj.table_column_widths[col_idx].to!string ~ "%\" " ~ _align ~ ">"; +                      _table ~= "<" +                        ~ _col_is +                        ~ " width=\"" +                        ~ obj.table_column_widths[col_idx].to!string +                        ~ "%\" " +                        ~ _align +                        ~ ">";                        _table ~= cell; -                      _table ~= "</" ~ _col_is ~ ">"; +                      _table ~= "</" +                        ~ _col_is +                        ~ ">";                      }                    }                    _table ~= "</tr>"; @@ -452,31 +533,41 @@ template SQLiteBuildTablesAndPopulate() {          return sqlite_format_and_load_objects();        }      } -    template SQLiteInstruct() { -      Statement SQLiteInstruct(I)( -        auto ref              I    doc_matters, -      ) { -        auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path, doc_matters.src.language); -        pth_sqlite.base.mkdirRecurse; -        auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); -        // auto db = Database(":memory:"); // open database in memory -        if (doc_matters.opt.action.sqlite_create) { -          if ((doc_matters.opt.action.verbose)) { writeln("sqlite create tables... "); } -          db.run(" +    template SQLiteTablesReCreate() { +      string SQLiteTablesReCreate()() { +        string _sql_instruct; +        _sql_instruct = format(q"¶ +          DROP INDEX IF EXISTS ocn; +          DROP INDEX IF EXISTS digest_clean; +          DROP INDEX IF EXISTS digest_all; +          DROP INDEX IF EXISTS clean; +          DROP INDEX IF EXISTS lev0; +          DROP INDEX IF EXISTS lev1; +          DROP INDEX IF EXISTS lev2; +          DROP INDEX IF EXISTS lev3; +          DROP INDEX IF EXISTS lev4; +          DROP INDEX IF EXISTS lev5; +          DROP INDEX IF EXISTS lev6; +          DROP INDEX IF EXISTS title; +          DROP INDEX IF EXISTS creator_author; +          DROP INDEX IF EXISTS src_filename; +          DROP INDEX IF EXISTS language_document_char; +          DROP INDEX IF EXISTS classify_topic_register;            DROP TABLE IF EXISTS metadata_and_text;            DROP TABLE IF EXISTS doc_objects;            DROP TABLE IF EXISTS urls;            CREATE TABLE metadata_and_text ( -            tid                              BIGINT            PRIMARY KEY, +            tid                              INTEGER           PRIMARY KEY, +          /* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */              title                            VARCHAR(800)  NOT NULL, -            title_main                       VARCHAR(400)      NULL, /*NOT*/ +            title_main                       VARCHAR(400)  NOT NULL,              title_sub                        VARCHAR(400)      NULL,              title_short                      VARCHAR(400)      NULL,              title_edition                    VARCHAR(10)       NULL,              title_note                       VARCHAR(2500)     NULL,              title_language                   VARCHAR(100)      NULL,              title_language_char              VARCHAR(6)        NULL, -            creator_author                   VARCHAR(600)      NULL, +            creator_author                   VARCHAR(600)  NOT NULL,              creator_author_email             VARCHAR(100)      NULL,              creator_author_hon               VARCHAR(100)      NULL,              creator_author_nationality       VARCHAR(100)      NULL, @@ -490,7 +581,7 @@ template SQLiteBuildTablesAndPopulate() {              creator_audio                    VARCHAR(600)      NULL,              creator_video                    VARCHAR(600)      NULL,              language_document                VARCHAR(100)      NULL, -            language_document_char           VARCHAR(6)        NULL, /*NOT*/ +            language_document_char           VARCHAR(6)    NOT NULL,              language_original                VARCHAR(100)      NULL,              language_original_char           VARCHAR(6)        NULL,              date_added_to_site               VARCHAR(10)       NULL, @@ -539,18 +630,20 @@ template SQLiteBuildTablesAndPopulate() {              notes_prefix_a                   TEXT              NULL,              notes_prefix_b                   TEXT              NULL,              notes_suffix                     TEXT              NULL, -            src_filename                     VARCHAR(256)      NULL, +            src_filename                     VARCHAR(256)  NOT NULL,              src_fingerprint                  VARCHAR(256)      NULL,              src_filesize                     VARCHAR(10)       NULL,              src_wordcount                    VARCHAR(10)       NULL, +            pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +            pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +            pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */              src_text                         TEXT              NULL,              fulltext                         TEXT              NULL,              links                            TEXT              NULL            ); -          /* */            CREATE TABLE doc_objects ( -            lid                              BIGINT PRIMARY KEY, -            metadata_tid                     BIGINT REFERENCES metadata_and_text, +            lid                              INTEGER PRIMARY KEY, +            metadata_tid                     INTEGER REFERENCES metadata_and_text,              ocn                              SMALLINT,              ocnd                             VARCHAR(6),              ocns                             VARCHAR(6), @@ -581,37 +674,50 @@ template SQLiteBuildTablesAndPopulate() {              digest_clean                     CHAR(256),              digest_all                       CHAR(256),              types                            CHAR(1) NULL -          ) -          "); -        } -        Statement insert_metadata = db.prepare(" +          ); +          CREATE INDEX idx_ocn ON doc_objects(ocn); +          CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); +          CREATE INDEX idx_digest_all ON doc_objects(digest_all); +          CREATE INDEX idx_clean ON doc_objects(clean); +          CREATE INDEX idx_lev0 ON doc_objects(lev0); +          CREATE INDEX idx_lev1 ON doc_objects(lev1); +          CREATE INDEX idx_lev2 ON doc_objects(lev2); +          CREATE INDEX idx_lev3 ON doc_objects(lev3); +          CREATE INDEX idx_lev4 ON doc_objects(lev4); +          CREATE INDEX idx_lev5 ON doc_objects(lev5); +          CREATE INDEX idx_lev6 ON doc_objects(lev6); +          CREATE INDEX idx_title ON metadata_and_text(title); +          CREATE INDEX idx_author ON metadata_and_text(creator_author); +          CREATE INDEX idx_filename ON metadata_and_text(src_filename); +          CREATE INDEX idx_language ON metadata_and_text(language_document_char); +          CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); +        ¶",); +        return _sql_instruct; +      } +    } +    template SQLiteInsertMetadata() { +      string SQLiteInsertMetadata(I)( +        auto ref       I    doc_matters, +      ) { +        string _insert_metadata; +        _insert_metadata = format(q"¶            INSERT INTO metadata_and_text ( -            tid, +            src_filename,              title,              title_main,              title_sub,              title_short,              title_edition, -            title_note,              title_language, -            title_language_char, +            classify_dewey, +            classify_keywords, +            classify_loc, +            classify_subject, +            classify_topic_register,              creator_author,              creator_author_email, -            creator_author_hon, -            creator_author_nationality, -            creator_editor, -            creator_contributor,              creator_illustrator, -            creator_photographer,              creator_translator, -            creator_prepared_by, -            creator_digitized_by, -            creator_audio, -            creator_video, -            language_document, -            language_document_char, -            language_original, -            language_original_char,              date_added_to_site,              date_available,              date_created, @@ -619,17 +725,18 @@ template SQLiteBuildTablesAndPopulate() {              date_modified,              date_published,              date_valid, -            date_translated, -            date_original_publication, -            date_generated, -            publisher, +            identifier_isbn, +            identifier_oclc, +            language_document, +            language_document_char, +            notes_abstract, +            notes_description,              original_publisher,              original_language,              original_language_char,              original_source, -            original_institution, -            original_nationality,              original_title, +            publisher,              rights_copyright,              rights_copyright_audio,              rights_copyright_cover, @@ -638,272 +745,144 @@ template SQLiteBuildTablesAndPopulate() {              rights_copyright_text,              rights_copyright_translation,              rights_copyright_video, -            rights_license, -            identifier_oclc, -            identifier_isbn, -            classify_topic_register, -            classify_subject, -            classify_loc, -            classify_dewey, -            classify_keywords, -            notes_abstract, -            notes_description, -            notes_comment, -            notes_coverage, -            notes_relation, -            notes_history, -            notes_type, -            notes_format, -            notes_prefix, -            notes_prefix_a, -            notes_prefix_b, -            notes_suffix, -            src_filename, -            src_fingerprint, -            src_filesize, -            src_wordcount, -            src_text, -            fulltext, -            links +            rights_license            )            VALUES ( -            :tid, -            :title, -            :title_main, -            :title_sub, -            :title_short, -            :title_edition, -            :title_note, -            :title_language, -            :title_language_char, -            :creator_author, -            :creator_author_email, -            :creator_author_hon, -            :creator_author_nationality, -            :creator_editor, -            :creator_contributor, -            :creator_illustrator, -            :creator_photographer, -            :creator_translator, -            :creator_prepared_by, -            :creator_digitized_by, -            :creator_audio, -            :creator_video, -            :language_document, -            :language_document_char, -            :language_original, -            :language_original_char, -            :date_added_to_site, -            :date_available, -            :date_created, -            :date_issued, -            :date_modified, -            :date_published, -            :date_valid, -            :date_translated, -            :date_original_publication, -            :date_generated, -            :publisher, -            :original_publisher, -            :original_language, -            :original_language_char, -            :original_source, -            :original_institution, -            :original_nationality, -            :original_title, -            :rights_copyright, -            :rights_copyright_audio, -            :rights_copyright_cover, -            :rights_copyright_illustrations, -            :rights_copyright_photographs, -            :rights_copyright_text, -            :rights_copyright_translation, -            :rights_copyright_video, -            :rights_license, -            :identifier_oclc, -            :identifier_isbn, -            :classify_topic_register, -            :classify_subject, -            :classify_loc, -            :classify_dewey, -            :classify_keywords, -            :notes_abstract, -            :notes_description, -            :notes_comment, -            :notes_coverage, -            :notes_relation, -            :notes_history, -            :notes_type, -            :notes_format, -            :notes_prefix, -            :notes_prefix_a, -            :notes_prefix_b, -            :notes_suffix, -            :src_filename, -            :src_fingerprint, -            :src_filesize, -            :src_wordcount, -            :src_text, -            :fulltext, -            :links -          ) -        "); -        // insert_metadata.bind(":tid",                               ); // unique identifier you could try use sha of title author & language? or auto increment?? -        insert_metadata.bind(":title",                             doc_matters.conf_make_meta.meta.title_full); -        insert_metadata.bind(":title_main",                        doc_matters.conf_make_meta.meta.title_main); -        insert_metadata.bind(":title_sub",                         doc_matters.conf_make_meta.meta.title_subtitle); -        insert_metadata.bind(":title_short",                       doc_matters.conf_make_meta.meta.title_short); // -        insert_metadata.bind(":title_edition",                     doc_matters.conf_make_meta.meta.title_edition); -        insert_metadata.bind(":title_language",                    doc_matters.conf_make_meta.meta.title_language); -        // insert_metadata.bind(":title_language_char",            doc_matters.conf_make_meta.meta.title_language_char); -        insert_metadata.bind(":classify_dewey",                    doc_matters.conf_make_meta.meta.classify_dewey); -        insert_metadata.bind(":classify_keywords",                 doc_matters.conf_make_meta.meta.classify_keywords); -        insert_metadata.bind(":classify_loc",                      doc_matters.conf_make_meta.meta.classify_loc); -        insert_metadata.bind(":classify_subject",                  doc_matters.conf_make_meta.meta.classify_subject); -        insert_metadata.bind(":classify_topic_register",           doc_matters.conf_make_meta.meta.classify_topic_register); -        insert_metadata.bind(":creator_author",                    doc_matters.conf_make_meta.meta.creator_author); -        insert_metadata.bind(":creator_author_email",              doc_matters.conf_make_meta.meta.creator_author_email); -        // insert_metadata.bind(":creator_editor",                 doc_matters.conf_make_meta.meta.creator_editor); -        // insert_metadata.bind(":creator_contributor",            doc_matters.conf_make_meta.meta.creator_contributor); -        insert_metadata.bind(":creator_illustrator",               doc_matters.conf_make_meta.meta.creator_illustrator); -        // insert_metadata.bind(":creator_photographer",           doc_matters.conf_make_meta.meta.creator_photographer); -        insert_metadata.bind(":creator_translator",                doc_matters.conf_make_meta.meta.creator_translator); -        // insert_metadata.bind(":creator_audio",                  doc_matters.conf_make_meta.meta.creator_audio); -        // insert_metadata.bind(":creator_video",                  doc_matters.conf_make_meta.meta.creator_video); -        insert_metadata.bind(":date_added_to_site",                doc_matters.conf_make_meta.meta.date_added_to_site); -        insert_metadata.bind(":date_available",                    doc_matters.conf_make_meta.meta.date_available); -        insert_metadata.bind(":date_created",                      doc_matters.conf_make_meta.meta.date_created); -        insert_metadata.bind(":date_issued",                       doc_matters.conf_make_meta.meta.date_issued); -        insert_metadata.bind(":date_modified",                     doc_matters.conf_make_meta.meta.date_modified); -        insert_metadata.bind(":date_published",                    doc_matters.conf_make_meta.meta.date_published); -        insert_metadata.bind(":date_valid",                        doc_matters.conf_make_meta.meta.date_valid); -        // insert_metadata.bind(":date_translated",                doc_matters.conf_make_meta.meta.date_translated); -        // insert_metadata.bind(":date_original_publication",      doc_matters.conf_make_meta.meta.date_original_publication); -        // insert_metadata.bind(":date_generated",                 doc_matters.conf_make_meta.meta.date_generated); -        insert_metadata.bind(":identifier_isbn",                   doc_matters.conf_make_meta.meta.identifier_isbn); -        insert_metadata.bind(":identifier_oclc",                   doc_matters.conf_make_meta.meta.identifier_oclc); -        insert_metadata.bind(":language_document",                 doc_matters.conf_make_meta.meta.language_document); -        insert_metadata.bind(":language_document_char",            doc_matters.conf_make_meta.meta.language_document_char); -        // insert_metadata.bind(":language_original",              doc_matters.conf_make_meta.meta.language_original); -        // insert_metadata.bind(":language_original_char",         doc_matters.conf_make_meta.meta.language_original_char); -        insert_metadata.bind(":notes_abstract",                    doc_matters.conf_make_meta.meta.notes_abstract); -        insert_metadata.bind(":notes_description",                 doc_matters.conf_make_meta.meta.notes_description); -        insert_metadata.bind(":original_publisher",                doc_matters.conf_make_meta.meta.original_publisher); -        insert_metadata.bind(":original_language",                 doc_matters.conf_make_meta.meta.original_language); -        insert_metadata.bind(":original_language_char",            doc_matters.conf_make_meta.meta.original_language_char); -        insert_metadata.bind(":original_source",                   doc_matters.conf_make_meta.meta.original_source); -        insert_metadata.bind(":original_title",                    doc_matters.conf_make_meta.meta.original_title); -        insert_metadata.bind(":publisher",                         doc_matters.conf_make_meta.meta.publisher); -        // insert_metadata.bind(":rights",                         doc_matters.conf_make_meta.meta.rights); -        insert_metadata.bind(":rights_copyright",                  doc_matters.conf_make_meta.meta.rights_copyright); -        insert_metadata.bind(":rights_copyright_audio",            doc_matters.conf_make_meta.meta.rights_copyright_audio); -        insert_metadata.bind(":rights_copyright_cover",            doc_matters.conf_make_meta.meta.rights_copyright_cover); -        insert_metadata.bind(":rights_copyright_illustrations",    doc_matters.conf_make_meta.meta.rights_copyright_illustrations); -        insert_metadata.bind(":rights_copyright_photographs",      doc_matters.conf_make_meta.meta.rights_copyright_photographs); -        // insert_metadata.bind(":rights_copyright_preparation",   doc_matters.conf_make_meta.meta.rights_preparation); -        insert_metadata.bind(":rights_copyright_text",      doc_matters.conf_make_meta.meta.rights_copyright_text); -        insert_metadata.bind(":rights_copyright_translation",      doc_matters.conf_make_meta.meta.rights_copyright_translation); -        insert_metadata.bind(":rights_copyright_video",            doc_matters.conf_make_meta.meta.rights_copyright_video); -        // insert_metadata.bind(":rights_copyright_digitization",  doc_matters.conf_make_meta.meta.rights_digitization); -        // insert_metadata.bind(":rights_copyright_audio",         doc_matters.conf_make_meta.meta.rights_audio); -        // insert_metadata.bind(":rights_copyright_video",         doc_matters.conf_make_meta.meta.rights_video); -        insert_metadata.bind(":rights_license",                    doc_matters.conf_make_meta.meta.rights_license); -        // insert_metadata.bind(":src_filename",                   doc_matters.conf_make_meta.meta.src_filename); -        // insert_metadata.bind(":src_fingerprint",                doc_matters.conf_make_meta.meta.src_fingerprint); -        // insert_metadata.bind(":src_filesize",                   doc_matters.conf_make_meta.meta.src_filesize); -        // insert_metadata.bind(":src_wordcount",                  doc_matters.conf_make_meta.meta.src_wordcount); -        // insert_metadata.bind(":src_text",                       doc_matters.conf_make_meta.meta.src_text); -        // insert_metadata.bind(":fulltext",                       doc_matters.conf_make_meta.meta.fulltext); -        // insert_metadata.bind(":links",                          doc_matters.conf_make_meta.meta.links); -        insert_metadata.execute(); insert_metadata.reset(); -        /+ watch +/ -        writeln(" ", pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); -        if ((doc_matters.opt.action.verbose)) { -          writeln("sql statement executed"); -        } -        assert(db.totalChanges == 1); -        // -        Statement insert_doc_objects = db.prepare(" -          INSERT INTO doc_objects ( -            lid, -            metadata_tid, -            ocn, -            ocnd, -            ocns, -            clean, -            body, -            book_idx, -            seg, -            lev_an, -            lev, -            lev0, -            lev1, -            lev2, -            lev3, -            lev4, -            lev5, -            lev6, -            lev7, -            en_a, -            en_z, -            en_a_asterisk, -            en_z_asterisk, -            en_a_plus, -            en_z_plus, -            t_of, -            t_is, -            node, -            parent, -            digest_clean, -            digest_all, -            types -          ) -          VALUES ( -            :lid, -            :metadata_tid, -            :ocn, -            :ocnd, -            :ocns, -            :clean, -            :body, -            :book_idx, -            :seg, -            :lev_an, -            :lev, -            :lev0, -            :lev1, -            :lev2, -            :lev3, -            :lev4, -            :lev5, -            :lev6, -            :lev7, -            :en_a, -            :en_z, -            :en_a_asterisk, -            :en_z_asterisk, -            :en_a_plus, -            :en_z_plus, -            :t_of, -            :t_is, -            :node, -            :parent, -            :digest_clean, -            :digest_all, -            :types -          ) -        "); -        return insert_doc_objects; +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s' +          ); +        ¶", +          SQLinsertDelimiter!()(doc_matters.src.filename), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) +        ); +        return _insert_metadata;        }      } -    template SQLiteObjectsLoop() { -      void SQLiteObjectsLoop(P)( -        auto ref P          doc_parts, +    template SQLiteInsertDocObjectsLoop() { +      string SQLiteInsertDocObjectsLoop(P,I)( +        auto ref       P    doc_parts, +        auto ref       I    _metadata_tid,        ) { -        Statement insert_doc_objects = SQLiteInstruct!()(doc_matters); +        string insertDocObjectsRow(O)(O obj) { +          string _insert_doc_objects_row; +          _insert_doc_objects_row = format(q"¶ +            INSERT INTO doc_objects ( +              metadata_tid, +              ocn, +              clean, +              body, +              lev, +              t_of, +              t_is +            ) +            VALUES ( +              %d, +              %s, +              '%s', +              '%s', +              %s, +              '%s', +              '%s' +            ); +          ¶", +            _metadata_tid, +            obj.ocn, +            SQLinsertDelimiter!()(obj_txt["text"]), +            SQLinsertDelimiter!()(obj_txt["html"]), +            obj.heading_lev_markup, +            obj.is_of, +            obj.is_a, +          ); +          return _insert_doc_objects_row; +        }          auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);          string[string] obj_txt;          string doc_text; +        string[] _insert_doc_objects;          foreach (part; doc_parts) {            foreach (obj; doc_abstraction[part]) {              switch (obj.of_part) { @@ -1048,43 +1027,192 @@ template SQLiteBuildTablesAndPopulate() {                  );                }              } -            insert_doc_objects.bind(":t_of",        obj.is_of); -            insert_doc_objects.bind(":t_is",        obj.is_a); -            insert_doc_objects.bind(":ocn",         obj.ocn); -            insert_doc_objects.bind(":clean",       obj_txt["text"]); // consider whether book index info should be made available within clear text for search -            insert_doc_objects.bind(":body",        obj_txt["html"]); -            // insert_doc_objects.bind(":book_idx",    ""); // not needed, but, consider whether should be made available within object for clear text search -            insert_doc_objects.bind(":lev",         obj.heading_lev_markup); -            // // insert_doc_objects.bind(":dom_markedup",    ""); // should make lev sequence below obsolete -            // // insert_doc_objects.bind(":dom_collapsed", ""); // should add info -            // insert_doc_objects.bind(":lev0",        ""); -            // insert_doc_objects.bind(":lev1",        ""); -            // insert_doc_objects.bind(":lev2",        ""); -            // insert_doc_objects.bind(":lev3",        ""); -            // insert_doc_objects.bind(":lev4",        ""); -            // insert_doc_objects.bind(":lev5",        ""); -            // insert_doc_objects.bind(":lev6",        ""); -            // insert_doc_objects.bind(":lev7",        ""); -            // insert_doc_objects.bind(":node",        ""); -            // insert_doc_objects.bind(":type",        ""); -            // insert_doc_objects.bind(":parent_ocn",                ""); -            // insert_doc_objects.bind(":ancestors",                 ""); -            // insert_doc_objects.bind(":heading_lev_markup",        ""); -            // insert_doc_objects.bind(":heading_lev_collapsed",     ""); -            // insert_doc_objects.bind(":parent_lev_markup",        ""); -            // insert_doc_objects.bind(":heading_ancestors",         ""); -            // insert_doc_objects.bind(":node",                      ""); -            insert_doc_objects.execute(); insert_doc_objects.reset(); -          } +            if (!(obj.is_a == "comment")) { +              _insert_doc_objects ~= insertDocObjectsRow(obj); +            } +          } // loop closes          } +        return _insert_doc_objects.join.to!(char[]).toUTF8;        }      } -    SQLiteObjectsLoop!()(doc_matters.xml.keys_seq.sql); +    SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);    }  }  template SQLiteTablesCreate() { -  void SQLiteTablesCreate()() { -    writeln("table create"); +  void SQLiteTablesCreate(E,O)(E env, O opt_action) { +    import d2sqlite3; +    template SQLiteTablesReCreate() { +      string SQLiteTablesReCreate()() { +        string _sql_instruct; +        _sql_instruct = format(q"¶ +          DROP INDEX IF EXISTS ocn; +          DROP INDEX IF EXISTS digest_clean; +          DROP INDEX IF EXISTS digest_all; +          DROP INDEX IF EXISTS clean; +          DROP INDEX IF EXISTS lev0; +          DROP INDEX IF EXISTS lev1; +          DROP INDEX IF EXISTS lev2; +          DROP INDEX IF EXISTS lev3; +          DROP INDEX IF EXISTS lev4; +          DROP INDEX IF EXISTS lev5; +          DROP INDEX IF EXISTS lev6; +          DROP INDEX IF EXISTS title; +          DROP INDEX IF EXISTS creator_author; +          DROP INDEX IF EXISTS src_filename; +          DROP INDEX IF EXISTS language_document_char; +          DROP INDEX IF EXISTS classify_topic_register; +          DROP TABLE IF EXISTS metadata_and_text; +          DROP TABLE IF EXISTS doc_objects; +          DROP TABLE IF EXISTS urls; +          CREATE TABLE metadata_and_text ( +            tid                              INTEGER           PRIMARY KEY, +          /* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ +            title                            VARCHAR(800)  NOT NULL, +            title_main                       VARCHAR(400)  NOT NULL, +            title_sub                        VARCHAR(400)      NULL, +            title_short                      VARCHAR(400)      NULL, +            title_edition                    VARCHAR(10)       NULL, +            title_note                       VARCHAR(2500)     NULL, +            title_language                   VARCHAR(100)      NULL, +            title_language_char              VARCHAR(6)        NULL, +            creator_author                   VARCHAR(600)  NOT NULL, +            creator_author_email             VARCHAR(100)      NULL, +            creator_author_hon               VARCHAR(100)      NULL, +            creator_author_nationality       VARCHAR(100)      NULL, +            creator_editor                   VARCHAR(600)      NULL, +            creator_contributor              VARCHAR(600)      NULL, +            creator_illustrator              VARCHAR(600)      NULL, +            creator_photographer             VARCHAR(600)      NULL, +            creator_translator               VARCHAR(600)      NULL, +            creator_prepared_by              VARCHAR(600)      NULL, +            creator_digitized_by             VARCHAR(600)      NULL, +            creator_audio                    VARCHAR(600)      NULL, +            creator_video                    VARCHAR(600)      NULL, +            language_document                VARCHAR(100)      NULL, +            language_document_char           VARCHAR(6)    NOT NULL, +            language_original                VARCHAR(100)      NULL, +            language_original_char           VARCHAR(6)        NULL, +            date_added_to_site               VARCHAR(10)       NULL, +            date_available                   VARCHAR(10)       NULL, +            date_created                     VARCHAR(10)       NULL, +            date_issued                      VARCHAR(10)       NULL, +            date_modified                    VARCHAR(10)       NULL, +            date_published                   VARCHAR(10)       NULL, +            date_valid                       VARCHAR(10)       NULL, +            date_translated                  VARCHAR(10)       NULL, +            date_original_publication        VARCHAR(10)       NULL, +            date_generated                   VARCHAR(10)       NULL, +            publisher                        VARCHAR(600)      NULL, +            original_publisher               VARCHAR(600)      NULL, +            original_language                VARCHAR(100)      NULL, +            original_language_char           VARCHAR(6)        NULL, +            original_source                  VARCHAR(600)      NULL, +            original_institution             VARCHAR(600)      NULL, +            original_nationality             VARCHAR(100)      NULL, +            original_title                   VARCHAR(800)      NULL, +            rights_copyright                 VARCHAR(2500)     NULL, +            rights_copyright_audio           VARCHAR(2500)     NULL, +            rights_copyright_cover           VARCHAR(2500)     NULL, +            rights_copyright_illustrations   VARCHAR(2500)     NULL, +            rights_copyright_photographs     VARCHAR(2500)     NULL, +            rights_copyright_text            VARCHAR(2500)     NULL, +            rights_copyright_translation     VARCHAR(2500)     NULL, +            rights_copyright_video           VARCHAR(2500)     NULL, +            rights_license                   VARCHAR(2500)     NULL, +            identifier_oclc                  VARCHAR(30)       NULL, +            identifier_isbn                  VARCHAR(16)       NULL, +            classify_topic_register          VARCHAR(2500)     NULL, +            classify_subject                 VARCHAR(600)      NULL, +            classify_loc                     VARCHAR(30)       NULL, +            classify_dewey                   VARCHAR(30)       NULL, +            classify_keywords                VARCHAR(600)      NULL, +            notes_abstract                   TEXT              NULL, +            notes_description                TEXT              NULL, +            notes_comment                    TEXT              NULL, +            notes_coverage                   VARCHAR(200)      NULL, +            notes_relation                   VARCHAR(200)      NULL, +            notes_history                    VARCHAR(600)      NULL, +            notes_type                       VARCHAR(600)      NULL, +            notes_format                     VARCHAR(600)      NULL, +            notes_prefix                     TEXT              NULL, +            notes_prefix_a                   TEXT              NULL, +            notes_prefix_b                   TEXT              NULL, +            notes_suffix                     TEXT              NULL, +            src_filename                     VARCHAR(256)  NOT NULL, +            src_fingerprint                  VARCHAR(256)      NULL, +            src_filesize                     VARCHAR(10)       NULL, +            src_wordcount                    VARCHAR(10)       NULL, +            pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +            pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +            pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */ +            src_text                         TEXT              NULL, +            fulltext                         TEXT              NULL, +            links                            TEXT              NULL +          ); +          CREATE TABLE doc_objects ( +            lid                              INTEGER PRIMARY KEY, +            metadata_tid                     INTEGER REFERENCES metadata_and_text, +            ocn                              SMALLINT, +            ocnd                             VARCHAR(6), +            ocns                             VARCHAR(6), +            clean                            TEXT NULL, +            body                             TEXT NULL, +            book_idx                         TEXT NULL, +            seg                              VARCHAR(256) NULL, +            lev_an                           VARCHAR(1), +            lev                              SMALLINT NULL, +            lev0                             SMALLINT, +            lev1                             SMALLINT, +            lev2                             SMALLINT, +            lev3                             SMALLINT, +            lev4                             SMALLINT, +            lev5                             SMALLINT, +            lev6                             SMALLINT, +            lev7                             SMALLINT, +            en_a                             SMALLINT NULL, +            en_z                             SMALLINT NULL, +            en_a_asterisk                    SMALLINT NULL, +            en_z_asterisk                    SMALLINT NULL, +            en_a_plus                        SMALLINT NULL, +            en_z_plus                        SMALLINT NULL, +            t_of                             VARCHAR(16), +            t_is                             VARCHAR(16), +            node                             VARCHAR(16) NULL, +            parent                           VARCHAR(16) NULL, +            digest_clean                     CHAR(256), +            digest_all                       CHAR(256), +            types                            CHAR(1) NULL +          ); +          CREATE INDEX idx_ocn ON doc_objects(ocn); +          CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); +          CREATE INDEX idx_digest_all ON doc_objects(digest_all); +          CREATE INDEX idx_clean ON doc_objects(clean); +          CREATE INDEX idx_lev0 ON doc_objects(lev0); +          CREATE INDEX idx_lev1 ON doc_objects(lev1); +          CREATE INDEX idx_lev2 ON doc_objects(lev2); +          CREATE INDEX idx_lev3 ON doc_objects(lev3); +          CREATE INDEX idx_lev4 ON doc_objects(lev4); +          CREATE INDEX idx_lev5 ON doc_objects(lev5); +          CREATE INDEX idx_lev6 ON doc_objects(lev6); +          CREATE INDEX idx_title ON metadata_and_text(title); +          CREATE INDEX idx_author ON metadata_and_text(creator_author); +          CREATE INDEX idx_filename ON metadata_and_text(src_filename); +          CREATE INDEX idx_language ON metadata_and_text(language_document_char); +          CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); +        ¶",); +        return _sql_instruct; +      } +    } +    if (opt_action.sqlite_create) { +      string _db_statement; +      auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE +      pth_sqlite.base.mkdirRecurse; +      auto db = Database(pth_sqlite.sqlite_file); // ISSUE +      { +        _db_statement ~= SQLiteTablesReCreate!()(); +      } +      SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE"); +    }    }  }  template SQLiteTablesDrop() { diff --git a/src/sdp/output/sqlite_discrete.d b/src/sdp/output/sqlite_discrete.d index 443fb3b..40e0b15 100644 --- a/src/sdp/output/sqlite_discrete.d +++ b/src/sdp/output/sqlite_discrete.d @@ -3,18 +3,75 @@ import sdp.output;  import    std.file,    std.conv : to; +template SQLiteDiscreteDbRun() { +  void SQLiteDiscreteDbRun(Db,St,O)( +    auto ref       Db   db, +    auto ref       St   db_statement, +    auto ref       O    opt_action, +               string   note, +  ) { +    debug(sql_statement) { +      writeln(db_statement); +    } +    try { +      db.run( +        "\nBEGIN;\n" ~ +        db_statement ~ +        "\nCOMMIT;\n" +      ); +    } catch (ErrnoException ex) { +      writeln("ERROR SQLite : ", ex); +    } catch (Exception ex) { +      writeln("ERROR SQLite : ", ex); +    } +    if (opt_action.debug_do) { +      writeln(note); +      if (opt_action.verbose) { +        writeln(db_statement); +      } +    } +  } +}  template SQLiteDiscreteBuildTablesAndPopulate() {    void SQLiteDiscreteBuildTablesAndPopulate(D,I)(      auto ref const D    doc_abstraction, -    auto ref I          doc_matters, +    auto ref       I    doc_matters,    ) {      import d2sqlite3;      import std.typecons : Nullable;      mixin SiSUoutputRgxInit;      static auto rgx = Rgx(); +    auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); +    pth_sqlite.base.mkdirRecurse; +    auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); +    template SQLiteDiscreteDbStatementComposite() { +      void SQLiteDiscreteDbStatementComposite(Db,D,I)( +        auto ref       Db   db, +        auto ref const D    doc_abstraction, +        auto ref       I    doc_matters, +      ) { +        { +          string _db_statement; +          { +            _db_statement ~= SQLiteDiscreteTablesReCreate!()(); +            _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters); +            _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX +            SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); +          } +          db.close; +        } +      } +    } +    template SQLinsertDelimiter() { +      auto SQLinsertDelimiter(string _txt) { +        _txt = _txt +          .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); +        return _txt; +      } +    }      template SQLiteFormatAndLoadObject() {        auto SQLiteFormatAndLoadObject(I)( -        auto ref              I    doc_matters, +        auto ref       I    doc_matters,        ) {          mixin SiSUoutputRgxInit;          struct sqlite_format_and_load_objects { @@ -459,27 +516,41 @@ template SQLiteDiscreteBuildTablesAndPopulate() {          return sqlite_format_and_load_objects();        }      } -    template SQLiteInstruct() { -      void SQLiteInstruct(Db,I)( -                              Db   db, -        auto ref              I    doc_matters, -      ) { -        db.run(" +    template SQLiteDiscreteTablesReCreate() { +      string SQLiteDiscreteTablesReCreate()() { +        string _sql_instruct; +        _sql_instruct = format(q"¶ +          DROP INDEX IF EXISTS ocn; +          DROP INDEX IF EXISTS digest_clean; +          DROP INDEX IF EXISTS digest_all; +          DROP INDEX IF EXISTS clean; +          DROP INDEX IF EXISTS lev0; +          DROP INDEX IF EXISTS lev1; +          DROP INDEX IF EXISTS lev2; +          DROP INDEX IF EXISTS lev3; +          DROP INDEX IF EXISTS lev4; +          DROP INDEX IF EXISTS lev5; +          DROP INDEX IF EXISTS lev6; +          DROP INDEX IF EXISTS title; +          DROP INDEX IF EXISTS creator_author; +          DROP INDEX IF EXISTS src_filename; +          DROP INDEX IF EXISTS language_document_char; +          DROP INDEX IF EXISTS classify_topic_register;            DROP TABLE IF EXISTS metadata_and_text;            DROP TABLE IF EXISTS doc_objects;            DROP TABLE IF EXISTS urls; -          BEGIN;            CREATE TABLE metadata_and_text ( -            tid                              BIGINT            PRIMARY KEY, +            tid                              INTEGER           PRIMARY KEY, +          /* src_filename_composite           VARCHAR(256)  NOT NULL UNIQUE, /* z pod name if any + src filename + language code */              title                            VARCHAR(800)  NOT NULL, -            title_main                       VARCHAR(400)      NULL, /*NOT*/ +            title_main                       VARCHAR(400)  NOT NULL,              title_sub                        VARCHAR(400)      NULL,              title_short                      VARCHAR(400)      NULL,              title_edition                    VARCHAR(10)       NULL,              title_note                       VARCHAR(2500)     NULL,              title_language                   VARCHAR(100)      NULL,              title_language_char              VARCHAR(6)        NULL, -            creator_author                   VARCHAR(600)      NULL, +            creator_author                   VARCHAR(600)  NOT NULL,              creator_author_email             VARCHAR(100)      NULL,              creator_author_hon               VARCHAR(100)      NULL,              creator_author_nationality       VARCHAR(100)      NULL, @@ -493,7 +564,7 @@ template SQLiteDiscreteBuildTablesAndPopulate() {              creator_audio                    VARCHAR(600)      NULL,              creator_video                    VARCHAR(600)      NULL,              language_document                VARCHAR(100)      NULL, -            language_document_char           VARCHAR(6)        NULL, /*NOT*/ +            language_document_char           VARCHAR(6)    NOT NULL,              language_original                VARCHAR(100)      NULL,              language_original_char           VARCHAR(6)        NULL,              date_added_to_site               VARCHAR(10)       NULL, @@ -542,18 +613,20 @@ template SQLiteDiscreteBuildTablesAndPopulate() {              notes_prefix_a                   TEXT              NULL,              notes_prefix_b                   TEXT              NULL,              notes_suffix                     TEXT              NULL, -            src_filename                     VARCHAR(256)      NULL, +            src_filename                     VARCHAR(256)  NOT NULL,              src_fingerprint                  VARCHAR(256)      NULL,              src_filesize                     VARCHAR(10)       NULL,              src_wordcount                    VARCHAR(10)       NULL, +            pod_name                         VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +            pod_fingerprint                  VARCHAR(256)      NULL, /* zipped pod, work to be done here */ +            pod_size                         VARCHAR(10)       NULL, /* zipped pod, work to be done here */              src_text                         TEXT              NULL,              fulltext                         TEXT              NULL,              links                            TEXT              NULL            ); -          /* */            CREATE TABLE doc_objects ( -            lid                              BIGINT PRIMARY KEY, -            metadata_tid                     BIGINT REFERENCES metadata_and_text, +            lid                              INTEGER PRIMARY KEY, +            metadata_tid                     INTEGER REFERENCES metadata_and_text,              ocn                              SMALLINT,              ocnd                             VARCHAR(6),              ocns                             VARCHAR(6), @@ -585,36 +658,49 @@ template SQLiteDiscreteBuildTablesAndPopulate() {              digest_all                       CHAR(256),              types                            CHAR(1) NULL            ); -          COMMIT -        "); -        Statement insert_metadata = db.prepare(" +          CREATE INDEX idx_ocn ON doc_objects(ocn); +          CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); +          CREATE INDEX idx_digest_all ON doc_objects(digest_all); +          CREATE INDEX idx_clean ON doc_objects(clean); +          CREATE INDEX idx_lev0 ON doc_objects(lev0); +          CREATE INDEX idx_lev1 ON doc_objects(lev1); +          CREATE INDEX idx_lev2 ON doc_objects(lev2); +          CREATE INDEX idx_lev3 ON doc_objects(lev3); +          CREATE INDEX idx_lev4 ON doc_objects(lev4); +          CREATE INDEX idx_lev5 ON doc_objects(lev5); +          CREATE INDEX idx_lev6 ON doc_objects(lev6); +          CREATE INDEX idx_title ON metadata_and_text(title); +          CREATE INDEX idx_author ON metadata_and_text(creator_author); +          CREATE INDEX idx_filename ON metadata_and_text(src_filename); +          CREATE INDEX idx_language ON metadata_and_text(language_document_char); +          CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); +        ¶",); +        return _sql_instruct; +      } +    } +    template SQLiteDiscreteInsertMetadata() { +      string SQLiteDiscreteInsertMetadata(I)( +        auto ref       I    doc_matters, +      ) { +        string _insert_metadata; +        _insert_metadata = format(q"¶            INSERT INTO metadata_and_text ( -            tid, +            src_filename,              title,              title_main,              title_sub,              title_short,              title_edition, -            title_note,              title_language, -            title_language_char, +            classify_dewey, +            classify_keywords, +            classify_loc, +            classify_subject, +            classify_topic_register,              creator_author,              creator_author_email, -            creator_author_hon, -            creator_author_nationality, -            creator_editor, -            creator_contributor,              creator_illustrator, -            creator_photographer,              creator_translator, -            creator_prepared_by, -            creator_digitized_by, -            creator_audio, -            creator_video, -            language_document, -            language_document_char, -            language_original, -            language_original_char,              date_added_to_site,              date_available,              date_created, @@ -622,17 +708,18 @@ template SQLiteDiscreteBuildTablesAndPopulate() {              date_modified,              date_published,              date_valid, -            date_translated, -            date_original_publication, -            date_generated, -            publisher, +            identifier_isbn, +            identifier_oclc, +            language_document, +            language_document_char, +            notes_abstract, +            notes_description,              original_publisher,              original_language,              original_language_char,              original_source, -            original_institution, -            original_nationality,              original_title, +            publisher,              rights_copyright,              rights_copyright_audio,              rights_copyright_cover, @@ -641,205 +728,113 @@ template SQLiteDiscreteBuildTablesAndPopulate() {              rights_copyright_text,              rights_copyright_translation,              rights_copyright_video, -            rights_license, -            identifier_oclc, -            identifier_isbn, -            classify_topic_register, -            classify_subject, -            classify_loc, -            classify_dewey, -            classify_keywords, -            notes_abstract, -            notes_description, -            notes_comment, -            notes_coverage, -            notes_relation, -            notes_history, -            notes_type, -            notes_format, -            notes_prefix, -            notes_prefix_a, -            notes_prefix_b, -            notes_suffix, -            src_filename, -            src_fingerprint, -            src_filesize, -            src_wordcount, -            src_text, -            fulltext, -            links +            rights_license            )            VALUES ( -            :tid, -            :title, -            :title_main, -            :title_sub, -            :title_short, -            :title_edition, -            :title_note, -            :title_language, -            :title_language_char, -            :creator_author, -            :creator_author_email, -            :creator_author_hon, -            :creator_author_nationality, -            :creator_editor, -            :creator_contributor, -            :creator_illustrator, -            :creator_photographer, -            :creator_translator, -            :creator_prepared_by, -            :creator_digitized_by, -            :creator_audio, -            :creator_video, -            :language_document, -            :language_document_char, -            :language_original, -            :language_original_char, -            :date_added_to_site, -            :date_available, -            :date_created, -            :date_issued, -            :date_modified, -            :date_published, -            :date_valid, -            :date_translated, -            :date_original_publication, -            :date_generated, -            :publisher, -            :original_publisher, -            :original_language, -            :original_language_char, -            :original_source, -            :original_institution, -            :original_nationality, -            :original_title, -            :rights_copyright, -            :rights_copyright_audio, -            :rights_copyright_cover, -            :rights_copyright_illustrations, -            :rights_copyright_photographs, -            :rights_copyright_text, -            :rights_copyright_translation, -            :rights_copyright_video, -            :rights_license, -            :identifier_oclc, -            :identifier_isbn, -            :classify_topic_register, -            :classify_subject, -            :classify_loc, -            :classify_dewey, -            :classify_keywords, -            :notes_abstract, -            :notes_description, -            :notes_comment, -            :notes_coverage, -            :notes_relation, -            :notes_history, -            :notes_type, -            :notes_format, -            :notes_prefix, -            :notes_prefix_a, -            :notes_prefix_b, -            :notes_suffix, -            :src_filename, -            :src_fingerprint, -            :src_filesize, -            :src_wordcount, -            :src_text, -            :fulltext, -            :links -          ) -        "); -        // insert_metadata.bind(":tid",                               ); // unique identifier you could try use sha of title author & language? or auto increment?? -        insert_metadata.bind(":title",                             doc_matters.conf_make_meta.meta.title_full); -        insert_metadata.bind(":title_main",                        doc_matters.conf_make_meta.meta.title_main); -        insert_metadata.bind(":title_sub",                         doc_matters.conf_make_meta.meta.title_subtitle); -        insert_metadata.bind(":title_short",                       doc_matters.conf_make_meta.meta.title_short); // -        insert_metadata.bind(":title_edition",                     doc_matters.conf_make_meta.meta.title_edition); -        insert_metadata.bind(":title_language",                    doc_matters.conf_make_meta.meta.title_language); -        // insert_metadata.bind(":title_language_char",            doc_matters.conf_make_meta.meta.title_language_char); -        insert_metadata.bind(":classify_dewey",                    doc_matters.conf_make_meta.meta.classify_dewey); -        insert_metadata.bind(":classify_keywords",                 doc_matters.conf_make_meta.meta.classify_keywords); -        insert_metadata.bind(":classify_loc",                      doc_matters.conf_make_meta.meta.classify_loc); -        insert_metadata.bind(":classify_subject",                  doc_matters.conf_make_meta.meta.classify_subject); -        insert_metadata.bind(":classify_topic_register",           doc_matters.conf_make_meta.meta.classify_topic_register); -        insert_metadata.bind(":creator_author",                    doc_matters.conf_make_meta.meta.creator_author); -        insert_metadata.bind(":creator_author_email",              doc_matters.conf_make_meta.meta.creator_author_email); -        // insert_metadata.bind(":creator_editor",                 doc_matters.conf_make_meta.meta.creator_editor); -        // insert_metadata.bind(":creator_contributor",            doc_matters.conf_make_meta.meta.creator_contributor); -        insert_metadata.bind(":creator_illustrator",               doc_matters.conf_make_meta.meta.creator_illustrator); -        // insert_metadata.bind(":creator_photographer",           doc_matters.conf_make_meta.meta.creator_photographer); -        insert_metadata.bind(":creator_translator",                doc_matters.conf_make_meta.meta.creator_translator); -        // insert_metadata.bind(":creator_audio",                  doc_matters.conf_make_meta.meta.creator_audio); -        // insert_metadata.bind(":creator_video",                  doc_matters.conf_make_meta.meta.creator_video); -        insert_metadata.bind(":date_added_to_site",                doc_matters.conf_make_meta.meta.date_added_to_site); -        insert_metadata.bind(":date_available",                    doc_matters.conf_make_meta.meta.date_available); -        insert_metadata.bind(":date_created",                      doc_matters.conf_make_meta.meta.date_created); -        insert_metadata.bind(":date_issued",                       doc_matters.conf_make_meta.meta.date_issued); -        insert_metadata.bind(":date_modified",                     doc_matters.conf_make_meta.meta.date_modified); -        insert_metadata.bind(":date_published",                    doc_matters.conf_make_meta.meta.date_published); -        insert_metadata.bind(":date_valid",                        doc_matters.conf_make_meta.meta.date_valid); -        // insert_metadata.bind(":date_translated",                doc_matters.conf_make_meta.meta.date_translated); -        // insert_metadata.bind(":date_original_publication",      doc_matters.conf_make_meta.meta.date_original_publication); -        // insert_metadata.bind(":date_generated",                 doc_matters.conf_make_meta.meta.date_generated); -        insert_metadata.bind(":identifier_isbn",                   doc_matters.conf_make_meta.meta.identifier_isbn); -        insert_metadata.bind(":identifier_oclc",                   doc_matters.conf_make_meta.meta.identifier_oclc); -        insert_metadata.bind(":language_document",                 doc_matters.conf_make_meta.meta.language_document); -        insert_metadata.bind(":language_document_char",            doc_matters.conf_make_meta.meta.language_document_char); -        // insert_metadata.bind(":language_original",              doc_matters.conf_make_meta.meta.language_original); -        // insert_metadata.bind(":language_original_char",         doc_matters.conf_make_meta.meta.language_original_char); -        insert_metadata.bind(":notes_abstract",                    doc_matters.conf_make_meta.meta.notes_abstract); -        insert_metadata.bind(":notes_description",                 doc_matters.conf_make_meta.meta.notes_description); -        insert_metadata.bind(":original_publisher",                doc_matters.conf_make_meta.meta.original_publisher); -        insert_metadata.bind(":original_language",                 doc_matters.conf_make_meta.meta.original_language); -        insert_metadata.bind(":original_language_char",            doc_matters.conf_make_meta.meta.original_language_char); -        insert_metadata.bind(":original_source",                   doc_matters.conf_make_meta.meta.original_source); -        insert_metadata.bind(":original_title",                    doc_matters.conf_make_meta.meta.original_title); -        insert_metadata.bind(":publisher",                         doc_matters.conf_make_meta.meta.publisher); -        // insert_metadata.bind(":rights",                         doc_matters.conf_make_meta.meta.rights); -        insert_metadata.bind(":rights_copyright",                  doc_matters.conf_make_meta.meta.rights_copyright); -        insert_metadata.bind(":rights_copyright_audio",            doc_matters.conf_make_meta.meta.rights_copyright_audio); -        insert_metadata.bind(":rights_copyright_cover",            doc_matters.conf_make_meta.meta.rights_copyright_cover); -        insert_metadata.bind(":rights_copyright_illustrations",    doc_matters.conf_make_meta.meta.rights_copyright_illustrations); -        insert_metadata.bind(":rights_copyright_photographs",      doc_matters.conf_make_meta.meta.rights_copyright_photographs); -        // insert_metadata.bind(":rights_copyright_preparation",   doc_matters.conf_make_meta.meta.rights_preparation); -        insert_metadata.bind(":rights_copyright_text",      doc_matters.conf_make_meta.meta.rights_copyright_text); -        insert_metadata.bind(":rights_copyright_translation",      doc_matters.conf_make_meta.meta.rights_copyright_translation); -        insert_metadata.bind(":rights_copyright_video",            doc_matters.conf_make_meta.meta.rights_copyright_video); -        // insert_metadata.bind(":rights_copyright_digitization",  doc_matters.conf_make_meta.meta.rights_digitization); -        // insert_metadata.bind(":rights_copyright_audio",         doc_matters.conf_make_meta.meta.rights_audio); -        // insert_metadata.bind(":rights_copyright_video",         doc_matters.conf_make_meta.meta.rights_video); -        insert_metadata.bind(":rights_license",                    doc_matters.conf_make_meta.meta.rights_license); -        // insert_metadata.bind(":src_filename",                   doc_matters.conf_make_meta.meta.src_filename); -        // insert_metadata.bind(":src_fingerprint",                doc_matters.conf_make_meta.meta.src_fingerprint); -        // insert_metadata.bind(":src_filesize",                   doc_matters.conf_make_meta.meta.src_filesize); -        // insert_metadata.bind(":src_wordcount",                  doc_matters.conf_make_meta.meta.src_wordcount); -        // insert_metadata.bind(":src_text",                       doc_matters.conf_make_meta.meta.src_text); -        // insert_metadata.bind(":fulltext",                       doc_matters.conf_make_meta.meta.fulltext); -        // insert_metadata.bind(":links",                          doc_matters.conf_make_meta.meta.links); -        insert_metadata.execute(); insert_metadata.reset(); -        /+ watch +/ -        if ((doc_matters.opt.action.verbose)) { -          writeln("sql statement executed"); -        } -        assert(db.totalChanges == 1); -        // +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s', +            '%s' +          ); +        ¶", +          SQLinsertDelimiter!()(doc_matters.src.filename), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), +          SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) +        ); +        return _insert_metadata;        }      } -    template SQLiteObjectsLoop() { -      void SQLiteObjectsLoop(P)( -        auto ref P          doc_parts, +    template SQLiteDiscreteInsertDocObjectsLoop() { +      string SQLiteDiscreteInsertDocObjectsLoop(P,I)( +        auto ref       P    doc_parts, +        auto ref       I    tid,        ) {          string insertDocObjectsRow(O)(O obj) { -          auto sql_insert_delimiter(string _txt) { -            _txt = _txt -              .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); -            return _txt; -          }            string _insert_doc_objects_row; -         _insert_doc_objects_row = format(q"¶ +          _insert_doc_objects_row = format(q"¶              INSERT INTO doc_objects ( +              metadata_tid,                ocn,                clean,                body, @@ -848,6 +843,7 @@ template SQLiteDiscreteBuildTablesAndPopulate() {                t_is              )              VALUES ( +              %d,                %s,                '%s',                '%s', @@ -856,24 +852,20 @@ template SQLiteDiscreteBuildTablesAndPopulate() {                '%s'              );            ¶", +            1,              obj.ocn, -            sql_insert_delimiter(obj_txt["text"]), -            sql_insert_delimiter(obj_txt["html"]), +            SQLinsertDelimiter!()(obj_txt["text"]), +            SQLinsertDelimiter!()(obj_txt["html"]),              obj.heading_lev_markup,              obj.is_of,              obj.is_a,            );            return _insert_doc_objects_row;          } -        auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); -        pth_sqlite.base.mkdirRecurse; -        auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); -        SQLiteInstruct!()(db, doc_matters); // consider best location, need to feed individual objects for sqlite table: doc_objects, possibly a separate template?          auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);          string[string] obj_txt;          string doc_text;          string[] _insert_doc_objects; -        _insert_doc_objects ~= "BEGIN;\n";          foreach (part; doc_parts) {            foreach (obj; doc_abstraction[part]) {              switch (obj.of_part) { @@ -1023,24 +1015,9 @@ template SQLiteDiscreteBuildTablesAndPopulate() {              }            } // loop closes          } -        _insert_doc_objects ~= "COMMIT"; -        debug(sql_statement) { -          writeln("#+BEGIN_SRC sql\n", _insert_doc_objects.join, "\n#+END_SRC"); -        } -        std.utf.validate(_insert_doc_objects.join); // TODO -        db.run(_insert_doc_objects.join.to!(char[]).toUTF8); +        return _insert_doc_objects.join.to!(char[]).toUTF8;        }      } -    SQLiteObjectsLoop!()(doc_matters.xml.keys_seq.sql); -  } -} -template SQLiteDiscreteTablesCreate() { -  void SQLiteDiscreteTablesCreate()() { -    writeln("table create"); -  } -} -template SQLiteDiscreteTablesDrop() { -  void SQLiteDiscreteTablesDrop()() { -    writeln("table drop"); +    SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);    }  } diff --git a/src/sdp/sdp.d b/src/sdp/sdp.d index a50b0e9..e5d73c0 100755 --- a/src/sdp/sdp.d +++ b/src/sdp/sdp.d @@ -103,7 +103,7 @@ void main(string[] args) {      std.getopt.config.passThrough,      "assert",             "--assert set optional assertions on",                        &opts["assertions"],      "concordance",        "--concordance file for document",                            &opts["concordance"], -    "debug",              "--debug only relevant when debug options compiled in",       &opts["debug"], +    "debug",              "--debug",                                                    &opts["debug"],      "digest",             "--digest hash digest for each object",                       &opts["digest"],      "docbook",            "--docbook process docbook output",                           &opts["docbook"],      "epub",               "--epub process epub output",                                 &opts["epub"], @@ -341,7 +341,7 @@ void main(string[] args) {      debug(steps) {        writeln("step0 commence → (without processing files)");      } -    outputHubOp!()(_opt_action); +    outputHubOp!()(_env, _opt_action);      debug(steps) {        writeln("- step0 complete");      } | 
