From fc2471c91966186fc7bc0a5ac2217496c9e43eea Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 16 Jul 2018 22:39:09 -0400 Subject: 0.26.5 sqlite output, focus --- src/sdp/meta/metadoc.d | 8 ++ src/sdp/meta/metadoc_from_src.d | 9 ++ src/sdp/output/epub3.d | 6 +- src/sdp/output/html.d | 9 +- src/sdp/output/hub.d | 21 ++-- src/sdp/output/paths_output.d | 9 +- src/sdp/output/sqlite.d | 225 ++++++++++++++++++++----------------- src/sdp/sdp.d | 64 ++++++++--- src/sdp/source/paths_source.d | 13 +++ src/sdp/source/read_config_files.d | 21 ++-- src/sdp/source/read_source_files.d | 9 +- 11 files changed, 236 insertions(+), 158 deletions(-) (limited to 'src') diff --git a/src/sdp/meta/metadoc.d b/src/sdp/meta/metadoc.d index 29af49c..e7c107a 100644 --- a/src/sdp/meta/metadoc.d +++ b/src/sdp/meta/metadoc.d @@ -138,6 +138,14 @@ template SiSUabstraction() { auto pod() { return _manifest.pod; } + auto sqlite() { + struct SQLite_ { + string filename() { + return _opt_action.sqlite_filename; + } + } + return SQLite_(); + } auto xml() { struct XML_ { auto keys_seq() { diff --git a/src/sdp/meta/metadoc_from_src.d b/src/sdp/meta/metadoc_from_src.d index 050089e..661e106 100644 --- a/src/sdp/meta/metadoc_from_src.d +++ b/src/sdp/meta/metadoc_from_src.d @@ -1707,6 +1707,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1735,6 +1736,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1795,6 +1797,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1838,6 +1841,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1873,6 +1877,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1911,6 +1916,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1954,6 +1960,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -1993,6 +2000,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2036,6 +2044,7 @@ template SiSUdocAbstraction() { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); diff --git a/src/sdp/output/epub3.d b/src/sdp/output/epub3.d index cde42d1..d10b7f2 100644 --- a/src/sdp/output/epub3.d +++ b/src/sdp/output/epub3.d @@ -758,8 +758,7 @@ template outputEPub3() { zip.addMember(zip_arc_member_file); createZipFile!()(fn_epub, zip.build()); } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } writeln(" ", fn_epub); @@ -771,8 +770,7 @@ template outputEPub3() { auto data = zipped.expand(member); writeln(filename, " length ", data.length); } - } - catch (ZipException ex) { + } catch (ZipException ex) { // Handle errors } } diff --git a/src/sdp/output/html.d b/src/sdp/output/html.d index 7024b0f..ba53eef 100644 --- a/src/sdp/output/html.d +++ b/src/sdp/output/html.d @@ -174,8 +174,7 @@ template outputHTML() { foreach (o; doc) { f.writeln(o); } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } writeln(" ", pth_html.fn_scroll(doc_matters.src.filename)); @@ -431,8 +430,7 @@ template outputHTML() { } f.writeln(xhtml_format.tail); } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // handle error } writeln(" ", pth_html.fn_seg(doc_matters.src.filename, "toc")); @@ -450,8 +448,7 @@ template outputHTML() { f.writeln(css.html_seg_css); f = File(pth_html.fn_scroll_css, "w"); f.writeln(css.html_scroll_css); - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } } diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d index b6caca9..47713f5 100644 --- a/src/sdp/output/hub.d +++ b/src/sdp/output/hub.d @@ -124,7 +124,14 @@ template outputHub() { } SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); } - if (doc_matters.opt.action.sqlite_update) { + if (doc_matters.opt.action.sqlite_insert + || doc_matters.opt.action.sqlite_update + ) { + if ((doc_matters.opt.action.verbose)) { + writeln("sqlite processing... "); + } + SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + } else if (doc_matters.opt.action.sqlite_delete) { if ((doc_matters.opt.action.verbose)) { writeln("sqlite processing... "); } @@ -148,17 +155,17 @@ template outputHubOp() { void outputHubOp(E,O)(E env, O opt_action) { mixin SiSUoutputRgxInit; static auto rgx = Rgx(); - if ((opt_action.sqlite_create)) { + if ((opt_action.sqlite_db_drop)) { if ((opt_action.verbose)) { - writeln("sqlite create table... "); + writeln("sqlite drop db... "); } - SQLiteTablesCreate!()(env, opt_action); + SQLiteDbDrop!()(opt_action); } - if ((opt_action.sqlite_drop)) { + if ((opt_action.sqlite_db_create)) { if ((opt_action.verbose)) { - writeln("sqlite drop table... "); + writeln("sqlite create table... "); } - SQLiteTablesDrop!()(); + SQLiteTablesCreate!()(env, opt_action); } } } diff --git a/src/sdp/output/paths_output.d b/src/sdp/output/paths_output.d index 8a4d623..3b18c75 100644 --- a/src/sdp/output/paths_output.d +++ b/src/sdp/output/paths_output.d @@ -243,7 +243,7 @@ template SiSUpathsSQLiteDiscrete() { return asNormalizedPath(base.chainPath(base_filename(fn_src))).array; } string sqlite_file(string fn_src) { - return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array; + return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sql.db")).array; } } return _PathsStruct(); @@ -252,7 +252,8 @@ template SiSUpathsSQLiteDiscrete() { template SiSUpathsSQLite() { mixin SiSUrgxInit; static auto rgx = Rgx(); - auto SiSUpathsSQLite(Po)( + auto SiSUpathsSQLite(DbN, Po)( + DbN db_name, Po output_pth_root, ) { struct _PathsStruct { @@ -264,8 +265,8 @@ template SiSUpathsSQLite() { string base_dir = "sqlite"; return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array; } - string sqlite_file(string fn_src = "sdp-niu") { - return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array; + string sqlite_file() { + return asNormalizedPath(base.chainPath(base_filename(db_name) ~ ".sql.db")).array; } } return _PathsStruct(); diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d index faff29b..41d504e 100644 --- a/src/sdp/output/sqlite.d +++ b/src/sdp/output/sqlite.d @@ -10,38 +10,44 @@ static auto rgx = Rgx(); long _metadata_tid_lastrowid; template SQLiteHubBuildTablesAndPopulate() { void SQLiteHubBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, - auto ref I doc_matters, + D doc_abstraction, + I doc_matters, ) { - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path); pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); + auto db = Database(pth_sqlite.sqlite_file); template SQLiteDbStatementComposite() { void SQLiteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, + Db db, + D doc_abstraction, + I doc_matters, ) { { string _db_statement; { - if ((doc_matters.opt.action.sqlite_create)) { - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + if ((doc_matters.opt.action.sqlite_db_create)) { + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path); pth_sqlite.base.mkdirRecurse; _db_statement ~= SQLiteTablesReCreate!()(); SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); } - if ((doc_matters.opt.action.sqlite_update)) { // TODO + if (doc_matters.opt.action.sqlite_delete) { + _db_statement ~= SQLiteDeleteDocument!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document"); + } + if (doc_matters.opt.action.sqlite_insert || doc_matters.opt.action.sqlite_update) { // sqlite_update is .opt.action sqlite_delete & sqlite_insert (set in sdp) + _db_statement ~= SQLiteDeleteDocument!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document"); _db_statement ~= SQLiteInsertMetadata!()(doc_matters); - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT MetaData"); /+ get tid (lastrowid or max) for use in doc_objects table +/ - _metadata_tid_lastrowid = db.lastInsertRowid(); - _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT DocObjects"); } } db.close; } + writeln(" ", pth_sqlite.sqlite_file); } } SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); @@ -49,28 +55,29 @@ template SQLiteHubBuildTablesAndPopulate() { } template SQLiteHubDiscreteBuildTablesAndPopulate() { void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, - auto ref I doc_matters, + D doc_abstraction, + I doc_matters, ) { auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); pth_sqlite.base.mkdirRecurse; auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); template SQLiteDiscreteDbStatementComposite() { void SQLiteDiscreteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, + Db db, + D doc_abstraction, + I doc_matters, ) { { string _db_statement; { _db_statement ~= SQLiteTablesReCreate!()(); _db_statement ~= SQLiteInsertMetadata!()(doc_matters); - _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters); SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); } db.close; } + writeln(" ", pth_sqlite.sqlite_file(doc_matters.src.filename)); } } SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); @@ -78,10 +85,10 @@ template SQLiteHubDiscreteBuildTablesAndPopulate() { } template SQLiteDbRun() { void SQLiteDbRun(Db,St,O)( - auto ref Db db, - auto ref St db_statement, - auto ref O opt_action, - string note, + Db db, + St db_statement, + O opt_action, + string note, ) { debug(sql_statement) { writeln(db_statement); @@ -93,10 +100,8 @@ template SQLiteDbRun() { "\nCOMMIT;\n" ); if (!(opt_action.sqlite_discrete) - && !(opt_action.sqlite_create) + && !(opt_action.sqlite_db_create) ) { - _metadata_tid_lastrowid = db.lastInsertRowid(); - writeln("last row id: ", _metadata_tid_lastrowid); } } catch (ErrnoException ex) { writeln("ERROR SQLite : ", ex); @@ -120,7 +125,7 @@ template SQLinsertDelimiter() { } template SQLiteFormatAndLoadObject() { auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, + I doc_matters, ) { mixin SiSUoutputRgxInit; struct sqlite_format_and_load_objects { @@ -569,29 +574,23 @@ template SQLiteTablesReCreate() { string SQLiteTablesReCreate()() { string _sql_instruct; _sql_instruct = format(q"¶ - DROP INDEX IF EXISTS ocn; - DROP INDEX IF EXISTS digest_clean; - DROP INDEX IF EXISTS digest_all; - DROP INDEX IF EXISTS clean; - DROP INDEX IF EXISTS lev0; - DROP INDEX IF EXISTS lev1; - DROP INDEX IF EXISTS lev2; - DROP INDEX IF EXISTS lev3; - DROP INDEX IF EXISTS lev4; - DROP INDEX IF EXISTS lev5; - DROP INDEX IF EXISTS lev6; - DROP INDEX IF EXISTS title; - DROP INDEX IF EXISTS creator_author; + DROP INDEX IF EXISTS idx_ocn; + DROP INDEX IF EXISTS idx_uid; + DROP INDEX IF EXISTS idx_digest_clean; + DROP INDEX IF EXISTS idx_digest_all; + DROP INDEX IF EXISTS idx_clean; + DROP INDEX IF EXISTS idx_title; + DROP INDEX IF EXISTS idx_creator_author; DROP INDEX IF EXISTS src_filename; - DROP INDEX IF EXISTS language_document_char; - DROP INDEX IF EXISTS classify_topic_register; + DROP INDEX IF EXISTS idx_language_document_char; + DROP INDEX IF EXISTS idx_classify_topic_register; DROP TABLE IF EXISTS metadata_and_text; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE metadata_and_text ( - tid INTEGER PRIMARY KEY, - src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ - src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */ + src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */ title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, @@ -675,24 +674,24 @@ template SQLiteTablesReCreate() { links TEXT NULL ); CREATE TABLE doc_objects ( - lid INTEGER PRIMARY KEY, - metadata_tid INTEGER REFERENCES metadata_and_text, + lid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, ocn SMALLINT, ocnd VARCHAR(6), ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - seg VARCHAR(256) NULL, + clean TEXT NULL, + body TEXT NULL, + seg VARCHAR(256) NULL, lev_an VARCHAR(1), - lev SMALLINT NULL, + lev SMALLINT NULL, is_of_type VARCHAR(16), is_a VARCHAR(16), - node VARCHAR(16) NULL, - parent VARCHAR(16) NULL, - last_decendant VARCHAR(16) NULL, /* headings only */ + node VARCHAR(16) NULL, + parent VARCHAR(16) NULL, + last_decendant VARCHAR(16) NULL, /* headings only */ digest_clean CHAR(256), digest_all CHAR(256), - types CHAR(1) NULL + types CHAR(1) NULL ); CREATE INDEX idx_ocn ON doc_objects(ocn); CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); @@ -700,6 +699,7 @@ template SQLiteTablesReCreate() { CREATE INDEX idx_clean ON doc_objects(clean); CREATE INDEX idx_title ON metadata_and_text(title); CREATE INDEX idx_author ON metadata_and_text(creator_author); + CREATE INDEX idx_uid ON metadata_and_text(uid); CREATE INDEX idx_filename ON metadata_and_text(src_filename); CREATE INDEX idx_language ON metadata_and_text(language_document_char); CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); @@ -707,13 +707,31 @@ template SQLiteTablesReCreate() { return _sql_instruct; } } +template SQLiteDeleteDocument() { + string SQLiteDeleteDocument(I)( + I doc_matters, + ) { + string _uid = doc_matters.src.doc_uid; + string _delete_uid = format(q"¶ + DELETE FROM metadata_and_text + WHERE uid = '%s'; + DELETE FROM doc_objects + WHERE uid_metadata_and_text = '%s'; + ¶", + _uid, + _uid, + ); + return _delete_uid; + } +} template SQLiteInsertMetadata() { string SQLiteInsertMetadata(I)( - auto ref I doc_matters, + I doc_matters, ) { - string _insert_metadata; - _insert_metadata = format(q"¶ + string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); + string _insert_metadata = format(q"¶ INSERT INTO metadata_and_text ( + uid, src_filename, src_composite_id_per_txt, src_composite_id_per_pod, @@ -807,9 +825,11 @@ template SQLiteInsertMetadata() { '%s', '%s', '%s', + '%s', '%s' ); ¶", + _uid, SQLinsertDelimiter!()(doc_matters.src.filename), SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), @@ -861,16 +881,15 @@ template SQLiteInsertMetadata() { } } template SQLiteInsertDocObjectsLoop() { - string SQLiteInsertDocObjectsLoop(D,I,X)( - auto ref const D doc_abstraction, - auto ref I doc_matters, - auto ref X _metadata_tid, + string SQLiteInsertDocObjectsLoop(D,I)( + D doc_abstraction, + I doc_matters, ) { + string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); string insertDocObjectsRow(O)(O obj) { - string _insert_doc_objects_row; - _insert_doc_objects_row = format(q"¶ + string _insert_doc_objects_row = format(q"¶ INSERT INTO doc_objects ( - metadata_tid, + uid_metadata_and_text, ocn, clean, body, @@ -879,7 +898,7 @@ template SQLiteInsertDocObjectsLoop() { is_a ) VALUES ( - %d, + '%s', %s, '%s', '%s', @@ -888,7 +907,7 @@ template SQLiteInsertDocObjectsLoop() { '%s' ); ¶", - _metadata_tid, + _uid, obj.metainfo.ocn, SQLinsertDelimiter!()(obj_txt["text"]), SQLinsertDelimiter!()(obj_txt["html"]), @@ -1058,29 +1077,23 @@ template SQLiteTablesCreate() { string SQLiteTablesReCreate()() { string _sql_instruct; _sql_instruct = format(q"¶ - DROP INDEX IF EXISTS ocn; - DROP INDEX IF EXISTS digest_clean; - DROP INDEX IF EXISTS digest_all; - DROP INDEX IF EXISTS clean; - DROP INDEX IF EXISTS lev0; - DROP INDEX IF EXISTS lev1; - DROP INDEX IF EXISTS lev2; - DROP INDEX IF EXISTS lev3; - DROP INDEX IF EXISTS lev4; - DROP INDEX IF EXISTS lev5; - DROP INDEX IF EXISTS lev6; - DROP INDEX IF EXISTS title; - DROP INDEX IF EXISTS creator_author; + DROP INDEX IF EXISTS idx_ocn; + DROP INDEX IF EXISTS idx_uid; + DROP INDEX IF EXISTS idx_digest_clean; + DROP INDEX IF EXISTS idx_digest_all; + DROP INDEX IF EXISTS idx_clean; + DROP INDEX IF EXISTS idx_title; + DROP INDEX IF EXISTS idx_creator_author; DROP INDEX IF EXISTS src_filename; - DROP INDEX IF EXISTS language_document_char; - DROP INDEX IF EXISTS classify_topic_register; + DROP INDEX IF EXISTS idx_language_document_char; + DROP INDEX IF EXISTS idx_classify_topic_register; DROP TABLE IF EXISTS metadata_and_text; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE metadata_and_text ( - tid INTEGER PRIMARY KEY, - src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ - src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */ + src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */ title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, @@ -1164,24 +1177,24 @@ template SQLiteTablesCreate() { links TEXT NULL ); CREATE TABLE doc_objects ( - lid INTEGER PRIMARY KEY, - metadata_tid INTEGER REFERENCES metadata_and_text, + lid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, ocn SMALLINT, ocnd VARCHAR(6), ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - seg VARCHAR(256) NULL, + clean TEXT NULL, + body TEXT NULL, + seg VARCHAR(256) NULL, lev_an VARCHAR(1), - lev SMALLINT NULL, + lev SMALLINT NULL, is_of_type VARCHAR(16), is_a VARCHAR(16), - node VARCHAR(16) NULL, - parent VARCHAR(16) NULL, - last_decendant VARCHAR(16) NULL, /* headings only */ + node VARCHAR(16) NULL, + parent VARCHAR(16) NULL, + last_decendant VARCHAR(16) NULL, /* headings only */ digest_clean CHAR(256), digest_all CHAR(256), - types CHAR(1) NULL + types CHAR(1) NULL ); CREATE INDEX idx_ocn ON doc_objects(ocn); CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); @@ -1189,6 +1202,7 @@ template SQLiteTablesCreate() { CREATE INDEX idx_clean ON doc_objects(clean); CREATE INDEX idx_title ON metadata_and_text(title); CREATE INDEX idx_author ON metadata_and_text(creator_author); + CREATE INDEX idx_uid ON metadata_and_text(uid); CREATE INDEX idx_filename ON metadata_and_text(src_filename); CREATE INDEX idx_language ON metadata_and_text(language_document_char); CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); @@ -1196,9 +1210,9 @@ template SQLiteTablesCreate() { return _sql_instruct; } } - if (opt_action.sqlite_create) { + if (opt_action.sqlite_db_create) { string _db_statement; - auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE + auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE pth_sqlite.base.mkdirRecurse; auto db = Database(pth_sqlite.sqlite_file); // ISSUE { @@ -1208,8 +1222,17 @@ template SQLiteTablesCreate() { } } } -template SQLiteTablesDrop() { - void SQLiteTablesDrop()() { - writeln("table drop"); +template SQLiteDbDrop() { + void SQLiteDbDrop(O)(O opt_action) { + writeln("db drop"); + if ((opt_action.sqlite_db_drop)) { + auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE + writeln("remove(", pth_sqlite.sqlite_file, ")"); + try { + remove(pth_sqlite.sqlite_file); + } catch (FileException ex) { + // handle error + } + } } } diff --git a/src/sdp/sdp.d b/src/sdp/sdp.d index b703838..50df3f2 100755 --- a/src/sdp/sdp.d +++ b/src/sdp/sdp.d @@ -76,9 +76,12 @@ void main(string[] args) { "sisupod" : false, "source" : false, "sqlite-discrete" : false, + "sqlite-db-create" : false, + "sqlite-db-drop" : false, + "sqlite-db-recreate" : false, + "sqlite-delete" : false, + "sqlite-insert" : false, "sqlite-update" : false, - "sqlite-create" : false, - "sqlite-drop" : false, "text" : false, "verbose" : false, "xhtml" : false, @@ -98,6 +101,7 @@ void main(string[] args) { "output-dir" : "", "site-config-dir" : "", "lang" : "all", + "sqlite-filename" : "documents", ]; auto helpInfo = getopt(args, std.getopt.config.passThrough, @@ -122,8 +126,11 @@ void main(string[] args) { "sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"], "source", "--source markup source text content", &opts["source"], "sqlite-discrete", "--sqlite process discrete sqlite output", &opts["sqlite-discrete"], - "sqlite-create", "--sqlite-create create db, create tables", &opts["sqlite-create"], - "sqlite-drop", "--sqlite-drop drop tables & db", &opts["sqlite-drop"], + "sqlite-db-create", "--sqlite-db-create create db, create tables", &opts["sqlite-db-create"], + "sqlite-db-drop", "--sqlite-db-drop drop tables & db", &opts["sqlite-db-drop"], + "sqlite-db-recreate", "--sqlite-db-recreate create db, create tables", &opts["sqlite-db-recreate"], + "sqlite-delete", "--sqlite process sqlite output", &opts["sqlite-delete"], + "sqlite-insert", "--sqlite process sqlite output", &opts["sqlite-insert"], "sqlite-update", "--sqlite process sqlite output", &opts["sqlite-update"], "text", "--text process text output", &opts["text"], "txt", "--txt process text output", &opts["text"], @@ -142,6 +149,7 @@ void main(string[] args) { "skip-output", "--skip-output", &opts["skip-output"], "output-dir", "--output-dir=[dir path]", &settings["output-dir"], "site-config-dir", "--site-config-dir=[dir path]", &settings["site-config-dir"], + "sqlite-filename", "--sqlite-filename=[filename].sqlite", &settings["sqlite-filename"], "lang", "--lang=[lang code e.g. =en or =en,es]", &settings["lang"], ); if (helpInfo.helpWanted) { @@ -224,14 +232,34 @@ void main(string[] args) { auto sqlite_discrete() { return opts["sqlite-discrete"]; } - auto sqlite_update() { - return opts["sqlite-update"]; + auto sqlite_db_drop() { + bool _is; + if ( + opts["sqlite-db-recreate"] + || opts["sqlite-db-drop"] + ) { + _is = true; + } else { _is = false; } + return _is; } - auto sqlite_create() { - return opts["sqlite-create"]; + auto sqlite_db_create() { + bool _is; + if ( + opts["sqlite-db-recreate"] + || opts["sqlite-db-create"] + ) { + _is = true; + } else { _is = false; } + return _is; + } + auto sqlite_insert() { + return opts["sqlite-insert"]; + } + auto sqlite_delete() { + return opts["sqlite-delete"]; } - auto sqlite_drop() { - return opts["sqlite-drop"]; + auto sqlite_update() { + return opts["sqlite-update"]; } auto text() { return opts["text"]; @@ -281,6 +309,9 @@ void main(string[] args) { auto output_dir_set() { return settings["output-dir"]; } + auto sqlite_filename() { + return settings["sqlite-filename"]; + } auto abstraction() { bool _is; if ( @@ -297,6 +328,8 @@ void main(string[] args) { || sisupod || source || sqlite_discrete + || sqlite_delete + || sqlite_insert || sqlite_update || text || xhtml @@ -336,10 +369,8 @@ void main(string[] args) { if (exists(sisudoc_txt_)) { contents_location_raw_ = sisudoc_txt_.readText; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { // Handle errors } if (contents_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+ @@ -369,14 +400,15 @@ void main(string[] args) { ) { auto _fns = (((tmp_dir_).chainPath(contents_location_pth_)).array).to!string; _manifest_matter = PathMatters!()(_opt_action, _env, arg, _fns, contents_locations_arr); - _manifests ~= _manifest_matter; // TODO how to capture? + _manifests ~= _manifest_matter; } } } else if (arg.match(rgx.src_pth_sst_or_ssm)) { if (exists(arg)==0) { writeln("ERROR >> Processing Skipped! File not found: ", arg); } else { - _manifests ~= PathMatters!()(_opt_action, _env, arg, arg); // gather input markup source file names for processing + _manifest_matter = PathMatters!()(_opt_action, _env, arg, arg); + _manifests ~= _manifest_matter; } } else if (arg.match(rgx.src_pth_zip)) { // fns_src ~= arg; // gather input markup source file names for processing diff --git a/src/sdp/source/paths_source.d b/src/sdp/source/paths_source.d index 8d3aefe..5637c80 100644 --- a/src/sdp/source/paths_source.d +++ b/src/sdp/source/paths_source.d @@ -180,6 +180,19 @@ template PathMatters() { } else {_k = "en"; } return _k; } + string doc_uid() { + string _uid; + if (is_pod && !(pod_name.empty)) { + if (pod_name.baseName == filename_base) { + _uid = filename_base ~ "." ~ filename_extension ~ _sep ~ lng; + } else { + _uid = pod_name.baseName ~ _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng; + } + } else { + _uid = _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng; + } + return _uid; + } string docname_composite_unique_per_src_doc() { /+ z pod name if any + src filename + lng code diff --git a/src/sdp/source/read_config_files.d b/src/sdp/source/read_config_files.d index 9d976a5..56ba065 100644 --- a/src/sdp/source/read_config_files.d +++ b/src/sdp/source/read_config_files.d @@ -32,10 +32,8 @@ static template configReadInSiteTOML() { config_file_str = conf_file.readText; break; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { } } return config_file_str; @@ -69,10 +67,8 @@ static template configReadInDocTOML() { config_file_str = conf_file.readText; break; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { } } return config_file_str; @@ -89,8 +85,7 @@ static template configTOML() { TOMLDocument _toml_conf; try { _toml_conf = parseTOML(configuration); // parseTOML(cast(string)(configuration)); - } - catch(ErrnoException e) { + } catch(ErrnoException e) { stderr.writeln("Toml problem with content for ", conf_toml_filename); stderr.writeln(e.msg); } @@ -172,10 +167,8 @@ static template readConfigDoc() { config_file_str = conf_file.readText; break; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { } } if (config_file_str.length > 0) { break; } diff --git a/src/sdp/source/read_source_files.d b/src/sdp/source/read_source_files.d index 28a660a..271adfa 100644 --- a/src/sdp/source/read_source_files.d +++ b/src/sdp/source/read_source_files.d @@ -85,13 +85,10 @@ static template SiSUrawMarkupContent() { } source_txt_str = fn_src.readText; } - } - catch (ErrnoException ex) { - } - catch (UTFException ex) { + } catch (ErrnoException ex) { + } catch (UTFException ex) { // Handle validation errors - } - catch (FileException ex) { + } catch (FileException ex) { // Handle errors } std.utf.validate(source_txt_str); -- cgit v1.2.3