diff options
Diffstat (limited to 'org')
| -rw-r--r-- | org/default_paths.org | 22 | ||||
| -rw-r--r-- | org/meta_abstraction.org | 9 | ||||
| -rw-r--r-- | org/output_hub.org | 30 | ||||
| -rw-r--r-- | org/output_sqlite.org | 264 | ||||
| -rw-r--r-- | org/output_xmls.org | 15 | ||||
| -rw-r--r-- | org/sdp.org | 100 | ||||
| -rw-r--r-- | org/source_files_read.org | 30 |
7 files changed, 289 insertions, 181 deletions
diff --git a/org/default_paths.org b/org/default_paths.org index 5b7141a..35dee32 100644 --- a/org/default_paths.org +++ b/org/default_paths.org @@ -233,6 +233,19 @@ template PathMatters() { } else {_k = "en"; } return _k; } + string doc_uid() { + string _uid; + if (is_pod && !(pod_name.empty)) { + if (pod_name.baseName == filename_base) { + _uid = filename_base ~ "." ~ filename_extension ~ _sep ~ lng; + } else { + _uid = pod_name.baseName ~ _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng; + } + } else { + _uid = _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng; + } + return _uid; + } string docname_composite_unique_per_src_doc() { /+ z pod name if any + src filename + lng code @@ -1181,7 +1194,7 @@ template SiSUpathsSQLiteDiscrete() { return asNormalizedPath(base.chainPath(base_filename(fn_src))).array; } string sqlite_file(string fn_src) { - return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array; + return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sql.db")).array; } } return _PathsStruct(); @@ -1196,7 +1209,8 @@ template SiSUpathsSQLiteDiscrete() { template SiSUpathsSQLite() { mixin SiSUrgxInit; static auto rgx = Rgx(); - auto SiSUpathsSQLite(Po)( + auto SiSUpathsSQLite(DbN, Po)( + DbN db_name, Po output_pth_root, ) { struct _PathsStruct { @@ -1208,8 +1222,8 @@ template SiSUpathsSQLite() { string base_dir = "sqlite"; return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array; } - string sqlite_file(string fn_src = "sdp-niu") { - return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array; + string sqlite_file() { + return asNormalizedPath(base.chainPath(base_filename(db_name) ~ ".sql.db")).array; } } return _PathsStruct(); diff --git a/org/meta_abstraction.org b/org/meta_abstraction.org index 00b87a3..010f8b5 100644 --- a/org/meta_abstraction.org +++ b/org/meta_abstraction.org @@ -2268,6 +2268,7 @@ foreach (ref obj; the_document_head_section) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2302,6 +2303,7 @@ if (the_table_of_contents_section["scroll"].length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2368,6 +2370,7 @@ if (the_document_body_section.length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2419,6 +2422,7 @@ if (the_endnotes_section.length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2462,6 +2466,7 @@ if (the_glossary_section.length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2508,6 +2513,7 @@ if (the_bibliography_section.length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2559,6 +2565,7 @@ if (the_bookindex_section["scroll"].length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2598,6 +2605,7 @@ if (the_bookindex_section["scroll"].length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); @@ -2647,6 +2655,7 @@ if (the_blurb_section.length > 1) { || (opt_action.html_seg) || (opt_action.epub) || (opt_action.sqlite_discrete) + || (opt_action.sqlite_insert) || (opt_action.sqlite_update) || (opt_action.postgresql)) { obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup); diff --git a/org/output_hub.org b/org/output_hub.org index b90d637..2f905e1 100644 --- a/org/output_hub.org +++ b/org/output_hub.org @@ -227,7 +227,14 @@ if (doc_matters.opt.action.sqlite_discrete) { #+name: output_options #+BEGIN_SRC d -if (doc_matters.opt.action.sqlite_update) { +if (doc_matters.opt.action.sqlite_insert + || doc_matters.opt.action.sqlite_update +) { + if ((doc_matters.opt.action.verbose)) { + writeln("sqlite processing... "); + } + SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters); +} else if (doc_matters.opt.action.sqlite_delete) { if ((doc_matters.opt.action.verbose)) { writeln("sqlite processing... "); } @@ -235,21 +242,28 @@ if (doc_matters.opt.action.sqlite_update) { } #+END_SRC -***** create, drop +***** no markup source files to process +****** drop #+name: output_options_op #+BEGIN_SRC d -if ((opt_action.sqlite_create)) { +if ((opt_action.sqlite_db_drop)) { if ((opt_action.verbose)) { - writeln("sqlite create table... "); + writeln("sqlite drop db... "); } - SQLiteTablesCreate!()(env, opt_action); + SQLiteDbDrop!()(opt_action); } -if ((opt_action.sqlite_drop)) { +#+END_SRC + +****** create + +#+name: output_options_op +#+BEGIN_SRC d +if ((opt_action.sqlite_db_create)) { if ((opt_action.verbose)) { - writeln("sqlite drop table... "); + writeln("sqlite create table... "); } - SQLiteTablesDrop!()(); + SQLiteTablesCreate!()(env, opt_action); } #+END_SRC diff --git a/org/output_sqlite.org b/org/output_sqlite.org index dc76da4..eeff4f1 100644 --- a/org/output_sqlite.org +++ b/org/output_sqlite.org @@ -33,19 +33,20 @@ static auto rgx = Rgx(); long _metadata_tid_lastrowid; template SQLiteHubBuildTablesAndPopulate() { void SQLiteHubBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, - auto ref I doc_matters, + D doc_abstraction, + I doc_matters, ) { - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path); pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); + auto db = Database(pth_sqlite.sqlite_file); template SQLiteDbStatementComposite() { void SQLiteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, + Db db, + D doc_abstraction, + I doc_matters, ) { <<sqlite_db_statement_composite_collection>> + writeln(" ", pth_sqlite.sqlite_file); } } SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); @@ -58,19 +59,20 @@ template SQLiteHubBuildTablesAndPopulate() { #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteHubDiscreteBuildTablesAndPopulate() { void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, - auto ref I doc_matters, + D doc_abstraction, + I doc_matters, ) { auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); pth_sqlite.base.mkdirRecurse; auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); template SQLiteDiscreteDbStatementComposite() { void SQLiteDiscreteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, + Db db, + D doc_abstraction, + I doc_matters, ) { <<sqlite_db_statement_composite_discrete>> + writeln(" ", pth_sqlite.sqlite_file(doc_matters.src.filename)); } } SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); @@ -83,10 +85,10 @@ template SQLiteHubDiscreteBuildTablesAndPopulate() { #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteDbRun() { void SQLiteDbRun(Db,St,O)( - auto ref Db db, - auto ref St db_statement, - auto ref O opt_action, - string note, + Db db, + St db_statement, + O opt_action, + string note, ) { debug(sql_statement) { writeln(db_statement); @@ -98,10 +100,8 @@ template SQLiteDbRun() { "\nCOMMIT;\n" ); if (!(opt_action.sqlite_discrete) - && !(opt_action.sqlite_create) + && !(opt_action.sqlite_db_create) ) { - _metadata_tid_lastrowid = db.lastInsertRowid(); - writeln("last row id: ", _metadata_tid_lastrowid); } } catch (ErrnoException ex) { writeln("ERROR SQLite : ", ex); @@ -130,7 +130,7 @@ template SQLinsertDelimiter() { } template SQLiteFormatAndLoadObject() { auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, + I doc_matters, ) { mixin SiSUoutputRgxInit; struct sqlite_format_and_load_objects { @@ -146,6 +146,7 @@ template SQLiteFormatAndLoadObject() { #+END_SRC *** sqlite instructions +**** create #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteTablesReCreate() { @@ -161,20 +162,43 @@ template SQLiteTablesReCreate() { return _sql_instruct; } } +#+END_SRC + +**** delete + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLiteDeleteDocument() { + string SQLiteDeleteDocument(I)( + I doc_matters, + ) { + <<sqlite_formatted_delete>> + return _delete_uid; + } +} +#+END_SRC + +**** insert + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteInsertMetadata() { string SQLiteInsertMetadata(I)( - auto ref I doc_matters, + I doc_matters, ) { <<sqlite_formatted_insertions_doc_matters_metadata>> return _insert_metadata; } } +#+END_SRC + +**** insert doc objects loop + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteInsertDocObjectsLoop() { - string SQLiteInsertDocObjectsLoop(D,I,X)( - auto ref const D doc_abstraction, - auto ref I doc_matters, - auto ref X _metadata_tid, + string SQLiteInsertDocObjectsLoop(D,I)( + D doc_abstraction, + I doc_matters, ) { + string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); string insertDocObjectsRow(O)(O obj) { <<sqlite_formatted_insertions_doc_objects>> return _insert_doc_objects_row; @@ -182,6 +206,11 @@ template SQLiteInsertDocObjectsLoop() { <<sqlite_objects_loop>> } } +#+END_SRC + +**** tables create + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteTablesCreate() { void SQLiteTablesCreate(E,O)(E env, O opt_action) { import d2sqlite3; @@ -198,9 +227,9 @@ template SQLiteTablesCreate() { return _sql_instruct; } } - if (opt_action.sqlite_create) { + if (opt_action.sqlite_db_create) { string _db_statement; - auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE + auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE pth_sqlite.base.mkdirRecurse; auto db = Database(pth_sqlite.sqlite_file); // ISSUE { @@ -210,9 +239,23 @@ template SQLiteTablesCreate() { } } } -template SQLiteTablesDrop() { - void SQLiteTablesDrop()() { - writeln("table drop"); +#+END_SRC + +**** tables drop + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLiteDbDrop() { + void SQLiteDbDrop(O)(O opt_action) { + writeln("db drop"); + if ((opt_action.sqlite_db_drop)) { + auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE + writeln("remove(", pth_sqlite.sqlite_file, ")"); + try { + remove(pth_sqlite.sqlite_file); + } catch (FileException ex) { + // handle error + } + } } } #+END_SRC @@ -225,19 +268,24 @@ template SQLiteTablesDrop() { { string _db_statement; { - if ((doc_matters.opt.action.sqlite_create)) { - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + if ((doc_matters.opt.action.sqlite_db_create)) { + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path); pth_sqlite.base.mkdirRecurse; _db_statement ~= SQLiteTablesReCreate!()(); SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); } - if ((doc_matters.opt.action.sqlite_update)) { // TODO + if (doc_matters.opt.action.sqlite_delete) { + _db_statement ~= SQLiteDeleteDocument!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document"); + } + if (doc_matters.opt.action.sqlite_insert || doc_matters.opt.action.sqlite_update) { // sqlite_update is .opt.action sqlite_delete & sqlite_insert (set in sdp) + _db_statement ~= SQLiteDeleteDocument!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document"); _db_statement ~= SQLiteInsertMetadata!()(doc_matters); - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT MetaData"); /+ get tid (lastrowid or max) for use in doc_objects table +/ - _metadata_tid_lastrowid = db.lastInsertRowid(); - _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT DocObjects"); } } db.close; @@ -253,7 +301,7 @@ template SQLiteTablesDrop() { { _db_statement ~= SQLiteTablesReCreate!()(); _db_statement ~= SQLiteInsertMetadata!()(doc_matters); - _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters); SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); } db.close; @@ -646,44 +694,7 @@ auto html_table(O)( } #+END_SRC -*** 2. _sqlite_ (loop preparation & pre-loop action) -**** prepare insert statements and do pre-loop inserts -***** 1. _metadata & src_ text -****** TODO sql remove selected -******* d { sql statement - -##+name: sqlite_remove_selected -#+BEGIN_SRC d -Statement insert_metadata = db.prepare(" -#+END_SRC - -******** remove selected - -identify and remove, identify by -- filename & language code -- (in ruby get_first_value and remove by fn & ln) - -##+name: sqlite_remove_selected -#+BEGIN_SRC sql - SELECT tid - FROM metadata_and_text - WHERE src_filename = '#{doc_matters.conf_make_meta.meta.src_filename}' - AND metadata_and_text.language_document_char = '#{doc_matters.conf_make_meta.meta.language_document_char}' -#+END_SRC - -******* d } - -##+name: sqlite_remove_selected -#+BEGIN_SRC d -"); -#+END_SRC - -***** 2. _doc objects_ (used with doc_objects in document loop) -****** TODO (within loop not here - insert doc objects - -work out - -*** 3. hub (sqlite_format_and_load_objects) +*** 2. hub (sqlite_format_and_load_objects) **** sql related #+name: sqlite_load_object @@ -1088,22 +1099,16 @@ return _insert_doc_objects.join.to!(char[]).toUTF8; #+name: sqlite_statement_drop_existing_index #+BEGIN_SRC sql -DROP INDEX IF EXISTS ocn; -DROP INDEX IF EXISTS digest_clean; -DROP INDEX IF EXISTS digest_all; -DROP INDEX IF EXISTS clean; -DROP INDEX IF EXISTS lev0; -DROP INDEX IF EXISTS lev1; -DROP INDEX IF EXISTS lev2; -DROP INDEX IF EXISTS lev3; -DROP INDEX IF EXISTS lev4; -DROP INDEX IF EXISTS lev5; -DROP INDEX IF EXISTS lev6; -DROP INDEX IF EXISTS title; -DROP INDEX IF EXISTS creator_author; +DROP INDEX IF EXISTS idx_ocn; +DROP INDEX IF EXISTS idx_uid; +DROP INDEX IF EXISTS idx_digest_clean; +DROP INDEX IF EXISTS idx_digest_all; +DROP INDEX IF EXISTS idx_clean; +DROP INDEX IF EXISTS idx_title; +DROP INDEX IF EXISTS idx_creator_author; DROP INDEX IF EXISTS src_filename; -DROP INDEX IF EXISTS language_document_char; -DROP INDEX IF EXISTS classify_topic_register; +DROP INDEX IF EXISTS idx_language_document_char; +DROP INDEX IF EXISTS idx_classify_topic_register; #+END_SRC **** DROP TABLE IF EXISTS @@ -1121,9 +1126,9 @@ DROP TABLE IF EXISTS urls; #+name: sqlite_statement_create_table_metadata_and_src_txt #+BEGIN_SRC sql CREATE TABLE metadata_and_text ( - tid INTEGER PRIMARY KEY, - src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ - src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */ + src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */ title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, @@ -1213,24 +1218,24 @@ CREATE TABLE metadata_and_text ( #+name: sqlite_statement_create_table_objects #+BEGIN_SRC sql CREATE TABLE doc_objects ( - lid INTEGER PRIMARY KEY, - metadata_tid INTEGER REFERENCES metadata_and_text, + lid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, ocn SMALLINT, ocnd VARCHAR(6), ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - seg VARCHAR(256) NULL, + clean TEXT NULL, + body TEXT NULL, + seg VARCHAR(256) NULL, lev_an VARCHAR(1), - lev SMALLINT NULL, + lev SMALLINT NULL, is_of_type VARCHAR(16), is_a VARCHAR(16), - node VARCHAR(16) NULL, - parent VARCHAR(16) NULL, - last_decendant VARCHAR(16) NULL, /* headings only */ + node VARCHAR(16) NULL, + parent VARCHAR(16) NULL, + last_decendant VARCHAR(16) NULL, /* headings only */ digest_clean CHAR(256), digest_all CHAR(256), - types CHAR(1) NULL + types CHAR(1) NULL ); #+END_SRC @@ -1244,11 +1249,44 @@ CREATE INDEX idx_digest_all ON doc_objects(digest_all); CREATE INDEX idx_clean ON doc_objects(clean); CREATE INDEX idx_title ON metadata_and_text(title); CREATE INDEX idx_author ON metadata_and_text(creator_author); +CREATE INDEX idx_uid ON metadata_and_text(uid); CREATE INDEX idx_filename ON metadata_and_text(src_filename); CREATE INDEX idx_language ON metadata_and_text(language_document_char); CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); #+END_SRC +*** TODO local site link & info + +*** delete rows (delete document) +**** DELETE uid rows doc matters & metadata +***** sql statement: dlang format + +#+name: sqlite_formatted_delete +#+BEGIN_SRC d +string _uid = doc_matters.src.doc_uid; +string _delete_uid = format(q"¶ +#+END_SRC + +***** DELETE FROM ... WHERE + +#+name: sqlite_formatted_delete +#+BEGIN_SRC sql +DELETE FROM metadata_and_text +WHERE uid = '%s'; +DELETE FROM doc_objects +WHERE uid_metadata_and_text = '%s'; +#+END_SRC + +***** VALUES + +#+name: sqlite_formatted_delete +#+BEGIN_SRC d +¶", + _uid, + _uid, +); +#+END_SRC + *** inserts **** INSERT doc objects @@ -1263,8 +1301,7 @@ either: #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC d -string _insert_doc_objects_row; -_insert_doc_objects_row = format(q"¶ +string _insert_doc_objects_row = format(q"¶ #+END_SRC ***** INSERT INTO @@ -1272,7 +1309,7 @@ _insert_doc_objects_row = format(q"¶ #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC sql INSERT INTO doc_objects ( - metadata_tid, + uid_metadata_and_text, ocn, clean, body, @@ -1287,7 +1324,7 @@ _insert_doc_objects_row = format(q"¶ #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC sql VALUES ( - %d, + '%s', %s, '%s', '%s', @@ -1302,7 +1339,7 @@ _insert_doc_objects_row = format(q"¶ #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC d ¶", - _metadata_tid, + _uid, obj.metainfo.ocn, SQLinsertDelimiter!()(obj_txt["text"]), SQLinsertDelimiter!()(obj_txt["html"]), @@ -1312,13 +1349,13 @@ _insert_doc_objects_row = format(q"¶ ); #+END_SRC -**** INSERT doc matters metadata +**** INSERT doc matters & metadata ***** sql statement: dlang format #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC d -string _insert_metadata; -_insert_metadata = format(q"¶ +string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); +string _insert_metadata = format(q"¶ #+END_SRC ***** INSERT INTO @@ -1326,6 +1363,7 @@ _insert_metadata = format(q"¶ #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC sql INSERT INTO metadata_and_text ( + uid, src_filename, src_composite_id_per_txt, src_composite_id_per_pod, @@ -1425,6 +1463,7 @@ _insert_metadata = format(q"¶ '%s', '%s', '%s', + '%s', '%s' ); #+END_SRC @@ -1434,6 +1473,7 @@ _insert_metadata = format(q"¶ #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC d ¶", + _uid, SQLinsertDelimiter!()(doc_matters.src.filename), SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), diff --git a/org/output_xmls.org b/org/output_xmls.org index 0aa50d5..6854f4d 100644 --- a/org/output_xmls.org +++ b/org/output_xmls.org @@ -1521,8 +1521,7 @@ void scroll_write_output(M,C)( foreach (o; doc) { f.writeln(o); } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } writeln(" ", pth_html.fn_scroll(doc_matters.src.filename)); @@ -1791,8 +1790,7 @@ void seg_write_output(M,D,E)( } f.writeln(xhtml_format.tail); } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // handle error } writeln(" ", pth_html.fn_seg(doc_matters.src.filename, "toc")); @@ -1816,8 +1814,7 @@ void css(M)( f.writeln(css.html_seg_css); f = File(pth_html.fn_scroll_css, "w"); f.writeln(css.html_scroll_css); - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } } @@ -2687,8 +2684,7 @@ void epub3_write_output_files(M,D,E,Mt,Mic,Otnx,Otn,Oc)( zip.addMember(zip_arc_member_file); createZipFile!()(fn_epub, zip.build()); } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } writeln(" ", fn_epub); @@ -2706,8 +2702,7 @@ void epub3_write_output_files(M,D,E,Mt,Mic,Otnx,Otn,Oc)( auto data = zipped.expand(member); writeln(filename, " length ", data.length); } - } - catch (ZipException ex) { + } catch (ZipException ex) { // Handle errors } } diff --git a/org/sdp.org b/org/sdp.org index b26f317..e85d040 100644 --- a/org/sdp.org +++ b/org/sdp.org @@ -26,7 +26,7 @@ struct Version { int minor; int patch; } -enum ver = Version(0, 26, 4); +enum ver = Version(0, 26, 5); #+END_SRC ** compilation restrictions (supported compilers) @@ -217,14 +217,13 @@ scope(failure) { } #+END_SRC -*** config files and command line arguements -**** getopt args for loop :args:getopt: +*** getopt args for loop :args:getopt: look into using getopt [[http://dlang.org/phobos/std_getopt.html][getopt]] [[http://dlang.org/library/std/getopt.html][getopt]] -***** getopt +**** getopt #+NAME: sdp_args #+BEGIN_SRC d @@ -248,9 +247,12 @@ bool[string] opts = [ "sisupod" : false, "source" : false, "sqlite-discrete" : false, + "sqlite-db-create" : false, + "sqlite-db-drop" : false, + "sqlite-db-recreate" : false, + "sqlite-delete" : false, + "sqlite-insert" : false, "sqlite-update" : false, - "sqlite-create" : false, - "sqlite-drop" : false, "text" : false, "verbose" : false, "xhtml" : false, @@ -270,6 +272,7 @@ string[string] settings = [ "output-dir" : "", "site-config-dir" : "", "lang" : "all", + "sqlite-filename" : "documents", ]; auto helpInfo = getopt(args, std.getopt.config.passThrough, @@ -294,8 +297,11 @@ auto helpInfo = getopt(args, "sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"], "source", "--source markup source text content", &opts["source"], "sqlite-discrete", "--sqlite process discrete sqlite output", &opts["sqlite-discrete"], - "sqlite-create", "--sqlite-create create db, create tables", &opts["sqlite-create"], - "sqlite-drop", "--sqlite-drop drop tables & db", &opts["sqlite-drop"], + "sqlite-db-create", "--sqlite-db-create create db, create tables", &opts["sqlite-db-create"], + "sqlite-db-drop", "--sqlite-db-drop drop tables & db", &opts["sqlite-db-drop"], + "sqlite-db-recreate", "--sqlite-db-recreate create db, create tables", &opts["sqlite-db-recreate"], + "sqlite-delete", "--sqlite process sqlite output", &opts["sqlite-delete"], + "sqlite-insert", "--sqlite process sqlite output", &opts["sqlite-insert"], "sqlite-update", "--sqlite process sqlite output", &opts["sqlite-update"], "text", "--text process text output", &opts["text"], "txt", "--txt process text output", &opts["text"], @@ -314,6 +320,7 @@ auto helpInfo = getopt(args, "skip-output", "--skip-output", &opts["skip-output"], "output-dir", "--output-dir=[dir path]", &settings["output-dir"], "site-config-dir", "--site-config-dir=[dir path]", &settings["site-config-dir"], + "sqlite-filename", "--sqlite-filename=[filename].sqlite", &settings["sqlite-filename"], "lang", "--lang=[lang code e.g. =en or =en,es]", &settings["lang"], ); if (helpInfo.helpWanted) { @@ -321,7 +328,7 @@ if (helpInfo.helpWanted) { } #+END_SRC -***** getopt hash to struct +**** getopt hash to struct #+NAME: sdp_args #+BEGIN_SRC d @@ -402,14 +409,34 @@ struct OptActions { auto sqlite_discrete() { return opts["sqlite-discrete"]; } - auto sqlite_update() { - return opts["sqlite-update"]; + auto sqlite_db_drop() { + bool _is; + if ( + opts["sqlite-db-recreate"] + || opts["sqlite-db-drop"] + ) { + _is = true; + } else { _is = false; } + return _is; + } + auto sqlite_db_create() { + bool _is; + if ( + opts["sqlite-db-recreate"] + || opts["sqlite-db-create"] + ) { + _is = true; + } else { _is = false; } + return _is; } - auto sqlite_create() { - return opts["sqlite-create"]; + auto sqlite_insert() { + return opts["sqlite-insert"]; } - auto sqlite_drop() { - return opts["sqlite-drop"]; + auto sqlite_delete() { + return opts["sqlite-delete"]; + } + auto sqlite_update() { + return opts["sqlite-update"]; } auto text() { return opts["text"]; @@ -459,6 +486,9 @@ struct OptActions { auto output_dir_set() { return settings["output-dir"]; } + auto sqlite_filename() { + return settings["sqlite-filename"]; + } auto abstraction() { bool _is; if ( @@ -475,6 +505,8 @@ struct OptActions { || sisupod || source || sqlite_discrete + || sqlite_delete + || sqlite_insert || sqlite_update || text || xhtml @@ -487,7 +519,7 @@ struct OptActions { auto _opt_action = OptActions(); #+END_SRC -***** getopt processing path, _manifest[] >> +**** env #+NAME: sdp_args #+BEGIN_SRC d @@ -495,6 +527,12 @@ auto _env = [ "pwd" : environment["PWD"], "home" : environment["HOME"], ]; +#+END_SRC + +*** opt actions on processing files & file paths (pods, src etc.) + +#+NAME: sdp_args +#+BEGIN_SRC d auto _manifest_start = PodManifest!()(""); auto _manifest_matter = PathMatters!()(_opt_action, _env, ""); auto _manifests = [ _manifest_matter ]; @@ -520,10 +558,8 @@ foreach(arg; args[1..$]) { if (exists(sisudoc_txt_)) { contents_location_raw_ = sisudoc_txt_.readText; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { // Handle errors } if (contents_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+ @@ -553,14 +589,15 @@ foreach(arg; args[1..$]) { ) { auto _fns = (((tmp_dir_).chainPath(contents_location_pth_)).array).to!string; _manifest_matter = PathMatters!()(_opt_action, _env, arg, _fns, contents_locations_arr); - _manifests ~= _manifest_matter; // TODO how to capture? + _manifests ~= _manifest_matter; } } } else if (arg.match(rgx.src_pth_sst_or_ssm)) { if (exists(arg)==0) { writeln("ERROR >> Processing Skipped! File not found: ", arg); } else { - _manifests ~= PathMatters!()(_opt_action, _env, arg, arg); // gather input markup source file names for processing + _manifest_matter = PathMatters!()(_opt_action, _env, arg, arg); + _manifests ~= _manifest_matter; } } else if (arg.match(rgx.src_pth_zip)) { // fns_src ~= arg; // gather input markup source file names for processing @@ -570,7 +607,7 @@ foreach(arg; args[1..$]) { } #+END_SRC -**** config files load & read +*** config files load & read #+NAME: sdp_conf_files #+BEGIN_SRC d @@ -581,7 +618,8 @@ _make_and_meta_struct = configParseTOMLreturnSiSUstruct!()(_make_and_meta_struct _make_and_meta_struct = configParseTOMLreturnSiSUstruct!()(_make_and_meta_struct, _config_local_site_struct); #+END_SRC -** 2a. actions independent of processing files +*** opt actions independent of processing files (no files no processing loop) + #+NAME: sdp_do_selected #+BEGIN_SRC d if (!(_opt_action.skip_output)) { @@ -595,7 +633,7 @@ if (!(_opt_action.skip_output)) { } #+END_SRC -** _2b. processing: (loop each file)_ [+2] :loop:files: +** _2. processing: (loop each file)_ [+2] :loop:files: *** scope (loop) :scope: #+NAME: sdp_each_file_do_scope @@ -911,6 +949,14 @@ struct DocumentMatters { auto pod() { return _manifest.pod; } + auto sqlite() { + struct SQLite_ { + string filename() { + return _opt_action.sqlite_filename; + } + } + return SQLite_(); + } auto xml() { struct XML_ { auto keys_seq() { @@ -1392,8 +1438,8 @@ provide the result as a single set of make instructions for each document parsed | | | | | sisupod | | | | | | source | | | | | | sqlite | -| | | | | sqlite-create | -| | | | | sqlite-drop | +| | | | | sqlite-db-create | +| | | | | sqlite-db-drop | | | | | | text | | | | | | verbose | | | | | | xhtml | diff --git a/org/source_files_read.org b/org/source_files_read.org index 899e628..fb53f54 100644 --- a/org/source_files_read.org +++ b/org/source_files_read.org @@ -119,10 +119,8 @@ static template readConfigDoc() { config_file_str = conf_file.readText; break; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { } } if (config_file_str.length > 0) { break; } @@ -173,10 +171,8 @@ static template configReadInSiteTOML() { config_file_str = conf_file.readText; break; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { } } return config_file_str; @@ -212,10 +208,8 @@ static template configReadInDocTOML() { config_file_str = conf_file.readText; break; } - } - catch (ErrnoException ex) { - } - catch (FileException ex) { + } catch (ErrnoException ex) { + } catch (FileException ex) { } } return config_file_str; @@ -234,8 +228,7 @@ static template configTOML() { TOMLDocument _toml_conf; try { _toml_conf = parseTOML(configuration); // parseTOML(cast(string)(configuration)); - } - catch(ErrnoException e) { + } catch(ErrnoException e) { stderr.writeln("Toml problem with content for ", conf_toml_filename); stderr.writeln(e.msg); } @@ -401,13 +394,10 @@ final private string readInMarkupSource(in char[] fn_src) { } source_txt_str = fn_src.readText; } - } - catch (ErrnoException ex) { - } - catch (UTFException ex) { + } catch (ErrnoException ex) { + } catch (UTFException ex) { // Handle validation errors - } - catch (FileException ex) { + } catch (FileException ex) { // Handle errors } std.utf.validate(source_txt_str); |