From fc2471c91966186fc7bc0a5ac2217496c9e43eea Mon Sep 17 00:00:00 2001
From: Ralph Amissah <ralph@amissah.com>
Date: Mon, 16 Jul 2018 22:39:09 -0400
Subject: 0.26.5 sqlite output, focus
---
org/default_paths.org | 22 +++-
org/meta_abstraction.org | 9 ++
org/output_hub.org | 30 ++++--
org/output_sqlite.org | 264 ++++++++++++++++++++++++++--------------------
org/output_xmls.org | 15 +--
org/sdp.org | 100 +++++++++++++-----
org/source_files_read.org | 30 ++----
7 files changed, 289 insertions(+), 181 deletions(-)
(limited to 'org')
diff --git a/org/default_paths.org b/org/default_paths.org
index 5b7141a..35dee32 100644
--- a/org/default_paths.org
+++ b/org/default_paths.org
@@ -233,6 +233,19 @@ template PathMatters() {
} else {_k = "en"; }
return _k;
}
+ string doc_uid() {
+ string _uid;
+ if (is_pod && !(pod_name.empty)) {
+ if (pod_name.baseName == filename_base) {
+ _uid = filename_base ~ "." ~ filename_extension ~ _sep ~ lng;
+ } else {
+ _uid = pod_name.baseName ~ _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng;
+ }
+ } else {
+ _uid = _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng;
+ }
+ return _uid;
+ }
string docname_composite_unique_per_src_doc() {
/+
z pod name if any + src filename + lng code
@@ -1181,7 +1194,7 @@ template SiSUpathsSQLiteDiscrete() {
return asNormalizedPath(base.chainPath(base_filename(fn_src))).array;
}
string sqlite_file(string fn_src) {
- return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array;
+ return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sql.db")).array;
}
}
return _PathsStruct();
@@ -1196,7 +1209,8 @@ template SiSUpathsSQLiteDiscrete() {
template SiSUpathsSQLite() {
mixin SiSUrgxInit;
static auto rgx = Rgx();
- auto SiSUpathsSQLite(Po)(
+ auto SiSUpathsSQLite(DbN, Po)(
+ DbN db_name,
Po output_pth_root,
) {
struct _PathsStruct {
@@ -1208,8 +1222,8 @@ template SiSUpathsSQLite() {
string base_dir = "sqlite";
return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array;
}
- string sqlite_file(string fn_src = "sdp-niu") {
- return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array;
+ string sqlite_file() {
+ return asNormalizedPath(base.chainPath(base_filename(db_name) ~ ".sql.db")).array;
}
}
return _PathsStruct();
diff --git a/org/meta_abstraction.org b/org/meta_abstraction.org
index 00b87a3..010f8b5 100644
--- a/org/meta_abstraction.org
+++ b/org/meta_abstraction.org
@@ -2268,6 +2268,7 @@ foreach (ref obj; the_document_head_section) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2302,6 +2303,7 @@ if (the_table_of_contents_section["scroll"].length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2368,6 +2370,7 @@ if (the_document_body_section.length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2419,6 +2422,7 @@ if (the_endnotes_section.length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2462,6 +2466,7 @@ if (the_glossary_section.length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2508,6 +2513,7 @@ if (the_bibliography_section.length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2559,6 +2565,7 @@ if (the_bookindex_section["scroll"].length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2598,6 +2605,7 @@ if (the_bookindex_section["scroll"].length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2647,6 +2655,7 @@ if (the_blurb_section.length > 1) {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
diff --git a/org/output_hub.org b/org/output_hub.org
index b90d637..2f905e1 100644
--- a/org/output_hub.org
+++ b/org/output_hub.org
@@ -227,7 +227,14 @@ if (doc_matters.opt.action.sqlite_discrete) {
#+name: output_options
#+BEGIN_SRC d
-if (doc_matters.opt.action.sqlite_update) {
+if (doc_matters.opt.action.sqlite_insert
+ || doc_matters.opt.action.sqlite_update
+) {
+ if ((doc_matters.opt.action.verbose)) {
+ writeln("sqlite processing... ");
+ }
+ SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
+} else if (doc_matters.opt.action.sqlite_delete) {
if ((doc_matters.opt.action.verbose)) {
writeln("sqlite processing... ");
}
@@ -235,21 +242,28 @@ if (doc_matters.opt.action.sqlite_update) {
}
#+END_SRC
-***** create, drop
+***** no markup source files to process
+****** drop
#+name: output_options_op
#+BEGIN_SRC d
-if ((opt_action.sqlite_create)) {
+if ((opt_action.sqlite_db_drop)) {
if ((opt_action.verbose)) {
- writeln("sqlite create table... ");
+ writeln("sqlite drop db... ");
}
- SQLiteTablesCreate!()(env, opt_action);
+ SQLiteDbDrop!()(opt_action);
}
-if ((opt_action.sqlite_drop)) {
+#+END_SRC
+
+****** create
+
+#+name: output_options_op
+#+BEGIN_SRC d
+if ((opt_action.sqlite_db_create)) {
if ((opt_action.verbose)) {
- writeln("sqlite drop table... ");
+ writeln("sqlite create table... ");
}
- SQLiteTablesDrop!()();
+ SQLiteTablesCreate!()(env, opt_action);
}
#+END_SRC
diff --git a/org/output_sqlite.org b/org/output_sqlite.org
index dc76da4..eeff4f1 100644
--- a/org/output_sqlite.org
+++ b/org/output_sqlite.org
@@ -33,19 +33,20 @@ static auto rgx = Rgx();
long _metadata_tid_lastrowid;
template SQLiteHubBuildTablesAndPopulate() {
void SQLiteHubBuildTablesAndPopulate(D,I)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ D doc_abstraction,
+ I doc_matters,
) {
- auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
+ auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path);
pth_sqlite.base.mkdirRecurse;
- auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName));
+ auto db = Database(pth_sqlite.sqlite_file);
template SQLiteDbStatementComposite() {
void SQLiteDbStatementComposite(Db,D,I)(
- auto ref Db db,
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ Db db,
+ D doc_abstraction,
+ I doc_matters,
) {
<<sqlite_db_statement_composite_collection>>
+ writeln(" ", pth_sqlite.sqlite_file);
}
}
SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);
@@ -58,19 +59,20 @@ template SQLiteHubBuildTablesAndPopulate() {
#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
template SQLiteHubDiscreteBuildTablesAndPopulate() {
void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ D doc_abstraction,
+ I doc_matters,
) {
auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language);
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename));
template SQLiteDiscreteDbStatementComposite() {
void SQLiteDiscreteDbStatementComposite(Db,D,I)(
- auto ref Db db,
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ Db db,
+ D doc_abstraction,
+ I doc_matters,
) {
<<sqlite_db_statement_composite_discrete>>
+ writeln(" ", pth_sqlite.sqlite_file(doc_matters.src.filename));
}
}
SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);
@@ -83,10 +85,10 @@ template SQLiteHubDiscreteBuildTablesAndPopulate() {
#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
template SQLiteDbRun() {
void SQLiteDbRun(Db,St,O)(
- auto ref Db db,
- auto ref St db_statement,
- auto ref O opt_action,
- string note,
+ Db db,
+ St db_statement,
+ O opt_action,
+ string note,
) {
debug(sql_statement) {
writeln(db_statement);
@@ -98,10 +100,8 @@ template SQLiteDbRun() {
"\nCOMMIT;\n"
);
if (!(opt_action.sqlite_discrete)
- && !(opt_action.sqlite_create)
+ && !(opt_action.sqlite_db_create)
) {
- _metadata_tid_lastrowid = db.lastInsertRowid();
- writeln("last row id: ", _metadata_tid_lastrowid);
}
} catch (ErrnoException ex) {
writeln("ERROR SQLite : ", ex);
@@ -130,7 +130,7 @@ template SQLinsertDelimiter() {
}
template SQLiteFormatAndLoadObject() {
auto SQLiteFormatAndLoadObject(I)(
- auto ref I doc_matters,
+ I doc_matters,
) {
mixin SiSUoutputRgxInit;
struct sqlite_format_and_load_objects {
@@ -146,6 +146,7 @@ template SQLiteFormatAndLoadObject() {
#+END_SRC
*** sqlite instructions
+**** create
#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
template SQLiteTablesReCreate() {
@@ -161,20 +162,43 @@ template SQLiteTablesReCreate() {
return _sql_instruct;
}
}
+#+END_SRC
+
+**** delete
+
+#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
+template SQLiteDeleteDocument() {
+ string SQLiteDeleteDocument(I)(
+ I doc_matters,
+ ) {
+ <<sqlite_formatted_delete>>
+ return _delete_uid;
+ }
+}
+#+END_SRC
+
+**** insert
+
+#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
template SQLiteInsertMetadata() {
string SQLiteInsertMetadata(I)(
- auto ref I doc_matters,
+ I doc_matters,
) {
<<sqlite_formatted_insertions_doc_matters_metadata>>
return _insert_metadata;
}
}
+#+END_SRC
+
+**** insert doc objects loop
+
+#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
template SQLiteInsertDocObjectsLoop() {
- string SQLiteInsertDocObjectsLoop(D,I,X)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
- auto ref X _metadata_tid,
+ string SQLiteInsertDocObjectsLoop(D,I)(
+ D doc_abstraction,
+ I doc_matters,
) {
+ string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
string insertDocObjectsRow(O)(O obj) {
<<sqlite_formatted_insertions_doc_objects>>
return _insert_doc_objects_row;
@@ -182,6 +206,11 @@ template SQLiteInsertDocObjectsLoop() {
<<sqlite_objects_loop>>
}
}
+#+END_SRC
+
+**** tables create
+
+#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
template SQLiteTablesCreate() {
void SQLiteTablesCreate(E,O)(E env, O opt_action) {
import d2sqlite3;
@@ -198,9 +227,9 @@ template SQLiteTablesCreate() {
return _sql_instruct;
}
}
- if (opt_action.sqlite_create) {
+ if (opt_action.sqlite_db_create) {
string _db_statement;
- auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE
+ auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file); // ISSUE
{
@@ -210,9 +239,23 @@ template SQLiteTablesCreate() {
}
}
}
-template SQLiteTablesDrop() {
- void SQLiteTablesDrop()() {
- writeln("table drop");
+#+END_SRC
+
+**** tables drop
+
+#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d
+template SQLiteDbDrop() {
+ void SQLiteDbDrop(O)(O opt_action) {
+ writeln("db drop");
+ if ((opt_action.sqlite_db_drop)) {
+ auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE
+ writeln("remove(", pth_sqlite.sqlite_file, ")");
+ try {
+ remove(pth_sqlite.sqlite_file);
+ } catch (FileException ex) {
+ // handle error
+ }
+ }
}
}
#+END_SRC
@@ -225,19 +268,24 @@ template SQLiteTablesDrop() {
{
string _db_statement;
{
- if ((doc_matters.opt.action.sqlite_create)) {
- auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
+ if ((doc_matters.opt.action.sqlite_db_create)) {
+ auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path);
pth_sqlite.base.mkdirRecurse;
_db_statement ~= SQLiteTablesReCreate!()();
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE");
}
- if ((doc_matters.opt.action.sqlite_update)) { // TODO
+ if (doc_matters.opt.action.sqlite_delete) {
+ _db_statement ~= SQLiteDeleteDocument!()(doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document");
+ }
+ if (doc_matters.opt.action.sqlite_insert || doc_matters.opt.action.sqlite_update) { // sqlite_update is .opt.action sqlite_delete & sqlite_insert (set in sdp)
+ _db_statement ~= SQLiteDeleteDocument!()(doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document");
_db_statement ~= SQLiteInsertMetadata!()(doc_matters);
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData");
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT MetaData");
/+ get tid (lastrowid or max) for use in doc_objects table +/
- _metadata_tid_lastrowid = db.lastInsertRowid();
- _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects");
+ _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT DocObjects");
}
}
db.close;
@@ -253,7 +301,7 @@ template SQLiteTablesDrop() {
{
_db_statement ~= SQLiteTablesReCreate!()();
_db_statement ~= SQLiteInsertMetadata!()(doc_matters);
- _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX
+ _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects");
}
db.close;
@@ -646,44 +694,7 @@ auto html_table(O)(
}
#+END_SRC
-*** 2. _sqlite_ (loop preparation & pre-loop action)
-**** prepare insert statements and do pre-loop inserts
-***** 1. _metadata & src_ text
-****** TODO sql remove selected
-******* d { sql statement
-
-##+name: sqlite_remove_selected
-#+BEGIN_SRC d
-Statement insert_metadata = db.prepare("
-#+END_SRC
-
-******** remove selected
-
-identify and remove, identify by
-- filename & language code
-- (in ruby get_first_value and remove by fn & ln)
-
-##+name: sqlite_remove_selected
-#+BEGIN_SRC sql
- SELECT tid
- FROM metadata_and_text
- WHERE src_filename = '#{doc_matters.conf_make_meta.meta.src_filename}'
- AND metadata_and_text.language_document_char = '#{doc_matters.conf_make_meta.meta.language_document_char}'
-#+END_SRC
-
-******* d }
-
-##+name: sqlite_remove_selected
-#+BEGIN_SRC d
-");
-#+END_SRC
-
-***** 2. _doc objects_ (used with doc_objects in document loop)
-****** TODO (within loop not here - insert doc objects
-
-work out
-
-*** 3. hub (sqlite_format_and_load_objects)
+*** 2. hub (sqlite_format_and_load_objects)
**** sql related
#+name: sqlite_load_object
@@ -1088,22 +1099,16 @@ return _insert_doc_objects.join.to!(char[]).toUTF8;
#+name: sqlite_statement_drop_existing_index
#+BEGIN_SRC sql
-DROP INDEX IF EXISTS ocn;
-DROP INDEX IF EXISTS digest_clean;
-DROP INDEX IF EXISTS digest_all;
-DROP INDEX IF EXISTS clean;
-DROP INDEX IF EXISTS lev0;
-DROP INDEX IF EXISTS lev1;
-DROP INDEX IF EXISTS lev2;
-DROP INDEX IF EXISTS lev3;
-DROP INDEX IF EXISTS lev4;
-DROP INDEX IF EXISTS lev5;
-DROP INDEX IF EXISTS lev6;
-DROP INDEX IF EXISTS title;
-DROP INDEX IF EXISTS creator_author;
+DROP INDEX IF EXISTS idx_ocn;
+DROP INDEX IF EXISTS idx_uid;
+DROP INDEX IF EXISTS idx_digest_clean;
+DROP INDEX IF EXISTS idx_digest_all;
+DROP INDEX IF EXISTS idx_clean;
+DROP INDEX IF EXISTS idx_title;
+DROP INDEX IF EXISTS idx_creator_author;
DROP INDEX IF EXISTS src_filename;
-DROP INDEX IF EXISTS language_document_char;
-DROP INDEX IF EXISTS classify_topic_register;
+DROP INDEX IF EXISTS idx_language_document_char;
+DROP INDEX IF EXISTS idx_classify_topic_register;
#+END_SRC
**** DROP TABLE IF EXISTS
@@ -1121,9 +1126,9 @@ DROP TABLE IF EXISTS urls;
#+name: sqlite_statement_create_table_metadata_and_src_txt
#+BEGIN_SRC sql
CREATE TABLE metadata_and_text (
- tid INTEGER PRIMARY KEY,
- src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
- src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */
+ src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */
+ src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */
title VARCHAR(800) NOT NULL,
title_main VARCHAR(400) NOT NULL,
title_sub VARCHAR(400) NULL,
@@ -1213,24 +1218,24 @@ CREATE TABLE metadata_and_text (
#+name: sqlite_statement_create_table_objects
#+BEGIN_SRC sql
CREATE TABLE doc_objects (
- lid INTEGER PRIMARY KEY,
- metadata_tid INTEGER REFERENCES metadata_and_text,
+ lid BIGINT PRIMARY KEY,
+ uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
ocn SMALLINT,
ocnd VARCHAR(6),
ocns VARCHAR(6),
- clean TEXT NULL,
- body TEXT NULL,
- seg VARCHAR(256) NULL,
+ clean TEXT NULL,
+ body TEXT NULL,
+ seg VARCHAR(256) NULL,
lev_an VARCHAR(1),
- lev SMALLINT NULL,
+ lev SMALLINT NULL,
is_of_type VARCHAR(16),
is_a VARCHAR(16),
- node VARCHAR(16) NULL,
- parent VARCHAR(16) NULL,
- last_decendant VARCHAR(16) NULL, /* headings only */
+ node VARCHAR(16) NULL,
+ parent VARCHAR(16) NULL,
+ last_decendant VARCHAR(16) NULL, /* headings only */
digest_clean CHAR(256),
digest_all CHAR(256),
- types CHAR(1) NULL
+ types CHAR(1) NULL
);
#+END_SRC
@@ -1244,11 +1249,44 @@ CREATE INDEX idx_digest_all ON doc_objects(digest_all);
CREATE INDEX idx_clean ON doc_objects(clean);
CREATE INDEX idx_title ON metadata_and_text(title);
CREATE INDEX idx_author ON metadata_and_text(creator_author);
+CREATE INDEX idx_uid ON metadata_and_text(uid);
CREATE INDEX idx_filename ON metadata_and_text(src_filename);
CREATE INDEX idx_language ON metadata_and_text(language_document_char);
CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);
#+END_SRC
+*** TODO local site link & info
+
+*** delete rows (delete document)
+**** DELETE uid rows doc matters & metadata
+***** sql statement: dlang format
+
+#+name: sqlite_formatted_delete
+#+BEGIN_SRC d
+string _uid = doc_matters.src.doc_uid;
+string _delete_uid = format(q"¶
+#+END_SRC
+
+***** DELETE FROM ... WHERE
+
+#+name: sqlite_formatted_delete
+#+BEGIN_SRC sql
+DELETE FROM metadata_and_text
+WHERE uid = '%s';
+DELETE FROM doc_objects
+WHERE uid_metadata_and_text = '%s';
+#+END_SRC
+
+***** VALUES
+
+#+name: sqlite_formatted_delete
+#+BEGIN_SRC d
+¶",
+ _uid,
+ _uid,
+);
+#+END_SRC
+
*** inserts
**** INSERT doc objects
@@ -1263,8 +1301,7 @@ either:
#+name: sqlite_formatted_insertions_doc_objects
#+BEGIN_SRC d
-string _insert_doc_objects_row;
-_insert_doc_objects_row = format(q"¶
+string _insert_doc_objects_row = format(q"¶
#+END_SRC
***** INSERT INTO
@@ -1272,7 +1309,7 @@ _insert_doc_objects_row = format(q"¶
#+name: sqlite_formatted_insertions_doc_objects
#+BEGIN_SRC sql
INSERT INTO doc_objects (
- metadata_tid,
+ uid_metadata_and_text,
ocn,
clean,
body,
@@ -1287,7 +1324,7 @@ _insert_doc_objects_row = format(q"¶
#+name: sqlite_formatted_insertions_doc_objects
#+BEGIN_SRC sql
VALUES (
- %d,
+ '%s',
%s,
'%s',
'%s',
@@ -1302,7 +1339,7 @@ _insert_doc_objects_row = format(q"¶
#+name: sqlite_formatted_insertions_doc_objects
#+BEGIN_SRC d
¶",
- _metadata_tid,
+ _uid,
obj.metainfo.ocn,
SQLinsertDelimiter!()(obj_txt["text"]),
SQLinsertDelimiter!()(obj_txt["html"]),
@@ -1312,13 +1349,13 @@ _insert_doc_objects_row = format(q"¶
);
#+END_SRC
-**** INSERT doc matters metadata
+**** INSERT doc matters & metadata
***** sql statement: dlang format
#+name: sqlite_formatted_insertions_doc_matters_metadata
#+BEGIN_SRC d
-string _insert_metadata;
-_insert_metadata = format(q"¶
+string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
+string _insert_metadata = format(q"¶
#+END_SRC
***** INSERT INTO
@@ -1326,6 +1363,7 @@ _insert_metadata = format(q"¶
#+name: sqlite_formatted_insertions_doc_matters_metadata
#+BEGIN_SRC sql
INSERT INTO metadata_and_text (
+ uid,
src_filename,
src_composite_id_per_txt,
src_composite_id_per_pod,
@@ -1425,6 +1463,7 @@ _insert_metadata = format(q"¶
'%s',
'%s',
'%s',
+ '%s',
'%s'
);
#+END_SRC
@@ -1434,6 +1473,7 @@ _insert_metadata = format(q"¶
#+name: sqlite_formatted_insertions_doc_matters_metadata
#+BEGIN_SRC d
¶",
+ _uid,
SQLinsertDelimiter!()(doc_matters.src.filename),
SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc),
SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod),
diff --git a/org/output_xmls.org b/org/output_xmls.org
index 0aa50d5..6854f4d 100644
--- a/org/output_xmls.org
+++ b/org/output_xmls.org
@@ -1521,8 +1521,7 @@ void scroll_write_output(M,C)(
foreach (o; doc) {
f.writeln(o);
}
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// Handle error
}
writeln(" ", pth_html.fn_scroll(doc_matters.src.filename));
@@ -1791,8 +1790,7 @@ void seg_write_output(M,D,E)(
}
f.writeln(xhtml_format.tail);
}
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// handle error
}
writeln(" ", pth_html.fn_seg(doc_matters.src.filename, "toc"));
@@ -1816,8 +1814,7 @@ void css(M)(
f.writeln(css.html_seg_css);
f = File(pth_html.fn_scroll_css, "w");
f.writeln(css.html_scroll_css);
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// Handle error
}
}
@@ -2687,8 +2684,7 @@ void epub3_write_output_files(M,D,E,Mt,Mic,Otnx,Otn,Oc)(
zip.addMember(zip_arc_member_file);
createZipFile!()(fn_epub, zip.build());
}
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// Handle error
}
writeln(" ", fn_epub);
@@ -2706,8 +2702,7 @@ void epub3_write_output_files(M,D,E,Mt,Mic,Otnx,Otn,Oc)(
auto data = zipped.expand(member);
writeln(filename, " length ", data.length);
}
- }
- catch (ZipException ex) {
+ } catch (ZipException ex) {
// Handle errors
}
}
diff --git a/org/sdp.org b/org/sdp.org
index b26f317..e85d040 100644
--- a/org/sdp.org
+++ b/org/sdp.org
@@ -26,7 +26,7 @@ struct Version {
int minor;
int patch;
}
-enum ver = Version(0, 26, 4);
+enum ver = Version(0, 26, 5);
#+END_SRC
** compilation restrictions (supported compilers)
@@ -217,14 +217,13 @@ scope(failure) {
}
#+END_SRC
-*** config files and command line arguements
-**** getopt args for loop :args:getopt:
+*** getopt args for loop :args:getopt:
look into using getopt
[[http://dlang.org/phobos/std_getopt.html][getopt]]
[[http://dlang.org/library/std/getopt.html][getopt]]
-***** getopt
+**** getopt
#+NAME: sdp_args
#+BEGIN_SRC d
@@ -248,9 +247,12 @@ bool[string] opts = [
"sisupod" : false,
"source" : false,
"sqlite-discrete" : false,
+ "sqlite-db-create" : false,
+ "sqlite-db-drop" : false,
+ "sqlite-db-recreate" : false,
+ "sqlite-delete" : false,
+ "sqlite-insert" : false,
"sqlite-update" : false,
- "sqlite-create" : false,
- "sqlite-drop" : false,
"text" : false,
"verbose" : false,
"xhtml" : false,
@@ -270,6 +272,7 @@ string[string] settings = [
"output-dir" : "",
"site-config-dir" : "",
"lang" : "all",
+ "sqlite-filename" : "documents",
];
auto helpInfo = getopt(args,
std.getopt.config.passThrough,
@@ -294,8 +297,11 @@ auto helpInfo = getopt(args,
"sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"],
"source", "--source markup source text content", &opts["source"],
"sqlite-discrete", "--sqlite process discrete sqlite output", &opts["sqlite-discrete"],
- "sqlite-create", "--sqlite-create create db, create tables", &opts["sqlite-create"],
- "sqlite-drop", "--sqlite-drop drop tables & db", &opts["sqlite-drop"],
+ "sqlite-db-create", "--sqlite-db-create create db, create tables", &opts["sqlite-db-create"],
+ "sqlite-db-drop", "--sqlite-db-drop drop tables & db", &opts["sqlite-db-drop"],
+ "sqlite-db-recreate", "--sqlite-db-recreate create db, create tables", &opts["sqlite-db-recreate"],
+ "sqlite-delete", "--sqlite process sqlite output", &opts["sqlite-delete"],
+ "sqlite-insert", "--sqlite process sqlite output", &opts["sqlite-insert"],
"sqlite-update", "--sqlite process sqlite output", &opts["sqlite-update"],
"text", "--text process text output", &opts["text"],
"txt", "--txt process text output", &opts["text"],
@@ -314,6 +320,7 @@ auto helpInfo = getopt(args,
"skip-output", "--skip-output", &opts["skip-output"],
"output-dir", "--output-dir=[dir path]", &settings["output-dir"],
"site-config-dir", "--site-config-dir=[dir path]", &settings["site-config-dir"],
+ "sqlite-filename", "--sqlite-filename=[filename].sqlite", &settings["sqlite-filename"],
"lang", "--lang=[lang code e.g. =en or =en,es]", &settings["lang"],
);
if (helpInfo.helpWanted) {
@@ -321,7 +328,7 @@ if (helpInfo.helpWanted) {
}
#+END_SRC
-***** getopt hash to struct
+**** getopt hash to struct
#+NAME: sdp_args
#+BEGIN_SRC d
@@ -402,14 +409,34 @@ struct OptActions {
auto sqlite_discrete() {
return opts["sqlite-discrete"];
}
- auto sqlite_update() {
- return opts["sqlite-update"];
+ auto sqlite_db_drop() {
+ bool _is;
+ if (
+ opts["sqlite-db-recreate"]
+ || opts["sqlite-db-drop"]
+ ) {
+ _is = true;
+ } else { _is = false; }
+ return _is;
+ }
+ auto sqlite_db_create() {
+ bool _is;
+ if (
+ opts["sqlite-db-recreate"]
+ || opts["sqlite-db-create"]
+ ) {
+ _is = true;
+ } else { _is = false; }
+ return _is;
}
- auto sqlite_create() {
- return opts["sqlite-create"];
+ auto sqlite_insert() {
+ return opts["sqlite-insert"];
}
- auto sqlite_drop() {
- return opts["sqlite-drop"];
+ auto sqlite_delete() {
+ return opts["sqlite-delete"];
+ }
+ auto sqlite_update() {
+ return opts["sqlite-update"];
}
auto text() {
return opts["text"];
@@ -459,6 +486,9 @@ struct OptActions {
auto output_dir_set() {
return settings["output-dir"];
}
+ auto sqlite_filename() {
+ return settings["sqlite-filename"];
+ }
auto abstraction() {
bool _is;
if (
@@ -475,6 +505,8 @@ struct OptActions {
|| sisupod
|| source
|| sqlite_discrete
+ || sqlite_delete
+ || sqlite_insert
|| sqlite_update
|| text
|| xhtml
@@ -487,7 +519,7 @@ struct OptActions {
auto _opt_action = OptActions();
#+END_SRC
-***** getopt processing path, _manifest[] >>
+**** env
#+NAME: sdp_args
#+BEGIN_SRC d
@@ -495,6 +527,12 @@ auto _env = [
"pwd" : environment["PWD"],
"home" : environment["HOME"],
];
+#+END_SRC
+
+*** opt actions on processing files & file paths (pods, src etc.)
+
+#+NAME: sdp_args
+#+BEGIN_SRC d
auto _manifest_start = PodManifest!()("");
auto _manifest_matter = PathMatters!()(_opt_action, _env, "");
auto _manifests = [ _manifest_matter ];
@@ -520,10 +558,8 @@ foreach(arg; args[1..$]) {
if (exists(sisudoc_txt_)) {
contents_location_raw_ = sisudoc_txt_.readText;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
// Handle errors
}
if (contents_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+
@@ -553,14 +589,15 @@ foreach(arg; args[1..$]) {
) {
auto _fns = (((tmp_dir_).chainPath(contents_location_pth_)).array).to!string;
_manifest_matter = PathMatters!()(_opt_action, _env, arg, _fns, contents_locations_arr);
- _manifests ~= _manifest_matter; // TODO how to capture?
+ _manifests ~= _manifest_matter;
}
}
} else if (arg.match(rgx.src_pth_sst_or_ssm)) {
if (exists(arg)==0) {
writeln("ERROR >> Processing Skipped! File not found: ", arg);
} else {
- _manifests ~= PathMatters!()(_opt_action, _env, arg, arg); // gather input markup source file names for processing
+ _manifest_matter = PathMatters!()(_opt_action, _env, arg, arg);
+ _manifests ~= _manifest_matter;
}
} else if (arg.match(rgx.src_pth_zip)) {
// fns_src ~= arg; // gather input markup source file names for processing
@@ -570,7 +607,7 @@ foreach(arg; args[1..$]) {
}
#+END_SRC
-**** config files load & read
+*** config files load & read
#+NAME: sdp_conf_files
#+BEGIN_SRC d
@@ -581,7 +618,8 @@ _make_and_meta_struct = configParseTOMLreturnSiSUstruct!()(_make_and_meta_struct
_make_and_meta_struct = configParseTOMLreturnSiSUstruct!()(_make_and_meta_struct, _config_local_site_struct);
#+END_SRC
-** 2a. actions independent of processing files
+*** opt actions independent of processing files (no files no processing loop)
+
#+NAME: sdp_do_selected
#+BEGIN_SRC d
if (!(_opt_action.skip_output)) {
@@ -595,7 +633,7 @@ if (!(_opt_action.skip_output)) {
}
#+END_SRC
-** _2b. processing: (loop each file)_ [+2] :loop:files:
+** _2. processing: (loop each file)_ [+2] :loop:files:
*** scope (loop) :scope:
#+NAME: sdp_each_file_do_scope
@@ -911,6 +949,14 @@ struct DocumentMatters {
auto pod() {
return _manifest.pod;
}
+ auto sqlite() {
+ struct SQLite_ {
+ string filename() {
+ return _opt_action.sqlite_filename;
+ }
+ }
+ return SQLite_();
+ }
auto xml() {
struct XML_ {
auto keys_seq() {
@@ -1392,8 +1438,8 @@ provide the result as a single set of make instructions for each document parsed
| | | | | sisupod |
| | | | | source |
| | | | | sqlite |
-| | | | | sqlite-create |
-| | | | | sqlite-drop |
+| | | | | sqlite-db-create |
+| | | | | sqlite-db-drop |
| | | | | text |
| | | | | verbose |
| | | | | xhtml |
diff --git a/org/source_files_read.org b/org/source_files_read.org
index 899e628..fb53f54 100644
--- a/org/source_files_read.org
+++ b/org/source_files_read.org
@@ -119,10 +119,8 @@ static template readConfigDoc() {
config_file_str = conf_file.readText;
break;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
}
}
if (config_file_str.length > 0) { break; }
@@ -173,10 +171,8 @@ static template configReadInSiteTOML() {
config_file_str = conf_file.readText;
break;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
}
}
return config_file_str;
@@ -212,10 +208,8 @@ static template configReadInDocTOML() {
config_file_str = conf_file.readText;
break;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
}
}
return config_file_str;
@@ -234,8 +228,7 @@ static template configTOML() {
TOMLDocument _toml_conf;
try {
_toml_conf = parseTOML(configuration); // parseTOML(cast(string)(configuration));
- }
- catch(ErrnoException e) {
+ } catch(ErrnoException e) {
stderr.writeln("Toml problem with content for ", conf_toml_filename);
stderr.writeln(e.msg);
}
@@ -401,13 +394,10 @@ final private string readInMarkupSource(in char[] fn_src) {
}
source_txt_str = fn_src.readText;
}
- }
- catch (ErrnoException ex) {
- }
- catch (UTFException ex) {
+ } catch (ErrnoException ex) {
+ } catch (UTFException ex) {
// Handle validation errors
- }
- catch (FileException ex) {
+ } catch (FileException ex) {
// Handle errors
}
std.utf.validate(source_txt_str);
--
cgit v1.2.3