aboutsummaryrefslogtreecommitdiffhomepage
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/sdp/meta/metadoc.d8
-rw-r--r--src/sdp/meta/metadoc_from_src.d9
-rw-r--r--src/sdp/output/epub3.d6
-rw-r--r--src/sdp/output/html.d9
-rw-r--r--src/sdp/output/hub.d21
-rw-r--r--src/sdp/output/paths_output.d9
-rw-r--r--src/sdp/output/sqlite.d225
-rwxr-xr-xsrc/sdp/sdp.d64
-rw-r--r--src/sdp/source/paths_source.d13
-rw-r--r--src/sdp/source/read_config_files.d21
-rw-r--r--src/sdp/source/read_source_files.d9
11 files changed, 236 insertions, 158 deletions
diff --git a/src/sdp/meta/metadoc.d b/src/sdp/meta/metadoc.d
index 29af49c..e7c107a 100644
--- a/src/sdp/meta/metadoc.d
+++ b/src/sdp/meta/metadoc.d
@@ -138,6 +138,14 @@ template SiSUabstraction() {
auto pod() {
return _manifest.pod;
}
+ auto sqlite() {
+ struct SQLite_ {
+ string filename() {
+ return _opt_action.sqlite_filename;
+ }
+ }
+ return SQLite_();
+ }
auto xml() {
struct XML_ {
auto keys_seq() {
diff --git a/src/sdp/meta/metadoc_from_src.d b/src/sdp/meta/metadoc_from_src.d
index 050089e..661e106 100644
--- a/src/sdp/meta/metadoc_from_src.d
+++ b/src/sdp/meta/metadoc_from_src.d
@@ -1707,6 +1707,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1735,6 +1736,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1795,6 +1797,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1838,6 +1841,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1873,6 +1877,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1911,6 +1916,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1954,6 +1960,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -1993,6 +2000,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
@@ -2036,6 +2044,7 @@ template SiSUdocAbstraction() {
|| (opt_action.html_seg)
|| (opt_action.epub)
|| (opt_action.sqlite_discrete)
+ || (opt_action.sqlite_insert)
|| (opt_action.sqlite_update)
|| (opt_action.postgresql)) {
obj = obj_dom_structure_set_markup_tags(obj, dom_structure_markedup_tags_status, obj.metainfo.heading_lev_markup);
diff --git a/src/sdp/output/epub3.d b/src/sdp/output/epub3.d
index cde42d1..d10b7f2 100644
--- a/src/sdp/output/epub3.d
+++ b/src/sdp/output/epub3.d
@@ -758,8 +758,7 @@ template outputEPub3() {
zip.addMember(zip_arc_member_file);
createZipFile!()(fn_epub, zip.build());
}
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// Handle error
}
writeln(" ", fn_epub);
@@ -771,8 +770,7 @@ template outputEPub3() {
auto data = zipped.expand(member);
writeln(filename, " length ", data.length);
}
- }
- catch (ZipException ex) {
+ } catch (ZipException ex) {
// Handle errors
}
}
diff --git a/src/sdp/output/html.d b/src/sdp/output/html.d
index 7024b0f..ba53eef 100644
--- a/src/sdp/output/html.d
+++ b/src/sdp/output/html.d
@@ -174,8 +174,7 @@ template outputHTML() {
foreach (o; doc) {
f.writeln(o);
}
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// Handle error
}
writeln(" ", pth_html.fn_scroll(doc_matters.src.filename));
@@ -431,8 +430,7 @@ template outputHTML() {
}
f.writeln(xhtml_format.tail);
}
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// handle error
}
writeln(" ", pth_html.fn_seg(doc_matters.src.filename, "toc"));
@@ -450,8 +448,7 @@ template outputHTML() {
f.writeln(css.html_seg_css);
f = File(pth_html.fn_scroll_css, "w");
f.writeln(css.html_scroll_css);
- }
- catch (ErrnoException ex) {
+ } catch (ErrnoException ex) {
// Handle error
}
}
diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d
index b6caca9..47713f5 100644
--- a/src/sdp/output/hub.d
+++ b/src/sdp/output/hub.d
@@ -124,7 +124,14 @@ template outputHub() {
}
SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
}
- if (doc_matters.opt.action.sqlite_update) {
+ if (doc_matters.opt.action.sqlite_insert
+ || doc_matters.opt.action.sqlite_update
+ ) {
+ if ((doc_matters.opt.action.verbose)) {
+ writeln("sqlite processing... ");
+ }
+ SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
+ } else if (doc_matters.opt.action.sqlite_delete) {
if ((doc_matters.opt.action.verbose)) {
writeln("sqlite processing... ");
}
@@ -148,17 +155,17 @@ template outputHubOp() {
void outputHubOp(E,O)(E env, O opt_action) {
mixin SiSUoutputRgxInit;
static auto rgx = Rgx();
- if ((opt_action.sqlite_create)) {
+ if ((opt_action.sqlite_db_drop)) {
if ((opt_action.verbose)) {
- writeln("sqlite create table... ");
+ writeln("sqlite drop db... ");
}
- SQLiteTablesCreate!()(env, opt_action);
+ SQLiteDbDrop!()(opt_action);
}
- if ((opt_action.sqlite_drop)) {
+ if ((opt_action.sqlite_db_create)) {
if ((opt_action.verbose)) {
- writeln("sqlite drop table... ");
+ writeln("sqlite create table... ");
}
- SQLiteTablesDrop!()();
+ SQLiteTablesCreate!()(env, opt_action);
}
}
}
diff --git a/src/sdp/output/paths_output.d b/src/sdp/output/paths_output.d
index 8a4d623..3b18c75 100644
--- a/src/sdp/output/paths_output.d
+++ b/src/sdp/output/paths_output.d
@@ -243,7 +243,7 @@ template SiSUpathsSQLiteDiscrete() {
return asNormalizedPath(base.chainPath(base_filename(fn_src))).array;
}
string sqlite_file(string fn_src) {
- return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array;
+ return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sql.db")).array;
}
}
return _PathsStruct();
@@ -252,7 +252,8 @@ template SiSUpathsSQLiteDiscrete() {
template SiSUpathsSQLite() {
mixin SiSUrgxInit;
static auto rgx = Rgx();
- auto SiSUpathsSQLite(Po)(
+ auto SiSUpathsSQLite(DbN, Po)(
+ DbN db_name,
Po output_pth_root,
) {
struct _PathsStruct {
@@ -264,8 +265,8 @@ template SiSUpathsSQLite() {
string base_dir = "sqlite";
return asNormalizedPath((out_pth.output_root).chainPath(base_dir)).array;
}
- string sqlite_file(string fn_src = "sdp-niu") {
- return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".sqlite")).array;
+ string sqlite_file() {
+ return asNormalizedPath(base.chainPath(base_filename(db_name) ~ ".sql.db")).array;
}
}
return _PathsStruct();
diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d
index faff29b..41d504e 100644
--- a/src/sdp/output/sqlite.d
+++ b/src/sdp/output/sqlite.d
@@ -10,38 +10,44 @@ static auto rgx = Rgx();
long _metadata_tid_lastrowid;
template SQLiteHubBuildTablesAndPopulate() {
void SQLiteHubBuildTablesAndPopulate(D,I)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ D doc_abstraction,
+ I doc_matters,
) {
- auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
+ auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path);
pth_sqlite.base.mkdirRecurse;
- auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName));
+ auto db = Database(pth_sqlite.sqlite_file);
template SQLiteDbStatementComposite() {
void SQLiteDbStatementComposite(Db,D,I)(
- auto ref Db db,
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ Db db,
+ D doc_abstraction,
+ I doc_matters,
) {
{
string _db_statement;
{
- if ((doc_matters.opt.action.sqlite_create)) {
- auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
+ if ((doc_matters.opt.action.sqlite_db_create)) {
+ auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.sqlite.filename, doc_matters.output_path);
pth_sqlite.base.mkdirRecurse;
_db_statement ~= SQLiteTablesReCreate!()();
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE");
}
- if ((doc_matters.opt.action.sqlite_update)) { // TODO
+ if (doc_matters.opt.action.sqlite_delete) {
+ _db_statement ~= SQLiteDeleteDocument!()(doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document");
+ }
+ if (doc_matters.opt.action.sqlite_insert || doc_matters.opt.action.sqlite_update) { // sqlite_update is .opt.action sqlite_delete & sqlite_insert (set in sdp)
+ _db_statement ~= SQLiteDeleteDocument!()(doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document");
_db_statement ~= SQLiteInsertMetadata!()(doc_matters);
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData");
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT MetaData");
/+ get tid (lastrowid or max) for use in doc_objects table +/
- _metadata_tid_lastrowid = db.lastInsertRowid();
- _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects");
+ _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT DocObjects");
}
}
db.close;
}
+ writeln(" ", pth_sqlite.sqlite_file);
}
}
SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);
@@ -49,28 +55,29 @@ template SQLiteHubBuildTablesAndPopulate() {
}
template SQLiteHubDiscreteBuildTablesAndPopulate() {
void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ D doc_abstraction,
+ I doc_matters,
) {
auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language);
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename));
template SQLiteDiscreteDbStatementComposite() {
void SQLiteDiscreteDbStatementComposite(Db,D,I)(
- auto ref Db db,
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ Db db,
+ D doc_abstraction,
+ I doc_matters,
) {
{
string _db_statement;
{
_db_statement ~= SQLiteTablesReCreate!()();
_db_statement ~= SQLiteInsertMetadata!()(doc_matters);
- _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX
+ _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects");
}
db.close;
}
+ writeln(" ", pth_sqlite.sqlite_file(doc_matters.src.filename));
}
}
SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);
@@ -78,10 +85,10 @@ template SQLiteHubDiscreteBuildTablesAndPopulate() {
}
template SQLiteDbRun() {
void SQLiteDbRun(Db,St,O)(
- auto ref Db db,
- auto ref St db_statement,
- auto ref O opt_action,
- string note,
+ Db db,
+ St db_statement,
+ O opt_action,
+ string note,
) {
debug(sql_statement) {
writeln(db_statement);
@@ -93,10 +100,8 @@ template SQLiteDbRun() {
"\nCOMMIT;\n"
);
if (!(opt_action.sqlite_discrete)
- && !(opt_action.sqlite_create)
+ && !(opt_action.sqlite_db_create)
) {
- _metadata_tid_lastrowid = db.lastInsertRowid();
- writeln("last row id: ", _metadata_tid_lastrowid);
}
} catch (ErrnoException ex) {
writeln("ERROR SQLite : ", ex);
@@ -120,7 +125,7 @@ template SQLinsertDelimiter() {
}
template SQLiteFormatAndLoadObject() {
auto SQLiteFormatAndLoadObject(I)(
- auto ref I doc_matters,
+ I doc_matters,
) {
mixin SiSUoutputRgxInit;
struct sqlite_format_and_load_objects {
@@ -569,29 +574,23 @@ template SQLiteTablesReCreate() {
string SQLiteTablesReCreate()() {
string _sql_instruct;
_sql_instruct = format(q"¶
- DROP INDEX IF EXISTS ocn;
- DROP INDEX IF EXISTS digest_clean;
- DROP INDEX IF EXISTS digest_all;
- DROP INDEX IF EXISTS clean;
- DROP INDEX IF EXISTS lev0;
- DROP INDEX IF EXISTS lev1;
- DROP INDEX IF EXISTS lev2;
- DROP INDEX IF EXISTS lev3;
- DROP INDEX IF EXISTS lev4;
- DROP INDEX IF EXISTS lev5;
- DROP INDEX IF EXISTS lev6;
- DROP INDEX IF EXISTS title;
- DROP INDEX IF EXISTS creator_author;
+ DROP INDEX IF EXISTS idx_ocn;
+ DROP INDEX IF EXISTS idx_uid;
+ DROP INDEX IF EXISTS idx_digest_clean;
+ DROP INDEX IF EXISTS idx_digest_all;
+ DROP INDEX IF EXISTS idx_clean;
+ DROP INDEX IF EXISTS idx_title;
+ DROP INDEX IF EXISTS idx_creator_author;
DROP INDEX IF EXISTS src_filename;
- DROP INDEX IF EXISTS language_document_char;
- DROP INDEX IF EXISTS classify_topic_register;
+ DROP INDEX IF EXISTS idx_language_document_char;
+ DROP INDEX IF EXISTS idx_classify_topic_register;
DROP TABLE IF EXISTS metadata_and_text;
DROP TABLE IF EXISTS doc_objects;
DROP TABLE IF EXISTS urls;
CREATE TABLE metadata_and_text (
- tid INTEGER PRIMARY KEY,
- src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
- src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */
+ src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */
+ src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */
title VARCHAR(800) NOT NULL,
title_main VARCHAR(400) NOT NULL,
title_sub VARCHAR(400) NULL,
@@ -675,24 +674,24 @@ template SQLiteTablesReCreate() {
links TEXT NULL
);
CREATE TABLE doc_objects (
- lid INTEGER PRIMARY KEY,
- metadata_tid INTEGER REFERENCES metadata_and_text,
+ lid BIGINT PRIMARY KEY,
+ uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
ocn SMALLINT,
ocnd VARCHAR(6),
ocns VARCHAR(6),
- clean TEXT NULL,
- body TEXT NULL,
- seg VARCHAR(256) NULL,
+ clean TEXT NULL,
+ body TEXT NULL,
+ seg VARCHAR(256) NULL,
lev_an VARCHAR(1),
- lev SMALLINT NULL,
+ lev SMALLINT NULL,
is_of_type VARCHAR(16),
is_a VARCHAR(16),
- node VARCHAR(16) NULL,
- parent VARCHAR(16) NULL,
- last_decendant VARCHAR(16) NULL, /* headings only */
+ node VARCHAR(16) NULL,
+ parent VARCHAR(16) NULL,
+ last_decendant VARCHAR(16) NULL, /* headings only */
digest_clean CHAR(256),
digest_all CHAR(256),
- types CHAR(1) NULL
+ types CHAR(1) NULL
);
CREATE INDEX idx_ocn ON doc_objects(ocn);
CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);
@@ -700,6 +699,7 @@ template SQLiteTablesReCreate() {
CREATE INDEX idx_clean ON doc_objects(clean);
CREATE INDEX idx_title ON metadata_and_text(title);
CREATE INDEX idx_author ON metadata_and_text(creator_author);
+ CREATE INDEX idx_uid ON metadata_and_text(uid);
CREATE INDEX idx_filename ON metadata_and_text(src_filename);
CREATE INDEX idx_language ON metadata_and_text(language_document_char);
CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);
@@ -707,13 +707,31 @@ template SQLiteTablesReCreate() {
return _sql_instruct;
}
}
+template SQLiteDeleteDocument() {
+ string SQLiteDeleteDocument(I)(
+ I doc_matters,
+ ) {
+ string _uid = doc_matters.src.doc_uid;
+ string _delete_uid = format(q"¶
+ DELETE FROM metadata_and_text
+ WHERE uid = '%s';
+ DELETE FROM doc_objects
+ WHERE uid_metadata_and_text = '%s';
+ ¶",
+ _uid,
+ _uid,
+ );
+ return _delete_uid;
+ }
+}
template SQLiteInsertMetadata() {
string SQLiteInsertMetadata(I)(
- auto ref I doc_matters,
+ I doc_matters,
) {
- string _insert_metadata;
- _insert_metadata = format(q"¶
+ string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
+ string _insert_metadata = format(q"¶
INSERT INTO metadata_and_text (
+ uid,
src_filename,
src_composite_id_per_txt,
src_composite_id_per_pod,
@@ -807,9 +825,11 @@ template SQLiteInsertMetadata() {
'%s',
'%s',
'%s',
+ '%s',
'%s'
);
¶",
+ _uid,
SQLinsertDelimiter!()(doc_matters.src.filename),
SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc),
SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod),
@@ -861,16 +881,15 @@ template SQLiteInsertMetadata() {
}
}
template SQLiteInsertDocObjectsLoop() {
- string SQLiteInsertDocObjectsLoop(D,I,X)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
- auto ref X _metadata_tid,
+ string SQLiteInsertDocObjectsLoop(D,I)(
+ D doc_abstraction,
+ I doc_matters,
) {
+ string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
string insertDocObjectsRow(O)(O obj) {
- string _insert_doc_objects_row;
- _insert_doc_objects_row = format(q"¶
+ string _insert_doc_objects_row = format(q"¶
INSERT INTO doc_objects (
- metadata_tid,
+ uid_metadata_and_text,
ocn,
clean,
body,
@@ -879,7 +898,7 @@ template SQLiteInsertDocObjectsLoop() {
is_a
)
VALUES (
- %d,
+ '%s',
%s,
'%s',
'%s',
@@ -888,7 +907,7 @@ template SQLiteInsertDocObjectsLoop() {
'%s'
);
¶",
- _metadata_tid,
+ _uid,
obj.metainfo.ocn,
SQLinsertDelimiter!()(obj_txt["text"]),
SQLinsertDelimiter!()(obj_txt["html"]),
@@ -1058,29 +1077,23 @@ template SQLiteTablesCreate() {
string SQLiteTablesReCreate()() {
string _sql_instruct;
_sql_instruct = format(q"¶
- DROP INDEX IF EXISTS ocn;
- DROP INDEX IF EXISTS digest_clean;
- DROP INDEX IF EXISTS digest_all;
- DROP INDEX IF EXISTS clean;
- DROP INDEX IF EXISTS lev0;
- DROP INDEX IF EXISTS lev1;
- DROP INDEX IF EXISTS lev2;
- DROP INDEX IF EXISTS lev3;
- DROP INDEX IF EXISTS lev4;
- DROP INDEX IF EXISTS lev5;
- DROP INDEX IF EXISTS lev6;
- DROP INDEX IF EXISTS title;
- DROP INDEX IF EXISTS creator_author;
+ DROP INDEX IF EXISTS idx_ocn;
+ DROP INDEX IF EXISTS idx_uid;
+ DROP INDEX IF EXISTS idx_digest_clean;
+ DROP INDEX IF EXISTS idx_digest_all;
+ DROP INDEX IF EXISTS idx_clean;
+ DROP INDEX IF EXISTS idx_title;
+ DROP INDEX IF EXISTS idx_creator_author;
DROP INDEX IF EXISTS src_filename;
- DROP INDEX IF EXISTS language_document_char;
- DROP INDEX IF EXISTS classify_topic_register;
+ DROP INDEX IF EXISTS idx_language_document_char;
+ DROP INDEX IF EXISTS idx_classify_topic_register;
DROP TABLE IF EXISTS metadata_and_text;
DROP TABLE IF EXISTS doc_objects;
DROP TABLE IF EXISTS urls;
CREATE TABLE metadata_and_text (
- tid INTEGER PRIMARY KEY,
- src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
- src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */
+ src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */
+ src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */
title VARCHAR(800) NOT NULL,
title_main VARCHAR(400) NOT NULL,
title_sub VARCHAR(400) NULL,
@@ -1164,24 +1177,24 @@ template SQLiteTablesCreate() {
links TEXT NULL
);
CREATE TABLE doc_objects (
- lid INTEGER PRIMARY KEY,
- metadata_tid INTEGER REFERENCES metadata_and_text,
+ lid BIGINT PRIMARY KEY,
+ uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
ocn SMALLINT,
ocnd VARCHAR(6),
ocns VARCHAR(6),
- clean TEXT NULL,
- body TEXT NULL,
- seg VARCHAR(256) NULL,
+ clean TEXT NULL,
+ body TEXT NULL,
+ seg VARCHAR(256) NULL,
lev_an VARCHAR(1),
- lev SMALLINT NULL,
+ lev SMALLINT NULL,
is_of_type VARCHAR(16),
is_a VARCHAR(16),
- node VARCHAR(16) NULL,
- parent VARCHAR(16) NULL,
- last_decendant VARCHAR(16) NULL, /* headings only */
+ node VARCHAR(16) NULL,
+ parent VARCHAR(16) NULL,
+ last_decendant VARCHAR(16) NULL, /* headings only */
digest_clean CHAR(256),
digest_all CHAR(256),
- types CHAR(1) NULL
+ types CHAR(1) NULL
);
CREATE INDEX idx_ocn ON doc_objects(ocn);
CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);
@@ -1189,6 +1202,7 @@ template SQLiteTablesCreate() {
CREATE INDEX idx_clean ON doc_objects(clean);
CREATE INDEX idx_title ON metadata_and_text(title);
CREATE INDEX idx_author ON metadata_and_text(creator_author);
+ CREATE INDEX idx_uid ON metadata_and_text(uid);
CREATE INDEX idx_filename ON metadata_and_text(src_filename);
CREATE INDEX idx_language ON metadata_and_text(language_document_char);
CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);
@@ -1196,9 +1210,9 @@ template SQLiteTablesCreate() {
return _sql_instruct;
}
}
- if (opt_action.sqlite_create) {
+ if (opt_action.sqlite_db_create) {
string _db_statement;
- auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE
+ auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file); // ISSUE
{
@@ -1208,8 +1222,17 @@ template SQLiteTablesCreate() {
}
}
}
-template SQLiteTablesDrop() {
- void SQLiteTablesDrop()() {
- writeln("table drop");
+template SQLiteDbDrop() {
+ void SQLiteDbDrop(O)(O opt_action) {
+ writeln("db drop");
+ if ((opt_action.sqlite_db_drop)) {
+ auto pth_sqlite = SiSUpathsSQLite!()(opt_action.sqlite_filename, opt_action.output_dir_set); // ISSUE
+ writeln("remove(", pth_sqlite.sqlite_file, ")");
+ try {
+ remove(pth_sqlite.sqlite_file);
+ } catch (FileException ex) {
+ // handle error
+ }
+ }
}
}
diff --git a/src/sdp/sdp.d b/src/sdp/sdp.d
index b703838..50df3f2 100755
--- a/src/sdp/sdp.d
+++ b/src/sdp/sdp.d
@@ -76,9 +76,12 @@ void main(string[] args) {
"sisupod" : false,
"source" : false,
"sqlite-discrete" : false,
+ "sqlite-db-create" : false,
+ "sqlite-db-drop" : false,
+ "sqlite-db-recreate" : false,
+ "sqlite-delete" : false,
+ "sqlite-insert" : false,
"sqlite-update" : false,
- "sqlite-create" : false,
- "sqlite-drop" : false,
"text" : false,
"verbose" : false,
"xhtml" : false,
@@ -98,6 +101,7 @@ void main(string[] args) {
"output-dir" : "",
"site-config-dir" : "",
"lang" : "all",
+ "sqlite-filename" : "documents",
];
auto helpInfo = getopt(args,
std.getopt.config.passThrough,
@@ -122,8 +126,11 @@ void main(string[] args) {
"sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"],
"source", "--source markup source text content", &opts["source"],
"sqlite-discrete", "--sqlite process discrete sqlite output", &opts["sqlite-discrete"],
- "sqlite-create", "--sqlite-create create db, create tables", &opts["sqlite-create"],
- "sqlite-drop", "--sqlite-drop drop tables & db", &opts["sqlite-drop"],
+ "sqlite-db-create", "--sqlite-db-create create db, create tables", &opts["sqlite-db-create"],
+ "sqlite-db-drop", "--sqlite-db-drop drop tables & db", &opts["sqlite-db-drop"],
+ "sqlite-db-recreate", "--sqlite-db-recreate create db, create tables", &opts["sqlite-db-recreate"],
+ "sqlite-delete", "--sqlite process sqlite output", &opts["sqlite-delete"],
+ "sqlite-insert", "--sqlite process sqlite output", &opts["sqlite-insert"],
"sqlite-update", "--sqlite process sqlite output", &opts["sqlite-update"],
"text", "--text process text output", &opts["text"],
"txt", "--txt process text output", &opts["text"],
@@ -142,6 +149,7 @@ void main(string[] args) {
"skip-output", "--skip-output", &opts["skip-output"],
"output-dir", "--output-dir=[dir path]", &settings["output-dir"],
"site-config-dir", "--site-config-dir=[dir path]", &settings["site-config-dir"],
+ "sqlite-filename", "--sqlite-filename=[filename].sqlite", &settings["sqlite-filename"],
"lang", "--lang=[lang code e.g. =en or =en,es]", &settings["lang"],
);
if (helpInfo.helpWanted) {
@@ -224,14 +232,34 @@ void main(string[] args) {
auto sqlite_discrete() {
return opts["sqlite-discrete"];
}
- auto sqlite_update() {
- return opts["sqlite-update"];
+ auto sqlite_db_drop() {
+ bool _is;
+ if (
+ opts["sqlite-db-recreate"]
+ || opts["sqlite-db-drop"]
+ ) {
+ _is = true;
+ } else { _is = false; }
+ return _is;
}
- auto sqlite_create() {
- return opts["sqlite-create"];
+ auto sqlite_db_create() {
+ bool _is;
+ if (
+ opts["sqlite-db-recreate"]
+ || opts["sqlite-db-create"]
+ ) {
+ _is = true;
+ } else { _is = false; }
+ return _is;
+ }
+ auto sqlite_insert() {
+ return opts["sqlite-insert"];
+ }
+ auto sqlite_delete() {
+ return opts["sqlite-delete"];
}
- auto sqlite_drop() {
- return opts["sqlite-drop"];
+ auto sqlite_update() {
+ return opts["sqlite-update"];
}
auto text() {
return opts["text"];
@@ -281,6 +309,9 @@ void main(string[] args) {
auto output_dir_set() {
return settings["output-dir"];
}
+ auto sqlite_filename() {
+ return settings["sqlite-filename"];
+ }
auto abstraction() {
bool _is;
if (
@@ -297,6 +328,8 @@ void main(string[] args) {
|| sisupod
|| source
|| sqlite_discrete
+ || sqlite_delete
+ || sqlite_insert
|| sqlite_update
|| text
|| xhtml
@@ -336,10 +369,8 @@ void main(string[] args) {
if (exists(sisudoc_txt_)) {
contents_location_raw_ = sisudoc_txt_.readText;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
// Handle errors
}
if (contents_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+
@@ -369,14 +400,15 @@ void main(string[] args) {
) {
auto _fns = (((tmp_dir_).chainPath(contents_location_pth_)).array).to!string;
_manifest_matter = PathMatters!()(_opt_action, _env, arg, _fns, contents_locations_arr);
- _manifests ~= _manifest_matter; // TODO how to capture?
+ _manifests ~= _manifest_matter;
}
}
} else if (arg.match(rgx.src_pth_sst_or_ssm)) {
if (exists(arg)==0) {
writeln("ERROR >> Processing Skipped! File not found: ", arg);
} else {
- _manifests ~= PathMatters!()(_opt_action, _env, arg, arg); // gather input markup source file names for processing
+ _manifest_matter = PathMatters!()(_opt_action, _env, arg, arg);
+ _manifests ~= _manifest_matter;
}
} else if (arg.match(rgx.src_pth_zip)) {
// fns_src ~= arg; // gather input markup source file names for processing
diff --git a/src/sdp/source/paths_source.d b/src/sdp/source/paths_source.d
index 8d3aefe..5637c80 100644
--- a/src/sdp/source/paths_source.d
+++ b/src/sdp/source/paths_source.d
@@ -180,6 +180,19 @@ template PathMatters() {
} else {_k = "en"; }
return _k;
}
+ string doc_uid() {
+ string _uid;
+ if (is_pod && !(pod_name.empty)) {
+ if (pod_name.baseName == filename_base) {
+ _uid = filename_base ~ "." ~ filename_extension ~ _sep ~ lng;
+ } else {
+ _uid = pod_name.baseName ~ _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng;
+ }
+ } else {
+ _uid = _sep ~ filename_base ~ "." ~ filename_extension ~ _sep ~ lng;
+ }
+ return _uid;
+ }
string docname_composite_unique_per_src_doc() {
/+
z pod name if any + src filename + lng code
diff --git a/src/sdp/source/read_config_files.d b/src/sdp/source/read_config_files.d
index 9d976a5..56ba065 100644
--- a/src/sdp/source/read_config_files.d
+++ b/src/sdp/source/read_config_files.d
@@ -32,10 +32,8 @@ static template configReadInSiteTOML() {
config_file_str = conf_file.readText;
break;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
}
}
return config_file_str;
@@ -69,10 +67,8 @@ static template configReadInDocTOML() {
config_file_str = conf_file.readText;
break;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
}
}
return config_file_str;
@@ -89,8 +85,7 @@ static template configTOML() {
TOMLDocument _toml_conf;
try {
_toml_conf = parseTOML(configuration); // parseTOML(cast(string)(configuration));
- }
- catch(ErrnoException e) {
+ } catch(ErrnoException e) {
stderr.writeln("Toml problem with content for ", conf_toml_filename);
stderr.writeln(e.msg);
}
@@ -172,10 +167,8 @@ static template readConfigDoc() {
config_file_str = conf_file.readText;
break;
}
- }
- catch (ErrnoException ex) {
- }
- catch (FileException ex) {
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
}
}
if (config_file_str.length > 0) { break; }
diff --git a/src/sdp/source/read_source_files.d b/src/sdp/source/read_source_files.d
index 28a660a..271adfa 100644
--- a/src/sdp/source/read_source_files.d
+++ b/src/sdp/source/read_source_files.d
@@ -85,13 +85,10 @@ static template SiSUrawMarkupContent() {
}
source_txt_str = fn_src.readText;
}
- }
- catch (ErrnoException ex) {
- }
- catch (UTFException ex) {
+ } catch (ErrnoException ex) {
+ } catch (UTFException ex) {
// Handle validation errors
- }
- catch (FileException ex) {
+ } catch (FileException ex) {
// Handle errors
}
std.utf.validate(source_txt_str);
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-02 03:59:01 +0000