From 1319336d6f8ce2de9a1a6319917bee6be278ad5a Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 8 Jul 2019 17:41:06 -0400 Subject: 0.7.4 sqlite related, also some xml paths --- org/default_paths.org | 199 +++++++++--------- org/doc_reform.org | 2 +- org/output_sqlite.org | 379 ++++++++++++++++++++--------------- org/output_xmls.org | 20 +- src/doc_reform/output/epub3.d | 20 +- src/doc_reform/output/paths_output.d | 192 ++++++++++-------- src/doc_reform/output/sqlite.d | 239 +++++++++++----------- views/version.txt | 2 +- 8 files changed, 570 insertions(+), 483 deletions(-) diff --git a/org/default_paths.org b/org/default_paths.org index dba021f..b1d6980 100644 --- a/org/default_paths.org +++ b/org/default_paths.org @@ -1026,47 +1026,6 @@ template DocReformOutPathsFnPd() { } #+END_SRC -** _url_ :url: - -#+name: template_paths_url -#+BEGIN_SRC d -template DocReformPathsUrl() { - import std.format; - mixin DocReformRgxInit; - static auto rgx = Rgx(); - auto DocReformPathsUrl(M)(M doc_matters) { - struct _UrlPathsStruct { - string doc_root() { - string _doc_root = (doc_matters.conf_make_meta.conf.webserv_url_doc_path.length > 0) - ? doc_matters.conf_make_meta.conf.webserv_url_doc_root - : "file://" ~ doc_matters.output_path; - return _doc_root; - } - string html_seg(string html_segname, string obj_id) { // TODO bespoke for sqlite - string _url = format(q"┃%s/%s/html/%s/%s.html#%s┃", - doc_root, - doc_matters.src.language, - doc_matters.src.filename_base, - html_segname, - obj_id - ); - if (html_segname.empty || obj_id.empty) { - if ( - doc_matters.opt.action.debug_do - && doc_matters.opt.action.verbose - ) { - writeln("sqlite object missing segname or object id: ", _url); - } - _url = ""; - } - return _url; - } - } - return _UrlPathsStruct(); - } -} -#+END_SRC - ** _html_ :html: *** relative @@ -1075,7 +1034,7 @@ template DocReformPathsUrl() { template DocReformDocRootTreeHTML() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformDocRootTreeHTML(Lng)(Lng lng) { + auto DocReformDocRootTreeHTML()(string lng) { auto lng_pth = DocReformOutPaths!()("", lng); string base_dir = "html"; string suffix = ".html"; @@ -1089,7 +1048,7 @@ template DocReformDocRootTreeHTML() { string base_filename_seg(string fn_src) { return base_filename(fn_src) ~ "." ~ lng; } - string docroot() { + string doc_root() { return asNormalizedPath(lng_pth.output_root).array; } string base() { @@ -1131,26 +1090,26 @@ template DocReformDocRootTreeHTML() { } #+END_SRC -*** absolute +*** absolute disk path #+name: template_paths_html #+BEGIN_SRC d template DocReformPathsHTML() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformPathsHTML(Po,Lng)( - Po root_pth, - Lng lng, + auto DocReformPathsHTML()( + string root_pth, + string lng, ) { auto doc_tree = DocReformDocRootTreeHTML!()(lng); string base_dir = "html"; string suffix = ".html"; struct _PathsStruct { - string docroot() { - return asNormalizedPath(root_pth.chainPath(doc_tree.docroot)).array; + string doc_root() { + return asNormalizedPath(root_pth.chainPath(doc_tree.doc_root)).array; } string harvest(string fn_harvest) { - return docroot ~ "/" ~ fn_harvest; + return doc_root ~ "/" ~ fn_harvest; } string base() { return asNormalizedPath(root_pth.chainPath(doc_tree.base)).array; @@ -1195,46 +1154,88 @@ template DocReformPathsHTML() { #+name: template_paths_html #+BEGIN_SRC d -template DocReformUrlPathsHTML() { +template DocReformUrlsHTML() { + import std.format; mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformUrlPathsHTML(Po,Lng)( - Po root_pth, - Lng lng, + auto DocReformUrlsHTML()( + string url_doc_root, + string lng, ) { auto doc_tree = DocReformDocRootTreeHTML!()(lng); string base_dir = "html"; string suffix = ".html"; struct _PathsStruct { - string docroot() { - return asNormalizedPath(root_pth.chainPath(doc_tree.docroot)).array; + string doc_root() { + return url_doc_root ~ asNormalizedPath(doc_tree.doc_root).array; } string harvest(string fn_harvest) { - return docroot ~ "/" ~ fn_harvest; + return format(q"┃%s/%s┃", + doc_root, + fn_harvest, + ); } string base() { - return asNormalizedPath(root_pth.chainPath(doc_tree.base)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.base).array, + ); } string image() { - return asNormalizedPath(root_pth.chainPath(doc_tree.image)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.image).array, + ); } string css() { - return asNormalizedPath(root_pth.chainPath(doc_tree.css)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.css).array, + ); } string fn_seg_css() { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_seg_css)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_seg_css).array, + ); } string fn_scroll_css() { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_scroll_css)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_scroll_css).array, + ); } string seg(string fn_src) { - return asNormalizedPath(root_pth.chainPath(doc_tree.seg(fn_src))).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.seg(fn_src)).array, + ); } string fn_scroll(string fn_src) { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_scroll(fn_src))).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_scroll(fn_src)).array, + ); } string fn_seg(string fn_src, string seg_filename) { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_seg(fn_src, seg_filename))).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_seg(fn_src, seg_filename)).array, + ); + } + string fn_scroll_obj_num(string fn_src, string obj_num) { + return format(q"┃%s/%s#%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_scroll(fn_src)).array, + obj_num, + ); + } + string fn_seg_obj_num(string fn_src, string seg_filename, string obj_num) { + return format(q"┃%s/%s#%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_seg(fn_src, seg_filename)).array, + obj_num, + ); } string tail_seg(string fn_src) { return doc_tree.tail_seg(fn_src); @@ -1258,9 +1259,9 @@ template DocReformUrlPathsHTML() { template DocReformPathsEPUB() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformPathsEPUB(Po,Lng)( - Po output_pth_root, - Lng lng, + auto DocReformPathsEPUB()( + string output_pth_root, + string lng, ) { auto out_pth = DocReformOutPaths!()(output_pth_root, lng); string base_dir = "epub"; @@ -1271,44 +1272,50 @@ template DocReformPathsEPUB() { string base_filename(string fn_src) { return fn_src.baseName.stripExtension ~ "." ~ lng; } - string epub_file(string fn_src) { - return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".epub")).array; + string base_filename(string fn_src) { + return fn_src.baseName.stripExtension; } - string dirtop() { - return "".chainPath("").array; + string base_filename_epub(string fn_src) { + return base_filename(fn_src) ~ "." ~ lng; } - string doc_meta_inf(string fn_src) { - return asNormalizedPath(dirtop.chainPath("META-INF")).array; + string doc_meta_inf() { + return asNormalizedPath("META-INF").array; } - string doc_oebps(string fn_src) { - return asNormalizedPath(dirtop.chainPath("OEBPS")).array; + string doc_oebps() { + return asNormalizedPath("OEBPS").array; } - string doc_oebps_css(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("css")).array; + string doc_oebps_css() { + return asNormalizedPath(doc_oebps.chainPath("css")).array; } - string doc_oebps_image(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("image")).array; + string doc_oebps_image() { + return asNormalizedPath(doc_oebps.chainPath("image")).array; } - string fn_mimetypes(string fn_src) { + string epub_file(string fn_src) { + return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".epub")).array; + } + string dirtop() { + return "".chainPath("").array; + } + string fn_mimetypes() { return asNormalizedPath(dirtop.chainPath("mimetypes")).array; } - string fn_dmi_container_xml(string fn_src) { - return asNormalizedPath(doc_meta_inf(fn_src).chainPath("container.xml")).array; + string fn_dmi_container_xml() { + return asNormalizedPath(doc_meta_inf.chainPath("container.xml")).array; } - string fn_oebps_toc_nav_xhtml(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("toc_nav.xhtml")).array; + string fn_oebps_toc_nav_xhtml() { + return asNormalizedPath(doc_oebps.chainPath("toc_nav.xhtml")).array; } - string fn_oebps_toc_ncx(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("toc.ncx")).array; + string fn_oebps_toc_ncx() { + return asNormalizedPath(doc_oebps.chainPath("toc.ncx")).array; } - string fn_oebps_content_opf(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("content.opf")).array; + string fn_oebps_content_opf() { + return asNormalizedPath(doc_oebps.chainPath("content.opf")).array; } - string fn_oebps_content_xhtml(string fn_src, string seg_filename) { - return asNormalizedPath(doc_oebps(fn_src).chainPath(seg_filename ~ ".xhtml")).array; + string fn_oebps_content_xhtml(string seg_filename) { + return asNormalizedPath(doc_oebps.chainPath(seg_filename ~ ".xhtml")).array; } - string fn_oebps_css(string fn_src) { - return asNormalizedPath(doc_oebps_css(fn_src).chainPath("epub.css")).array; + string fn_oebps_css() { + return asNormalizedPath(doc_oebps_css.chainPath("epub.css")).array; } /+ debug +/ string dbg_docdir(string fn_src) { @@ -1469,9 +1476,9 @@ template DocReformPathsLaTeX() { template DocReformPathsSQLiteDiscrete() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformPathsSQLiteDiscrete(Po,Lng)( - Po output_pth_root, - Lng lng, + auto DocReformPathsSQLiteDiscrete()( + string output_pth_root, + string lng, ) { struct _PathsStruct { string base_filename(string fn_src) { diff --git a/org/doc_reform.org b/org/doc_reform.org index 798d680..848291f 100644 --- a/org/doc_reform.org +++ b/org/doc_reform.org @@ -27,7 +27,7 @@ struct Version { int minor; int patch; } -enum _ver = Version(0, 7, 3); +enum _ver = Version(0, 7, 4); #+END_SRC ** compilation restrictions (supported compilers) diff --git a/org/output_sqlite.org b/org/output_sqlite.org index 2dc29f7..fc2603d 100644 --- a/org/output_sqlite.org +++ b/org/output_sqlite.org @@ -29,7 +29,9 @@ module doc_reform.output.sqlite; import d2sqlite3; import std.typecons : Nullable; mixin DocReformOutputRgxInit; +mixin InternalMarkup; static auto rgx = Rgx(); +static auto mkup = InlineMarkup(); long _metadata_tid_lastrowid; template SQLiteHubBuildTablesAndPopulate() { void SQLiteHubBuildTablesAndPopulate(D,M)( @@ -64,6 +66,7 @@ template SQLiteHubDiscreteBuildTablesAndPopulate() { const D doc_abstraction, M doc_matters, ) { + auto url_html = DocReformUrlsHTML!()(doc_matters.conf_make_meta.conf.webserv_url_doc_root, doc_matters.src.language); auto pth_sqlite = DocReformPathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); pth_sqlite.base.mkdirRecurse; auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); @@ -163,6 +166,7 @@ template SQLiteTablesReCreate() { <> <> <> + <> <> <> ┃",); @@ -192,6 +196,7 @@ template SQLiteInsertMetadata() { M doc_matters, ) { <> + <> return _insert_metadata; } } @@ -206,7 +211,7 @@ template SQLiteInsertDocObjectsLoop() { M doc_matters, ) { string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); - auto site_url = DocReformPathsUrl!()(doc_matters); + auto url_html = DocReformUrlsHTML!()(doc_matters.conf_make_meta.conf.webserv_url_doc_root, doc_matters.src.language); string insertDocObjectsRow(O)(O obj) { <> return _insert_doc_objects_row; @@ -229,6 +234,7 @@ template SQLiteTablesCreate() { <> <> <> + <> <> <> ┃",); @@ -1457,6 +1463,7 @@ DROP INDEX IF EXISTS idx_classify_topic_register; #+name: sqlite_statement_drop_existing_tables #+BEGIN_SRC sql DROP TABLE IF EXISTS metadata_and_text; +DROP TABLE IF EXISTS topic_register; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; #+END_SRC @@ -1467,15 +1474,14 @@ DROP TABLE IF EXISTS urls; #+name: sqlite_statement_create_table_metadata_and_src_txt #+BEGIN_SRC sql CREATE TABLE metadata_and_text ( - uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */ - src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */ - src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */ + uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) + src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code + src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, title_short VARCHAR(400) NULL, title_edition VARCHAR(10) NULL, - title_note VARCHAR(2500) NULL, title_language VARCHAR(100) NULL, title_language_char VARCHAR(6) NULL, creator_author VARCHAR(600) NOT NULL, @@ -1505,14 +1511,13 @@ CREATE TABLE metadata_and_text ( date_translated VARCHAR(10) NULL, date_original_publication VARCHAR(10) NULL, date_generated VARCHAR(10) NULL, - publisher VARCHAR(600) NULL, + original_title VARCHAR(800) NULL, original_publisher VARCHAR(600) NULL, original_language VARCHAR(100) NULL, original_language_char VARCHAR(6) NULL, original_source VARCHAR(600) NULL, original_institution VARCHAR(600) NULL, original_nationality VARCHAR(100) NULL, - original_title VARCHAR(800) NULL, rights_copyright VARCHAR(2500) NULL, rights_copyright_audio VARCHAR(2500) NULL, rights_copyright_cover VARCHAR(2500) NULL, @@ -1541,18 +1546,58 @@ CREATE TABLE metadata_and_text ( notes_prefix_a TEXT NULL, notes_prefix_b TEXT NULL, notes_suffix TEXT NULL, + publisher VARCHAR(600) NULL, src_filename VARCHAR(256) NOT NULL, src_fingerprint VARCHAR(256) NULL, src_filesize VARCHAR(10) NULL, src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ - site_url_doc_root VARCHAR(256) NULL, /* url path to doc root */ + pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL, + site_url_epub VARCHAR(256) NULL, links TEXT NULL ); #+END_SRC +**** CREATE TABLE topic_register + +#+name: sqlite_statement_create_table_topic_register +#+BEGIN_SRC sql +CREATE TABLE topic_register ( + -- tid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code + -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename + topic_register_lv0 VARCHAR(250) NOT NULL, + topic_register_lv1 VARCHAR(250) NULL, + topic_register_lv2 VARCHAR(250) NULL, + topic_register_lv3 VARCHAR(250) NULL, + topic_register_lv4 VARCHAR(250) NULL, + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL +); +#+END_SRC + + +**** CREATE TABLE site_urls ? + +#+name: sqlite_statement_create_table_site_urls +#+BEGIN_SRC sql +CREATE TABLE site_urls ( + -- tid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, - z pod name if any + src filename + language code + src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL +); +#+END_SRC + **** CREATE TABLE doc_objects #+name: sqlite_statement_create_table_objects @@ -1571,7 +1616,7 @@ CREATE TABLE doc_objects ( lev SMALLINT NULL, node VARCHAR(16) NULL, parent VARCHAR(16) NULL, - last_decendant VARCHAR(16) NULL, /* headings only */ + last_decendant VARCHAR(16) NULL, -- headings only digest_clean CHAR(256), digest_all CHAR(256), html_seg_url CHAR(256), @@ -1628,73 +1673,6 @@ WHERE uid_metadata_and_text = '%s'; #+END_SRC *** inserts -**** INSERT doc objects - -lid unique, increment by 1 per object, not ocn - -metadata tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -***** sql statement: dlang format - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC d -string _insert_doc_objects_row = format(q"┃ -#+END_SRC - -***** INSERT INTO - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC sql - INSERT INTO doc_objects ( - uid_metadata_and_text, - ocn, - obj_id, - clean, - body, - lev, - is_of_type, - is_a, - html_seg_url - ) -#+END_SRC - -***** VALUES - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC sql - VALUES ( - '%s', - %s, - '%s', - '%s', - '%s', - %s, - '%s', - '%s', - '%s' - ); -#+END_SRC - -***** dlang values for formatting - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC d -┃", - _uid, - obj.metainfo.ocn, - obj.metainfo.identifier, - SQLinsertDelimiter!()(obj_txt["text"]), - SQLinsertDelimiter!()(obj_txt["html"]), - obj.metainfo.heading_lev_markup, - obj.metainfo.is_of_type, - obj.metainfo.is_a, - site_url.html_seg(obj.tags.html_segment_anchor_tag_is, obj.metainfo.identifier), -); -#+END_SRC - **** INSERT doc matters & metadata ***** sql statement: dlang format @@ -1719,15 +1697,12 @@ string _insert_metadata = format(q"┃ title_short, title_edition, title_language, - classify_dewey, - classify_keywords, - classify_loc, - classify_subject, - classify_topic_register, creator_author, creator_author_email, creator_illustrator, creator_translator, + language_document, + language_document_char, date_added_to_site, date_available, date_created, @@ -1735,18 +1710,6 @@ string _insert_metadata = format(q"┃ date_modified, date_published, date_valid, - identifier_isbn, - identifier_oclc, - language_document, - language_document_char, - notes_abstract, - notes_description, - original_publisher, - original_language, - original_language_char, - original_source, - original_title, - publisher, rights_copyright, rights_copyright_audio, rights_copyright_cover, @@ -1756,6 +1719,21 @@ string _insert_metadata = format(q"┃ rights_copyright_translation, rights_copyright_video, rights_license, + identifier_oclc, + identifier_isbn, + classify_dewey, + classify_keywords, + classify_loc, + classify_subject, + classify_topic_register, + original_title, + original_publisher, + original_language, + original_language_char, + original_source, + notes_abstract, + notes_description, + publisher, site_url_doc_root ) #+END_SRC @@ -1765,54 +1743,7 @@ string _insert_metadata = format(q"┃ #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC sql VALUES ( - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s' + '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' ); #+END_SRC @@ -1831,15 +1762,12 @@ string _insert_metadata = format(q"┃ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), @@ -1847,18 +1775,6 @@ string _insert_metadata = format(q"┃ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), @@ -1868,8 +1784,149 @@ string _insert_metadata = format(q"┃ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), SQLinsertDelimiter!()(doc_matters.conf_make_meta.conf.webserv_url_doc_root) ); #+END_SRC +**** INSERT topic register + +writeln(doc_matters.conf_make_meta.meta.classify_topic_register_arr); + +***** { if topic register then loop topic register array + +#+name: sqlite_formatted_insertions_topic_register +#+BEGIN_SRC d +if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) { + + string _insert_topics; + foreach (topic; doc_matters.conf_make_meta.meta.classify_topic_register_arr) { + string[] subject_tree = topic.split(mkup.sep); +#+END_SRC + +***** sql statement: dlang format + +#+name: sqlite_formatted_insertions_topic_register +#+BEGIN_SRC d +_insert_topics ~= format(q"┃ +#+END_SRC + +***** INSERT INTO + +#+name: sqlite_formatted_insertions_topic_register +#+BEGIN_SRC sql + INSERT INTO topic_register ( + uid_metadata_and_text, + topic_register_lv0, + topic_register_lv1, + topic_register_lv2, + topic_register_lv3, + topic_register_lv4 + ) +#+END_SRC + +***** VALUES + +#+name: sqlite_formatted_insertions_topic_register +#+BEGIN_SRC sql + VALUES ( + '%s', '%s', '%s', '%s', '%s', '%s' + ); +#+END_SRC + +***** dlang values for formatting + +#+name: sqlite_formatted_insertions_topic_register +#+BEGIN_SRC d +┃", + _uid, + (subject_tree.length > 0) ? subject_tree[0] : "", + (subject_tree.length > 1) ? subject_tree[1] : "", + (subject_tree.length > 2) ? subject_tree[2] : "", + (subject_tree.length > 3) ? subject_tree[3] : "", + (subject_tree.length > 4) ? subject_tree[4] : "" +); +#+END_SRC + +***** } close topic register & loop topic register array + +#+name: sqlite_formatted_insertions_topic_register +#+BEGIN_SRC d + } +} +#+END_SRC + +**** INSERT doc objects + +lid unique, increment by 1 per object, not ocn + +metadata tid document number unique +either: +- increment by adding 1 for each document, +- make hash of document filename or url and use? + +***** sql statement: dlang format + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC d +string _insert_doc_objects_row = format(q"┃ +#+END_SRC + +***** INSERT INTO + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC sql + INSERT INTO doc_objects ( + uid_metadata_and_text, + ocn, + obj_id, + clean, + body, + lev, + is_of_type, + is_a, + html_seg_url + ) +#+END_SRC + +***** VALUES + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC sql + VALUES ( + '%s', %s, '%s', '%s', '%s', %s, '%s', '%s', '%s' + ); +#+END_SRC + +***** dlang values for formatting + +#+name: sqlite_formatted_insertions_doc_objects +#+BEGIN_SRC d +┃", + _uid, + obj.metainfo.ocn, + obj.metainfo.identifier, + SQLinsertDelimiter!()(obj_txt["text"]), + SQLinsertDelimiter!()(obj_txt["html"]), + obj.metainfo.heading_lev_markup, + obj.metainfo.is_of_type, + obj.metainfo.is_a, + url_html.fn_seg_obj_num(doc_matters.src.filename, obj.tags.html_segment_anchor_tag_is, obj.metainfo.identifier), +); +#+END_SRC + * __END__ diff --git a/org/output_xmls.org b/org/output_xmls.org index 2eb0e48..5e312f2 100644 --- a/org/output_xmls.org +++ b/org/output_xmls.org @@ -2217,7 +2217,7 @@ string epub3_oebps_content(D,M,P)(D doc_abstraction, M doc_matters, P parts) { _uuid, _uuid, _uuid, - (pth_epub3.fn_oebps_css(doc_matters.src.filename)).chompPrefix("OEBPS/"), + (pth_epub3.fn_oebps_css).chompPrefix("OEBPS/"), ); content ~= " " ~ "" ~ "\n "; content ~= parts["manifest_documents"]; @@ -2227,7 +2227,7 @@ string epub3_oebps_content(D,M,P)(D doc_abstraction, M doc_matters, P parts) { content ~= format(q"┃ ┃", image.baseName.stripExtension, - (pth_epub3.doc_oebps_image(doc_matters.src.filename)).chompPrefix("OEBPS/"), + (pth_epub3.doc_oebps_image).chompPrefix("OEBPS/"), image, image.extension.chompPrefix("."), ); @@ -2868,7 +2868,7 @@ void epub3_write_output_files(W,I)( } { /+ OEBPS/[segments].xhtml (the document contents) +/ foreach (seg_filename; doc_matters.has.segnames_lv_0_to_4) { - string fn = pth_epub3.fn_oebps_content_xhtml(doc_matters.src.filename, seg_filename); + string fn = pth_epub3.fn_oebps_content_xhtml(seg_filename); auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = fn; auto zip_data = new OutBuffer(); @@ -2908,7 +2908,7 @@ void epub3_write_output_files(W,I)( File(fn_dbg, "w").writeln(epub_write.mimetypes); } } - fn = pth_epub3.fn_mimetypes(doc_matters.src.filename); + fn = pth_epub3.fn_mimetypes; EPUBzip(epub_write.mimetypes, fn); } { /+ META-INF/container.xml (identify doc root) +/ @@ -2918,7 +2918,7 @@ void epub3_write_output_files(W,I)( File(fn_dbg, "w").writeln(epub_write.meta_inf_container_xml); } } - fn = pth_epub3.fn_dmi_container_xml(doc_matters.src.filename); + fn = pth_epub3.fn_dmi_container_xml; EPUBzip(epub_write.meta_inf_container_xml, fn); } { /+ OEBPS/toc_nav.xhtml (navigation toc epub3) +/ @@ -2928,7 +2928,7 @@ void epub3_write_output_files(W,I)( File(fn_dbg, "w").writeln(epub_write.oebps_toc_nav_xhtml); } } - fn = pth_epub3.fn_oebps_toc_nav_xhtml(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_toc_nav_xhtml; EPUBzip(epub_write.oebps_toc_nav_xhtml, fn); } { /+ OEBPS/toc.ncx (navigation toc epub2) +/ @@ -2938,7 +2938,7 @@ void epub3_write_output_files(W,I)( File(fn_dbg, "w").writeln(epub_write.oebps_toc_ncx); } } - fn = pth_epub3.fn_oebps_toc_ncx(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_toc_ncx; EPUBzip(epub_write.oebps_toc_ncx, fn); } { /+ OEBPS/content.opf (doc manifest) +/ @@ -2948,7 +2948,7 @@ void epub3_write_output_files(W,I)( File(fn_dbg, "w").writeln(epub_write.oebps_content_opf); } } - fn = pth_epub3.fn_oebps_content_opf(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_content_opf; EPUBzip(epub_write.oebps_content_opf, fn); } { /+ OEBPS/_sisu/image (images) +/ @@ -2968,7 +2968,7 @@ void epub3_write_output_files(W,I)( } } auto fn_src = doc_matters.src.image_dir_path ~ "/" ~ image; - auto fn_out = pth_epub3.doc_oebps_image(doc_matters.src.filename.to!string) ~ "/" ~ image; + auto fn_out = pth_epub3.doc_oebps_image ~ "/" ~ image; if (exists(fn_src)) { { auto zip_arc_member_file = new ArchiveMember(); @@ -2992,7 +2992,7 @@ void epub3_write_output_files(W,I)( : File(fn_dbg, "w").writeln(css.dark.epub); } } - fn = pth_epub3.fn_oebps_css(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_css; auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = fn; auto zip_data = new OutBuffer(); diff --git a/src/doc_reform/output/epub3.d b/src/doc_reform/output/epub3.d index cdfd2a6..09dea3d 100644 --- a/src/doc_reform/output/epub3.d +++ b/src/doc_reform/output/epub3.d @@ -79,7 +79,7 @@ template outputEPub3() { _uuid, _uuid, _uuid, - (pth_epub3.fn_oebps_css(doc_matters.src.filename)).chompPrefix("OEBPS/"), + (pth_epub3.fn_oebps_css).chompPrefix("OEBPS/"), ); content ~= " " ~ "" ~ "\n "; content ~= parts["manifest_documents"]; @@ -89,7 +89,7 @@ template outputEPub3() { content ~= format(q"┃ ┃", image.baseName.stripExtension, - (pth_epub3.doc_oebps_image(doc_matters.src.filename)).chompPrefix("OEBPS/"), + (pth_epub3.doc_oebps_image).chompPrefix("OEBPS/"), image, image.extension.chompPrefix("."), ); @@ -659,7 +659,7 @@ template outputEPub3() { } { /+ OEBPS/[segments].xhtml (the document contents) +/ foreach (seg_filename; doc_matters.has.segnames_lv_0_to_4) { - string fn = pth_epub3.fn_oebps_content_xhtml(doc_matters.src.filename, seg_filename); + string fn = pth_epub3.fn_oebps_content_xhtml(seg_filename); auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = fn; auto zip_data = new OutBuffer(); @@ -699,7 +699,7 @@ template outputEPub3() { File(fn_dbg, "w").writeln(epub_write.mimetypes); } } - fn = pth_epub3.fn_mimetypes(doc_matters.src.filename); + fn = pth_epub3.fn_mimetypes; EPUBzip(epub_write.mimetypes, fn); } { /+ META-INF/container.xml (identify doc root) +/ @@ -709,7 +709,7 @@ template outputEPub3() { File(fn_dbg, "w").writeln(epub_write.meta_inf_container_xml); } } - fn = pth_epub3.fn_dmi_container_xml(doc_matters.src.filename); + fn = pth_epub3.fn_dmi_container_xml; EPUBzip(epub_write.meta_inf_container_xml, fn); } { /+ OEBPS/toc_nav.xhtml (navigation toc epub3) +/ @@ -719,7 +719,7 @@ template outputEPub3() { File(fn_dbg, "w").writeln(epub_write.oebps_toc_nav_xhtml); } } - fn = pth_epub3.fn_oebps_toc_nav_xhtml(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_toc_nav_xhtml; EPUBzip(epub_write.oebps_toc_nav_xhtml, fn); } { /+ OEBPS/toc.ncx (navigation toc epub2) +/ @@ -729,7 +729,7 @@ template outputEPub3() { File(fn_dbg, "w").writeln(epub_write.oebps_toc_ncx); } } - fn = pth_epub3.fn_oebps_toc_ncx(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_toc_ncx; EPUBzip(epub_write.oebps_toc_ncx, fn); } { /+ OEBPS/content.opf (doc manifest) +/ @@ -739,7 +739,7 @@ template outputEPub3() { File(fn_dbg, "w").writeln(epub_write.oebps_content_opf); } } - fn = pth_epub3.fn_oebps_content_opf(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_content_opf; EPUBzip(epub_write.oebps_content_opf, fn); } { /+ OEBPS/_sisu/image (images) +/ @@ -759,7 +759,7 @@ template outputEPub3() { } } auto fn_src = doc_matters.src.image_dir_path ~ "/" ~ image; - auto fn_out = pth_epub3.doc_oebps_image(doc_matters.src.filename.to!string) ~ "/" ~ image; + auto fn_out = pth_epub3.doc_oebps_image ~ "/" ~ image; if (exists(fn_src)) { { auto zip_arc_member_file = new ArchiveMember(); @@ -783,7 +783,7 @@ template outputEPub3() { : File(fn_dbg, "w").writeln(css.dark.epub); } } - fn = pth_epub3.fn_oebps_css(doc_matters.src.filename); + fn = pth_epub3.fn_oebps_css; auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = fn; auto zip_data = new OutBuffer(); diff --git a/src/doc_reform/output/paths_output.d b/src/doc_reform/output/paths_output.d index 795da4c..0ca9fa7 100644 --- a/src/doc_reform/output/paths_output.d +++ b/src/doc_reform/output/paths_output.d @@ -77,45 +77,11 @@ template DocReformOutPathsFnPd() { return _PathsStruct(); } } -template DocReformPathsUrl() { - import std.format; - mixin DocReformRgxInit; - static auto rgx = Rgx(); - auto DocReformPathsUrl(M)(M doc_matters) { - struct _UrlPathsStruct { - string doc_root() { - string _doc_root = (doc_matters.conf_make_meta.conf.webserv_url_doc_path.length > 0) - ? doc_matters.conf_make_meta.conf.webserv_url_doc_root - : "file://" ~ doc_matters.output_path; - return _doc_root; - } - string html_seg(string html_segname, string obj_id) { // TODO bespoke for sqlite - string _url = format(q"┃%s/%s/html/%s/%s.html#%s┃", - doc_root, - doc_matters.src.language, - doc_matters.src.filename_base, - html_segname, - obj_id - ); - if (html_segname.empty || obj_id.empty) { - if ( - doc_matters.opt.action.debug_do - && doc_matters.opt.action.verbose - ) { - writeln("sqlite object missing segname or object id: ", _url); - } - _url = ""; - } - return _url; - } - } - return _UrlPathsStruct(); - } -} + template DocReformDocRootTreeHTML() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformDocRootTreeHTML(Lng)(Lng lng) { + auto DocReformDocRootTreeHTML()(string lng) { auto lng_pth = DocReformOutPaths!()("", lng); string base_dir = "html"; string suffix = ".html"; @@ -129,7 +95,7 @@ template DocReformDocRootTreeHTML() { string base_filename_seg(string fn_src) { return base_filename(fn_src) ~ "." ~ lng; } - string docroot() { + string doc_root() { return asNormalizedPath(lng_pth.output_root).array; } string base() { @@ -172,19 +138,19 @@ template DocReformDocRootTreeHTML() { template DocReformPathsHTML() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformPathsHTML(Po,Lng)( - Po root_pth, - Lng lng, + auto DocReformPathsHTML()( + string root_pth, + string lng, ) { auto doc_tree = DocReformDocRootTreeHTML!()(lng); string base_dir = "html"; string suffix = ".html"; struct _PathsStruct { - string docroot() { - return asNormalizedPath(root_pth.chainPath(doc_tree.docroot)).array; + string doc_root() { + return asNormalizedPath(root_pth.chainPath(doc_tree.doc_root)).array; } string harvest(string fn_harvest) { - return docroot ~ "/" ~ fn_harvest; + return doc_root ~ "/" ~ fn_harvest; } string base() { return asNormalizedPath(root_pth.chainPath(doc_tree.base)).array; @@ -223,46 +189,88 @@ template DocReformPathsHTML() { return _PathsStruct(); } } -template DocReformUrlPathsHTML() { +template DocReformUrlsHTML() { + import std.format; mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformUrlPathsHTML(Po,Lng)( - Po root_pth, - Lng lng, + auto DocReformUrlsHTML()( + string url_doc_root, + string lng, ) { auto doc_tree = DocReformDocRootTreeHTML!()(lng); string base_dir = "html"; string suffix = ".html"; struct _PathsStruct { - string docroot() { - return asNormalizedPath(root_pth.chainPath(doc_tree.docroot)).array; + string doc_root() { + return url_doc_root ~ asNormalizedPath(doc_tree.doc_root).array; } string harvest(string fn_harvest) { - return docroot ~ "/" ~ fn_harvest; + return format(q"┃%s/%s┃", + doc_root, + fn_harvest, + ); } string base() { - return asNormalizedPath(root_pth.chainPath(doc_tree.base)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.base).array, + ); } string image() { - return asNormalizedPath(root_pth.chainPath(doc_tree.image)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.image).array, + ); } string css() { - return asNormalizedPath(root_pth.chainPath(doc_tree.css)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.css).array, + ); } string fn_seg_css() { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_seg_css)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_seg_css).array, + ); } string fn_scroll_css() { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_scroll_css)).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_scroll_css).array, + ); } string seg(string fn_src) { - return asNormalizedPath(root_pth.chainPath(doc_tree.seg(fn_src))).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.seg(fn_src)).array, + ); } string fn_scroll(string fn_src) { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_scroll(fn_src))).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_scroll(fn_src)).array, + ); } string fn_seg(string fn_src, string seg_filename) { - return asNormalizedPath(root_pth.chainPath(doc_tree.fn_seg(fn_src, seg_filename))).array; + return format(q"┃%s/%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_seg(fn_src, seg_filename)).array, + ); + } + string fn_scroll_obj_num(string fn_src, string obj_num) { + return format(q"┃%s/%s#%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_scroll(fn_src)).array, + obj_num, + ); + } + string fn_seg_obj_num(string fn_src, string seg_filename, string obj_num) { + return format(q"┃%s/%s#%s┃", + url_doc_root, + asNormalizedPath(doc_tree.fn_seg(fn_src, seg_filename)).array, + obj_num, + ); } string tail_seg(string fn_src) { return doc_tree.tail_seg(fn_src); @@ -280,9 +288,9 @@ template DocReformUrlPathsHTML() { template DocReformPathsEPUB() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformPathsEPUB(Po,Lng)( - Po output_pth_root, - Lng lng, + auto DocReformPathsEPUB()( + string output_pth_root, + string lng, ) { auto out_pth = DocReformOutPaths!()(output_pth_root, lng); string base_dir = "epub"; @@ -293,44 +301,50 @@ template DocReformPathsEPUB() { string base_filename(string fn_src) { return fn_src.baseName.stripExtension ~ "." ~ lng; } - string epub_file(string fn_src) { - return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".epub")).array; + string base_filename(string fn_src) { + return fn_src.baseName.stripExtension; } - string dirtop() { - return "".chainPath("").array; + string base_filename_epub(string fn_src) { + return base_filename(fn_src) ~ "." ~ lng; } - string doc_meta_inf(string fn_src) { - return asNormalizedPath(dirtop.chainPath("META-INF")).array; + string doc_meta_inf() { + return asNormalizedPath("META-INF").array; } - string doc_oebps(string fn_src) { - return asNormalizedPath(dirtop.chainPath("OEBPS")).array; + string doc_oebps() { + return asNormalizedPath("OEBPS").array; } - string doc_oebps_css(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("css")).array; + string doc_oebps_css() { + return asNormalizedPath(doc_oebps.chainPath("css")).array; } - string doc_oebps_image(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("image")).array; + string doc_oebps_image() { + return asNormalizedPath(doc_oebps.chainPath("image")).array; } - string fn_mimetypes(string fn_src) { + string epub_file(string fn_src) { + return asNormalizedPath(base.chainPath(base_filename(fn_src) ~ ".epub")).array; + } + string dirtop() { + return "".chainPath("").array; + } + string fn_mimetypes() { return asNormalizedPath(dirtop.chainPath("mimetypes")).array; } - string fn_dmi_container_xml(string fn_src) { - return asNormalizedPath(doc_meta_inf(fn_src).chainPath("container.xml")).array; + string fn_dmi_container_xml() { + return asNormalizedPath(doc_meta_inf.chainPath("container.xml")).array; } - string fn_oebps_toc_nav_xhtml(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("toc_nav.xhtml")).array; + string fn_oebps_toc_nav_xhtml() { + return asNormalizedPath(doc_oebps.chainPath("toc_nav.xhtml")).array; } - string fn_oebps_toc_ncx(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("toc.ncx")).array; + string fn_oebps_toc_ncx() { + return asNormalizedPath(doc_oebps.chainPath("toc.ncx")).array; } - string fn_oebps_content_opf(string fn_src) { - return asNormalizedPath(doc_oebps(fn_src).chainPath("content.opf")).array; + string fn_oebps_content_opf() { + return asNormalizedPath(doc_oebps.chainPath("content.opf")).array; } - string fn_oebps_content_xhtml(string fn_src, string seg_filename) { - return asNormalizedPath(doc_oebps(fn_src).chainPath(seg_filename ~ ".xhtml")).array; + string fn_oebps_content_xhtml(string seg_filename) { + return asNormalizedPath(doc_oebps.chainPath(seg_filename ~ ".xhtml")).array; } - string fn_oebps_css(string fn_src) { - return asNormalizedPath(doc_oebps_css(fn_src).chainPath("epub.css")).array; + string fn_oebps_css() { + return asNormalizedPath(doc_oebps_css.chainPath("epub.css")).array; } /+ debug +/ string dbg_docdir(string fn_src) { @@ -472,9 +486,9 @@ template DocReformPathsLaTeX() { template DocReformPathsSQLiteDiscrete() { mixin DocReformRgxInit; static auto rgx = Rgx(); - auto DocReformPathsSQLiteDiscrete(Po,Lng)( - Po output_pth_root, - Lng lng, + auto DocReformPathsSQLiteDiscrete()( + string output_pth_root, + string lng, ) { struct _PathsStruct { string base_filename(string fn_src) { diff --git a/src/doc_reform/output/sqlite.d b/src/doc_reform/output/sqlite.d index 57c90e0..5bc1551 100644 --- a/src/doc_reform/output/sqlite.d +++ b/src/doc_reform/output/sqlite.d @@ -7,7 +7,9 @@ import import d2sqlite3; import std.typecons : Nullable; mixin DocReformOutputRgxInit; +mixin InternalMarkup; static auto rgx = Rgx(); +static auto mkup = InlineMarkup(); long _metadata_tid_lastrowid; template SQLiteHubBuildTablesAndPopulate() { void SQLiteHubBuildTablesAndPopulate(D,M)( @@ -61,6 +63,7 @@ template SQLiteHubDiscreteBuildTablesAndPopulate() { const D doc_abstraction, M doc_matters, ) { + auto url_html = DocReformUrlsHTML!()(doc_matters.conf_make_meta.conf.webserv_url_doc_root, doc_matters.src.language); auto pth_sqlite = DocReformPathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); pth_sqlite.base.mkdirRecurse; auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); @@ -867,18 +870,18 @@ template SQLiteTablesReCreate() { DROP INDEX IF EXISTS idx_language_document_char; DROP INDEX IF EXISTS idx_classify_topic_register; DROP TABLE IF EXISTS metadata_and_text; + DROP TABLE IF EXISTS topic_register; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE metadata_and_text ( - uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */ - src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */ - src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */ + uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) + src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code + src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, title_short VARCHAR(400) NULL, title_edition VARCHAR(10) NULL, - title_note VARCHAR(2500) NULL, title_language VARCHAR(100) NULL, title_language_char VARCHAR(6) NULL, creator_author VARCHAR(600) NOT NULL, @@ -908,14 +911,13 @@ template SQLiteTablesReCreate() { date_translated VARCHAR(10) NULL, date_original_publication VARCHAR(10) NULL, date_generated VARCHAR(10) NULL, - publisher VARCHAR(600) NULL, + original_title VARCHAR(800) NULL, original_publisher VARCHAR(600) NULL, original_language VARCHAR(100) NULL, original_language_char VARCHAR(6) NULL, original_source VARCHAR(600) NULL, original_institution VARCHAR(600) NULL, original_nationality VARCHAR(100) NULL, - original_title VARCHAR(800) NULL, rights_copyright VARCHAR(2500) NULL, rights_copyright_audio VARCHAR(2500) NULL, rights_copyright_cover VARCHAR(2500) NULL, @@ -944,16 +946,34 @@ template SQLiteTablesReCreate() { notes_prefix_a TEXT NULL, notes_prefix_b TEXT NULL, notes_suffix TEXT NULL, + publisher VARCHAR(600) NULL, src_filename VARCHAR(256) NOT NULL, src_fingerprint VARCHAR(256) NULL, src_filesize VARCHAR(10) NULL, src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ - site_url_doc_root VARCHAR(256) NULL, /* url path to doc root */ + pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL, + site_url_epub VARCHAR(256) NULL, links TEXT NULL ); + CREATE TABLE topic_register ( + -- tid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code + -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename + topic_register_lv0 VARCHAR(250) NOT NULL, + topic_register_lv1 VARCHAR(250) NULL, + topic_register_lv2 VARCHAR(250) NULL, + topic_register_lv3 VARCHAR(250) NULL, + topic_register_lv4 VARCHAR(250) NULL, + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL + ); CREATE TABLE doc_objects ( lid BIGINT PRIMARY KEY, uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, @@ -968,7 +988,7 @@ template SQLiteTablesReCreate() { lev SMALLINT NULL, node VARCHAR(16) NULL, parent VARCHAR(16) NULL, - last_decendant VARCHAR(16) NULL, /* headings only */ + last_decendant VARCHAR(16) NULL, -- headings only digest_clean CHAR(256), digest_all CHAR(256), html_seg_url CHAR(256), @@ -1022,15 +1042,12 @@ template SQLiteInsertMetadata() { title_short, title_edition, title_language, - classify_dewey, - classify_keywords, - classify_loc, - classify_subject, - classify_topic_register, creator_author, creator_author_email, creator_illustrator, creator_translator, + language_document, + language_document_char, date_added_to_site, date_available, date_created, @@ -1038,18 +1055,6 @@ template SQLiteInsertMetadata() { date_modified, date_published, date_valid, - identifier_isbn, - identifier_oclc, - language_document, - language_document_char, - notes_abstract, - notes_description, - original_publisher, - original_language, - original_language_char, - original_source, - original_title, - publisher, rights_copyright, rights_copyright_audio, rights_copyright_cover, @@ -1059,57 +1064,25 @@ template SQLiteInsertMetadata() { rights_copyright_translation, rights_copyright_video, rights_license, + identifier_oclc, + identifier_isbn, + classify_dewey, + classify_keywords, + classify_loc, + classify_subject, + classify_topic_register, + original_title, + original_publisher, + original_language, + original_language_char, + original_source, + notes_abstract, + notes_description, + publisher, site_url_doc_root ) VALUES ( - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s' + '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' ); ┃", _uid, @@ -1122,15 +1095,12 @@ template SQLiteInsertMetadata() { SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), @@ -1138,18 +1108,6 @@ template SQLiteInsertMetadata() { SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), @@ -1159,8 +1117,50 @@ template SQLiteInsertMetadata() { SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), SQLinsertDelimiter!()(doc_matters.conf_make_meta.conf.webserv_url_doc_root) ); + if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) { + + string _insert_topics; + foreach (topic; doc_matters.conf_make_meta.meta.classify_topic_register_arr) { + string[] subject_tree = topic.split(mkup.sep); + _insert_topics ~= format(q"┃ + INSERT INTO topic_register ( + uid_metadata_and_text, + topic_register_lv0, + topic_register_lv1, + topic_register_lv2, + topic_register_lv3, + topic_register_lv4 + ) + VALUES ( + '%s', '%s', '%s', '%s', '%s', '%s' + ); + ┃", + _uid, + (subject_tree.length > 0) ? subject_tree[0] : "", + (subject_tree.length > 1) ? subject_tree[1] : "", + (subject_tree.length > 2) ? subject_tree[2] : "", + (subject_tree.length > 3) ? subject_tree[3] : "", + (subject_tree.length > 4) ? subject_tree[4] : "" + ); + } + } return _insert_metadata; } } @@ -1170,7 +1170,7 @@ template SQLiteInsertDocObjectsLoop() { M doc_matters, ) { string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); - auto site_url = DocReformPathsUrl!()(doc_matters); + auto url_html = DocReformUrlsHTML!()(doc_matters.conf_make_meta.conf.webserv_url_doc_root, doc_matters.src.language); string insertDocObjectsRow(O)(O obj) { string _insert_doc_objects_row = format(q"┃ INSERT INTO doc_objects ( @@ -1185,15 +1185,7 @@ template SQLiteInsertDocObjectsLoop() { html_seg_url ) VALUES ( - '%s', - %s, - '%s', - '%s', - '%s', - %s, - '%s', - '%s', - '%s' + '%s', %s, '%s', '%s', '%s', %s, '%s', '%s', '%s' ); ┃", _uid, @@ -1204,7 +1196,7 @@ template SQLiteInsertDocObjectsLoop() { obj.metainfo.heading_lev_markup, obj.metainfo.is_of_type, obj.metainfo.is_a, - site_url.html_seg(obj.tags.html_segment_anchor_tag_is, obj.metainfo.identifier), + url_html.fn_seg_obj_num(doc_matters.src.filename, obj.tags.html_segment_anchor_tag_is, obj.metainfo.identifier), ); return _insert_doc_objects_row; } @@ -1403,18 +1395,18 @@ template SQLiteTablesCreate() { DROP INDEX IF EXISTS idx_language_document_char; DROP INDEX IF EXISTS idx_classify_topic_register; DROP TABLE IF EXISTS metadata_and_text; + DROP TABLE IF EXISTS topic_register; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE metadata_and_text ( - uid VARCHAR(256) UNIQUE, /* filename, language char, pod/txt (decide on delimiter [,;:/]) */ - src_composite_id_per_txt VARCHAR(256) NOT NULL, /* UNIQUE, /* z pod name if any + src filename + language code */ - src_composite_id_per_pod VARCHAR(256) NOT NULL, /* z pod name if any + src filename */ + uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) + src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code + src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, title_short VARCHAR(400) NULL, title_edition VARCHAR(10) NULL, - title_note VARCHAR(2500) NULL, title_language VARCHAR(100) NULL, title_language_char VARCHAR(6) NULL, creator_author VARCHAR(600) NOT NULL, @@ -1444,14 +1436,13 @@ template SQLiteTablesCreate() { date_translated VARCHAR(10) NULL, date_original_publication VARCHAR(10) NULL, date_generated VARCHAR(10) NULL, - publisher VARCHAR(600) NULL, + original_title VARCHAR(800) NULL, original_publisher VARCHAR(600) NULL, original_language VARCHAR(100) NULL, original_language_char VARCHAR(6) NULL, original_source VARCHAR(600) NULL, original_institution VARCHAR(600) NULL, original_nationality VARCHAR(100) NULL, - original_title VARCHAR(800) NULL, rights_copyright VARCHAR(2500) NULL, rights_copyright_audio VARCHAR(2500) NULL, rights_copyright_cover VARCHAR(2500) NULL, @@ -1480,16 +1471,34 @@ template SQLiteTablesCreate() { notes_prefix_a TEXT NULL, notes_prefix_b TEXT NULL, notes_suffix TEXT NULL, + publisher VARCHAR(600) NULL, src_filename VARCHAR(256) NOT NULL, src_fingerprint VARCHAR(256) NULL, src_filesize VARCHAR(10) NULL, src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ - site_url_doc_root VARCHAR(256) NULL, /* url path to doc root */ + pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here + pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL, + site_url_epub VARCHAR(256) NULL, links TEXT NULL ); + CREATE TABLE topic_register ( + -- tid BIGINT PRIMARY KEY, + uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, + -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code + -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename + topic_register_lv0 VARCHAR(250) NOT NULL, + topic_register_lv1 VARCHAR(250) NULL, + topic_register_lv2 VARCHAR(250) NULL, + topic_register_lv3 VARCHAR(250) NULL, + topic_register_lv4 VARCHAR(250) NULL, + site_url_doc_root VARCHAR(256) NULL, -- url path to doc root + site_url_html_toc VARCHAR(256) NULL, + site_url_html_scroll VARCHAR(256) NULL + ); CREATE TABLE doc_objects ( lid BIGINT PRIMARY KEY, uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, @@ -1504,7 +1513,7 @@ template SQLiteTablesCreate() { lev SMALLINT NULL, node VARCHAR(16) NULL, parent VARCHAR(16) NULL, - last_decendant VARCHAR(16) NULL, /* headings only */ + last_decendant VARCHAR(16) NULL, -- headings only digest_clean CHAR(256), digest_all CHAR(256), html_seg_url CHAR(256), diff --git a/views/version.txt b/views/version.txt index ace7cb6..311e123 100644 --- a/views/version.txt +++ b/views/version.txt @@ -4,7 +4,7 @@ struct Version { int minor; int patch; } -enum _ver = Version(0, 7, 3); +enum _ver = Version(0, 7, 4); version (Posix) { version (DigitalMars) { } else version (LDC) { -- cgit v1.2.3