-*- mode: org -*- #+TITLE: spine (doc_reform) hub #+DESCRIPTION: documents - structuring, various output representations & search #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 - 2020 Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t #+PROPERTY: header-args :exports code #+PROPERTY: header-args+ :noweb yes #+PROPERTY: header-args+ :eval no #+PROPERTY: header-args+ :results no #+PROPERTY: header-args+ :cache no #+PROPERTY: header-args+ :padline no [[./spine_info.org][spine_info.org]] [[./][org/]] [[./spine_build_scaffold.org][make/build]] VERSION * 0. version.txt (set version) :version: ** 0. set program tangle #+BEGIN_SRC txt :tangle "../views/version.txt" <<spine_version_struct>> <<spine_version_current_set>> <<spine_compiler_restrictions>> #+END_SRC ** program version struct #+NAME: spine_version_struct #+BEGIN_SRC d /+ obt - org-mode generated file +/ struct Version { int major; int minor; int patch; } #+END_SRC ** set program version VERSION :version:set:project: #+NAME: spine_version_current_set #+BEGIN_SRC d enum _ver = Version(0, 11, 1); #+END_SRC ** compilation restrictions (supported compilers) - set compilation restrictions http://dlang.org/spec/version.html#predefined-versions #+NAME: spine_compiler_restrictions #+BEGIN_SRC d version (Posix) { version (DigitalMars) { } else version (LDC) { } else version (GNU) { } else { static assert (0, "Unsupported D compiler"); } } else { static assert (0, "Unsupported D compiler"); } #+END_SRC * 1. spine (sisu document parser) :spine: ** notes - deal with imports - get options - get command line instructions - read config instructions - process files as instructed by options - read in file - process file - output ** 0. spine src/spine :template: - process files (act according to requirements of each type) - by sourcefilename - by sourcefiles contents identifier - by zip filename #+BEGIN_SRC d :tangle "../src/doc_reform/spine.d" :shebang "#!/usr/bin/env rdmd" <<doc_header_including_copyright_and_license>> module doc_reform.sisu_document_parser; /++ name "spine" description "A SiSU inspired document parser writen in D." homepage "http://sisudoc.org" +/ <<imports_spine>> <<mixin_spine_version>> <<mixin_pre_main>> string project_name = "spine"; string program_name = "spine"; @system void main(string[] args) { <<spine_mixin>> <<spine_init_0>> <<spine_init_1>> <<spine_init_2>> <<spine_args_0_args_init>> <<spine_args_1_args_get_options_aa>> <<spine_args_2_args_get_options_aa2str>> <<spine_args_3_program_info>> <<spine_args_4_get_configuration_details>> <<spine_args_5_call_output_hub>> <<spine_args_6_process_args>> if (_manifests.length > 1 // _manifests[0] initialized dummy element && _opt_action.abstraction) { if (_opt_action.parallelise) { // see else import std.parallelism; foreach(manifest; parallel(_manifests[1..$])) { if (!empty(manifest.src.filename)) { <<spine_each_file_do_scope>> <<spine_each_file_do_abstraction>> <<spine_each_file_do_debugs_checkdoc_0>> <<spine_each_file_do_debugs_checkdoc_1>> <<spine_each_file_do_debugs_checkdoc_2>> <<spine_each_file_do_debugs_checkdoc_3>> <<spine_each_file_do_debugs_checkdoc_4>> <<spine_each_file_do_debugs_checkdoc_5>> <<spine_each_file_do_selected_output>> <<spine_each_file_do_scope_exit>> } else { <<spine_no_filename_provided>> } } } else { // note cannot parallelise sqlite shared db foreach(manifest; _manifests[1..$]) { if (_opt_action.very_verbose) { writeln("parallelisation off: actions include sqlite shared db"); } if (!empty(manifest.src.filename)) { <<spine_each_file_do_scope>> <<spine_each_file_do_abstraction>> <<spine_each_file_do_debugs_checkdoc_0>> <<spine_each_file_do_debugs_checkdoc_1>> <<spine_each_file_do_debugs_checkdoc_2>> <<spine_each_file_do_debugs_checkdoc_3>> <<spine_each_file_do_debugs_checkdoc_4>> <<spine_each_file_do_debugs_checkdoc_5>> <<spine_each_file_do_selected_output>> <<spine_each_file_do_scope_exit>> } else { <<spine_no_filename_provided>> } } } } if (hvst.harvests.length > 0) { if (_opt_action.harvest_topics) { spineMetaDocHarvestsTopics!()(hvst, _make_and_meta_struct, _opt_action); } if (_opt_action.harvest_authors) { spineMetaDocHarvestsAuthors!()(hvst.harvests, _make_and_meta_struct, _opt_action); } if (!(_opt_action.quiet)) { import doc_reform.io_out.paths_output; auto out_pth = spinePathsHTML!()(_make_and_meta_struct.conf.output_path, ""); if (_opt_action.harvest_authors) { writeln("- ", out_pth.harvest("authors.html")); } if (_opt_action.harvest_topics) { writeln("- ", out_pth.harvest("topics.html")); } } } else { writeln("NO HARVESTS"); } } #+END_SRC *** document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+BEGIN_SRC txt /+ - Name: Spine, Doc Reform - Description: documents, structuring, processing, publishing, search - static content generator - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 - 2020 Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: Spine (SiSU), a framework for document structuring, publishing and search Copyright (C) Ralph Amissah This program is free software: you can redistribute it and/or modify it under the terms of the GNU AFERO General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see [http://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at these locations: [http://www.fsf.org/licensing/licenses/agpl.html] [http://www.gnu.org/licenses/agpl.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Hompages: [http://www.doc_reform.org] [http://www.sisudoc.org] - Git [http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary] [http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/html.rb;hb=HEAD] +/ #+END_SRC ** 1. pre-loop init :init: *** init **** imports :import: ***** spine :spine: #+NAME: imports_spine #+BEGIN_SRC d import std.algorithm, std.datetime, std.getopt, std.file, std.path, std.process; import doc_reform.conf.compile_time_info, doc_reform.meta, doc_reform.meta.metadoc, doc_reform.meta.metadoc_harvest, doc_reform.meta.metadoc_harvests_authors, doc_reform.meta.metadoc_harvests_topics, doc_reform.meta.metadoc_from_src, doc_reform.meta.conf_make_meta_structs, doc_reform.meta.conf_make_meta_json, doc_reform.meta.defaults, doc_reform.meta.doc_debugs, doc_reform.meta.rgx, doc_reform.io_in.paths_source, doc_reform.io_in.read_config_files, doc_reform.io_in.read_source_files, doc_reform.io_out.hub; #+END_SRC ***** spine metadoc :spine: #+NAME: imports_spine_metadoc #+BEGIN_SRC d import std.datetime; import doc_reform.meta, doc_reform.meta.metadoc_from_src, doc_reform.meta.conf_make_meta_structs, doc_reform.meta.conf_make_meta_json, doc_reform.meta.defaults, doc_reform.meta.rgx, doc_reform.io_in.paths_source, doc_reform.io_in.read_config_files, doc_reform.io_in.read_source_files, doc_reform.io_out.hub; #+END_SRC ****** notes ├── src │ ├── doc_reform.d │ └── spine │ ├── conf │ ├── meta │ └── output └── views └── version.txt [[./meta_abstraction.org][meta_abstraction]] [[./meta_conf_make_meta.org][meta_conf_make_meta]] [[./meta_defaults.org][meta_defaults]] [[./meta_output_debugs.org][meta_output_debugs]] [[./source_read_files.org][source_read_files]] [[./compile_time_info.org][compile time info]] [[./output.org][output]] [[./spine.org][spine]] keep up to date, configuration in [[./spine_build_scaffold.org][make/build]] **** mixins :mixin: ***** version.txt :version: #+NAME: mixin_spine_version #+BEGIN_SRC d mixin(import("version.txt")); #+END_SRC ***** pre main mixins #+NAME: mixin_pre_main #+BEGIN_SRC d mixin CompileTimeInfo; #+END_SRC ***** spine "main" mixins :spine: #+NAME: spine_mixin #+BEGIN_SRC d mixin spineRgxIn; mixin spineBiblio; mixin outputHub; #+END_SRC **** init :init: #+NAME: spine_init_0 #+BEGIN_SRC d auto hvst = spineHarvest!(); #+END_SRC **** args :args: #+NAME: spine_init_1 #+BEGIN_SRC d string flag_action; string arg_unrecognized; enum dAM { abstraction, matters } static auto rgx = RgxI(); #+END_SRC *** scope (run complete) :scope: #+NAME: spine_init_2 #+BEGIN_SRC d scope(success) { writefln( "~ run complete, ok ~ (%s-%s.%s.%s, %s D:%s, %s %s)", program_name, _ver.major, _ver.minor, _ver.patch, __VENDOR__, __VERSION__, bits, os, ); } scope(failure) { debug(checkdoc) { stderr.writefln( "run failure", ); } } #+END_SRC *** getopt args for loop :args:getopt: **** set getopt options - set getopt options ***** initialize #+NAME: spine_args_0_args_init #+BEGIN_SRC d bool[string] opts = [ "abstraction" : false, "allow-downloads" : false, "assertions" : false, "cgi-search-form-codegen" : false, "concordance" : false, "dark" : false, "debug" : false, "digest" : false, "epub" : false, "harvest" : false, "harvest-authors" : false, "harvest-topics" : false, "html" : false, "html-link-harvest" : false, "html-link-search" : false, "html-seg" : false, "html-scroll" : false, "latex" : false, "latex-color-links" : false, "light" : false, "manifest" : false, "hide-ocn" : false, "ocn-off" : false, "odf" : false, "odt" : false, "parallel" : false, "parallel-subprocesses" : false, "pdf" : false, "pdf-color-links" : false, "quiet" : false, "pod" : false, "serial" : false, "show-summary" : false, "show-metadata" : false, "show-make" : false, "show-config" : false, "source" : false, "sqlite-discrete" : false, "sqlite-db-create" : false, "sqlite-db-drop" : false, "sqlite-db-recreate" : false, "sqlite-delete" : false, "sqlite-insert" : false, "sqlite-update" : false, "text" : false, "verbose" : false, "very-verbose" : false, "xhtml" : false, "section_toc" : true, "section_body" : true, "section_endnotes" : true, "section_glossary" : true, "section_biblio" : true, "section_bookindex" : true, "section_blurb" : true, "backmatter" : true, "skip-output" : false, "theme-dark" : false, "theme-light" : false, "workon" : false, ]; string[string] settings = [ "cgi-search-title" : "", // config default: "spine-search" "cgi-sqlite-search-filename" : "", // config default: "spine-search" "config" : "", "output" : "", "lang" : "all", // which language to process "set-papersize" : "", // config default: a4 "sqlite-db-filename" : "", // config default: "spine-search.db" "webserv-domain" : "", // config default: localhost ]; #+END_SRC ***** set & describe #+NAME: spine_args_1_args_get_options_aa #+BEGIN_SRC d auto helpInfo = getopt(args, std.getopt.config.passThrough, "abstraction", "document abstraction", &opts["abstraction"], "allow-downloads", "allow downloads (includes cgi.d from github)", &opts["allow-downloads"], "assert", "set optional assertions on", &opts["assertions"], "cgi-search-form-codegen", "generates (pre-compiled) d code for search of specified db", &opts["cgi-search-form-codegen"], "concordance", "file for document", &opts["concordance"], "dark", "alternative dark theme", &opts["dark"], "debug", "debug", &opts["debug"], "digest", "hash digest for each object", &opts["digest"], "epub", "process epub output", &opts["epub"], "harvest", "extract info on authors & topics from document header metadata", &opts["harvest"], "harvest-authors", "extract info on authors from document header metadata", &opts["harvest-authors"], "harvest-topics", "extract info on topics from document header metadata", &opts["harvest-topics"], "hide-ocn", "object cite numbers", &opts["hide-ocn"], "html", "process html output", &opts["html"], "html-link-harvest", "place links back to harvest in segmented html", &opts["html-link-harvest"], "html-link-search", "html embedded search submission", &opts["html-link-search"], "html-seg", "process html output", &opts["html-seg"], "html-scroll", "process html output", &opts["html-scroll"], "latex", "output for pdfs", &opts["latex"], "latex-color-links", "mono or color links for pdfs", &opts["latex-color-links"], "light", "default light theme", &opts["light"], "manifest", "process manifest output", &opts["manifest"], "ocn-off", "object cite numbers", &opts["ocn-off"], "odf", "open document format text (--odt)", &opts["odf"], "odt", "open document format text", &opts["odt"], "parallel", "parallelisation", &opts["parallel"], "parallel-subprocesses", "nested parallelisation", &opts["parallel-subprocesses"], "pdf", "latex output for pdfs", &opts["pdf"], "pdf-color-links", "mono or color links for pdfs", &opts["pdf-color-links"], "pod", "spine (doc reform) pod source content bundled", &opts["pod"], "quiet|q", "output to terminal", &opts["quiet"], "section-backmatter", "document backmatter (default)" , &opts["backmatter"], "section-biblio", "document biblio (default)", &opts["section_biblio"], "section-blurb", "document blurb (default)", &opts["section_blurb"], "section-body", "document body (default)", &opts["section_body"], "section-bookindex", "document bookindex (default)", &opts["section_bookindex"], "section-endnotes", "document endnotes (default)", &opts["section_endnotes"], "section-glossary", "document glossary (default)", &opts["section_glossary"], "section-toc", "table of contents (default)", &opts["section_toc"], "serial", "serial processing", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-config", "show config", &opts["show-config"], "show-make", "show make", &opts["show-make"], "show-metadata", "show metadata", &opts["show-metadata"], "show-summary", "show summary", &opts["show-summary"], "source", "document markup source", &opts["source"], "sqlite-discrete", "process discrete sqlite output", &opts["sqlite-discrete"], "sqlite-db-create", "create db, create tables", &opts["sqlite-db-create"], "sqlite-db-drop", "drop tables & db", &opts["sqlite-db-drop"], "sqlite-db-recreate", "create db, create tables", &opts["sqlite-db-recreate"], "sqlite-delete", "sqlite output", &opts["sqlite-delete"], "sqlite-insert", "sqlite output", &opts["sqlite-insert"], "sqlite-update", "sqlite output", &opts["sqlite-update"], "text", "text output", &opts["text"], "theme-dark", "alternative dark theme", &opts["theme-dark"], "theme-light", "default light theme", &opts["theme-light"], "txt", "text output", &opts["text"], "verbose|v", "output to terminal", &opts["verbose"], "very-verbose", "output to terminal", &opts["very-verbose"], "workon", "(reserved for some matters under development & testing)", &opts["workon"], "xhtml", "xhtml output", &opts["xhtml"], "cgi-search-title", "if generating a cgi search form the title to use for it", &settings["cgi-search-title"], "cgi-sqlite-search-filename", "=[filename] default is spine-search", &settings["cgi-sqlite-search-filename"], "config", "=/path/to/config/file/including/filename", &settings["config"], "lang", "=[lang code e.g. =en or =en,es]", &settings["lang"], "output", "=/path/to/output/dir specify where to place output", &settings["output"], "set-papersize", "=\"a4,a5,b4,letter\"", &settings["set-papersize"], "sqlite-db-filename", "=[filename].sql.db", &settings["sqlite-db-filename"], "webserv-domain", "=[web-server.domain.name]", &settings["webserv-domain"], ); if (helpInfo.helpWanted) { defaultGetoptPrinter("Some information about the program.", helpInfo.options); } #+END_SRC **** getopt hash to struct - move getopt options to struct #+NAME: spine_args_2_args_get_options_aa2str #+BEGIN_SRC d enum outTask { source_or_pod, sqlite, sqlite_multi, latex, odt, epub, html_scroll, html_seg, html_stuff } struct OptActions { @trusted bool allow_downloads() { return opts["allow-downloads"]; } @trusted bool assertions() { return opts["assertions"]; } @trusted bool cgi_search_form_codegen() { return opts["cgi-search-form-codegen"]; } @trusted bool concordance() { return opts["concordance"]; } @trusted string config_path_set() { return settings["config"]; } @trusted bool css_theme_default() { bool _is_light; if (opts["light"] || opts["theme-light"]) { _is_light = true; } else if (opts["dark"] || opts["theme-dark"]) { _is_light = false; } else { _is_light = true; } return _is_light; } @trusted bool debug_do() { return opts["debug"]; } @trusted bool digest() { return opts["digest"]; } @trusted bool epub() { return opts["epub"]; } @trusted bool html_harvest_link() { return (opts["html-link-harvest"]) ? true : false; } @trusted bool html_search_link() { return (opts["html-link-search"]) ? true : false; } @trusted bool harvest() { return (opts["harvest"] || opts["harvest-authors"] || opts["harvest-topics"]) ? true : false; } @trusted bool harvest_authors() { return (opts["harvest"] || opts["harvest-authors"]) ? true : false; } @trusted bool harvest_topics() { return (opts["harvest"] || opts["harvest-topics"]) ? true : false; } @trusted bool html() { return (opts["html"] || opts["html-seg"] || opts["html-scroll"]) ? true : false; } @trusted bool html_seg() { return (opts["html"] || opts["html-seg"]) ? true : false; } @trusted bool html_scroll() { return (opts["html"] || opts["html-scroll"]) ? true : false; } @trusted bool html_stuff() { return (opts["html"] || opts["html-scroll"] || opts["html-seg"]) ? true : false; } @trusted bool latex() { return (opts["latex"] || opts["pdf"]) ? true : false; } @trusted bool latex_color_links() { return (opts["latex-color-links"] || opts["pdf-color-links"]) ? true : false; } @trusted bool odt() { return (opts["odf"] || opts["odt"]) ? true : false; } @trusted bool manifest() { return opts["manifest"]; } @trusted bool ocn_hidden() { return opts["hide-ocn"]; } @trusted bool ocn_off() { return opts["ocn-off"]; } @trusted bool quiet() { return opts["quiet"]; } @trusted bool pod() { return opts["pod"]; } @trusted bool show_summary() { return opts["show-summary"]; } @trusted bool show_make() { return opts["show-make"]; } @trusted bool show_metadata() { return opts["show-metadata"]; } @trusted bool show_config() { return opts["show-config"]; } @trusted bool source() { return opts["source"]; } @trusted bool source_or_pod() { return (opts["pod"] || opts["source"]) ? true : false; } @trusted bool sqlite_discrete() { return opts["sqlite-discrete"]; } @trusted bool sqlite_db_drop() { return (opts["sqlite-db-recreate"] || opts["sqlite-db-drop"]) ? true : false; } @trusted bool sqlite_db_create() { return (opts["sqlite-db-recreate"] || opts["sqlite-db-create"]) ? true : false; } @trusted bool sqlite_delete() { return opts["sqlite-delete"]; } @trusted bool sqlite_update() { return (opts["sqlite-update"] || opts["sqlite-insert"]) ? true : false; } @trusted bool sqlite_shared_db_action() { return ( opts["sqlite-db-recreate"] || opts["sqlite-db-create"] || opts["sqlite-delete"] || opts["sqlite-insert"] || opts["sqlite-update"] ) ? true : false; } @trusted bool text() { return opts["text"]; } @trusted bool verbose() { return (opts["verbose"] || opts["very-verbose"]) ? true : false; } @trusted bool very_verbose() { return opts["very-verbose"]; } @trusted bool xhtml() { return opts["xhtml"]; } @trusted bool section_toc() { return opts["section_toc"]; } @trusted bool section_body() { return opts["section_body"]; } @trusted bool section_endnotes() { return opts["section_endnotes"]; } @trusted bool section_glossary() { return opts["section_glossary"]; } @trusted bool section_biblio() { return opts["section_biblio"]; } @trusted bool section_bookindex() { return opts["section_bookindex"]; } @trusted bool section_blurb() { return opts["section_blurb"]; } @trusted bool backmatter() { return opts["backmatter"]; } @trusted bool skip_output() { return opts["skip-output"]; } @trusted bool workon() { return opts["workon"]; } @trusted string[] languages_set() { return settings["lang"].split(","); } @trusted string output_dir_set() { return settings["output"]; } @trusted string sqlite_filename() { return settings["sqlite-db-filename"]; } @trusted string cgi_search_title() { return settings["cgi-search-title"]; } @trusted string cgi_sqlite_search_filename() { return settings["cgi-sqlite-search-filename"]; } @trusted string cgi_sqlite_search_filename_d() { return (settings["cgi-sqlite-search-filename"].length > 0) ? (settings["cgi-sqlite-search-filename"].translate(['-' : "_"]) ~ ".d") : ""; } @trusted string latex_papersize() { return settings["set-papersize"]; } @trusted string webserver_domain_name() { return settings["webserv-domain"]; } @trusted bool parallelise() { bool _is; if (opts["parallel"] == true) { _is = true; if (sqlite_shared_db_action) { _is = false; } } else if (opts["parallel"] == false && opts["serial"] == true) { _is = false; } else if ( opts["abstraction"] || concordance || harvest || html || epub || odt || latex || manifest || source_or_pod || sqlite_discrete ) { _is = true; } else { _is = false; } return _is; } @trusted bool parallelise_subprocesses() { return opts["parallel-subprocesses"]; } auto output_task_scheduler() { int[] schedule; if (source_or_pod) { schedule ~= outTask.source_or_pod; } if (sqlite_discrete) { schedule ~= outTask.sqlite; } if (epub) { schedule ~= outTask.epub; } if (html_scroll) { schedule ~= outTask.html_scroll; } if (html_seg) { schedule ~= outTask.html_seg; } if (html_stuff) { schedule ~= outTask.html_stuff; } if (odt) { schedule ~= outTask.odt; } if (latex) { schedule ~= outTask.latex; } return schedule.sort().uniq; } @trusted bool abstraction() { return ( opts["abstraction"] || concordance || source_or_pod || harvest || html || epub || odt || latex || manifest || sqlite_discrete || sqlite_delete || sqlite_update ) ? true : false; } @trusted bool require_processing_files() { return ( opts["abstraction"] || epub || html || html_seg || html_scroll || latex || odt || manifest || pod || show_make || show_metadata || show_summary || source || source_or_pod || sqlite_discrete || sqlite_update || text || xhtml ) ? true : false; } @trusted bool meta_processing_general() { return ( opts["abstraction"] || html || epub || odt || latex || sqlite_discrete || sqlite_update ) ? true :false; } @trusted bool meta_processing_xml_dom() { return ( opts["abstraction"] || html || epub || odt || sqlite_discrete || sqlite_update ) ? true : false; } } OptActions _opt_action = OptActions(); #+END_SRC **** env - environmental info #+NAME: spine_args_3_program_info #+BEGIN_SRC d @safe auto program_info() { struct ProgramInfo { @safe string project() { return project_name; } @safe string name() { return program_name; } @safe string ver() { return format("%s.%s.%s", _ver.major, _ver.minor, _ver.patch, ); } @safe string compiler() { return format ("%s D:%s, %s %s", __VENDOR__, __VERSION__, bits, os, ); } } return ProgramInfo(); } auto _env = [ "pwd" : environment["PWD"], "home" : environment["HOME"], ]; #+END_SRC *** get/read site config #+NAME: spine_args_4_get_configuration_details #+BEGIN_SRC d auto _manifested = PathMatters!()(_opt_action, _env, ""); auto _manifests = [ _manifested ]; auto _conf_file_details = configFilePaths!()(_manifested, _env, _opt_action.config_path_set); ConfComposite _config; if ( _opt_action.require_processing_files && _opt_action.config_path_set.empty ) { foreach(arg; args[1..$]) { if (!(arg.match(rgx.flag_action))) { /+ cli markup source path +/ // get first input markup source file names for processing _manifested = PathMatters!()(_opt_action, _env, arg); { /+ local site config +/ _conf_file_details = configFilePaths!()(_manifested, _env, _opt_action.config_path_set); auto _config_local_site_struct = readConfigSite!()(_conf_file_details, _opt_action); import doc_reform.meta.conf_make_meta_yaml; _config = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_config, _manifested, _opt_action); // - get local site config break; } } } } else { /+ local site config +/ auto _config_local_site_struct = readConfigSite!()(_conf_file_details, _opt_action); import doc_reform.meta.conf_make_meta_yaml; _config = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_config, _manifested, _opt_action); // - get local site config } if (_opt_action.show_config) { import doc_reform.meta.metadoc_show_config; spineShowSiteConfig!()(_opt_action, _config); } #+END_SRC *** use config for operations that do not require file processing #+NAME: spine_args_5_call_output_hub #+BEGIN_SRC d if (!(_opt_action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step0 commence → (without processing files)"); } outputHubOp!()(_env, _opt_action, _config); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step0 complete"); } } #+END_SRC *** opt action on processing files (loop args) #+NAME: spine_args_6_process_args #+BEGIN_SRC d ConfComposite _make_and_meta_struct = _config; destroy(_config); foreach(arg; args[1..$]) { if (arg.match(rgx.flag_action)) { /+ cli instruction, flag do +/ flag_action ~= " " ~ arg; // flags not taken by getopt } else if (_opt_action.require_processing_files) { /+ cli, assumed to be path to source files +/ auto _manifest_start = PodManifest!()(_opt_action, arg); if ( /+ pod files +/ !(arg.match(rgx.src_pth_sst_or_ssm)) && _manifest_start.pod_manifest_file_with_path && _opt_action.abstraction ) { string pod_manifest_root_content_paths_to_markup_location_raw_; string markup_contents_location_; string sisudoc_txt_ = _manifest_start.pod_manifest_file_with_path; enforce( exists(sisudoc_txt_)!=0, "file not found: «" ~ sisudoc_txt_ ~ "»" ); if (exists(sisudoc_txt_)) { try { if (exists(sisudoc_txt_)) { import dyaml; try { Node pod_manifest_yaml; try { pod_manifest_yaml = Loader.fromFile(sisudoc_txt_).load(); } catch (ErrnoException ex) { } catch (FileException ex) { writeln("ERROR failed to read config file"); } catch (Throwable) { writeln("ERROR failed to read config file content, not parsed as yaml"); } if ("doc" in pod_manifest_yaml) { if (pod_manifest_yaml["doc"].type.mapping && pod_manifest_yaml["doc"].tag.match(rgx.yaml_tag_is_map) ) { if ("path" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_seq)) { foreach (string _path; pod_manifest_yaml["doc"]["path"]) { markup_contents_location_ ~= _path ~ "\n"; pod_manifest_root_content_paths_to_markup_location_raw_ ~= _path ~ "\n"; } } else if ( pod_manifest_yaml["doc"]["path"].type.string && pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_str) ) { markup_contents_location_ = pod_manifest_yaml["doc"]["path"].get!string; pod_manifest_root_content_paths_to_markup_location_raw_ = pod_manifest_yaml["doc"]["path"].get!string; } } if ("filename" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_seq)) { foreach (string _filename; pod_manifest_yaml["doc"]["filename"]) { if ("language" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) { foreach (string _lang; pod_manifest_yaml["doc"]["language"]) { markup_contents_location_ ~= "media/text/" ~ _lang ~ "/" ~ _filename ~ "\n"; } } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str) ) { markup_contents_location_ = "media/text/" ~ pod_manifest_yaml["doc"]["language"].get!string ~ "/" ~ _filename ~ "\n"; } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } } else if ( pod_manifest_yaml["doc"]["filename"].type.string && pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_str) ) { if ("language" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) { foreach (string _lang; pod_manifest_yaml["doc"]["language"]) { markup_contents_location_ ~= "media/text/" ~ _lang ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str)) { markup_contents_location_ = "media/text/" ~ pod_manifest_yaml["doc"]["language"].get!string ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } } } } } catch (ErrnoException ex) { } } } catch (ErrnoException ex) { } catch (FileException ex) { // Handle errors } } else { writeln("manifest not found: ", sisudoc_txt_); } auto markup_contents_locations_arr = (cast(char[]) markup_contents_location_).split; auto tmp_dir_ = (sisudoc_txt_).dirName.array; foreach (markup_contents_location; markup_contents_locations_arr) { assert(markup_contents_location.match(rgx.src_pth_sst_or_ssm), "not a recognised file: «" ~ markup_contents_location ~ "»" ); auto markup_contents_location_pth_ = (markup_contents_location).to!string; Regex!(char) lang_rgx_ = regex(r"/(" ~ _opt_action.languages_set.join("|") ~ ")/"); if (_opt_action.languages_set[0] == "all" || (markup_contents_location_pth_).match(lang_rgx_) ) { auto _fns = (((tmp_dir_).chainPath(markup_contents_location_pth_)).array).to!string; _manifested = PathMatters!()(_opt_action, _env, arg, _fns, markup_contents_locations_arr); _manifests ~= _manifested; } } } else if (arg.match(rgx.src_pth_sst_or_ssm)) { /+ markup txt files +/ if (exists(arg)==0) { writeln("ERROR >> Processing Skipped! File not found: ", arg); } else { _manifested = PathMatters!()(_opt_action, _env, arg, arg); _manifests ~= _manifested; } } else if (arg.match(rgx.src_pth_zip)) { // fns_src ~= arg; // gather input markup source file names for processing } else { // anything remaining, unused arg_unrecognized ~= " " ~ arg; } } } #+END_SRC *** config files load & read #+NAME: spine_conf_files_in_yaml #+BEGIN_SRC d { /+ document config/make file +/ auto _config_document_struct = readConfigDoc!()(_manifest, _env); import doc_reform.meta.conf_make_meta_yaml; _make_and_meta_struct = _config_document_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifest, _opt_action); } #+END_SRC ** _2. processing: (loop each file)_ [+2] :loop:files: *** scope (loop) :scope: #+NAME: spine_each_file_do_scope #+BEGIN_SRC d scope(success) { if (!(_opt_action.quiet)) { writefln( "%s", "-- ~ document complete, ok ~ ------------------------------------", ); } } scope(failure) { debug(checkdoc) { stderr.writefln( "~ document run failure ~ (%s v%s)\n\t%s\n%s", __VENDOR__, __VERSION__, manifest.src.filename, "------------------------------------------------------------------", ); } } enforce( manifest.src.filename.match(rgx.src_pth_types), "not a sisu markup filename: «" ~ manifest.src.filename ~ "»" ); #+END_SRC *** 1. _document abstraction_ [#A] - return tuple of: - doc_abstraction (the document) - doc_matters #+NAME: spine_each_file_do_abstraction #+BEGIN_SRC d if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("--->\nstepX commence → (document abstraction)"); } auto t = spineAbstraction!()(_env, program_info, _opt_action, manifest, _make_and_meta_struct); static assert(t.length==2); auto doc_abstraction = t[dAM.abstraction]; auto doc_matters = t[dAM.matters]; if ((doc_matters.opt.action.debug_do) || (doc_matters.opt.action.very_verbose) ) { writeln("- stepX complete"); } #+END_SRC *** 2. _output processing_ (post abstraction processing) **** 0. abstraction ***** show document summary - abstraction _print summary_ (to screen) :abstraction:summary: --show-summary (abstraction summary) #+NAME: spine_each_file_do_debugs_checkdoc_0 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.verbose || doc_matters.opt.action.show_summary ) { import doc_reform.meta.metadoc_show_summary; spineMetaDocSummary!()(doc_abstraction, doc_matters); } #+END_SRC ***** show metadata (doc headers metadata) --show-metadata #+NAME: spine_each_file_do_debugs_checkdoc_1 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_metadata) { import doc_reform.meta.metadoc_show_metadata; spineShowMetaData!()(doc_matters); } #+END_SRC ***** show make (make files & doc headers) --show-make #+NAME: spine_each_file_do_debugs_checkdoc_2 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_make) { import doc_reform.meta.metadoc_show_make; spineShowMake!()(doc_matters); } #+END_SRC ***** show config (config files & doc headers) --show-config #+NAME: spine_each_file_do_debugs_checkdoc_3 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_config) { import doc_reform.meta.metadoc_show_config; spineShowConfig!()(doc_matters); } #+END_SRC ***** abstraction harvest :abstraction:harvest: - abstraction harvest #+NAME: spine_each_file_do_debugs_checkdoc_4 #+BEGIN_SRC d if (doc_matters.opt.action.harvest) { auto _hvst = spineMetaDocHarvest!()(doc_matters, hvst); if ( _hvst.title.length > 0 && _hvst.author_surname_fn.length > 0 ) { hvst.harvests ~= _hvst; } else { if ((doc_matters.opt.action.debug_do) || (doc_matters.opt.action.very_verbose) ) { writeln("WARNING harvest: document header yaml does not contain information related to: title or author: ", _hvst.path_html_segtoc); } } } #+END_SRC **** 1. _debug_ (document parts, checkdoc) :debug:checkdoc: - [[./meta_output_debugs.org][meta_output_debugs]] #+NAME: spine_each_file_do_debugs_checkdoc_5 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.debug_do) { spineDebugs!()(doc_abstraction, doc_matters); } #+END_SRC **** 2. _process outputs_ :outputs: - [[./output_hub.org][output_hub]] #+NAME: spine_each_file_do_selected_output #+BEGIN_SRC d /+ ↓ output hub +/ if (!(doc_matters.opt.action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step5 commence → (process outputs)"); } doc_abstraction.outputHub!()(doc_matters); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step5 complete"); } } #+END_SRC *** scope (on loop exit) :scope:exit: #+NAME: spine_each_file_do_scope_exit #+BEGIN_SRC d scope(exit) { if (!(_opt_action.quiet)) { writefln( "processed file: %s [%s]", manifest.src.filename, manifest.src.language ); } destroy(manifest); } #+END_SRC ** +2c. no valid filename provided+ #+NAME: spine_no_filename_provided #+BEGIN_SRC d /+ no recognized filename provided +/ writeln("no recognized filename"); break; // terminate, stop #+END_SRC * 2. pre-processing ** Output _document abstraction functions_ :module:spine:abstraction: *** 0 module template - abstraction template #+BEGIN_SRC d :tangle "../src/doc_reform/meta/metadoc.d" module doc_reform.meta.metadoc; template spineAbstraction() { <<imports_spine_metadoc>> <<spine_mixin>> enum headBody { header, body_content, insert_file_list, image_list } enum makeMeta { make, meta } enum docAbst { doc_abstract_obj, doc_has } static auto rgx = RgxI(); @system auto spineAbstraction(E,P,O,M,S)( E _env, P program_info, O _opt_action, M _manifest, S _make_and_meta_struct ){ <<spine_conf_files_in_yaml>> <<spine_each_file_do_read_and_split_dr_markup_file_content_into_header_and_body>> <<spine_each_file_do_split_dr_markup_file_header_into_make_and_meta_structs>> <<spine_each_file_do_document_abstraction>> <<spine_each_file_do_document_matters_msg_step4_start>> <<spine_each_file_do_document_matters_0_struct_open>> <<spine_each_file_do_document_matters_1_detail_program_time>> <<spine_each_file_do_document_matters_2_meta>> <<spine_each_file_do_document_matters_3_env>> <<spine_each_file_do_document_matters_4_opt>> <<spine_each_file_do_document_matters_5_doc>> <<spine_each_file_do_document_matters_6_struct_close_gather>> <<spine_each_file_do_document_matters_msg_step4_end>> auto t = tuple(doc_abstraction, doc_matters); return t; } } #+END_SRC ** Output & Harvest pre-processing *** 1. raw file content split, doc: _header_, _content_ +(lists: subdocs? images?) - [[./source_read_files.org][source_read_files]] return tuple: header; body; insert file list; image list - read in the _marked up source document_ and - split the document into: - document header - document body - from markup source - if master document from sub documents content - if a master document - make a list of insert files - if build source pod requested - scan for list of images (action avoided if not needed at this stage) - _return a tuple of_: - header - body - insert file list - image list (if build source pod requested) if build source pod requested all information needed to build it available at this point - manifest related information _manifest - insert file list _header_body_insertfilelist_imagelist[headBody.insert_file_list] - image list _header_body_insertfilelist_imagelist[headBody.image_list] #+NAME: spine_each_file_do_read_and_split_dr_markup_file_content_into_header_and_body #+BEGIN_SRC d /+ ↓ read file (filename with path) +/ /+ ↓ file tuple of header and content +/ if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step1 commence → (get document header & body & insert file list & if needed image list)" ); } auto _header_body_insertfilelist_imagelist = spineRawMarkupContent!()(_opt_action, _manifest.src.path_and_fn); static assert(_header_body_insertfilelist_imagelist.length==4); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step1 complete"); } debug(header_and_body) { writeln(header); writeln(_header_body_insertfilelist_imagelist.length); writeln(_header_body_insertfilelist_imagelist.length[headBody.body_content][0]); } #+END_SRC *** 2. _document metadata_ & _make instructions_ (struct from yaml) - [[./meta_conf_make_meta.org][meta_conf_make_meta]] return tuple: document metadata; make instructions - read _document header_, split into: - metadata - make instructions - read config files - consolidate make instructions - _return tuple of_: - document metadata - make instructions (from configuration files & document header make instructions) #+NAME: spine_each_file_do_split_dr_markup_file_header_into_make_and_meta_structs #+BEGIN_SRC d /+ ↓ split header into make and meta +/ if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step2 commence → (read document header (yaml) return struct)"); } import doc_reform.meta.conf_make_meta_yaml; _make_and_meta_struct = docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct!()( _header_body_insertfilelist_imagelist[headBody.header], _make_and_meta_struct, _manifest, _opt_action, ); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step2 complete"); } #+END_SRC *** 3. _document abstraction, tuple_ (output-pre-processing) [#A] - [[./meta_abstraction.org][meta_abstraction]] return tuple: document abstraction; abstraction keys; segnames; image list - prepare the document abstraction used in downstream processing - _return tuple of_: - document abstraction (_the_document_ or doc_abstraction) - document abstraction keys - (head, toc, body, endnotes, glossary, bibliography, bookindex, blurb, tail) - (transfer to _doc_matters_) - segnames for html epub (transfer to _doc_matters_) - image list (transfer to _doc_matters_) #+NAME: spine_each_file_do_document_abstraction #+BEGIN_SRC d /+ ↓ document abstraction: process document, return abstraction as tuple +/ if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step3 commence → (document abstraction (da); da keys; segnames; doc_matters)"); } auto da = docAbstraction!()( _header_body_insertfilelist_imagelist[headBody.body_content], _make_and_meta_struct, _opt_action, _manifest, true, ); static assert(da.length==2); auto doc_abstraction = da[docAbst.doc_abstract_obj]; /+ head ~ toc ~ body ~ endnotes_seg ~ glossary ~ bibliography ~ bookindex ~ blurb; +/ auto _doc_has_struct = da[docAbst.doc_has]; if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step3 complete"); } #+END_SRC *** 4. _document matters_ (doc info gathered, various sources) [#A] - gather doc matters - prepare document_matters, miscellany about processing and the document of use in downstream processing **** verbose message #+NAME: spine_each_file_do_document_matters_msg_step4_start #+BEGIN_SRC d if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step4 commence → (doc_matters)"); } #+END_SRC **** DocumentMatters struct { #+NAME: spine_each_file_do_document_matters_0_struct_open #+BEGIN_SRC d struct DocumentMatters { #+END_SRC **** generator related #+NAME: spine_each_file_do_document_matters_1_detail_program_time #+BEGIN_SRC d @safe auto generator_program() { struct Prog_ { @safe string project_name() { return "spine"; } @safe string name() { return program_info.name; } @safe string ver() { return program_info.ver; } @trusted string name_and_version() { return format("%s-%s", name, ver, ); } @safe string url_home() { return "http://sisudoc.org"; } @safe string url_git() { return "https://git.sisudoc.org/software/sisu"; } @safe auto compiler() { return program_info.compiler; } @safe auto stime() { return Clock.currTime(UTC()).toSimpleString(); } } return Prog_(); } @safe auto generated_time() { auto _st = Clock.currTime(UTC()); auto _time = _st.year.to!string ~ "-" ~ _st.month.to!int.to!string // prefer as month number ~ "-" ~ _st.day.to!string ~ " [" ~ _st.isoWeek.to!string ~ "/" ~ _st.dayOfWeek.to!int.to!string ~ "]" ~ " " ~ _st.hour.to!string ~ ":" ~ _st.minute.to!string ~ ":" ~ _st.second.to!string; return _time; } #+END_SRC **** config make & meta #+NAME: spine_each_file_do_document_matters_2_meta #+BEGIN_SRC d @safe auto conf_make_meta() { return _make_and_meta_struct; } @safe auto has() { return _doc_has_struct; } #+END_SRC **** env related #+NAME: spine_each_file_do_document_matters_3_env #+BEGIN_SRC d @safe auto env() { struct Env_ { @safe auto pwd() { return _manifest.env.pwd; } @safe auto home() { return _manifest.env.home; } } return Env_(); } #+END_SRC **** opt #+NAME: spine_each_file_do_document_matters_4_opt #+BEGIN_SRC d @safe auto opt() { struct Opt_ { @safe auto action() { /+ getopt options, commandline instructions, raw - processing instructions --epub --html etc. - command line config instructions --output +/ return _opt_action; } } return Opt_(); } #+END_SRC **** output related #+NAME: spine_each_file_do_document_matters_5_doc #+BEGIN_SRC d @safe auto src() { return _manifest.src; } @safe auto src_path_info() { return spinePathsSRC!()(_manifest.env.pwd, _manifest.src.file_with_absolute_path); // would like (to have and use) relative path } @safe auto pod() { return _manifest.pod; } @safe auto sqlite() { struct SQLite_ { @safe string filename() { string _fn = ""; if (_opt_action.sqlite_filename.length > 0) { _fn = _opt_action.sqlite_filename; } else if (_make_and_meta_struct.conf.w_srv_db_sqlite.length > 0) { _fn = _make_and_meta_struct.conf.w_srv_db_sqlite; } return _fn; } @safe string cgi_filename() { string _fn = ""; if (_opt_action.cgi_sqlite_search_filename.length > 0) { _fn = _opt_action.cgi_sqlite_search_filename; } else if (_make_and_meta_struct.conf.w_srv_cgi_search_script.length > 0) { _fn = _make_and_meta_struct.conf.w_srv_cgi_search_script; } return _fn; } @safe string cgi_filename_d() { string _fn = ""; if (_opt_action.cgi_sqlite_search_filename_d.length > 0) { _fn = _opt_action.cgi_sqlite_search_filename_d; } else if (_make_and_meta_struct.conf.w_srv_cgi_search_script_raw_fn_d.length > 0) { _fn = _make_and_meta_struct.conf.w_srv_cgi_search_script_raw_fn_d; } return _fn; } } return SQLite_(); } @safe auto output_path() { return _make_and_meta_struct.conf.output_path; } @safe auto srcs() { struct SRC_ { auto file_insert_list() { return _header_body_insertfilelist_imagelist[headBody.insert_file_list]; } auto image_list() { return _doc_has_struct.imagelist; } } return SRC_(); } #+END_SRC **** } close #+NAME: spine_each_file_do_document_matters_6_struct_close_gather #+BEGIN_SRC d } auto doc_matters = DocumentMatters(); #+END_SRC **** step complete message #+NAME: spine_each_file_do_document_matters_msg_step4_end #+BEGIN_SRC d if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step4 complete"); } #+END_SRC * __END__ dev notes ** the document notes *** document sections (table) |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | section | part | opt. | | objects | ocn | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | front matter | head | * | | | no | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | toc | toc | | | generated from headings | no | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | body | body | * | | default section | yes | | | | | | | | - headings | | | | | | | | | - paras | | | | | | | | | - code | | | | | | | | | - poem | | | | | | | | | - group | | | | | | | | | - block | | | | | | | | | - quote | | | | | | | | | - table | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | back matter | endnote | | | generated from inline note markup | no (each endnote belongs to | | | | | | | | | a (body) object) | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | glossary | | | identified section, limited markup | possibly, to make searchable | | | | | | | | - heading | hidden | | | | | | | | - paras | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | bibliography | | | generated from inline special markup | possibly, to make searchable | | | | | | | | appended to paragraphs contained in body section | hidden | | | | | | | | - heading | | | | | | | | | - paras | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | book index | | | generated from inline special markup | possibly, special numbering or | | | | | | | | - heading | could use term as anchor? | | | | | | | | - paras | to make searchable | | | | | | | | | hidden | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | blurb | | | identified section, limited markup | no (unless non-substantive | | | | | | | | - heading | given special numbering) | | | | | | | | - paras | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| *** document objects (table) - check, keep up to date |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | doc object | doc object | attributes | inline | appended | structure | delimiters | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | is_a | is_of_type | | | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | heading | para | - level | - font face | - object number off | - level | - two newlines | | | | - object number | - endnotes | - book index meta | (document structure) | | | | | - object number off | | | | | | | | - dummy (toc & seg) | | | | | | | | - tags (internal links) | | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | toc | para | - level | - font face | | | - auto generated from headings | | | | (auto-indent) | - links (auto) | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | para | para | - bullet | - font face | - object number off | | - two newlines | | | | - indent | - links/urls * | - book index meta | | | | | | - object number | - images* | | | | | | | - object number off | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | bookindex | para | - auto indent | - font face | | | - two newlines | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | blurb | para | - bullet | - font face | - object number off | | - two newlines | | | | - indent | - links/urls * | - book index meta | | | | | | - object number | - images* | | | | | | | - object number off | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | group | block | - object number | - font face | - book index meta | - para break | - block tags | | | | - object number off | - links/urls * | | | (group) | | | | | - images* | | | | | | | | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | block | block | - object number | - font face | - book index meta | - new line | - block tags | | | | - object number off | - links/urls * | | | (block) | | | | | - images* | | | | | | | | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | quote | block | - object number | - font face | - book index meta | | - block tags | | | | | - endnotes | | | (quote) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | poem (see verse) | block | | | - book index meta | | - block tags | | | | | | | | (poem) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | verse (of poem) | | - object number | - font face | | - new line | - (see poem delimiter) | | | | | - endnotes | | - preceeding spaces | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | code | block | - syntax | | | - new line | - block tags | | | | - numbered | | | - preceeding spaces | (code) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | table | block | - object number | | | | - block tags (table) | | | | | | | | (table) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | endnote | | | - font face | | | (generated from | | | | | | | | inline markup tags) | | | | | | | | - two newlines | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| - consider special treatment for links/urls (& for images?) take them out of document munge (for various outputs), by storing in own array (within each object struct), and providing info on where in array to extract them from, debating whether necessary or even worthwhile as is extra work **** check |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | | identified by | object notes | attributes | inline | embedded | special | | | | | | | | appended | characters | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | para | heading | level markers | | | - italics | - endnotes | | | | | at start of line | | | | - bibliography | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | paragraph | delimited by two new lines | default object | - indent | - bold | - endnotes | | | | | | [discard leading & | - bullet | - italics | - bibliography | | | | | | newline whitespace] | | - underscore | | | | | | | | | - strikethrough | | | | | | | | | - superscript | | | | | | | | | - subscript | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | block | | open and close tags | | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | TODO | quote | | | - language? | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | TODO | group | | - inline markup applied | - language? | as paragraph | - endnotes | | | | | | - [discard leading & | | | - bibliography | | | | | | newline whitespace] | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | TODO | block | | - inline markup applied | | as paragraph | - endnotes | | | | | | - whitespace indentation | | | - bibliography | | | | | | & newlines | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | poem / verse | open and close tags | verse is the object | | | - endnotes | | | | | | - inline markup applied | | | - bibliography | | | | | (for poem) | - whitespace indentation | | | | | | | | | & newlines | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | code | | - contents untouched | - syntax | | | | | | | | - whitespace indentation | - numbered | | | | | | | | & newlines | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | table | | | - column width | | | | | | | | | - heading row | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| *** on abstraction - abstract for downstream processing - identify document structure and objects - identify document structure (headings/levels/sections) - identify objects (headings, paragraphs, tables, code blocks, verse ...) - set document, generate common abstraction for downstream parsing - set different _document sections_: - _head_, toc, _body_, endnotes, glossary, bibliography, book index, blurb - _object numbers_, heading/ chapter numbering etc, endnote numbers - _regular ocn_ - body objects - glossary objects - bibliography objects - _special ocn_ - non substantive text (provide special numbers) - blurb objects - book index - special (_exceptions_) - endnotes - unify object representations - multiple markups for same object type given single representation - extract object attributes - unify inline markup on objects - inline markup made easier to identify - simplify downstream parsing *** ocn |-------------+-----------------------+-----------------------+----------------+------| | objects | section / part | ocn described | how used | type | |-------------+-----------------------+-----------------------+----------------+------| | regular ocn | | | | | |-------------+-----------------------+-----------------------+----------------+------| | | body objects | seq. digit | anchor | ocn | | | | [0-9]+ | visible | | |-------------+-----------------------+-----------------------+----------------+------| | | glossary objects | seq. digit | anchor | ocn | | | | [0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | | bibliography objects | seq. digit | anchor | ocn | | | | [0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | special ocn | | | | | |-------------+-----------------------+-----------------------+----------------+------| | | non-substantive text | x char + seq. digit | anchor | non | | | (within body & blurb) | x[0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | | book index | i char + seq. digit | anchor | idx | | | | i[0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | without ocn | | | | | |-------------+-----------------------+-----------------------+----------------+------| | | endnotes | ocn of parent object | no ocn | fn | | | | + footnote seq. digit | anchor visible | | |-------------+-----------------------+-----------------------+----------------+------| ** spine glossary / terms |-------+-------------------------------------| | spine | sisu document parser | |-------+-------------------------------------| | dmso | document markup, structure, objects | |-------+-------------------------------------| | meta | meta document, document abstraction | | mda | meta, meta document abstraction | | adr | abstract document representation | | dar | document abstract representation | | (da) | (document abstraction) | | | (code representation of document) | |-------+-------------------------------------| | ao | abstract objects | | | (code representation of objects) | |-------+-------------------------------------| consider |-------+----------------------------------------------| | dao | document abstraction, objects | |-------+----------------------------------------------| | daso | document abstraction, structure, objects | |-------+----------------------------------------------| | drso | document representation, structure, objects | |-------+----------------------------------------------| | daows | document abstraction, objects with structure | |-------+----------------------------------------------| ** make config - _composite make_ work on composite make a unification of make instructions for each document run extract instructions from all config files, unify the make instructions and provide the result as a single set of make instructions for each document parsed - 1. general, document_make config file (to be applied to all documents unless overridden by document or command line instruction) - 2. local, site specific (site local instructions such as the site's url, cgi location etc.) - 3. each document header, make (the document header contains metadata and may include make instructions for that document) - make - meta - 4. command line instruction, make (some make instructions may be passed through the command line) *** instruction sources |----+---------------------------------+----------------------------------------+---------------------+---| | | make instruction source | | varies (applies to) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 0. | unify the following as a single | take into account all the instructions | | | | | set of make instructions | provided below, provide interface | | | |----+---------------------------------+----------------------------------------+---------------------+---| | 1. | document_make file | to be applied to all documents | per directory | | | | "config_document" | (unless subsequently overridden) | (all docs within) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 2. | config file | local site specific | per directory | | | | "config_local_site" | | (all docs within) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 3. | document header make | make instructions contained | per document | | | | | in document header | (single doc) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 4. | command line instruction | make instruction passed | each command | | | | | | (all docs within) | | |----+---------------------------------+----------------------------------------+---------------------+---| *** config & metadata (from instruction sources) |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | 1. document make file | 2. config file | 3. document header | 4. command line instruction | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | comment, fixed: | per dir (pod) | per dir | per document (pod) | per command instruction | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | config_document | config_local_site | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | local site specific | | * | | *? | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | webserv | | | | | | - url_root | | | | | | - path | | | | | | - images | | | | | | - cgi | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | webserv_cgi | | | | | | - host | | | | | | - base_path | | | | | | - port | | | | | | - user | | | | | | - file_links | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | processing | | | | | | - path | | | | | | - dir | | | | | | - concord_max | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | flag (configure) | | (call) | | | | - act0 | | act0 | | | | - act1 | | act1 | | | | - act2 | | act2 | | | | - act3 | | act3 | | | | - act4 | | act4 | | | | - act5 | | act5 | | | | - act6 | | act6 | | | | - act7 | | act7 | | | | - act8 | | act8 | | | | - act9 | | act9 | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | default | | | | | | - papersize | | | | | | - text_wrap | | | | | | - emphasis | | | | | | - language | | | | | | - digest | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | permission | | | | | | - share_source | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | program_select | | | | | | - editor | | | | | | - epub_viewer | | | | | | - html_viewer | | | | | | - odf_viewer | | | | | | - pdf_viewer | | | | | | - xml_viewer | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | search | | | | | | - flag | | | | | | - action | | | | | | - db | | | | | | - title | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | make instruction | ** | omit or override share? | ** | *? | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | make | make | make | | | | - bold | - bold | - bold | | | | - breaks | - breaks | - breaks | | | | - cover_image | - cover_image | - cover_image | | | | - css | - css | - css | | | | - emphasis | - emphasis | - emphasis | | | | - footer | - footer | - footer | | | | - headings | - headings | - headings | | | | - home_button_image | - home_button_image | - home_button_image | | | | - home_button_text | - home_button_text | - home_button_text | | | | - italics | - italics | - italics | | | | - num_top | - num_top | - num_top | | | | - auto_num_depth | - auto_num_depth | - auto_num_depth | | | | - substitute | - substitute | - substitute | | | | - texpdf_font | - texpdf_font | - texpdf_font | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | actions | | | | * | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | | assertions | | | | | | concordance | | | | | | debug | | | | | | digest | | | | | | docbook | | | | | | epub | | | | | | html | | | | | | html-seg | | | | | | html-scroll | | | | | | manifest | | | | | | ocn | | | | | | odt | | | | | | pdf | | | | | | postgresql | | | | | | qrcode | | | | | | pod | | | | | | source | | | | | | sqlite | | | | | | sqlite-db-create | | | | | | sqlite-db-drop | | | | | | text | | | | | | verbose | | | | | | xhtml | | | | | | xml-dom | | | | | | xml-sax | | | | | | section_toc | | | | | | section_body | | | | | | section_endnotes | | | | | | section_glossary | | | | | | section_biblio | | | | | | section_bookindex | | | | | | section_blurb | | | | | | backmatter | | | | | | skip-output | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | metadata | | | * | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | classify | | | | | | - dewey | | | | | | - keywords | | | | | | - loc | | | | | | - subject | | | | | | - topic_register | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | creator | | | | | | - author | | | | | | - author_email | | | | | | - illustrator | | | | | | - translator | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | date | | | | | | - added_to_site | | | | | | - available | | | | | | - created | | | | | | - issued | | | | | | - modified | | | | | | - published | | | | | | - valid | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | identifier | | | | | | - isbn | | | | | | - oclc | | | | | | - pg | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | links | | | | | | - link | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | notes | | | | | | - abstract | | | | | | - description | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | original | | | | | | - language | | | | | | - source | | | | | | - title | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | publisher | | | | | | - name | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | rights | | | | | | - copyright | | | | | | - cover | | | | | | - illustrations | | | | | | - license | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | title | | | | | | - edition | | | | | | - full | | | | | | - language | | | | | | - main | | | | | | - note | | | | | | - sub | | | | | | - subtitle | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------|