0.2.0 renames pod from sisupod

3 files changed, 285 insertions, 7 deletions

diff --git a/src/doc_reform/output/hub.d b/src/doc_reform/output/hub.d
index 409f2bc..5e036dd 100644
--- a/src/doc_reform/output/hub.d
+++ b/src/doc_reform/output/hub.d
@@ -9,7 +9,7 @@ template outputHub() {
     doc_reform.output.html,
     doc_reform.output.sqlite,
     doc_reform.output.xmls,
-    doc_reform.output.source_sisupod,
+    doc_reform.output.source_pod,
     doc_reform.output.create_zip_file,
     doc_reform.output.paths_output;
   import std.parallelism;
@@ -18,10 +18,10 @@ template outputHub() {
     mixin Msg;
     auto msg = Msg!()(doc_matters);
     static auto rgx = Rgx();
-    enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
+    enum outTask { pod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
     void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) {
       auto msg = Msg!()(doc_matters);
-      if (sched == outTask.sisupod) {
+      if (sched == outTask.pod) {
         msg.v("sisu source processing... ");
         DocReformPod!()(doc_matters);
         msg.vv("sisu source done");
@@ -78,7 +78,7 @@ template outputHubOp() {
     doc_reform.output.html,
     doc_reform.output.sqlite,
     doc_reform.output.xmls,
-    doc_reform.output.source_sisupod,
+    doc_reform.output.source_pod,
     doc_reform.output.create_zip_file,
     doc_reform.output.paths_output;
   void outputHubOp(E,O)(E env, O opt_action) {
diff --git a/src/doc_reform/output/rgx.d b/src/doc_reform/output/rgx.d
index 6896237..02c3cbb 100644
--- a/src/doc_reform/output/rgx.d
+++ b/src/doc_reform/output/rgx.d
@@ -18,12 +18,12 @@ static template DocReformOutputRgxInit() {
     static special_markup_chars                           = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg");
     static src_pth_sst_or_ssm                             = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);
     static src_pth_pod_sst_or_ssm                         = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);
-    static src_pth_contents                               = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/sisupod[.]manifest$`);
-    static src_pth_pod_root                               = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`);
+    static src_pth_contents                               = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/pod[.]manifest$`);
+    static src_pth_pod_root                               = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(pod))$`);
     static src_pth_zip                                    = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`);
     static src_pth_unzip_pod                              = ctRegex!(`^(?P<path>media/text/[a-z]{2}/)*(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`);
     static src_pth_types                                  =
-      ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/sisupod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`);
+      ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`);
     static pod_content_location                           =
       ctRegex!(`^(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])(?P<languages>(?:\s+[a-z]{2}(?:,|$))+)`, "mg");
     static src_fn                                         =
diff --git a/src/doc_reform/output/source_pod.d b/src/doc_reform/output/source_pod.d
new file mode 100644
index 0000000..4303c32
--- /dev/null
+++ b/src/doc_reform/output/source_pod.d
@@ -0,0 +1,278 @@
+module doc_reform.output.source_pod;
+template DocReformPod() {
+  import doc_reform.output;
+  import
+    std.digest.sha,
+    std.file,
+    std.outbuffer,
+    std.zip,
+    std.conv : to;
+  import
+    doc_reform.output.create_zip_file,
+    doc_reform.output.xmls;
+  void DocReformPod(T)(T doc_matters) {
+    debug(asserts) {
+      // static assert(is(typeof(doc_matters) == tuple));
+    }
+    mixin DocReformOutputRgxInit;
+    string pwd = doc_matters.env.pwd;
+    auto src_path_info = doc_matters.src_path_info;
+    auto pth_sisudoc_src = doc_matters.src_path_info;
+    auto pths_pod = DocReformPathsPods!()(doc_matters);
+    mixin DocReformLanguageCodes;
+    auto lang = Lang();
+    static auto rgx = Rgx();
+    assert (doc_matters.src.filename.match(rgx.src_fn));
+    try {
+      /+ create directory structure +/
+      if (!exists(pths_pod.pod_dir_())) {
+        // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
+        pths_pod.pod_dir_().mkdirRecurse;
+      }
+      if (doc_matters.opt.action.source) {
+        if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) {
+          pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
+        }
+        if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) {
+          pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
+        }
+        if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) {
+          pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
+        }
+        if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) {
+          pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
+        }
+        if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) {
+          pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
+        }
+        if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) {
+          pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse;
+        }
+      }
+      debug(pod) {
+        writeln(__LINE__, ": ",
+          doc_matters.src.filename, " -> ",
+          pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod
+        );
+      }
+      auto zip = new ZipArchive();
+      auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
+      { /+ bundle images +/
+        foreach (image; doc_matters.srcs.image_list) {
+          debug(podimages) {
+            writeln(
+              pth_sisudoc_src.image_root.to!string, "/", image, " -> ",
+              pths_pod.image_root(doc_matters.src.filename).zpod, "/", image
+            );
+          }
+          auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image;
+          auto fn_src_out_pod_zip_base
+            = pths_pod.image_root(doc_matters.src.filename).zpod.to!string
+            ~ "/" ~ image;
+          auto fn_src_out_filesystem
+            = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string
+            ~ "/" ~ image;
+          if (exists(fn_src_in)) {
+            debug(io) {
+              writeln("(io debug) src out found: ", fn_src_in);
+            }
+            if (doc_matters.opt.action.source) {
+              fn_src_in.copy(fn_src_out_filesystem);
+            }
+            if (doc_matters.opt.action.pod) {
+              auto zip_arc_member_file = new ArchiveMember();
+              zip_arc_member_file.name = fn_src_out_pod_zip_base;
+              auto zip_data = new OutBuffer();
+              zip_data.write(cast(char[]) ((fn_src_in).read));
+              zip_arc_member_file.expandedData = zip_data.toBytes();
+              zip.addMember(zip_arc_member_file);
+            }
+          } else {
+            if (doc_matters.opt.action.verbose) {
+              writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
+            }
+          }
+        }
+      } { /+ bundle sisu_document_make +/
+        auto fn_src_in = ((doc_matters.src.is_pod)
+          ? doc_matters.src.conf_dir_path
+          : pth_sisudoc_src.conf_root).to!string
+          ~ "/" ~ "sisu_document_make";
+        auto fn_src_out_pod_zip_base
+          = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "sisu_document_make";
+        auto fn_src_out_filesystem
+          = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string
+          ~ "/" ~ "sisu_document_make";
+        if (exists(fn_src_in)) {
+          debug(io) {
+            writeln("(io debug) src out found: ", fn_src_in);
+          }
+          if (doc_matters.opt.action.source) {
+            fn_src_in.copy(fn_src_out_filesystem);
+          }
+          if (doc_matters.opt.action.pod) {
+            auto zip_arc_member_file = new ArchiveMember();
+            zip_arc_member_file.name = fn_src_out_pod_zip_base;
+            auto zip_data = new OutBuffer();
+            zip_data.write((fn_src_in).readText);
+            zip_arc_member_file.expandedData = zip_data.toBytes();
+            zip.addMember(zip_arc_member_file);
+          }
+        } else {
+          if (doc_matters.opt.action.verbose
+          || doc_matters.opt.action.debug_do) {
+            writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
+          }
+        }
+      } { /+ bundle primary file +/
+        auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
+        auto fn_src_out_pod_zip_base
+          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
+        auto fn_src_out_filesystem
+          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // without root path:
+        auto fn_src_out_inside_pod
+          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // without root path:
+        string[] filelist_src_out_pod_arr;
+        string[] filelist_src_zpod_arr;
+        if (exists(fn_src_in)) {
+          debug(io) {
+            writeln("(io debug) src in found: ", fn_src_in);
+          }
+          filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
+          filelist_src_zpod_arr ~= fn_src_out_inside_pod;
+          if (doc_matters.opt.action.source) {
+            auto filelist
+              = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
+            foreach (source_pth_and_fn; filelist_src_zpod_arr) {
+              filelist.writeln(source_pth_and_fn);
+            }
+            fn_src_in.copy(fn_src_out_filesystem);
+          }
+          if (doc_matters.opt.action.pod) {
+            auto zip_arc_member_file = new ArchiveMember();
+            zip_arc_member_file.name = fn_src_out_pod_zip_base;
+            auto zip_data = new OutBuffer();
+            zip_data.write((fn_src_in).readText);
+            zip_arc_member_file.expandedData = zip_data.toBytes();
+            zip.addMember(zip_arc_member_file);
+          }
+        } else {
+          if (doc_matters.opt.action.verbose
+          || doc_matters.opt.action.debug_do) {
+            writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
+          }
+        }
+      } { /+ bundle insert files +/
+        if (doc_matters.srcs.file_insert_list.length > 0) {
+          foreach (insert_file; doc_matters.srcs.file_insert_list) {
+            debug(pod) {
+              writeln(
+                insert_file, " -> ",
+                pths_pod.fn_doc_insert(
+                  doc_matters.src.filename,
+                  insert_file,
+                  doc_matters.src.language,
+                ).zpod
+              );
+            }
+            auto fn_src_in = insert_file;
+            auto fn_src_out_pod_zip_base
+              = pths_pod.fn_doc_insert(
+                doc_matters.src.filename,
+                insert_file,
+                doc_matters.src.language,
+              ).zpod.to!string;
+            auto fn_src_out_filesystem
+              = pths_pod.fn_doc_insert(
+                doc_matters.src.filename,
+                insert_file,
+                doc_matters.src.language,
+              ).filesystem_open_zpod.to!string;
+            if (exists(fn_src_in)) {
+              debug(io) {
+                writeln("(io debug) src out found: ", fn_src_in);
+              }
+              if (doc_matters.opt.action.source) {
+                fn_src_in.copy(fn_src_out_filesystem);
+              }
+              if (doc_matters.opt.action.pod) {
+                auto zip_arc_member_file = new ArchiveMember();
+                zip_arc_member_file.name = fn_src_out_pod_zip_base;
+                auto zip_data = new OutBuffer();
+                zip_data.write((fn_src_in).readText);
+                zip_arc_member_file.expandedData = zip_data.toBytes();
+                zip.addMember(zip_arc_member_file);
+                createZipFile!()(fn_pod, zip.build());
+              }
+            } else {
+              if (doc_matters.opt.action.verbose
+              || doc_matters.opt.action.debug_do) {
+                writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
+              }
+            }
+          }
+        }
+      } {
+        auto fn_src_in = doc_matters.src.filename;
+        if (doc_matters.opt.action.pod) {
+          if (exists(doc_matters.src.file_with_absolute_path)) {
+            createZipFile!()(fn_pod, zip.build());
+          } else {
+            writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod);
+          }
+          if (!(exists(fn_pod))) {
+            writeln("WARNING failed to create pod zip archive: ", fn_pod);
+          }
+        }
+      }
+      if (exists(fn_pod)) {
+        try {
+          if (doc_matters.opt.action.verbose) {
+            auto data = (cast(byte[]) (fn_pod).read);
+            writeln(doc_matters.src.filename, " >> ");
+            writefln("%-(%02x%) %s", data.sha256Of, fn_pod);
+          }
+          debug(pod) {
+            try {
+              auto zipped = new ZipArchive((fn_pod).read);
+              foreach (filename, member; zipped.directory) {
+                auto data = zipped.expand(member);
+                writeln("> ", filename, " length ", data.length);
+              }
+            }
+            catch (ZipException ex) {
+              // Handle errors
+            }
+            if (doc_matters.src.filename == "sisudoc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") {
+              assert(
+                ((data).sha256Of).toHexString
+                == "626F83A31ED82F42CF528E922C1643498A137ABA3F2E5AFF8A379EA79EA22A1E",
+                "\npod: sha256 value for "
+                ~ doc_matters.src.filename
+                ~ " has changed, is now: "
+                ~ ((data).sha256Of).toHexString
+              );
+            }
+            if (doc_matters.src.filename == "sisudoc/media/text/en/sisu_markup_stress_test.sst") {
+              assert(
+                ((data).sha256Of).toHexString
+                == "AAE0C87AB3F6D5F7385AEEA6EE661F56D40475CFE87AD930C78C9FE07FFB0D91",
+                "\npod: sha256 value for "
+                ~ doc_matters.src.filename
+                ~ " has changed, is now: "
+                ~ ((data).sha256Of).toHexString
+              );
+            }
+          }
+        }
+        catch (ErrnoException ex) {
+          // Handle errors
+        }
+      }
+      
+    }
+    catch (ErrnoException ex) {
+      // Handle error
+    }
+  }
+}