nested parallelisation, output task scheduler

- default off, --pp2 to use

4 files changed, 144 insertions, 108 deletions

diff --git a/org/output_hub.org b/org/output_hub.org
index 89be2ca..08d47fe 100644
--- a/org/output_hub.org
+++ b/org/output_hub.org
@@ -27,12 +27,27 @@
 module sdp.output.hub;
 template outputHub() {
   <<output_imports>>
+  import std.parallelism;
   void outputHub(D,I)(D doc_abstraction, I doc_matters) {
     mixin SiSUoutputRgxInit;
     mixin Msg;
     auto msg = Msg!()(doc_matters);
     static auto rgx = Rgx();
-    <<output_options>>
+    enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
+    void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) {
+      auto msg = Msg!()(doc_matters);
+      <<output_scheduled_task>>
+    }
+    if (!(doc_matters.opt.action.pp2)) {
+      foreach(schedule; doc_matters.opt.action.output_task_scheduler) {
+        Scheduled!()(schedule, doc_abstraction, doc_matters);
+      }
+    } else {
+      foreach(schedule; parallel(doc_matters.opt.action.output_task_scheduler)) {
+        Scheduled!()(schedule, doc_abstraction, doc_matters);
+      }
+    }
+    <<output_shared_sqlite_db>>
   }
 }
 template outputHubOp() {
@@ -59,92 +74,50 @@ import sdp.output,
   sdp.output.paths_output;
 #+END_SRC
 
-** output options
-*** verbose
-
-#+name: output_options
-#+BEGIN_SRC d
-if (doc_matters.opt.action.verbose) {
-  writeln(doc_matters.xml.keys_seq.seg);
-}
-#+END_SRC
-
-*** output source: _sisusrc_ &/or _sisupod_
-- [[./output_sisupod.org][output_sisupod]]
-
-#+name: output_options
+#+name: output_scheduled_task
 #+BEGIN_SRC d
-if ((doc_matters.opt.action.source)
-|| (doc_matters.opt.action.sisupod)) {
-  if (doc_matters.opt.action.source) {
-    msg.v("sisu source processing...");
-  }
-  if (doc_matters.opt.action.sisupod) {
-    msg.v("sisupod source processing...");
-  }
+if (sched == outTask.sisupod) {
+  msg.v("sisu source processing... ");
   SiSUpod!()(doc_matters);
-  if (doc_matters.opt.action.source) {
-    msg.vv("sisu source done");
-  }
-  if (doc_matters.opt.action.sisupod) {
-    msg.v("sisupod source done");
-  }
+  msg.vv("sisu source done");
 }
-#+END_SRC
-
-*** html
-- [[./output_xmls.org][output_xmls]]
-
-#+name: output_options
-#+BEGIN_SRC d
-if (doc_matters.opt.action.html_seg) {
-  msg.v("html seg source processing...");
-  outputHTML!().seg(doc_abstraction, doc_matters);
-  msg.vv("html seg done");
+if (sched == outTask.sqlite) {
+  msg.v("sqlite processing... ");
+  SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
+  msg.vv("sqlite done");
 }
-if (doc_matters.opt.action.html_scroll) {
-  msg.v("html scroll source processing...");
+if (sched == outTask.epub) {
+  msg.v("epub3 processing... ");
+  outputEPub3!()(doc_abstraction, doc_matters);
+  msg.vv("epub3 done");
+}
+if (sched == outTask.html_scroll) {
+  msg.v("html scroll processing... ");
   outputHTML!().scroll(doc_abstraction, doc_matters);
   msg.vv("html scroll done");
 }
-if (doc_matters.opt.action.html_stuff) {
-  msg.v("html copying images & css...");
+if (sched == outTask.html_seg) {
+  msg.v("html seg processing... ");
+  outputHTML!().seg(doc_abstraction, doc_matters);
+  msg.vv("html seg done");
+}
+if (sched == outTask.html_stuff) {
   outputHTML!().css(doc_matters);
   outputHTML!().images_cp(doc_matters);
   msg.vv("html css & images done");
 }
 #+END_SRC
 
-*** epub
-- [[./output_xmls.org][output_xmls]]
-
-#+name: output_options
-#+BEGIN_SRC d
-if (doc_matters.opt.action.epub) {
-  msg.v("epub3 source processing...");
-  outputEPub3!()(doc_abstraction, doc_matters);
-  msg.vv("epub3 done");
-}
-#+END_SRC
-
-*** sqlite
-- [[./output_sqlite.org][output_sqlite]]
-
-**** discrete
+** output options
+*** verbose
 
-#+name: output_options
-#+BEGIN_SRC d
-if (doc_matters.opt.action.sqlite_discrete) {
-  msg.v("sqlite processing...");
-  SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
-  msg.vv("sqlite done");
-}
-#+END_SRC
+*** output source: _sisusrc_ &/or _sisupod_
+- [[./output_sisupod.org][output_sisupod]]
 
 **** collection
 ***** update / populate
 
-#+name: output_options
+#+name: output_shared_sqlite_db
 #+BEGIN_SRC d
 if (doc_matters.opt.action.sqlite_insert
   || doc_matters.opt.action.sqlite_update
diff --git a/org/sdp.org b/org/sdp.org
index 5a0160d..499224a 100644
--- a/org/sdp.org
+++ b/org/sdp.org
@@ -70,6 +70,7 @@ import
   sdp.conf.compile_time_info,
   sdp.meta.metadoc;
 <<imports_sdp>>
+import std.algorithm;
 import std.parallelism;
 <<mixin_sdp_version>>
 <<mixin_pre_main>>
@@ -238,6 +239,7 @@ bool[string] opts = [
   "html-scroll"        : false,
   "manifest"           : false,
   "ocn"                : true,
+  "pp2"                : false,
   "quiet"              : false,
   "sisupod"            : false,
   "source"             : false,
@@ -281,6 +283,7 @@ auto helpInfo = getopt(args,
   "html-scroll",        "--html-seg process html output",                                           &opts["html-scroll"],
   "manifest",           "--manifest process manifest output",                                       &opts["manifest"],
   "ocn",                "--ocn object cite numbers (default)",                                      &opts["ocn"],
+  "pp2",                "--pp2 nested parallelisation",                                             &opts["pp2"],
   "quiet",              "--quiet output to terminal",                                               &opts["quiet"],
   "sisupod",            "--sisupod sisupod source content bundled",                                 &opts["sisupod"],
   "source",             "--source markup source text content",                                      &opts["source"],
@@ -320,6 +323,7 @@ if (helpInfo.helpWanted) {
 
 #+NAME: sdp_args
 #+BEGIN_SRC d
+enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
 struct OptActions {
   auto assertions() {
     return opts["assertions"];
@@ -473,6 +477,34 @@ struct OptActions {
   auto sqlite_filename() {
     return settings["sqlite-filename"];
   }
+  auto pp2() {
+    return opts["pp2"];
+  }
+  auto output_task_scheduler() {
+    int[] schedule;
+    if (sisupod) {
+      schedule ~= outTask.sisupod;
+    }
+    if (source) {
+      schedule ~= outTask.source;
+    }
+    if (sqlite_discrete) {
+      schedule ~= outTask.sqlite;
+    }
+    if (epub) {
+      schedule ~= outTask.epub;
+    }
+    if (html_scroll) {
+      schedule ~= outTask.html_scroll;
+    }
+    if (html_seg) {
+      schedule ~= outTask.html_seg;
+    }
+    if (html_stuff) {
+      schedule ~= outTask.html_stuff;
+    }
+    return schedule.sort().uniq;
+  }
   auto abstraction() {
     bool _is = (
       opts["abstraction"]
diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d
index 75be424..832cda8 100644
--- a/src/sdp/output/hub.d
+++ b/src/sdp/output/hub.d
@@ -12,55 +12,54 @@ template outputHub() {
     sdp.output.source_sisupod,
     sdp.output.create_zip_file,
     sdp.output.paths_output;
+  import std.parallelism;
   void outputHub(D,I)(D doc_abstraction, I doc_matters) {
     mixin SiSUoutputRgxInit;
     mixin Msg;
     auto msg = Msg!()(doc_matters);
     static auto rgx = Rgx();
-    if (doc_matters.opt.action.verbose) {
-      writeln(doc_matters.xml.keys_seq.seg);
-    }
-    if ((doc_matters.opt.action.source)
-    || (doc_matters.opt.action.sisupod)) {
-      if (doc_matters.opt.action.source) {
-        msg.v("sisu source processing...");
+    enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
+    void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) {
+      auto msg = Msg!()(doc_matters);
+      if (sched == outTask.sisupod) {
+        msg.v("sisu source processing... ");
+        SiSUpod!()(doc_matters);
+        msg.vv("sisu source done");
       }
-      if (doc_matters.opt.action.sisupod) {
-        msg.v("sisupod source processing...");
+      if (sched == outTask.sqlite) {
+        msg.v("sqlite processing... ");
+        SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
+        msg.vv("sqlite done");
       }
-      SiSUpod!()(doc_matters);
-      if (doc_matters.opt.action.source) {
-        msg.vv("sisu source done");
+      if (sched == outTask.epub) {
+        msg.v("epub3 processing... ");
+        outputEPub3!()(doc_abstraction, doc_matters);
+        msg.vv("epub3 done");
       }
-      if (doc_matters.opt.action.sisupod) {
-        msg.v("sisupod source done");
+      if (sched == outTask.html_scroll) {
+        msg.v("html scroll processing... ");
+        outputHTML!().scroll(doc_abstraction, doc_matters);
+        msg.vv("html scroll done");
+      }
+      if (sched == outTask.html_seg) {
+        msg.v("html seg processing... ");
+        outputHTML!().seg(doc_abstraction, doc_matters);
+        msg.vv("html seg done");
+      }
+      if (sched == outTask.html_stuff) {
+        outputHTML!().css(doc_matters);
+        outputHTML!().images_cp(doc_matters);
+        msg.vv("html css & images done");
       }
     }
-    if (doc_matters.opt.action.html_seg) {
-      msg.v("html seg source processing...");
-      outputHTML!().seg(doc_abstraction, doc_matters);
-      msg.vv("html seg done");
-    }
-    if (doc_matters.opt.action.html_scroll) {
-      msg.v("html scroll source processing...");
-      outputHTML!().scroll(doc_abstraction, doc_matters);
-      msg.vv("html scroll done");
-    }
-    if (doc_matters.opt.action.html_stuff) {
-      msg.v("html copying images & css...");
-      outputHTML!().css(doc_matters);
-      outputHTML!().images_cp(doc_matters);
-      msg.vv("html css & images done");
-    }
-    if (doc_matters.opt.action.epub) {
-      msg.v("epub3 source processing...");
-      outputEPub3!()(doc_abstraction, doc_matters);
-      msg.vv("epub3 done");
-    }
-    if (doc_matters.opt.action.sqlite_discrete) {
-      msg.v("sqlite processing...");
-      SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
-      msg.vv("sqlite done");
+    if (!(doc_matters.opt.action.pp2)) {
+      foreach(schedule; doc_matters.opt.action.output_task_scheduler) {
+        Scheduled!()(schedule, doc_abstraction, doc_matters);
+      }
+    } else {
+      foreach(schedule; parallel(doc_matters.opt.action.output_task_scheduler)) {
+        Scheduled!()(schedule, doc_abstraction, doc_matters);
+      }
     }
     if (doc_matters.opt.action.sqlite_insert
       || doc_matters.opt.action.sqlite_update
diff --git a/src/sdp/sdp.d b/src/sdp/sdp.d
index 5a4c2f8..a135423 100755
--- a/src/sdp/sdp.d
+++ b/src/sdp/sdp.d
@@ -22,6 +22,7 @@ import
   sdp.source.read_config_files,
   sdp.source.read_source_files,
   sdp.output.hub;
+import std.algorithm;
 import std.parallelism;
 mixin(import("version.txt"));
 mixin CompileTimeInfo;
@@ -69,6 +70,7 @@ void main(string[] args) {
     "html-scroll"        : false,
     "manifest"           : false,
     "ocn"                : true,
+    "pp2"                : false,
     "quiet"              : false,
     "sisupod"            : false,
     "source"             : false,
@@ -112,6 +114,7 @@ void main(string[] args) {
     "html-scroll",        "--html-seg process html output",                                           &opts["html-scroll"],
     "manifest",           "--manifest process manifest output",                                       &opts["manifest"],
     "ocn",                "--ocn object cite numbers (default)",                                      &opts["ocn"],
+    "pp2",                "--pp2 nested parallelisation",                                             &opts["pp2"],
     "quiet",              "--quiet output to terminal",                                               &opts["quiet"],
     "sisupod",            "--sisupod sisupod source content bundled",                                 &opts["sisupod"],
     "source",             "--source markup source text content",                                      &opts["source"],
@@ -145,6 +148,7 @@ void main(string[] args) {
   if (helpInfo.helpWanted) {
     defaultGetoptPrinter("Some information about the program.", helpInfo.options);
   }
+  enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
   struct OptActions {
     auto assertions() {
       return opts["assertions"];
@@ -298,6 +302,34 @@ void main(string[] args) {
     auto sqlite_filename() {
       return settings["sqlite-filename"];
     }
+    auto pp2() {
+      return opts["pp2"];
+    }
+    auto output_task_scheduler() {
+      int[] schedule;
+      if (sisupod) {
+        schedule ~= outTask.sisupod;
+      }
+      if (source) {
+        schedule ~= outTask.source;
+      }
+      if (sqlite_discrete) {
+        schedule ~= outTask.sqlite;
+      }
+      if (epub) {
+        schedule ~= outTask.epub;
+      }
+      if (html_scroll) {
+        schedule ~= outTask.html_scroll;
+      }
+      if (html_seg) {
+        schedule ~= outTask.html_seg;
+      }
+      if (html_stuff) {
+        schedule ~= outTask.html_stuff;
+      }
+      return schedule.sort().uniq;
+    }
     auto abstraction() {
       bool _is = (
         opts["abstraction"]