biblio (fixes)

6 files changed, 228 insertions, 85 deletions

diff --git a/org/ao_abstract_doc_source.org b/org/ao_abstract_doc_source.org
index 1239a50..b570f83 100644
--- a/org/ao_abstract_doc_source.org
+++ b/org/ao_abstract_doc_source.org
@@ -71,6 +71,7 @@ enum DocStructCollapsedHeading { lv0, lv1, lv2, lv3, lv4, lv5, lv6, lv7 } // not
 /+ biblio variables +/
 string biblio_tag_name, biblio_tag_entry, st;
 string[] biblio_arr_json;
+string biblio_entry_str_json;
 JSONValue[] bib_arr_json;
 int bib_entry;
 /+ counters +/
@@ -282,12 +283,16 @@ if (type["code"] == TriState.on) {
 
 #+name: abs_in_loop_body_non_code_obj
 #+BEGIN_SRC d
-if (((matchFirst(line, rgx.heading_biblio)
-|| (type["heading_biblio"] == State.on)))
+if ((matchFirst(line, rgx.heading_biblio)
+|| (type["heading_biblio"] == State.on))
 && (!matchFirst(line, rgx.heading))
 && (!matchFirst(line, rgx.comment))) {
   /+ within block object: biblio +/
-  biblio_block(line, type, bib_entry, biblio_arr_json);
+  biblio_block(line, type, bib_entry, biblio_entry_str_json, biblio_arr_json);
+  debug(bibliobuild) {
+    writeln("-  ", biblio_entry_str_json);
+    writeln("-> ", biblio_arr_json.length);
+  }
   continue;
 #+END_SRC
 
@@ -1243,36 +1248,65 @@ final string biblio_tag_map_(string abr) {
 
 #+name: abs_functions
 #+BEGIN_SRC d
-auto biblio_block(
+void biblio_block(
   char[] line,
   ref int[string] type,
   ref int bib_entry,
+  ref string biblio_entry_str_json,
   ref string[] biblio_arr_json
 ) {
   if (matchFirst(line, rgx.heading_biblio)) {
     type["heading_biblio"] = TriState.on;
   }
-  if (empty(line) && (bib_entry == TriState.off)) {
-    biblio_arr_json ~= biblio_entry_tags_jsonstr;
-    bib_entry = TriState.on;
-  }
-  debug(biblio) {
-    writefln(
-      "* %s",
-      line
-    );
-  }
-  if (matchFirst(line, rgx.biblio_tags)) {
+  if (line.empty) {
+    debug {
+      debug(biblioblock) {
+        writeln("---");
+      }
+      debug(biblioblockinclude) {
+        writeln(biblio_entry_str_json.length);
+      }
+    }
+    if ((bib_entry == State.off)
+    && (biblio_entry_str_json.empty)) {
+      bib_entry = State.on;
+      biblio_entry_str_json = biblio_entry_tags_jsonstr;
+    } else if (!(biblio_entry_str_json.empty)) {
+      bib_entry = State.off;
+      if (!(biblio_entry_str_json == biblio_entry_tags_jsonstr)) {
+        auto biblio_entry = parseJSON(biblio_entry_str_json);
+        if (biblio_entry["fulltitle"].str.empty) {
+          writeln("check problem entry (Title missing): ", biblio_entry_str_json);
+        } else if ((biblio_entry["author_raw"].str.empty) && (biblio_entry["editor_raw"].str.empty)) {
+          writeln("check problem entry (No author and no editor): ", biblio_entry_str_json);
+        // } else if (biblio_entry["sortby_deemed_author_year_title"].str.empty) {
+        //   writeln("check problem entry (Sort Field missing): ", biblio_entry_str_json);
+        } else {
+          biblio_arr_json ~= biblio_entry_str_json;
+        }
+        biblio_entry_str_json = biblio_entry_tags_jsonstr;
+      }
+    } else { // CHECK ERROR
+      writeln("?? 2. ERROR ", biblio_entry_str_json, "??");
+      biblio_entry_str_json = "";
+    }
+  } else if (matchFirst(line, rgx.biblio_tags)) {
+    debug(biblioblock) {
+      writeln(line);
+    }
     auto bt = match(line, rgx.biblio_tags);
-    bib_entry = 0;
+    bib_entry = State.off;
     st=to!string(bt.captures[1]);
-    biblio_tag_entry=to!string(bt.captures[2]);
-    JSONValue j = parseJSON(biblio_arr_json[$-1]);
+    auto header_tag_value=to!string(bt.captures[2]);
+    JSONValue j = parseJSON(biblio_entry_str_json);
     biblio_tag_name = (match(st, rgx.biblio_abbreviations))
       ? (biblio_tag_map(st))
       : st;
-    j.object[biblio_tag_name] = biblio_tag_entry;
-    auto header_tag_value=to!string(bt.captures[2]);
+    j.object[biblio_tag_name] = header_tag_value;
+    debug(bibliounsortedcheckduplicates) {
+      writeln(biblio_tag_name, ": ", header_tag_value);
+      writeln("--");
+    }
     switch (biblio_tag_name) {
     case "author_raw": // author_arr author (fn sn)
       j["author_arr"] =
@@ -1289,7 +1323,7 @@ auto biblio_block(
       tmp = replace(tmp, rgx.trailing_comma, "");
       // tmp = replace(tmp, regex(r"(,[ ]*)$","g"), "");
       j["author"].str = tmp;
-      break;
+      goto default;
     case "editor_raw": // editor_arr editor (fn sn)
       j["editor_arr"] =
         split(header_tag_value, rgx.arr_delimiter);
@@ -1305,16 +1339,14 @@ auto biblio_block(
       tmp = replace(tmp, rgx.trailing_comma, "");
       // tmp = replace(tmp, regex(r"(,[ ]*)$","g"), "");
       j["editor"].str = tmp;
-      break;
+      goto default;
     case "fulltitle": // title & subtitle
-      break;
+      goto default;
     default:
       break;
     }
-    // header_tag_value="";
-    auto s = to!string(j);
-    s = j.toString();
-    debug(biblio) {
+    auto s = j.toString();
+    debug(biblio1) {
       writefln(
         "* %s: %s\n%s",
         biblio_tag_name,
@@ -1322,10 +1354,20 @@ auto biblio_block(
         j[biblio_tag_name]
       );
     }
-    biblio_arr_json ~= s;
-    biblio_tag_entry="";
+    if ((match(line, rgx.comment))) {
+      writeln("ERROR", line, "COMMENT");
+      writeln("ERROR", s, "%%");
+    }
+    if (!(match(line, rgx.comment))) {
+      debug(biblioblockinclude) {
+        writeln(line);
+      }
+      biblio_entry_str_json = s;
+    } else {
+      biblio_entry_str_json = "";
+    }
+    header_tag_value="";
   }
-  return 0;
 }
 #+END_SRC
 
@@ -3269,9 +3311,23 @@ struct Bibliography {
   body {
     JSONValue[] biblio_unsorted =
       biblio_unsorted_complete(biblio_unsorted_incomplete, bib_arr_json);
-    JSONValue[] biblio_sorted = biblio_sort(biblio_unsorted);
-    biblio_debug(biblio_sorted);
-    return biblio_sorted;
+    JSONValue[] biblio_sorted__ = biblio_sort(biblio_unsorted);
+    biblio_debug(biblio_sorted__);
+    debug(biblio0) {
+      writeln("---");
+      writeln("unsorted incomplete: ", biblio_unsorted_incomplete.length);
+      writeln("json:                ", bib_arr_json.length);
+      writeln("unsorted:            ", biblio_unsorted.length);
+      writeln("sorted:              ", biblio_sorted__.length);
+      //  writeln("0: ", biblio_sorted__[0]);
+      int counter;
+      int[7] x;
+      while (counter < x.length) {
+        writeln(counter, ": ", biblio_sorted__[counter]["fulltitle"]);
+        counter++;
+      }
+    }
+    return biblio_sorted__;
   }
   final private JSONValue[] biblio_unsorted_complete(
     string[] biblio_unordered,
@@ -3303,23 +3359,23 @@ struct Bibliography {
     return biblio_unsorted_array_of_json_objects;
   }
   final private JSONValue[] biblio_sort(JSONValue[] biblio_unordered) {
-    JSONValue[] biblio_sorted;
-    biblio_sorted =
+    JSONValue[] biblio_sorted_;
+    biblio_sorted_ =
       sort!((a, b){
         return ((a["sortby_deemed_author_year_title"].str) < (b["sortby_deemed_author_year_title"].str));
       })(biblio_unordered).array;
     debug(bibliosorted) {
-      foreach (j; biblio_sorted) {
+      foreach (j; biblio_sorted_) {
         if (!empty(j["fulltitle"].str)) {
           writeln(j["sortby_deemed_author_year_title"]);
           // writeln(j["deemed_author"], " (", j["author"], ") ",  j["fulltitle"]);
         }
       }
     }
-    return biblio_sorted;
+    return biblio_sorted_;
   }
-  auto biblio_debug(JSONValue[] biblio_sorted) {
-    debug(biblio) {
+  void biblio_debug(JSONValue[] biblio_sorted) {
+    debug(biblio0) {
       foreach (j; biblio_sorted) {
         if (!empty(j["fulltitle"].str)) {
           writeln(j["sortby_deemed_author_year_title"]);
diff --git a/org/ao_output_debugs.org b/org/ao_output_debugs.org
index d5d8177..c52b622 100644
--- a/org/ao_output_debugs.org
+++ b/org/ao_output_debugs.org
@@ -302,6 +302,23 @@ debug(bookindex) {
 }
 #+END_SRC
 
+*** biblio (biblio)
+
+#+name: ao_output_debugs
+#+BEGIN_SRC d
+
+debug(biblio) {
+  foreach (entry; biblio) {
+    writefln(
+      "%s; (%s)",
+      entry["sortby_deemed_author_year_title"],
+      entry["author"],
+    );
+  }
+}
+
+#+END_SRC
+
 ** (summary) [+1]                                                   :summary:
 
 #+name: ao_output_debugs
@@ -359,10 +376,12 @@ writefln(
   fn_src,
   "length contents array: ",
   contents.length,
-  "last obj_cite_number: ",
+  "last obj_cite_number:  ",
   check["last_obj_cite_number"],
-  "length bookindex: ",
+  "length bookindex:      ",
   bookindex_unordered_hashes.length,
+  "length biblio:         ",
+  biblio.length,
   __FILE__,
   __LINE__,
 );
diff --git a/org/sdp.org b/org/sdp.org
index 7c87a8e..0539f09 100644
--- a/org/sdp.org
+++ b/org/sdp.org
@@ -24,7 +24,7 @@ struct Version {
   int minor;
   int patch;
 }
-enum ver = Version(0, 6, 4);
+enum ver = Version(0, 6, 5);
 #+END_SRC
 
 * sdp.d   sisu document parser                                       :sdp.d:
@@ -77,7 +77,6 @@ import
 [[./ao_conf_make_meta_sdlang.org][ao_onf_make_meta_sdlang]]
 [[./ao_defaults.org][ao_defaults]]
 [[./ao_output_debugs.org][ao_output_debugs]]
-[[./ao_output_hub.org][ao_output_hub]]
 [[./ao_read_config_files.org][ao_read_config_files]]
 [[./ao_read_source_files.org][ao_read_source_files]]
 [[./ao_read_source_files.org][ao_read_source_files]]
diff --git a/src/sdp/ao_abstract_doc_source.d b/src/sdp/ao_abstract_doc_source.d
index eb87535..3b9dbe6 100644
--- a/src/sdp/ao_abstract_doc_source.d
+++ b/src/sdp/ao_abstract_doc_source.d
@@ -44,6 +44,7 @@ template SiSUdocAbstraction() {
     /+ biblio variables +/
     string biblio_tag_name, biblio_tag_entry, st;
     string[] biblio_arr_json;
+    string biblio_entry_str_json;
     JSONValue[] bib_arr_json;
     int bib_entry;
     /+ counters +/
@@ -230,12 +231,16 @@ template SiSUdocAbstraction() {
         } else if (!matchFirst(line, rgx.skip_code_block_from_regular_parse)) {
           /+ object other than "code block" object
              (includes regular text paragraph, headings & blocks other than code) +/
-          if (((matchFirst(line, rgx.heading_biblio)
-          || (type["heading_biblio"] == State.on)))
+          if ((matchFirst(line, rgx.heading_biblio)
+          || (type["heading_biblio"] == State.on))
           && (!matchFirst(line, rgx.heading))
           && (!matchFirst(line, rgx.comment))) {
             /+ within block object: biblio +/
-            biblio_block(line, type, bib_entry, biblio_arr_json);
+            biblio_block(line, type, bib_entry, biblio_entry_str_json, biblio_arr_json);
+            debug(bibliobuild) {
+              writeln("-  ", biblio_entry_str_json);
+              writeln("-> ", biblio_arr_json.length);
+            }
             continue;
           } else if (type["poem"] == TriState.on) {
             /+ within block object: poem +/
@@ -888,36 +893,65 @@ template SiSUdocAbstraction() {
       ];
       return btm[abr];
     }
-    auto biblio_block(
+    void biblio_block(
       char[] line,
       ref int[string] type,
       ref int bib_entry,
+      ref string biblio_entry_str_json,
       ref string[] biblio_arr_json
     ) {
       if (matchFirst(line, rgx.heading_biblio)) {
         type["heading_biblio"] = TriState.on;
       }
-      if (empty(line) && (bib_entry == TriState.off)) {
-        biblio_arr_json ~= biblio_entry_tags_jsonstr;
-        bib_entry = TriState.on;
-      }
-      debug(biblio) {
-        writefln(
-          "* %s",
-          line
-        );
-      }
-      if (matchFirst(line, rgx.biblio_tags)) {
+      if (line.empty) {
+        debug {
+          debug(biblioblock) {
+            writeln("---");
+          }
+          debug(biblioblockinclude) {
+            writeln(biblio_entry_str_json.length);
+          }
+        }
+        if ((bib_entry == State.off)
+        && (biblio_entry_str_json.empty)) {
+          bib_entry = State.on;
+          biblio_entry_str_json = biblio_entry_tags_jsonstr;
+        } else if (!(biblio_entry_str_json.empty)) {
+          bib_entry = State.off;
+          if (!(biblio_entry_str_json == biblio_entry_tags_jsonstr)) {
+            auto biblio_entry = parseJSON(biblio_entry_str_json);
+            if (biblio_entry["fulltitle"].str.empty) {
+              writeln("check problem entry (Title missing): ", biblio_entry_str_json);
+            } else if ((biblio_entry["author_raw"].str.empty) && (biblio_entry["editor_raw"].str.empty)) {
+              writeln("check problem entry (No author and no editor): ", biblio_entry_str_json);
+            // } else if (biblio_entry["sortby_deemed_author_year_title"].str.empty) {
+            //   writeln("check problem entry (Sort Field missing): ", biblio_entry_str_json);
+            } else {
+              biblio_arr_json ~= biblio_entry_str_json;
+            }
+            biblio_entry_str_json = biblio_entry_tags_jsonstr;
+          }
+        } else { // CHECK ERROR
+          writeln("?? 2. ERROR ", biblio_entry_str_json, "??");
+          biblio_entry_str_json = "";
+        }
+      } else if (matchFirst(line, rgx.biblio_tags)) {
+        debug(biblioblock) {
+          writeln(line);
+        }
         auto bt = match(line, rgx.biblio_tags);
-        bib_entry = 0;
+        bib_entry = State.off;
         st=to!string(bt.captures[1]);
-        biblio_tag_entry=to!string(bt.captures[2]);
-        JSONValue j = parseJSON(biblio_arr_json[$-1]);
+        auto header_tag_value=to!string(bt.captures[2]);
+        JSONValue j = parseJSON(biblio_entry_str_json);
         biblio_tag_name = (match(st, rgx.biblio_abbreviations))
           ? (biblio_tag_map(st))
           : st;
-        j.object[biblio_tag_name] = biblio_tag_entry;
-        auto header_tag_value=to!string(bt.captures[2]);
+        j.object[biblio_tag_name] = header_tag_value;
+        debug(bibliounsortedcheckduplicates) {
+          writeln(biblio_tag_name, ": ", header_tag_value);
+          writeln("--");
+        }
         switch (biblio_tag_name) {
         case "author_raw": // author_arr author (fn sn)
           j["author_arr"] =
@@ -934,7 +968,7 @@ template SiSUdocAbstraction() {
           tmp = replace(tmp, rgx.trailing_comma, "");
           // tmp = replace(tmp, regex(r"(,[ ]*)$","g"), "");
           j["author"].str = tmp;
-          break;
+          goto default;
         case "editor_raw": // editor_arr editor (fn sn)
           j["editor_arr"] =
             split(header_tag_value, rgx.arr_delimiter);
@@ -950,16 +984,14 @@ template SiSUdocAbstraction() {
           tmp = replace(tmp, rgx.trailing_comma, "");
           // tmp = replace(tmp, regex(r"(,[ ]*)$","g"), "");
           j["editor"].str = tmp;
-          break;
+          goto default;
         case "fulltitle": // title & subtitle
-          break;
+          goto default;
         default:
           break;
         }
-        // header_tag_value="";
-        auto s = to!string(j);
-        s = j.toString();
-        debug(biblio) {
+        auto s = j.toString();
+        debug(biblio1) {
           writefln(
             "* %s: %s\n%s",
             biblio_tag_name,
@@ -967,10 +999,20 @@ template SiSUdocAbstraction() {
             j[biblio_tag_name]
           );
         }
-        biblio_arr_json ~= s;
-        biblio_tag_entry="";
+        if ((match(line, rgx.comment))) {
+          writeln("ERROR", line, "COMMENT");
+          writeln("ERROR", s, "%%");
+        }
+        if (!(match(line, rgx.comment))) {
+          debug(biblioblockinclude) {
+            writeln(line);
+          }
+          biblio_entry_str_json = s;
+        } else {
+          biblio_entry_str_json = "";
+        }
+        header_tag_value="";
       }
-      return 0;
     }
     auto poem_block(
       char[] line,
@@ -2773,9 +2815,23 @@ template SiSUdocAbstraction() {
       body {
         JSONValue[] biblio_unsorted =
           biblio_unsorted_complete(biblio_unsorted_incomplete, bib_arr_json);
-        JSONValue[] biblio_sorted = biblio_sort(biblio_unsorted);
-        biblio_debug(biblio_sorted);
-        return biblio_sorted;
+        JSONValue[] biblio_sorted__ = biblio_sort(biblio_unsorted);
+        biblio_debug(biblio_sorted__);
+        debug(biblio0) {
+          writeln("---");
+          writeln("unsorted incomplete: ", biblio_unsorted_incomplete.length);
+          writeln("json:                ", bib_arr_json.length);
+          writeln("unsorted:            ", biblio_unsorted.length);
+          writeln("sorted:              ", biblio_sorted__.length);
+          //  writeln("0: ", biblio_sorted__[0]);
+          int counter;
+          int[7] x;
+          while (counter < x.length) {
+            writeln(counter, ": ", biblio_sorted__[counter]["fulltitle"]);
+            counter++;
+          }
+        }
+        return biblio_sorted__;
       }
       final private JSONValue[] biblio_unsorted_complete(
         string[] biblio_unordered,
@@ -2807,23 +2863,23 @@ template SiSUdocAbstraction() {
         return biblio_unsorted_array_of_json_objects;
       }
       final private JSONValue[] biblio_sort(JSONValue[] biblio_unordered) {
-        JSONValue[] biblio_sorted;
-        biblio_sorted =
+        JSONValue[] biblio_sorted_;
+        biblio_sorted_ =
           sort!((a, b){
             return ((a["sortby_deemed_author_year_title"].str) < (b["sortby_deemed_author_year_title"].str));
           })(biblio_unordered).array;
         debug(bibliosorted) {
-          foreach (j; biblio_sorted) {
+          foreach (j; biblio_sorted_) {
             if (!empty(j["fulltitle"].str)) {
               writeln(j["sortby_deemed_author_year_title"]);
               // writeln(j["deemed_author"], " (", j["author"], ") ",  j["fulltitle"]);
             }
           }
         }
-        return biblio_sorted;
+        return biblio_sorted_;
       }
-      auto biblio_debug(JSONValue[] biblio_sorted) {
-        debug(biblio) {
+      void biblio_debug(JSONValue[] biblio_sorted) {
+        debug(biblio0) {
           foreach (j; biblio_sorted) {
             if (!empty(j["fulltitle"].str)) {
               writeln(j["sortby_deemed_author_year_title"]);
diff --git a/src/sdp/ao_output_debugs.d b/src/sdp/ao_output_debugs.d
index be08ebe..a17f8ea 100644
--- a/src/sdp/ao_output_debugs.d
+++ b/src/sdp/ao_output_debugs.d
@@ -261,6 +261,17 @@ template SiSUoutputDebugs() {
         auto bookindex = BookIndexReport();
         bookindex.bookindex_report_sorted(bookindex_unordered_hashes);
       }
+      
+      debug(biblio) {
+        foreach (entry; biblio) {
+          writefln(
+            "%s; (%s)",
+            entry["sortby_deemed_author_year_title"],
+            entry["author"],
+          );
+        }
+      }
+      
       debug(summary) {
         string[string] check = [
           "last_obj_cite_number" : "NA [debug \"checkdoc\" not run]",
@@ -302,10 +313,12 @@ template SiSUoutputDebugs() {
           fn_src,
           "length contents array: ",
           contents.length,
-          "last obj_cite_number: ",
+          "last obj_cite_number:  ",
           check["last_obj_cite_number"],
-          "length bookindex: ",
+          "length bookindex:      ",
           bookindex_unordered_hashes.length,
+          "length biblio:         ",
+          biblio.length,
           __FILE__,
           __LINE__,
         );
diff --git a/views/version.txt b/views/version.txt
index 28dc40a..ccda9ec 100644
--- a/views/version.txt
+++ b/views/version.txt
@@ -4,4 +4,4 @@ struct Version {
   int minor;
   int patch;
 }
-enum ver = Version(0, 6, 4);
+enum ver = Version(0, 6, 5);