path: root/org/out_src_pod.org

-*- mode: org -*-
#+TITLE:       spine (doc_reform) output pod
#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
#+FILETAGS:    :spine:output:source:pod:
#+AUTHOR:      Ralph Amissah
#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT:   Copyright (C) 2015 - 2019 Ralph Amissah
#+LANGUAGE:    en
#+STARTUP:     indent content hideblocks hidestars
#+OPTIONS:     H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
#+OPTIONS:     TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc
#+OPTIONS:     author:nil email:nil creator:nil timestamp:nil
#+PROPERTY:    header-args :padline no :exports code :cache no :noweb yes
#+EXPORT_SELECT_TAGS:  export
#+EXPORT_EXCLUDE_TAGS: noexport
#+TAGS: assert(a) class(c) debug(d) mixin(m) spine(s) tangle(T) template(t) WEB(W) noexport(n)

- [[./spine.org][spine]]  [[./][org/]]
- [[./output_hub.org][output_hub]]

* pod
** _module template_                                                :module:

#+BEGIN_SRC d :tangle "../src/doc_reform/io_out/source_pod.d"
module doc_reform.io_out.source_pod;
template spinePod() {
  <<output_imports>>
  void spinePod(T)(T doc_matters) {
    <<source_pod_init>>
    <<source_pod_archive>>
    try {
      <<source_pod_mkdirs>>
      <<source_pod_copy>>
      <<source_pod_zip>>
    } catch (ErrnoException ex) {
      // Handle error
    }
  }
}
#+END_SRC

** init
*** imports

#+name: output_imports
#+BEGIN_SRC d
import doc_reform.io_out;
import
  std.digest.sha,
  std.file,
  std.outbuffer,
  std.zip,
  std.conv : to;
import
  doc_reform.io_out.create_zip_file,
  doc_reform.io_out.xmls;
#+END_SRC

*** init

#+name: source_pod_init
#+BEGIN_SRC d
debug(asserts) {
  // static assert(is(typeof(doc_matters) == tuple));
}
mixin spineOutputRgxInit;
string pwd = doc_matters.env.pwd;
auto src_path_info = doc_matters.src_path_info;
auto pth_dr_doc_src = doc_matters.src_path_info;
auto pths_pod = spinePathsPods!()(doc_matters);
mixin spineLanguageCodes;
auto lang = Lang();
static auto rgx = Rgx();
assert (doc_matters.src.filename.match(rgx.src_fn));
#+END_SRC

*** pod zip archive

#+name: source_pod_archive
#+BEGIN_SRC d
auto pod_archive(Z)(
  string _source_type,
  string _data_in,
  string _pth_out,
  Z zip
) {
  auto zip_arc_member_file = new ArchiveMember();
  zip_arc_member_file.name = _pth_out;
  auto zip_data = new OutBuffer();
  switch (_source_type) {
  case "file_path_bin":
    zip_data.write(cast(char[]) ((_data_in).read));
    goto default;
  case "file_path_text":
    zip_data.write((_data_in).readText);
    goto default;
  case "string":
    zip_data.write(_data_in);
    goto default;
  default:
    zip_arc_member_file.expandedData = zip_data.toBytes();
    zip.addMember(zip_arc_member_file);
  }
  return zip;
}
#+END_SRC

** mkdir                                                             :mkdir:

#+name: source_pod_mkdirs
#+BEGIN_SRC d
/+ create directory structure +/
if (!exists(pths_pod.pod_dir_())) {
  // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
  pths_pod.pod_dir_().mkdirRecurse;
}
if (doc_matters.opt.action.source) {
  if (!(doc_matters.opt.action.quiet)) {
    writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod);
  }
  if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) {
    pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
  }
  if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) {
    pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
  }
  if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) {
    pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
  }
  if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) {
    pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
  }
  if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) {
    pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
  }
  if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) {
    pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse;
  }
}
#+END_SRC

** copy                                                               :copy:

#+name: source_pod_copy
#+BEGIN_SRC d
debug(pod) {
  writeln(__LINE__, ": ",
    doc_matters.src.filename, " -> ",
    pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod
  );
}
auto zip = new ZipArchive();
auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
{ /+ bundle images +/
  foreach (image; doc_matters.srcs.image_list) {
    debug(podimages) {
      writeln(
        pth_dr_doc_src.image_root.to!string, "/", image, " -> ",
        pths_pod.image_root(doc_matters.src.filename).zpod, "/", image
      );
    }
    auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image;
    auto fn_src_out_pod_zip_base
      = pths_pod.image_root(doc_matters.src.filename).zpod.to!string
      ~ "/" ~ image;
    auto fn_src_out_filesystem
      = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string
      ~ "/" ~ image;
    if (exists(fn_src_in)) {
      debug(io) {
        writeln("(io debug) src out found: ", fn_src_in);
      }
      if (doc_matters.opt.action.source) {
        fn_src_in.copy(fn_src_out_filesystem);
      }
      if (doc_matters.opt.action.pod) {
        zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip);
      }
    } else {
      if (doc_matters.opt.action.debug_do
      && doc_matters.opt.action.verbose) {
        writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
      }
    }
  }
} { /+ bundle dr_document_make +/
  auto fn_src_in = ((doc_matters.src.is_pod)
    ? doc_matters.src.conf_dir_path
    : pth_dr_doc_src.conf_root).to!string
    ~ "/" ~ "dr_document_make";
  auto fn_src_out_pod_zip_base
    = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make";
  auto fn_src_out_filesystem
    = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string
    ~ "/" ~ "dr_document_make";
  if (exists(fn_src_in)) {
    debug(io) {
      writeln("(io debug) src out found: ", fn_src_in);
    }
    if (doc_matters.opt.action.source) {
      fn_src_in.copy(fn_src_out_filesystem);
    }
    if (doc_matters.opt.action.pod) {
      zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
    }
  } else {
    if (doc_matters.opt.action.debug_do
    && doc_matters.opt.action.verbose) {
      writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
    }
  }
} { /+ pod manifest +/
  auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
  auto fn_src_out_pod_zip_base
    = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string;
  auto fn_src_out_filesystem
    = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path
  auto fn_src_out_inside_pod
    = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path
  string[] filelist_src_out_pod_arr;
  string[] filelist_src_zpod_arr;
  if (exists(fn_src_in)) {
    debug(io) {
      writeln("(io debug) src in found: ", fn_src_in);
    }
    filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
    filelist_src_zpod_arr ~= fn_src_out_inside_pod;
    {
      import dyaml;
      auto pod_filelist_yaml_string
        = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
      Node _pmy;
      string _pm = "doc:\n  filename: " ~ doc_matters.src.filename ~ "\n  language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n";
      if (doc_matters.opt.action.debug_do
      && doc_matters.opt.action.verbose) {
        _pmy = Loader.fromString(_pm).load();
        writeln("pod filename:  ", _pmy["doc"]["filename"].get!string);
        writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string);
        writeln("pod languages: ", doc_matters.src.language);
        // foreach(string _l; _pmy["doc"]["language"]) {
        //   writeln("language:      ", _l);
        // }
      }
      if (doc_matters.opt.action.source) {
        pod_filelist_yaml_string.writeln(_pm);
      }
      if (doc_matters.opt.action.pod) {
        zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip);
      }
    }
  }
} { /+ bundle primary file (.ssm/.sst) +/
  auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
  auto fn_src_out_pod_zip_base
    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
  auto fn_src_out_filesystem
    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path:
  auto fn_src_out_inside_pod
    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path:
  string[] filelist_src_out_pod_arr;
  string[] filelist_src_zpod_arr;
  if (exists(fn_src_in)) {
    debug(io) {
      writeln("(io debug) src in found: ", fn_src_in);
    }
    filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
    filelist_src_zpod_arr ~= fn_src_out_inside_pod;
    string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename;
    if (doc_matters.opt.action.source) {
      fn_src_in.copy(fn_src_out_filesystem);
    }
    if (doc_matters.opt.action.pod) {
      auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])");
      if (auto _x = fn_src_in.match(_rgx)){
        if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
          string _path_to_pod = _x.captures["path_to_pod"];
          string _podname = _x.captures["podname"];
          string _root_to_lang = _x.captures["from_root"];
          string _language = _x.captures["language"];
          string _filename = _x.captures["filename"];
          foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
            string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
            string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
            zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
          }
        }
      } else {
        zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
      }
    }
  } else {
    if (doc_matters.opt.action.debug_do
    && doc_matters.opt.action.verbose) {
      writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
    }
  }
} { /+ bundle insert files (.ssi) +/
  if (doc_matters.srcs.file_insert_list.length > 0) {
    auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])");
    foreach (insert_file; doc_matters.srcs.file_insert_list) {
      debug(pod) {
        writeln(
          insert_file, " -> ",
          pths_pod.fn_doc_insert(
            doc_matters.src.filename,
            insert_file,
            doc_matters.src.language,
          ).zpod
        );
      }
      if (auto _x = insert_file.match(_rgx)){
        if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
          string _path_to_pod = _x.captures["path_to_pod"];
          string _podname = _x.captures["podname"];
          string _root_to_lang = _x.captures["from_root"];
          string _language = _x.captures["language"];
          string _filename = _x.captures["filename"];
          foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
            string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
            string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
            if (exists(_pth_mkup_src_in)) {
              if (doc_matters.opt.action.source) {
                auto fn_src_out_filesystem // you need to change language sources
                  = pths_pod.fn_doc_insert(
                    doc_matters.src.filename, // doc_matters.src.filename
                    _pth_mkup_src_in, // insert_file
                    _lang,
                  ).filesystem_open_zpod.to!string;
                _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere
              }
              if (doc_matters.opt.action.pod) {
                 zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
              }
            } else {
              if (doc_matters.opt.action.debug_do
              && doc_matters.opt.action.verbose) {
                writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in);
              }
            }
          }
        }
      } else {
        auto fn_src_in = insert_file;
        auto fn_src_out_pod_zip_base
          = pths_pod.fn_doc_insert(
            doc_matters.src.filename,
            insert_file,
            doc_matters.src.language,
          ).zpod.to!string;
        auto fn_src_out_filesystem
          = pths_pod.fn_doc_insert(
            doc_matters.src.filename,
            insert_file,
            doc_matters.src.language,
          ).filesystem_open_zpod.to!string;
        if (exists(fn_src_in)) {
          debug(io) {
            writeln("(io debug) src out found: ", fn_src_in);
          }
          if (doc_matters.opt.action.source) {
            fn_src_in.copy(fn_src_out_filesystem);
          }
          if (doc_matters.opt.action.pod) {
            zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
          }
        } else {
          if (doc_matters.opt.action.debug_do
          && doc_matters.opt.action.verbose) {
            writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
          }
        }
      }
    }
  }
} {
  auto fn_src_in = doc_matters.src.filename;
  if (doc_matters.opt.action.pod) {
    if (exists(doc_matters.src.file_with_absolute_path)) {
      createZipFile!()(fn_pod, zip.build());
    } else {
      writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod);
    }
    if (!(exists(fn_pod))) {
      writeln("WARNING failed to create pod zip archive: ", fn_pod);
    }
  }
}
#+END_SRC

** sha256 of pod.zip, zip debug, read zip archive

#+name: source_pod_copy
#+BEGIN_SRC d
if (exists(fn_pod)) {
  try {
    if (!(doc_matters.opt.action.quiet)
    && doc_matters.opt.action.pod) {
      auto data = (cast(byte[]) (fn_pod).read);
      if (doc_matters.opt.action.verbose) {
        writeln(" ", doc_matters.src.filename, " > ");
      }
      if (doc_matters.opt.action.pod) {
        writefln(" %s\n   %s %-(%02x%)", fn_pod, "sha256", data.sha256Of);
      }
    }
    debug(pod) {
      try {
        auto zipped = new ZipArchive((fn_pod).read);
        foreach (filename, member; zipped.directory) {
          auto data = zipped.expand(member);
          writeln("> ", filename, " length ", data.length);
        }
      } catch (ZipException ex) {
        // Handle errors
      }
      if (doc_matters.src.filename == "dr_doc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") {
        assert(
          ((data).sha256Of).toHexString
          == "626F83A31ED82F42CF528E922C1643498A137ABA3F2E5AFF8A379EA79EA22A1E",
          "\npod: sha256 value for "
          ~ doc_matters.src.filename
          ~ " has changed, is now: "
          ~ ((data).sha256Of).toHexString
        );
      }
      if (doc_matters.src.filename == "dr_doc/media/text/en/sisu_markup_stress_test.sst") {
        assert(
          ((data).sha256Of).toHexString
          == "AAE0C87AB3F6D5F7385AEEA6EE661F56D40475CFE87AD930C78C9FE07FFB0D91",
          "\npod: sha256 value for "
          ~ doc_matters.src.filename
          ~ " has changed, is now: "
          ~ ((data).sha256Of).toHexString
        );
      }
    }
  } catch (ErrnoException ex) {
    // Handle errors
  }
}
#+END_SRC

* __END__