From a22df0d4e7df79da6943bd1a37d6e3f74728165f Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Sat, 13 Oct 2018 22:05:22 -0400 Subject: 0.2.0 renames pod from sisupod --- src/doc_reform/output/hub.d | 8 +- src/doc_reform/output/rgx.d | 6 +- src/doc_reform/output/source_pod.d | 278 +++++++++++++++++++++++++++++++++++++ 3 files changed, 285 insertions(+), 7 deletions(-) create mode 100644 src/doc_reform/output/source_pod.d (limited to 'src/doc_reform/output') diff --git a/src/doc_reform/output/hub.d b/src/doc_reform/output/hub.d index 409f2bc..5e036dd 100644 --- a/src/doc_reform/output/hub.d +++ b/src/doc_reform/output/hub.d @@ -9,7 +9,7 @@ template outputHub() { doc_reform.output.html, doc_reform.output.sqlite, doc_reform.output.xmls, - doc_reform.output.source_sisupod, + doc_reform.output.source_pod, doc_reform.output.create_zip_file, doc_reform.output.paths_output; import std.parallelism; @@ -18,10 +18,10 @@ template outputHub() { mixin Msg; auto msg = Msg!()(doc_matters); static auto rgx = Rgx(); - enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff } + enum outTask { pod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff } void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) { auto msg = Msg!()(doc_matters); - if (sched == outTask.sisupod) { + if (sched == outTask.pod) { msg.v("sisu source processing... "); DocReformPod!()(doc_matters); msg.vv("sisu source done"); @@ -78,7 +78,7 @@ template outputHubOp() { doc_reform.output.html, doc_reform.output.sqlite, doc_reform.output.xmls, - doc_reform.output.source_sisupod, + doc_reform.output.source_pod, doc_reform.output.create_zip_file, doc_reform.output.paths_output; void outputHubOp(E,O)(E env, O opt_action) { diff --git a/src/doc_reform/output/rgx.d b/src/doc_reform/output/rgx.d index 6896237..02c3cbb 100644 --- a/src/doc_reform/output/rgx.d +++ b/src/doc_reform/output/rgx.d @@ -18,12 +18,12 @@ static template DocReformOutputRgxInit() { static special_markup_chars = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg"); static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.](?Pss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); - static src_pth_contents = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+)/sisupod[.]manifest$`); - static src_pth_pod_root = ctRegex!(`^(?P(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); + static src_pth_contents = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+)/pod[.]manifest$`); + static src_pth_pod_root = ctRegex!(`^(?P(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(pod))$`); static src_pth_zip = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.]zip)$`); static src_pth_unzip_pod = ctRegex!(`^(?Pmedia/text/[a-z]{2}/)*(?P[a-zA-Z0-9._-]+[.]ss[im])$`); static src_pth_types = - ctRegex!(`^(?P[/]?[a-zA-Z0-9._-]+/)*(?P(?P[a-zA-Z0-9._-]+[.]ss[tm])|(?P[a-zA-Z0-9._-]+/sisupod[.]manifest)|(?P[a-zA-Z0-9._-]+[.]zip))$`); + ctRegex!(`^(?P[/]?[a-zA-Z0-9._-]+/)*(?P(?P[a-zA-Z0-9._-]+[.]ss[tm])|(?P[a-zA-Z0-9._-]+/pod[.]manifest)|(?P[a-zA-Z0-9._-]+[.]zip))$`); static pod_content_location = ctRegex!(`^(?P[a-zA-Z0-9._-]+[.]ss[tm])(?P(?:\s+[a-z]{2}(?:,|$))+)`, "mg"); static src_fn = diff --git a/src/doc_reform/output/source_pod.d b/src/doc_reform/output/source_pod.d new file mode 100644 index 0000000..4303c32 --- /dev/null +++ b/src/doc_reform/output/source_pod.d @@ -0,0 +1,278 @@ +module doc_reform.output.source_pod; +template DocReformPod() { + import doc_reform.output; + import + std.digest.sha, + std.file, + std.outbuffer, + std.zip, + std.conv : to; + import + doc_reform.output.create_zip_file, + doc_reform.output.xmls; + void DocReformPod(T)(T doc_matters) { + debug(asserts) { + // static assert(is(typeof(doc_matters) == tuple)); + } + mixin DocReformOutputRgxInit; + string pwd = doc_matters.env.pwd; + auto src_path_info = doc_matters.src_path_info; + auto pth_sisudoc_src = doc_matters.src_path_info; + auto pths_pod = DocReformPathsPods!()(doc_matters); + mixin DocReformLanguageCodes; + auto lang = Lang(); + static auto rgx = Rgx(); + assert (doc_matters.src.filename.match(rgx.src_fn)); + try { + /+ create directory structure +/ + if (!exists(pths_pod.pod_dir_())) { + // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) + pths_pod.pod_dir_().mkdirRecurse; + } + if (doc_matters.opt.action.source) { + if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { + pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; + } + } + debug(pod) { + writeln(__LINE__, ": ", + doc_matters.src.filename, " -> ", + pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod + ); + } + auto zip = new ZipArchive(); + auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; + { /+ bundle images +/ + foreach (image; doc_matters.srcs.image_list) { + debug(podimages) { + writeln( + pth_sisudoc_src.image_root.to!string, "/", image, " -> ", + pths_pod.image_root(doc_matters.src.filename).zpod, "/", image + ); + } + auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; + auto fn_src_out_pod_zip_base + = pths_pod.image_root(doc_matters.src.filename).zpod.to!string + ~ "/" ~ image; + auto fn_src_out_filesystem + = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string + ~ "/" ~ image; + if (exists(fn_src_in)) { + debug(io) { + writeln("(io debug) src out found: ", fn_src_in); + } + if (doc_matters.opt.action.source) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = fn_src_out_pod_zip_base; + auto zip_data = new OutBuffer(); + zip_data.write(cast(char[]) ((fn_src_in).read)); + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + } + } else { + if (doc_matters.opt.action.verbose) { + writeln("WARNING (io) src out NOT found (image): ", fn_src_in); + } + } + } + } { /+ bundle sisu_document_make +/ + auto fn_src_in = ((doc_matters.src.is_pod) + ? doc_matters.src.conf_dir_path + : pth_sisudoc_src.conf_root).to!string + ~ "/" ~ "sisu_document_make"; + auto fn_src_out_pod_zip_base + = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "sisu_document_make"; + auto fn_src_out_filesystem + = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string + ~ "/" ~ "sisu_document_make"; + if (exists(fn_src_in)) { + debug(io) { + writeln("(io debug) src out found: ", fn_src_in); + } + if (doc_matters.opt.action.source) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = fn_src_out_pod_zip_base; + auto zip_data = new OutBuffer(); + zip_data.write((fn_src_in).readText); + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + } + } else { + if (doc_matters.opt.action.verbose + || doc_matters.opt.action.debug_do) { + writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); + } + } + } { /+ bundle primary file +/ + auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // without root path: + auto fn_src_out_inside_pod + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // without root path: + string[] filelist_src_out_pod_arr; + string[] filelist_src_zpod_arr; + if (exists(fn_src_in)) { + debug(io) { + writeln("(io debug) src in found: ", fn_src_in); + } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; + if (doc_matters.opt.action.source) { + auto filelist + = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); + foreach (source_pth_and_fn; filelist_src_zpod_arr) { + filelist.writeln(source_pth_and_fn); + } + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = fn_src_out_pod_zip_base; + auto zip_data = new OutBuffer(); + zip_data.write((fn_src_in).readText); + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + } + } else { + if (doc_matters.opt.action.verbose + || doc_matters.opt.action.debug_do) { + writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); + } + } + } { /+ bundle insert files +/ + if (doc_matters.srcs.file_insert_list.length > 0) { + foreach (insert_file; doc_matters.srcs.file_insert_list) { + debug(pod) { + writeln( + insert_file, " -> ", + pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).zpod + ); + } + auto fn_src_in = insert_file; + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).filesystem_open_zpod.to!string; + if (exists(fn_src_in)) { + debug(io) { + writeln("(io debug) src out found: ", fn_src_in); + } + if (doc_matters.opt.action.source) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = fn_src_out_pod_zip_base; + auto zip_data = new OutBuffer(); + zip_data.write((fn_src_in).readText); + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + createZipFile!()(fn_pod, zip.build()); + } + } else { + if (doc_matters.opt.action.verbose + || doc_matters.opt.action.debug_do) { + writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); + } + } + } + } + } { + auto fn_src_in = doc_matters.src.filename; + if (doc_matters.opt.action.pod) { + if (exists(doc_matters.src.file_with_absolute_path)) { + createZipFile!()(fn_pod, zip.build()); + } else { + writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); + } + if (!(exists(fn_pod))) { + writeln("WARNING failed to create pod zip archive: ", fn_pod); + } + } + } + if (exists(fn_pod)) { + try { + if (doc_matters.opt.action.verbose) { + auto data = (cast(byte[]) (fn_pod).read); + writeln(doc_matters.src.filename, " >> "); + writefln("%-(%02x%) %s", data.sha256Of, fn_pod); + } + debug(pod) { + try { + auto zipped = new ZipArchive((fn_pod).read); + foreach (filename, member; zipped.directory) { + auto data = zipped.expand(member); + writeln("> ", filename, " length ", data.length); + } + } + catch (ZipException ex) { + // Handle errors + } + if (doc_matters.src.filename == "sisudoc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") { + assert( + ((data).sha256Of).toHexString + == "626F83A31ED82F42CF528E922C1643498A137ABA3F2E5AFF8A379EA79EA22A1E", + "\npod: sha256 value for " + ~ doc_matters.src.filename + ~ " has changed, is now: " + ~ ((data).sha256Of).toHexString + ); + } + if (doc_matters.src.filename == "sisudoc/media/text/en/sisu_markup_stress_test.sst") { + assert( + ((data).sha256Of).toHexString + == "AAE0C87AB3F6D5F7385AEEA6EE661F56D40475CFE87AD930C78C9FE07FFB0D91", + "\npod: sha256 value for " + ~ doc_matters.src.filename + ~ " has changed, is now: " + ~ ((data).sha256Of).toHexString + ); + } + } + } + catch (ErrnoException ex) { + // Handle errors + } + } + + } + catch (ErrnoException ex) { + // Handle error + } + } +} -- cgit v1.2.3