-*- mode: org -*- #+TITLE: sisudoc spine (doc_reform) output xmls #+DESCRIPTION: documents - structuring, publishing in multiple formats & search #+FILETAGS: :spine:output:text: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code #+PROPERTY: header-args+ :noweb yes :padline no #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - magic single double-quote → " ← FIX changes hilighting behavior (occuring after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] - [[./output_hub.org][output_hub]] * Markdown (Text) ** outputText template #+HEADER: :tangle "../src/sisudoc/outputs/io_out/markdown.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> module sisudoc.outputs.io_out.markdown; @safe: /+ ↓ the markdown output: the document as CommonMark. . An alternative to --text, and the better one where a reader will render it: the text file has the object numbers, the markdown has the object numbers *and the graph between them* - every citation, every note, every index entry and every line of the table of contents is a working link, and the headings are headings. . The dialect is **CommonMark**, plus pipe tables, which every renderer worth using understands. Raw inline html is used in exactly two places, both for the object number, because markdown has no anchor syntax and nothing else will do: . The text of the paragraph.[12](#12) . The anchor is at the head of the object so that a link to it lands on the object and not past it; the number is at the end, as a link to itself, so that a reader can take the citation out of the rendered page. That is what the html output does with an object number. . **Markdown's own footnote syntax is not used.** It is an extension rather than CommonMark, and a renderer numbers footnotes itself, in the order it meets them. The document's note numbers are part of its citation and are not a renderer's to choose. So a note is written where it is referred to, with the document's own number and a link back to the object - the same decision the typst output makes. . A port of tools/sisudoc-ocda-writers/dlang/src/write/markdown.d, which is held over the 36 document reference collection against the html writer's objects word for word, and against the Gleam writer byte for byte. +/ template outputMarkdown() { import sisudoc.outputs.io_out; import sisudoc.outputs.io_out.rgx; import sisudoc.outputs.io_out.paths_output; import std.algorithm : canFind, max; import std.array : appender, array, join, replace; import std.array : asplit = split; import std.conv : to; import std.exception : ErrnoException; import std.file; import std.regex : matchAll, matchFirst, replaceAll, split; import std.stdio; import std.string : strip; mixin spineRgxOut; static auto rgx = RgxO(); enum newline = "\n"; enum newlines = "\n\n"; /+ ↓ the markers that protect the markdown this writer emits from the escaping that follows. . The escaping has to come *last*, and this is why: the abstraction's own markers are built from the characters markdown reserves - "⑆*┨" for emphasis, "⑆_┨" for an underscore face - so escaping first turns them into "⑆\*┨" and no face is ever recognised. +/ enum keep_open = ""; enum keep_close = ""; /+ ↓ a third marker of the same kind, standing for the directory the images are in. . An image reference is written while an object is written, and *where this file sits* is known only to spinePathsMarkdown. Naming the path here as well would be two places agreeing about one path until one of them changed, which is the defect the pdf links had. So the reference carries a marker and the marker is substituted once, where the document is written, out of the one place that knows. +/ enum image_dir_mark = ""; string keep(string markdown_source) { return keep_open ~ markdown_source ~ keep_close; } string unprotect(string txt) { return txt.replace(keep_open, "").replace(keep_close, ""); } /+ ↓ the characters markdown reads as syntax, escaped wherever they occur and only outside a protected run. . The same blunt rule the typst output uses, and for the same reason: a closed set is checkable and a rule about positions is not. The three html characters become entities rather than backslash escapes, because markdown passes raw html through and " 0) { depth--; } continue; } if (depth > 0) { _out ~= c; continue; } switch (c) { case '&': _out ~= "&"; break; case '<': _out ~= "<"; break; case '>': _out ~= ">"; break; case '\\': case '`': case '*': case '_': case '[': case ']': case '#': case '|': case '~': _out ~= '\\'; _out ~= c; break; default: _out ~= c; break; } } return _out.data; } /+ ↓ the document's own text, escaped for a protected run: the same rule, run early, and then flattened so that the result is inert +/ string inert(string txt) { return unprotect(escape(txt)); } /+ ↓ a "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a list item, an enumeration item, a blockquote. The writer uses the first of them itself, and a line of the document's that begins the same way would be read as one - which is what a verse or a group can do. Guarded rather than escaped everywhere, because these are special only in that position. +/ string guardLineStarts(string s) { auto _out = appender!string; bool at_start = true; for (size_t i = 0; i < s.length; i++) { if (at_start) { size_t j = i; while (j < s.length && s[j] == ' ') { j++; } if (j < s.length) { char c = s[j]; bool marker = (c == '-' || c == '+' || c == '>') && j + 1 < s.length && (s[j + 1] == ' ' || s[j + 1] == '\t'); size_t k = j; while (k < s.length && s[k] >= '0' && s[k] <= '9') { k++; } bool enumerated = k > j && k + 1 < s.length && (s[k] == '.' || s[k] == ')') && s[k + 1] == ' '; if (marker || enumerated) { out_put(_out, s[i .. (enumerated ? k : j)]); _out ~= '\\'; i = (enumerated ? k : j) - 1; at_start = false; continue; } } at_start = false; } _out ~= s[i]; if (s[i] == '\n') { at_start = true; } } return _out.data; } void out_put(T)(ref T sink, string s) { sink ~= s; } /+ ↓ a yaml scalar. Quoted always, so that a colon or a leading marker in a title cannot change the shape of the document's own metadata. +/ string yaml(string s) { return "\"" ~ s.replace("\\", "\\\\").replace("\"", "\\\"") .replace("\n", " ") ~ "\""; } /+ ↓ which sections, in which order: the latex sequence, less the endnotes, because a note is written where it is referred to +/ string[] markdownSections(M)(M doc_matters) { string[] _seq; foreach (part; doc_matters.has.keys_seq.latex) { if (part == "endnotes") { continue; } _seq ~= part; } return _seq; } /+ ↓ the table of contents depth of each indent_hang the document uses. . spine's indent_hang is not a depth: it reserves 1 to 3 for levels above the body, so a toc uses 1 and then jumps to 4, 5, 6. A markdown list nests by two spaces a level and cannot take a jump, so the values the document actually uses are ranked. +/ size_t[int] tocDepths(D,M)(const D doc_abstraction, M doc_matters) { bool[int] _seen; foreach (part; markdownSections!()(doc_matters)) { if (part != "toc") { continue; } foreach (obj; doc_abstraction[part]) { if (obj.metainfo.is_of_type == "comment") { continue; } _seen[obj.attrib.indent_hang.to!int] = true; } } import std.algorithm : sort; auto _levels = _seen.keys.sort.array; size_t[int] _depth; foreach (i, level; _levels) { _depth[level] = i; } return _depth; } /+ ↓ the document's metadata as yaml front matter, which is how a markdown file carries metadata and what every static site generator reads +/ string markdownHead(M)(M doc_matters) { auto _out = appender!string; _out ~= "---" ~ newline; void put(string key, string value) { if (value.length > 0) { _out ~= key ~ ": " ~ yaml(value) ~ newline; } } put("title", doc_matters.conf_make_meta.meta.title_full); put("author", doc_matters.conf_make_meta.meta.creator_author); put("date", doc_matters.conf_make_meta.meta.date_published); put("language", doc_matters.src.language); put("copyright", doc_matters.conf_make_meta.meta.rights_copyright); put("license", doc_matters.conf_make_meta.meta.rights_license); _out ~= "---" ~ newlines; return _out.data; } /+ ↓ an image, with its alt text where the document gave one +/ string images(string txt) { return replaceAll!((m) { string _rest = m["post"].to!string; string _alt; if (auto a = _rest.matchFirst(rgx.inline_image_alt)) { _alt = a["alt"].to!string; _rest = a.post.to!string; } return m["pre"].to!string ~ keep("![" ~ inert(_alt) ~ "](" ~ image_dir_mark ~ "/" ~ m["img"].to!string ~ ")") ~ _rest; })(txt, rgx.inline_image); } /+ ↓ a fragment that names an object of this document +/ bool isObjectNumber(string s) { if (s.length == 0 || s == "0") { return false; } foreach (c; s) { if (c < '0' || c > '9') { return false; } } return true; } /+ ↓ a link target is a url or a fragment and not prose, so the escaping the text went through comes off it again +/ string unescapeTarget(string target) { auto _out = appender!string; for (size_t i = 0; i < target.length; i++) { if (target[i] == 0x5c && i + 1 < target.length) { _out ~= target[++i]; continue; } _out ~= target[i]; } return _out.data.replace("&", "&").replace("<", "<").replace(">", ">"); } /+ ↓ the font faces. Emphasis and strong are markdown's own; the rest have no markdown and take the html element markdown itself would produce. . The spaces inside a run are dropped: the abstraction's marker takes in the space that was beside the run, the text before it already ends in one, and markdown's emphasis will not open on a space - "* lex *" is three words and two asterisks, not an italic. +/ string fontFace(string txt) { string wrap(string open, string close, string inner) { size_t a = 0, b = inner.length; while (a < b && inner[a] == ' ') { a++; } while (b > a && inner[b - 1] == ' ') { b--; } return keep(open) ~ inner[a .. b] ~ keep(close); } string apply(alias pattern)(string s, string open, string close) { return replaceAll!((m) => wrap(open, close, m.captures[1].to!string))(s, pattern); } txt = apply!(rgx.inline_emphasis)(txt, "**", "**"); txt = apply!(rgx.inline_bold)(txt, "**", "**"); txt = apply!(rgx.inline_italics)(txt, "*", "*"); txt = apply!(rgx.inline_underscore)(txt, "", ""); txt = apply!(rgx.inline_superscript)(txt, "", ""); txt = apply!(rgx.inline_subscript)(txt, "", ""); txt = apply!(rgx.inline_mono)(txt, "`", "`"); txt = apply!(rgx.inline_strike)(txt, "~~", "~~"); txt = apply!(rgx.inline_insert)(txt, "", ""); txt = apply!(rgx.inline_cite)(txt, "", ""); return txt; } /+ ↓ a link. An internal target becomes a fragment, which is the object number, and so resolves against the anchor every object carries. . And *only* an object number: those are the anchors this writer defines, so a target that names something else - the endnotes section, an anchor tag the markup placed - would be a link to nothing. The text stands on its own instead, which is what a reader sees anyway. +/ string links(O)(string txt, const O obj) { auto _stow = obj.stow.link; txt = replaceAll!((m) { size_t _num = m["num"].to!size_t; string _url = (_num < _stow.length) ? _stow[_num].to!string : ""; return m["linked_text"].to!string ~ "┤" ~ _url ~ "├"; })(txt, rgx.inline_link_number_only); return replaceAll!((m) { string _text = m.captures[1].to!string; string _target = m.captures[2].to!string; if (auto im = _target.matchFirst(rgx.inline_link_seg_and_hash)) { _target = "#" ~ im["hash"].to!string; } string _shown = inert(fontFace(_text)); if (_target.length == 0) { return keep(_shown); } string _clean = unescapeTarget(_target); if (_clean.length > 0 && _clean[0] == '#' && !isObjectNumber(_clean[1 .. $])) { return keep(_shown); } return keep("[" ~ _shown ~ "](" ~ _clean ~ ")"); })(txt, rgx.inline_link); } /+ ↓ a note reference: a superscript link to the note, which sits under the object that refers to it +/ string noteRefs(string txt) { return replaceAll!((m) { string _mark = m["num"].to!string; return keep("[" ~ inert(_mark) ~ "](#note-" ~ _mark ~ ")"); })(txt, rgx.inline_notes_al_all_note); } /+ ↓ the line breaks the markup asked for, and the grouped indents +/ string breaks(O)(string txt, const O obj) { if (obj.metainfo.is_a == "code") { return txt; } if (obj.metainfo.is_a == "group" || obj.metainfo.is_a == "block") { txt = txt .replaceAll(rgx.grouped_para_indent_hang, "$2$2") .replaceAll(rgx.grouped_para_bullet_indent, "$1● ") .replaceAll(rgx.grouped_para_bullet, "● ") .replaceAll(rgx.grouped_para_indent, "$1$1"); } return txt .replaceAll(rgx.nbsp_char, " ") .replaceAll(rgx.br_line, newline) .replaceAll(rgx.br_line_inline, newline) .replaceAll(rgx.br_line_spaced, newlines) .replaceAll(rgx.line_break, newline) .replaceAll(rgx.mark_internal_site_lnk, ""); } /+ ↓ an object's text as markdown. . The order is the one every writer here uses: images before links, because an image sits inside a link; then the note references, the faces, the anchors and the breaks. The escaping is last. +/ string inlineText(O)(string txt, const O obj) { txt = images(txt); txt = links!()(txt, obj); txt = noteRefs(txt); txt = fontFace(txt); txt = txt.replaceAll(rgx.inline_link_anchor, ""); txt = breaks!()(txt, obj); return guardLineStarts(unprotect(escape(txt))); } /+ ↓ six levels of heading, which is all markdown has. A document may be eight deep; the two deepest take the sixth, and the object number, the anchor and the table of contents carry the true depth. +/ int headingLevel(string level) { switch (level) { case "A": return 1; case "B": return 2; case "C": return 3; case "D": return 4; case "1": return 5; case "2": case "3": case "4": return 6; default: return 1; } } /+ ↓ the object's anchor, so that a citation can point at it +/ string anchor(O)(const O obj) { return (obj.metainfo.ocn == 0) ? "" : ""; } /+ ↓ the object's number, at the end of it, as a link to itself +/ string ocnMark(O)(const O obj) { if (obj.metainfo.ocn == 0) { return ""; } string _n = obj.metainfo.ocn.to!string; return "[" ~ _n ~ "](#" ~ _n ~ ")"; } /+ ↓ the notes the object referred to, each where it was referred to, with the document's own number and a link back to the object +/ string notes(O)(const O obj) { auto _out = appender!string; string _back = (obj.metainfo.ocn == 0) ? "" : " [↩](#" ~ obj.metainfo.ocn.to!string ~ ")"; foreach (m; obj.text.matchAll(rgx.inline_notes_al_all_note)) { string _mark = m["num"].to!string; _out ~= newlines ~ "" ~ inert(_mark) ~ ". " ~ inlineText!()(m["note"].to!string, obj) ~ _back; } /+ ↓ and a blank line after the last of them, or the next object begins on the same line and markdown reads the two as one paragraph +/ return (_out.data.length > 0) ? _out.data ~ newlines : ""; } /+ ↓ the object a table of contents entry points at, where that is an object number +/ string internalTarget(O)(const O obj) { foreach (m; obj.text.matchAll(rgx.any_internal_target)) { string _frag = m["frag"].to!string; if (isObjectNumber(_frag)) { return _frag; } } return ""; } /+ ↓ one object, and then the notes it referred to. . The notes are gathered here rather than inside each kind, because a reference can be anywhere - the markup manual has one in a heading - and a reference whose note was never written is a link to nothing. +/ string object(O,M)(const O obj, M doc_matters, size_t[int] toc_depth) { return objectBody!()(obj, doc_matters, toc_depth) ~ notes!()(obj); } string objectBody(O,M)(const O obj, M doc_matters, size_t[int] toc_depth) { switch (obj.metainfo.is_a) { case "heading": string _hashes; foreach (_; 0 .. headingLevel(obj.metainfo.marked_up_level.to!string)) { _hashes ~= "#"; } return _hashes ~ " " ~ anchor!()(obj) ~ inlineText!()(obj.text, obj) ~ ocnMark!()(obj) ~ newlines; case "toc": /+ ↓ a list item at its depth, the heading as a link, and the object number as the locator. The number and not a page: a page is a fact about one typesetting, and the object number is the same reference in every output of the document. +/ string _indent; if (auto d = obj.attrib.indent_hang.to!int in toc_depth) { foreach (_; 0 .. *d) { _indent ~= " "; } } string _n = internalTarget!()(obj); string _locator = (_n.length > 0) ? (" [" ~ _n ~ "](#" ~ _n ~ ")") : ""; return _indent ~ "- " ~ inlineText!()(obj.text, obj) ~ _locator ~ newline; case "table": return table!()(obj, doc_matters); case "code": /+ ↓ a fenced block, long enough that the code cannot close it +/ string _txt = obj.text.replaceAll(rgx.nbsp_char, " "); string _fence = "```"; while (_txt.canFind(_fence)) { _fence ~= "`"; } return anchor!()(obj) ~ newline ~ _fence ~ obj.metainfo.syntax.to!string ~ newline ~ _txt ~ newline ~ _fence ~ newline ~ ocnMark!()(obj) ~ newlines; case "quote": return "> " ~ anchor!()(obj) ~ inlineText!()(obj.text, obj) ~ ocnMark!()(obj) ~ newlines; case "verse": case "poem": case "group": case "block": /+ ↓ every line kept, with CommonMark's own hard line break - a backslash at the end of a line - rather than the two trailing spaces an editor will strip +/ auto _rows = inlineText!()(obj.text, obj).asplit(newline); auto _out = appender!string; _out ~= anchor!()(obj); foreach (i, row; _rows) { _out ~= row; if (i + 1 < _rows.length) { _out ~= "\\" ~ newline; } } return _out.data ~ ocnMark!()(obj) ~ newlines; case "bookindex": /+ ↓ a list item, and every object it names a link +/ string _txt = obj.text.replaceAll(rgx.trailing_backslash, ""); return "- " ~ anchor!()(obj) ~ inlineText!()(_txt, obj) ~ ocnMark!()(obj) ~ newlines; case "para": case "blurb": case "glossary": case "bibliography": string _bullet = obj.attrib.bullet ? "- " : ""; string _indent; foreach (_; 0 .. obj.attrib.indent_base) { _indent ~= " "; } return _indent ~ _bullet ~ anchor!()(obj) ~ inlineText!()(obj.text, obj) ~ ocnMark!()(obj) ~ newlines; default: return ""; } } /+ ↓ a pipe table, which needs a header row: markdown has no table without one, so a table the document did not give a header gets an empty row and keeps all of its own. +/ string table(O,M)(const O obj, M doc_matters) { auto _rows = obj.text.split(rgx.table_delimiter_row); int _cols = max(obj.table.number_of_columns.to!int, 1); string[][] _body; foreach (row; _rows) { if (row.strip.length == 0) { continue; } string[] _cells; foreach (cell; row.split(rgx.table_delimiter_col)) { _cells ~= inlineText!()(cell, obj).replace("|", "\\|"); } _body ~= _cells; } string rule() { auto r = appender!string; r ~= "|"; foreach (i; 0 .. _cols) { string _a = (i < obj.table.column_aligns.length) ? obj.table.column_aligns[i].to!string : "l"; r ~= (_a == "c") ? " :---: |" : (_a == "r") ? " ---: |" : " :--- |"; } return r.data ~ newline; } string line(string[] cells) { auto l = appender!string; l ~= "|"; foreach (i; 0 .. _cols) { l ~= " " ~ ((i < cells.length) ? cells[i] : "") ~ " |"; } return l.data ~ newline; } auto _out = appender!string; _out ~= anchor!()(obj) ~ newline; if (_body.length > 0 && obj.table.heading) { _out ~= line(_body[0]) ~ rule(); foreach (row; _body[1 .. $]) { _out ~= line(row); } } else { _out ~= line([]) ~ rule(); foreach (row; _body) { _out ~= line(row); } } return _out.data ~ ocnMark!()(obj) ~ newlines; } string markdownBody(D,M)(const D doc_abstraction, M doc_matters) { auto _toc_depth = tocDepths!()(doc_abstraction, doc_matters); auto _out = appender!string; foreach (part; markdownSections!()(doc_matters)) { foreach (obj; doc_abstraction[part]) { if (obj.metainfo.is_of_type == "comment") { continue; } if (obj.metainfo.dummy_heading) { continue; } if (obj.text.strip.length == 0) { continue; } _out ~= object!()(obj, doc_matters, _toc_depth); } } return _out.data; } void outputMarkdown(D,M)( const D doc_abstraction, M doc_matters, ) { auto pth_md = spinePathsMarkdown(doc_matters); try { if (!exists(pth_md.base_pth)) { (pth_md.base_pth).mkdirRecurse; } } catch (ErrnoException ex) { } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", pth_md.markdown_file); } { auto f = File(pth_md.markdown_file, "w"); f.write(markdownHead!()(doc_matters)); /+ ↓ the image directory, named once, here, out of the paths module that decided where this file goes +/ f.write(markdownBody!()(doc_abstraction, doc_matters) .replace(image_dir_mark, pth_md.images_rel)); } /+ ↓ the images the document names, into the shared image directory at the output root, because markdown names them by path and does not carry them. . The same set the html references, and not a copy of it: a file already there is left alone, so this costs nothing when the html output has written them and still works when --markdown is asked for on its own. +/ if (doc_matters.srcs.image_list.length > 0) { try { if (!exists(pth_md.images)) { (pth_md.images).mkdirRecurse; } foreach (image; doc_matters.srcs.image_list) { string _in = doc_matters.src.image_dir_path ~ "/" ~ image; string _out = pth_md.images ~ "/" ~ image; if (exists(_in) && !exists(_out)) { _in.copy(_out); } } } catch (Exception ex) { } } } } #+END_SRC * org includes ** spine project VERSION #+NAME: spine_version #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> #+END_SRC ** year #+NAME: year #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> #+END_SRC ** document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC * __END__