diff options
Diffstat (limited to 'src/sisudoc/ocda')
| -rw-r--r-- | src/sisudoc/ocda/abstraction/load.d | 7 | ||||
| -rw-r--r-- | src/sisudoc/ocda/io_in/read_zip_pod.d | 26 | ||||
| -rw-r--r-- | src/sisudoc/ocda/meta/rgx.d | 4 | ||||
| -rw-r--r-- | src/sisudoc/ocda/meta/rgx_files.d | 4 | ||||
| -rw-r--r-- | src/sisudoc/ocda/zstd.d | 195 |
5 files changed, 228 insertions, 8 deletions
diff --git a/src/sisudoc/ocda/abstraction/load.d b/src/sisudoc/ocda/abstraction/load.d index ea387d1..2072aae 100644 --- a/src/sisudoc/ocda/abstraction/load.d +++ b/src/sisudoc/ocda/abstraction/load.d @@ -120,7 +120,12 @@ template spineAbstractionLoad() { if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } - if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } + /+ ↓ .sisupod is the pod archive spine writes; .zip is what it wrote before, + and every pod already published carries that name, so both are read. + What is inside decides how it is opened, not the name. + +/ + if (_path.endsWith(".sisupod") || _path.endsWith(".zip")) + { return AbstractionSource.pod_zip; } if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } return AbstractionSource.unknown; } diff --git a/src/sisudoc/ocda/io_in/read_zip_pod.d b/src/sisudoc/ocda/io_in/read_zip_pod.d index 49429be..d0eb65a 100644 --- a/src/sisudoc/ocda/io_in/read_zip_pod.d +++ b/src/sisudoc/ocda/io_in/read_zip_pod.d @@ -63,6 +63,7 @@ template spineExtractZipPod() { import std.regex; import std.stdio; import std.string : indexOf; + import sisudoc.ocda.zstd; /+ security limits for zip extraction +/ enum size_t MAX_ENTRY_SIZE = 50 * 1024 * 1024; /+ 50 MB per entry +/ @@ -122,10 +123,29 @@ template spineExtractZipPod() { } /+ ↓ derive pod name from zip filename +/ string zip_basename = zip_path.baseName.stripExtension; - /+ ↓ read and parse zip archive +/ + /+ ↓ read and parse the pod archive. + a .sisupod is one zstd frame wrapping the archive spine writes, so + the bytes are unwrapped before the archive is parsed. Determined from + the first four bytes and not from the filename: a pod published as a + plain .zip, or one a reader has renamed, is read either way. Everything + after this point sees the same archive it always did, guards included. + +/ + ubyte[] _archive_bytes; + try { + _archive_bytes = cast(ubyte[]) read(zip_path); + if (zstdIsFrame(_archive_bytes)) { + _archive_bytes = zstdDecompress(_archive_bytes, MAX_TOTAL_SIZE); + } + } catch (ZstdException ex) { + result.error_msg = "failed to decompress pod archive: " ~ zip_path ~ " - " ~ ex.msg; + return result; + } catch (Exception ex) { + result.error_msg = "error reading pod archive: " ~ zip_path ~ " - " ~ ex.msg; + return result; + } ZipArchive zip; try { - zip = new ZipArchive(read(zip_path)); + zip = new ZipArchive(_archive_bytes); } catch (ZipException ex) { result.error_msg = "failed to read zip archive: " ~ zip_path ~ " - " ~ ex.msg; return result; @@ -274,7 +294,7 @@ template spineExtractZipPod() { its own, carrying its images. A .ssp is not here: it describes its images without carrying them, so it would arrive without them. +/ - static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]zip|[.]ocda[.]db)$`); + static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]sisupod|[.]zip|[.]ocda[.]db)$`); struct DownloadResult { string local_path; /+ path to downloaded temp file +/ diff --git a/src/sisudoc/ocda/meta/rgx.d b/src/sisudoc/ocda/meta/rgx.d index 4cc40ae..d3c9aa1 100644 --- a/src/sisudoc/ocda/meta/rgx.d +++ b/src/sisudoc/ocda/meta/rgx.d @@ -211,8 +211,8 @@ static template spineRgxIn() { static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*?[.]ss[tm])$`); static src_pth_contents = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*)/pod[.]manifest$`); - static src_pth_zip = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`); - static src_pth_types = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`); + static src_pth_zip = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod))$`); + static src_pth_types = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod)))$`); static src_fn = ctRegex!(`^([/]?(?:[a-zA-Z0-9._-]+/)*)(?P<fn_src>(?P<fn_base>[a-zA-Z0-9._-]+)[.](?P<fn_src_suffix>ss[tm]))$`); static src_fn_master = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ssm)$`); static src_fn_find_inserts = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`); diff --git a/src/sisudoc/ocda/meta/rgx_files.d b/src/sisudoc/ocda/meta/rgx_files.d index b26422f..13c7e46 100644 --- a/src/sisudoc/ocda/meta/rgx_files.d +++ b/src/sisudoc/ocda/meta/rgx_files.d @@ -57,8 +57,8 @@ static template spineRgxFiles() { static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*?[.]ss[tm])$`); static src_pth_contents = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*)/pod[.]manifest$`); - static src_pth_zip = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`); - static src_pth_types = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`); + static src_pth_zip = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod))$`); + static src_pth_types = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod)))$`); static src_fn = ctRegex!(`^([/]?(?:[a-zA-Z0-9._-]+/)*)(?P<fn_src>(?P<fn_base>[a-zA-Z0-9._-]+)[.](?P<fn_src_suffix>ss[tm]))$`); static src_fn_master = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ssm)$`); static src_fn_find_inserts = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`); diff --git a/src/sisudoc/ocda/zstd.d b/src/sisudoc/ocda/zstd.d new file mode 100644 index 0000000..2c8186f --- /dev/null +++ b/src/sisudoc/ocda/zstd.d @@ -0,0 +1,195 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ +/+ + module zstd;<BR> + - the few libzstd entry points spine uses, and a buffer in / buffer out + wrapper over each direction<BR> + - one frame per call: what is compressed is a whole pod archive, not a + stream ++/ +module sisudoc.ocda.zstd; +@safe: +/+ ↓ libzstd, declared rather than bound by a generated header + Provides the fifteen declarations that spine needs (there is nothing + here to generate and no vendored tree to track): the C library is linked + (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these + prototypes say what is being called. That is the same arrangement as + sqlite3, which is declared in the vendored d2sqlite3 and linked from the + system. + . + It sits under ocda/ rather than outputs/ because the pod *reader* needs it, + and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part, + which the outputs depend on and not the other way round. + A compression primitive is not an abstraction concern; the dependency + direction is what puts it here. ++/ +extern (C) @nogc nothrow private { + size_t ZSTD_compressBound(size_t srcSize); + size_t ZSTD_compress(void* dst, size_t dstCapacity, + const(void)* src, size_t srcSize, int compressionLevel); + size_t ZSTD_decompress(void* dst, size_t dstCapacity, + const(void)* src, size_t compressedSize); + ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); + uint ZSTD_isError(size_t code); + const(char)* ZSTD_getErrorName(size_t code); + int ZSTD_minCLevel(); + int ZSTD_maxCLevel(); + uint ZSTD_versionNumber(); +} +/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/ +private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/ +private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/ +/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian. + A pod is recognised by this rather than by its name, so that a pod + already published as a plain zip keeps being read whatever it is called. ++/ +enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD]; +/+ ↓ the ceiling on what one frame may be allowed to become. + A frame declares its uncompressed size in its header and that number is + attacker-controlled: a .sisupod fetched over https could claim a size no + machine can allocate. The declared size is checked against this before a + buffer is asked for, so a hostile declaration is refused rather than + attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what + the archive inside may extract to. ++/ +enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024; +/+ ↓ thrown rather than returned: every caller here wants the whole buffer or + nothing, and there is no partial result worth handing back ++/ +class ZstdException : Exception { + this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow { + super(_msg, _file, _line); + } +} +/+ ↓ does this begin a zstd frame? +/ +bool zstdIsFrame(const(ubyte)[] _bytes) { + if (_bytes.length < zstd_frame_magic.length) { + return false; + } + foreach (_i, _b; zstd_frame_magic) { + if (_bytes[_i] != _b) { + return false; + } + } + return true; +} +/+ ↓ the library's version, for a run that wants to record what compressed a + pod. The bytes of a frame are a function of the library and the level, so + two zstd versions may compress the same input differently; nothing is + promised about archive bytes ++/ +@trusted uint zstdVersion() { + return ZSTD_versionNumber(); +} +/+ ↓ the level clamped to what this library actually supports +/ +@trusted int zstdLevelClamped(int _level) { + int _min = ZSTD_minCLevel(); + int _max = ZSTD_maxCLevel(); + if (_level < _min) { return _min; } + if (_level > _max) { return _max; } + return _level; +} +private @trusted string _zstdError(size_t _code) { + import std.string : fromStringz; + return ZSTD_getErrorName(_code).fromStringz.idup; +} +/+ ↓ one buffer to one frame +/ +@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) { + size_t _bound = ZSTD_compressBound(_src.length); + auto _dst = new ubyte[_bound]; + size_t _got = ZSTD_compress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + zstdLevelClamped(_level), + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not compress: " ~ _zstdError(_got)); + } + return _dst[0 .. _got]; +} +/+ ↓ one frame back to one buffer, refusing a frame that will not say how + large it is or that says something absurd ++/ +@trusted ubyte[] zstdDecompress(const(ubyte)[] _src, + size_t _max = ZSTD_MAX_FRAME_CONTENT +) { + import std.conv : to; + if (!(zstdIsFrame(_src))) { + throw new ZstdException("not a zstd frame"); + } + ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length); + if (_declared == ZSTD_CONTENTSIZE_ERROR) { + throw new ZstdException("zstd frame header will not read"); + } + if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) { + /+ every frame spine writes is one whole buffer, so its size is always + declared; a frame without one was made by something else + +/ + throw new ZstdException("zstd frame does not declare its size"); + } + if (_declared > _max) { + throw new ZstdException("zstd frame declares " + ~ _declared.to!string ~ " bytes, over the " + ~ _max.to!string ~ " allowed"); + } + auto _dst = new ubyte[_declared.to!size_t]; + size_t _got = ZSTD_decompress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got)); + } + if (_got != _dst.length) { + throw new ZstdException("zstd frame gave " ~ _got.to!string + ~ " bytes where its header declared " ~ _dst.length.to!string); + } + return _dst; +} |