1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
/+
- Name: SisuDoc Spine, Doc Reform [a part of]
- Description: documents, structuring, processing, publishing, search
- static content generator
- Author: Ralph Amissah
[ralph.amissah@gmail.com]
- Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
- License: AGPL 3 or later:
Spine (SiSU), a framework for document structuring, publishing and
search
Copyright (C) Ralph Amissah
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU AFERO General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
You should have received a copy of the GNU General Public License along with
this program. If not, see [https://www.gnu.org/licenses/].
If you have Internet connection, the latest version of the AGPL should be
available at these locations:
[https://www.fsf.org/licensing/licenses/agpl.html]
[https://www.gnu.org/licenses/agpl.html]
- Spine (by Doc Reform, related to SiSU) uses standard:
- docReform markup syntax
- standard SiSU markup syntax with modified headers and minor modifications
- docReform object numbering
- standard SiSU object citation numbering & system
- Homepages:
[https://www.sisudoc.org]
[https://www.doc-reform.org]
- Git
[https://git.sisudoc.org/]
+/
/+
module zstd;<BR>
- the few libzstd entry points spine uses, and a buffer in / buffer out
wrapper over each direction<BR>
- one frame per call: what is compressed is a whole pod archive, not a
stream
+/
module sisudoc.ocda.zstd;
@safe:
/+ ↓ libzstd, declared rather than bound by a generated header
Provides the fifteen declarations that spine needs (there is nothing
here to generate and no vendored tree to track): the C library is linked
(dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these
prototypes say what is being called. That is the same arrangement as
sqlite3, which is declared in the vendored d2sqlite3 and linked from the
system.
.
It sits under ocda/ rather than outputs/ because the pod *reader* needs it,
and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part,
which the outputs depend on and not the other way round.
A compression primitive is not an abstraction concern; the dependency
direction is what puts it here.
+/
extern (C) @nogc nothrow private {
size_t ZSTD_compressBound(size_t srcSize);
size_t ZSTD_compress(void* dst, size_t dstCapacity,
const(void)* src, size_t srcSize, int compressionLevel);
size_t ZSTD_decompress(void* dst, size_t dstCapacity,
const(void)* src, size_t compressedSize);
ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
uint ZSTD_isError(size_t code);
const(char)* ZSTD_getErrorName(size_t code);
int ZSTD_minCLevel();
int ZSTD_maxCLevel();
uint ZSTD_versionNumber();
}
/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/
private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/
private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/
/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian.
A pod is recognised by this rather than by its name, so that a pod
already published as a plain zip keeps being read whatever it is called.
+/
enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD];
/+ ↓ the ceiling on what one frame may be allowed to become.
A frame declares its uncompressed size in its header and that number is
attacker-controlled: a .sisupod fetched over https could claim a size no
machine can allocate. The declared size is checked against this before a
buffer is asked for, so a hostile declaration is refused rather than
attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what
the archive inside may extract to.
+/
enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024;
/+ ↓ thrown rather than returned: every caller here wants the whole buffer or
nothing, and there is no partial result worth handing back
+/
class ZstdException : Exception {
this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow {
super(_msg, _file, _line);
}
}
/+ ↓ does this begin a zstd frame? +/
bool zstdIsFrame(const(ubyte)[] _bytes) {
if (_bytes.length < zstd_frame_magic.length) {
return false;
}
foreach (_i, _b; zstd_frame_magic) {
if (_bytes[_i] != _b) {
return false;
}
}
return true;
}
/+ ↓ the library's version, for a run that wants to record what compressed a
pod. The bytes of a frame are a function of the library and the level, so
two zstd versions may compress the same input differently; nothing is
promised about archive bytes
+/
@trusted uint zstdVersion() {
return ZSTD_versionNumber();
}
/+ ↓ the level clamped to what this library actually supports +/
@trusted int zstdLevelClamped(int _level) {
int _min = ZSTD_minCLevel();
int _max = ZSTD_maxCLevel();
if (_level < _min) { return _min; }
if (_level > _max) { return _max; }
return _level;
}
private @trusted string _zstdError(size_t _code) {
import std.string : fromStringz;
return ZSTD_getErrorName(_code).fromStringz.idup;
}
/+ ↓ one buffer to one frame +/
@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) {
size_t _bound = ZSTD_compressBound(_src.length);
auto _dst = new ubyte[_bound];
size_t _got = ZSTD_compress(
_dst.ptr, _dst.length,
_src.ptr, _src.length,
zstdLevelClamped(_level),
);
if (ZSTD_isError(_got)) {
throw new ZstdException("zstd could not compress: " ~ _zstdError(_got));
}
return _dst[0 .. _got];
}
/+ ↓ one frame back to one buffer, refusing a frame that will not say how
large it is or that says something absurd
+/
@trusted ubyte[] zstdDecompress(const(ubyte)[] _src,
size_t _max = ZSTD_MAX_FRAME_CONTENT
) {
import std.conv : to;
if (!(zstdIsFrame(_src))) {
throw new ZstdException("not a zstd frame");
}
ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length);
if (_declared == ZSTD_CONTENTSIZE_ERROR) {
throw new ZstdException("zstd frame header will not read");
}
if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) {
/+ every frame spine writes is one whole buffer, so its size is always
declared; a frame without one was made by something else
+/
throw new ZstdException("zstd frame does not declare its size");
}
if (_declared > _max) {
throw new ZstdException("zstd frame declares "
~ _declared.to!string ~ " bytes, over the "
~ _max.to!string ~ " allowed");
}
auto _dst = new ubyte[_declared.to!size_t];
size_t _got = ZSTD_decompress(
_dst.ptr, _dst.length,
_src.ptr, _src.length,
);
if (ZSTD_isError(_got)) {
throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got));
}
if (_got != _dst.length) {
throw new ZstdException("zstd frame gave " ~ _got.to!string
~ " bytes where its header declared " ~ _dst.length.to!string);
}
return _dst;
}
|
