1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
-*- mode: org -*-
#+TITLE: sisudoc spine (doc_reform) output zip
#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
#+FILETAGS: :spine:zip:
#+AUTHOR: Ralph Amissah
#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
#+LANGUAGE: en
#+STARTUP: content hideblocks hidestars noindent entitiespretty
#+PROPERTY: header-args+ :eval never-export :exports code
#+PROPERTY: header-args+ :noweb yes :padline no
#+PROPERTY: header-args+ :results silent :cache no
#+PROPERTY: header-args+ :mkdirp yes
#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
- magic single double-quote → " ← FIX changes hilighting behavior (occuring
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
- [[./output_hub.org][output_hub]]
* _zstd_ :module:spine:compression:
#+HEADER: :tangle "../src/sisudoc/ocda/zstd.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/+
module zstd;<BR>
- the few libzstd entry points spine uses, and a buffer in / buffer out
wrapper over each direction<BR>
- one frame per call: what is compressed is a whole pod archive, not a
stream
+/
module sisudoc.ocda.zstd;
@safe:
/+ ↓ libzstd, declared rather than bound by a generated header
Provides the fifteen declarations that spine needs (there is nothing
here to generate and no vendored tree to track): the C library is linked
(dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these
prototypes say what is being called. That is the same arrangement as
sqlite3, which is declared in the vendored d2sqlite3 and linked from the
system.
.
It sits under ocda/ rather than outputs/ because the pod *reader* needs it,
and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part,
which the outputs depend on and not the other way round.
A compression primitive is not an abstraction concern; the dependency
direction is what puts it here.
+/
extern (C) @nogc nothrow private {
size_t ZSTD_compressBound(size_t srcSize);
size_t ZSTD_compress(void* dst, size_t dstCapacity,
const(void)* src, size_t srcSize, int compressionLevel);
size_t ZSTD_decompress(void* dst, size_t dstCapacity,
const(void)* src, size_t compressedSize);
ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
uint ZSTD_isError(size_t code);
const(char)* ZSTD_getErrorName(size_t code);
int ZSTD_minCLevel();
int ZSTD_maxCLevel();
uint ZSTD_versionNumber();
}
/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/
private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/
private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/
/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian.
A pod is recognised by this rather than by its name, so that a pod
already published as a plain zip keeps being read whatever it is called.
+/
enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD];
/+ ↓ the ceiling on what one frame may be allowed to become.
A frame declares its uncompressed size in its header and that number is
attacker-controlled: a .sisupod fetched over https could claim a size no
machine can allocate. The declared size is checked against this before a
buffer is asked for, so a hostile declaration is refused rather than
attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what
the archive inside may extract to.
+/
enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024;
/+ ↓ thrown rather than returned: every caller here wants the whole buffer or
nothing, and there is no partial result worth handing back
+/
class ZstdException : Exception {
this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow {
super(_msg, _file, _line);
}
}
/+ ↓ does this begin a zstd frame? +/
bool zstdIsFrame(const(ubyte)[] _bytes) {
if (_bytes.length < zstd_frame_magic.length) {
return false;
}
foreach (_i, _b; zstd_frame_magic) {
if (_bytes[_i] != _b) {
return false;
}
}
return true;
}
/+ ↓ the library's version, for a run that wants to record what compressed a
pod. The bytes of a frame are a function of the library and the level, so
two zstd versions may compress the same input differently; nothing is
promised about archive bytes
+/
@trusted uint zstdVersion() {
return ZSTD_versionNumber();
}
/+ ↓ the level clamped to what this library actually supports +/
@trusted int zstdLevelClamped(int _level) {
int _min = ZSTD_minCLevel();
int _max = ZSTD_maxCLevel();
if (_level < _min) { return _min; }
if (_level > _max) { return _max; }
return _level;
}
private @trusted string _zstdError(size_t _code) {
import std.string : fromStringz;
return ZSTD_getErrorName(_code).fromStringz.idup;
}
/+ ↓ one buffer to one frame +/
@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) {
size_t _bound = ZSTD_compressBound(_src.length);
auto _dst = new ubyte[_bound];
size_t _got = ZSTD_compress(
_dst.ptr, _dst.length,
_src.ptr, _src.length,
zstdLevelClamped(_level),
);
if (ZSTD_isError(_got)) {
throw new ZstdException("zstd could not compress: " ~ _zstdError(_got));
}
return _dst[0 .. _got];
}
/+ ↓ one frame back to one buffer, refusing a frame that will not say how
large it is or that says something absurd
+/
@trusted ubyte[] zstdDecompress(const(ubyte)[] _src,
size_t _max = ZSTD_MAX_FRAME_CONTENT
) {
import std.conv : to;
if (!(zstdIsFrame(_src))) {
throw new ZstdException("not a zstd frame");
}
ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length);
if (_declared == ZSTD_CONTENTSIZE_ERROR) {
throw new ZstdException("zstd frame header will not read");
}
if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) {
/+ every frame spine writes is one whole buffer, so its size is always
declared; a frame without one was made by something else
+/
throw new ZstdException("zstd frame does not declare its size");
}
if (_declared > _max) {
throw new ZstdException("zstd frame declares "
~ _declared.to!string ~ " bytes, over the "
~ _max.to!string ~ " allowed");
}
auto _dst = new ubyte[_declared.to!size_t];
size_t _got = ZSTD_decompress(
_dst.ptr, _dst.length,
_src.ptr, _src.length,
);
if (ZSTD_isError(_got)) {
throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got));
}
if (_got != _dst.length) {
throw new ZstdException("zstd frame gave " ~ _got.to!string
~ " bytes where its header declared " ~ _dst.length.to!string);
}
return _dst;
}
#+END_SRC
* org includes
** project version
#+NAME: spine_version
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
#+END_SRC
** year
#+NAME: year
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
#+END_SRC
** document header including copyright & license
#+NAME: doc_header_including_copyright_and_license
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
#+END_SRC
* __END__
|
