test: epubcheck (epub-validity) over sample set

(assisted by Claude-Code)

1 files changed, 127 insertions, 3 deletions

diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org
index 690144f..baf7fa4 100644
--- a/org/tests_for_document_abstraction_shell_scripts.org
+++ b/org/tests_for_document_abstraction_shell_scripts.org
@@ -45,6 +45,7 @@
 #   2 test-abstraction-ssp-roundtrip.sh  the .ssp can be read back whole
 #   3 test-abstraction-db.sh             the two serialisations agree
 #   4 test-abstraction-db-roundtrip.sh   the .db can be read back whole
+#   5 test-epub-validity.sh              the epubs a reader can actually open
 #
 # None of them calls another, and each can be run on its own. The order
 # matters only for reading the result:
@@ -72,8 +73,8 @@ fi
 
 if ! command -v sqlite3 > /dev/null 2>&1; then
   if command -v nix > /dev/null 2>&1; then
-    echo "sqlite3 not on PATH, re-running inside nix shell \"nixpkgs#sqlite\""
-    exec nix shell "nixpkgs#sqlite" -c "$0" "$@"
+    echo "sqlite3 not on PATH, re-running inside nix shell (sqlite, epubcheck)"
+    exec nix shell "nixpkgs#sqlite" "nixpkgs#epubcheck" -c "$0" "$@"
   fi
   echo "ERROR: sqlite3 not on PATH and nix not available." >&2
   exit 2
@@ -82,7 +83,8 @@ fi
 TESTS="test-abstraction-ssp.sh
 test-abstraction-ssp-roundtrip.sh
 test-abstraction-db.sh
-test-abstraction-db-roundtrip.sh"
+test-abstraction-db-roundtrip.sh
+test-epub-validity.sh"
 
 PASSED=0
 FAILED=0
@@ -1234,3 +1236,125 @@ else
   exit 1
 fi
 #+END_SRC
+
+** test epub validity
+
+#+HEADER: :tangle ../test/test-epub-validity.sh
+#+HEADER: :tangle-mode (identity #o755)
+#+HEADER: :shebang "#!/usr/bin/env sh"
+#+BEGIN_SRC shell
+# epubcheck over the sample set, as a regression guard.
+#
+# Every epub spine writes is checked with epubcheck. A *fatal* is never
+# acceptable: it means a reader cannot parse the file at all. Errors are
+# compared against a recorded allowance, so that the ones known to be
+# outstanding do not fail the run while any new one does.
+#
+# The allowance is deliberately a single number rather than a list of
+# messages: a list invites adding to it. When the number goes down, lower it
+# here in the same commit that lowers it in the output.
+#
+# Usage:
+#   SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \
+#     ./test/test-epub-validity.sh ./bin/spine-ldc
+#
+# Exit codes:
+#   0  no fatals, and errors within the allowance
+#   1  a fatal, or more errors than allowed
+#   2  set-up problem (no $SpinePOD, no binary, no epubcheck, no samples)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# What is known to be outstanding over the whole sample set. Both numbers are
+# a ceiling, so a regression fails and an improvement does not; lower them in
+# the commit that lowers the output, and say in that commit what went.
+#
+# The one fatal is the_public_domain, chapter_2: a bare URL at the end of a
+# footnote body leaves a stray </a> in the paragraph, so the <p> never closes.
+# It predates this test, it is in the html output too, and it is one
+# occurrence in 35 documents. It wants its own fix; until then it is named
+# here rather than tolerated silently.
+FATALS_ALLOWED=${SpineEpubFatalsAllowed:-1}
+ERRORS_ALLOWED=${SpineEpubErrorsAllowed:-1371}
+
+if [ -z "$SpinePOD" ]; then
+  echo "ERROR: \$SpinePOD is not set." >&2
+  echo "  e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2
+  exit 2
+fi
+SAMPLES_RAW="$SPINE_DIR/$SpinePOD"
+if [ -d "$SAMPLES_RAW" ]; then
+  SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)"
+else
+  SAMPLES_DIR="$SAMPLES_RAW"
+fi
+# SPINE_BIN="${1:-}"
+# if [ -z "$SPINE_BIN" ] || [ ! -x "$SPINE_BIN" ]; then
+#   echo "ERROR: give the spine binary, e.g. ./bin/spine-ldc" >&2
+#   exit 2
+# fi
+SPINE_BIN="${1:-}"
+# find spine binary
+if [ -z "$SPINE_BIN" ]; then
+  if [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine"
+  elif [ -x "$SPINE_DIR/bin/spine-ldc" ]; then SPINE_BIN="$SPINE_DIR/bin/spine-ldc"
+  elif [ -x "$SPINE_DIR/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/bin/spine"
+  else
+    echo "ERROR: spine binary not found. Specify path as argument." >&2
+    exit 2
+  fi
+fi
+if [ ! -d "$SAMPLES_DIR" ]; then
+  echo "ERROR: samples not found: $SAMPLES_DIR" >&2
+  exit 2
+fi
+if ! command -v epubcheck >/dev/null 2>&1; then
+  echo "ERROR: epubcheck not on PATH." >&2
+  echo "  e.g. nix shell \"nixpkgs#epubcheck\" -c $0 $SPINE_BIN" >&2
+  exit 2
+fi
+
+echo "spine binary: $SPINE_BIN"
+echo "samples: $SAMPLES_DIR"
+
+OUT_DIR="$(mktemp -d)"
+trap 'rm -rf "$OUT_DIR"' EXIT
+
+echo "Generating epubs ..."
+"$SPINE_BIN" --epub --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1
+
+FATALS=0
+ERRORS=0
+DOCS=0
+for f in $(find "$OUT_DIR" -name '*.epub' | sort); do
+  DOCS=$((DOCS + 1))
+  msg=$(epubcheck "$f" 2>&1 | grep -E '^Messages:' || true)
+  ff=$(echo "$msg" | sed -n 's/.*: \([0-9]*\) fatal.*/\1/p')
+  ee=$(echo "$msg" | sed -n 's/.*\/ \([0-9]*\) error.*/\1/p')
+  ff=${ff:-0}; ee=${ee:-0}
+  FATALS=$((FATALS + ff))
+  ERRORS=$((ERRORS + ee))
+  if [ "$ff" -gt 0 ]; then
+    echo "  FATAL in $(basename "$f"): $ff"
+    epubcheck "$f" 2>&1 | grep -i 'FATAL' | head -3 | sed 's/^/    /'
+  fi
+done
+
+echo "checked $DOCS epubs: $FATALS fatal (allowed $FATALS_ALLOWED), $ERRORS errors (allowed $ERRORS_ALLOWED)"
+
+if [ "$FATALS" -gt "$FATALS_ALLOWED" ]; then
+  echo "FAIL: $FATALS fatal(s), more than the $FATALS_ALLOWED allowed;"
+  echo "  a fatal means a reader cannot parse that file at all"
+  exit 1
+fi
+if [ "$ERRORS" -gt "$ERRORS_ALLOWED" ]; then
+  echo "FAIL: $ERRORS errors, more than the $ERRORS_ALLOWED allowed"
+  echo "  if this is a deliberate change, say why and adjust ERRORS_ALLOWED"
+  exit 1
+fi
+echo "PASS: $FATALS fatal and $ERRORS errors, both within what is recorded"
+exit 0
+#+END_SRC