diff options
Diffstat (limited to 'test/test-abstraction-db.sh')
| -rwxr-xr-x | test/test-abstraction-db.sh | 228 |
1 files changed, 228 insertions, 0 deletions
diff --git a/test/test-abstraction-db.sh b/test/test-abstraction-db.sh new file mode 100755 index 0000000..91748a3 --- /dev/null +++ b/test/test-abstraction-db.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env sh +# test-abstraction-db.sh +# +# Cross-check for spine's two serialisations of the document abstraction: +# the .ssp text file and the <doc>.abstraction.db sqlite database. +# +# It generates both for every sample document and asserts that they agree +# object for object and field for field. There is no committed reference +# set: the two artefacts check each other, which is what catches the thing +# that went wrong before, one writer gaining a field and the other not. +# +# Usage: +# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ +# ./test/test-abstraction-db.sh ./bin/spine-ldc +# +# Exit codes: +# 0 the two agree for every document +# 1 disagreements found (printed) +# 2 set-up problem (no $SpinePOD, no binary, no sqlite3, no samples) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ -z "$SpinePOD" ]; then + echo "ERROR: \$SpinePOD is not set." >&2 + echo " Set it to the sample pod directory, relative to $SPINE_DIR" >&2 + echo " e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2 + exit 2 +fi + +SAMPLES_RAW="$SPINE_DIR/$SpinePOD" +if [ -d "$SAMPLES_RAW" ]; then + SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)" +else + SAMPLES_DIR="$SAMPLES_RAW" +fi + +SPINE_BIN="${1:-}" +if [ -z "$SPINE_BIN" ]; then + if [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine" + elif [ -x "$SPINE_DIR/bin/spine-ldc" ]; then SPINE_BIN="$SPINE_DIR/bin/spine-ldc" + elif [ -x "$SPINE_DIR/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/bin/spine" + else + echo "ERROR: spine binary not found. Specify path as argument." >&2 + exit 2 + fi +fi + +if ! command -v sqlite3 > /dev/null 2>&1; then + echo "ERROR: sqlite3 not on PATH." >&2 + echo " e.g. nix shell \"nixpkgs#sqlite\" -c ./test/test-abstraction-db.sh $SPINE_BIN" >&2 + exit 2 +fi + +if [ ! -d "$SAMPLES_DIR" ]; then + echo "ERROR: sample documents not found at $SAMPLES_DIR" >&2 + exit 2 +fi + +OUT_DIR="$SCRIPT_DIR/current-db" + +echo "spine binary: $SPINE_BIN" +echo "samples: $SAMPLES_DIR" +echo "Generating .ssp and .abstraction.db ..." +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" +$SPINE_BIN --show-abstraction --show-abstraction-db --skip-output \ + --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 + +FAILURES=0 +DOCS=0 + +# ssp property -> sqlite predicate counting the same thing +# "<property>|<predicate>" +CHECKS=" +children|children IS NOT NULL +ancestors|ancestors IS NOT NULL +ancestors_collapsed|ancestors_collapsed IS NOT NULL +dom_status|dom_status IS NOT NULL +dom_status_collapsed|dom_status_collapsed IS NOT NULL +last_descendant|last_descendant_ocn != 0 +parent_lev|parent_lev != 0 +heading_lev_collapsed|heading_lev_collapsed IS NOT NULL +section|is_of_section IS NOT NULL AND is_of_section != section +part|is_of_part IS NOT NULL +is_of_type|is_of_type IS NOT NULL +attrib|attrib IS NOT NULL +meta_lang|meta_lang IS NOT NULL +meta_syntax|meta_syntax IS NOT NULL +sha256|sha256 IS NOT NULL +heading_ancestors_text|heading_ancestors_text IS NOT NULL +segment|segment IS NOT NULL +segment_prev|segment_prev IS NOT NULL +segment_next|segment_next IS NOT NULL +segment_epub|segment_epub IS NOT NULL +anchor|anchor IS NOT NULL AND anchor != segment +heading_lev_anchor|heading_lev_anchor IS NOT NULL +table_cols|table_cols IS NOT NULL +table_widths|table_widths IS NOT NULL +table_aligns|table_aligns IS NOT NULL +dummy: true|dummy_heading = 1 +ocn_off: true|object_number_off = 1 +bullet: true|bullet = 1 +table_header: true|table_header = 1 +code_linenumbers: true|code_linenumbers = 1 +lang|lang IS NOT NULL +indent|indent_base != 0 OR indent_hang != 0 +" + +# repeatable ssp property -> child table +LISTS=" +image|object_images +stow_link|object_links +anchor_tag|object_anchors +lev4_subtoc|object_subtoc +" + +report() { echo " MISMATCH $1: .ssp $2, db $3"; } + +for ssp in "$OUT_DIR"/*/abstraction/*.ssp; do + [ -f "$ssp" ] || continue + db="${ssp%.ssp}.abstraction.db" + base=$(basename "$ssp" .ssp) + if [ ! -f "$db" ]; then + echo "MISSING db for $base" + FAILURES=$((FAILURES + 1)) + continue + fi + DOCS=$((DOCS + 1)) + doc_fail=0 + + # object population + s_objs=$(grep -c '^\[' "$ssp" || true) + d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR) + if [ "$s_objs" != "$d_objs" ]; then + echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1 + fi + + # per section + for sec in head toc body endnotes glossary bibliography bookindex blurb; do + s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp") + d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR) + if [ "$s_sec" != "$d_sec" ]; then + [ $doc_fail -eq 0 ] && echo "$base" + report "section $sec" "$s_sec" "$d_sec"; doc_fail=1 + fi + done + + # single valued properties + echo "$CHECKS" | while IFS='|' read -r prop pred; do + [ -z "$prop" ] && continue + case "$prop" in + *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; + *) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; + esac + d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR) + if [ "$s_n" != "$d_n" ]; then + echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n" + fi + done > "$OUT_DIR/.props.$$" + if [ -s "$OUT_DIR/.props.$$" ]; then + [ $doc_fail -eq 0 ] && echo "$base" + sed 's/^[^:]*: / /' "$OUT_DIR/.props.$$" + doc_fail=1 + fi + rm -f "$OUT_DIR/.props.$$" + + # repeatable properties against their tables + echo "$LISTS" | while IFS='|' read -r prop table; do + [ -z "$prop" ] && continue + s_n=$(grep -c "^\.$prop: " "$ssp" || true) + d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR) + if [ "$s_n" != "$d_n" ]; then + echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n" + fi + done > "$OUT_DIR/.lists.$$" + if [ -s "$OUT_DIR/.lists.$$" ]; then + [ $doc_fail -eq 0 ] && echo "$base" + sed 's/^[^:]*: / /' "$OUT_DIR/.lists.$$" + doc_fail=1 + fi + rm -f "$OUT_DIR/.lists.$$" + + # every image the abstraction referred to and found is carried in the file + want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR) + have=$(sqlite3 "$db" "select count(*) from files f where f.role='image' + and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR) + if [ "$want" != "$have" ]; then + [ $doc_fail -eq 0 ] && echo "$base" + report "images carried" "$want referenced" "$have stored"; doc_fail=1 + fi + + # the text itself, on the first and last citable object + for pick in "min" "max"; do + ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "") + [ -z "$ocn" ] && continue + sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true + awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \ + "$ssp" > "$OUT_DIR/.ssptxt.$$" + if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then + [ $doc_fail -eq 0 ] && echo "$base" + echo " MISMATCH text of ocn $ocn" + doc_fail=1 + fi + rm -f "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" + done + + # the file says what it is + sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '') + if [ -z "$sv" ]; then + [ $doc_fail -eq 0 ] && echo "$base" + echo " MISSING metadata schema.version"; doc_fail=1 + fi + + FAILURES=$((FAILURES + doc_fail)) +done + +rm -rf "$OUT_DIR" + +if [ "$FAILURES" -eq 0 ]; then + echo "PASS: .ssp and .abstraction.db agree for all $DOCS documents" + exit 0 +else + echo "FAIL: $FAILURES document(s) disagree" + exit 1 +fi |