Initial committesting

10 files changed, 291 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..984ece2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/data/
+/libyaml/
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..6f6dd31
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,30 @@
+LIBYAML_HEAD_COMMIT ?= 0032321756cee86a67171de425267c1d0d406092
+LIBYAML_MAIN_BRANCH ?= master
+LIBYAML_REPO_URL ?= git@github.com:yaml/libyaml
+
+TEST_SUITE_DATA_COMMIT ?= data-2020-02-11
+TEST_SUITE_DATA_BRANCH ?= data
+TEST_SUITE_REPO_URL ?= git@github.com:yaml/yaml-test-suite
+
+.PHONY: test
+test: libyaml/tests/run-parser-test-suite data
+	prove -lv test/
+
+libyaml/tests/run-parser-test-suite: libyaml
+	( \
+	    cd $< && \
+	    ./bootstrap && \
+	    ./configure && \
+	    make \
+	)
+
+libyaml:
+	git clone --branch=$(LIBYAML_MAIN_BRANCH) $(LIBYAML_REPO_URL) $@
+	(cd $@ && git reset --hard $(LIBYAML_HEAD_COMMIT))
+
+data:
+	git clone --branch=$(TEST_SUITE_DATA_BRANCH) $(TEST_SUITE_REPO_URL) $@
+	(cd $@ && git reset --hard $(TEST_SUITE_DATA_COMMIT))
+
+clean:
+	rm -fr libyaml data
diff --git a/ReadMe.md b/ReadMe.md
new file mode 100644
index 0000000..b021089
--- /dev/null
+++ b/ReadMe.md
@@ -0,0 +1,11 @@
+testing
+=======
+
+Branch to test libyaml against the test suite
+
+# Synopsis
+
+```
+git checkout testing
+make test
+```
diff --git a/blacklist/libyaml-emitter b/blacklist/libyaml-emitter
new file mode 100644
index 0000000..b52000f
--- /dev/null
+++ b/blacklist/libyaml-emitter
@@ -0,0 +1,72 @@
+26DV: Whitespace around colon in mappings
+2EBW: Allowed characters in keys
+2JQS: Block Mapping with Missing Keys
+2LFX: Spec Example 6.13. Reserved Directives [1.3]
+2SXE: Anchors With Colon in Name
+2XXW: Spec Example 2.25. Unordered Sets
+3MYT: Plain Scalar looking like key, comment, anchor and tag
+4ABK: Spec Example 7.17. Flow Mapping Separate Values
+4MUZ: Flow mapping colon on line after key
+4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3]
+52DL: Explicit Non-Specific Tag [1.3]
+565N: Construct Binary
+5TYM: Spec Example 6.21. Local Tag Prefix
+5WE3: Spec Example 8.17. Explicit Block Mapping Entries
+6CK3: Spec Example 6.26. Tag Shorthands
+6FWR: Block Scalar Keep
+6KGN: Anchor for empty node
+6M2F: Aliases in Explicit Block Mapping
+6PBE: Zero-indented sequences in explicit mapping keys
+6SLA: Allowed characters in quoted mapping key
+6WLZ: Spec Example 6.18. Primary Tag Handle [1.3]
+6WPF: Spec Example 6.8. Flow Folding [1.3]
+6XDY: Two document start markers
+6ZKB: Spec Example 9.6. Stream
+7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines
+7W2P: Block Mapping with Missing Values
+7Z25: Bare document after document end marker
+8KB6: Multiline plain flow mapping key without value
+8XYN: Anchor with unicode character
+9BXH: Multiline doublequoted flow mapping key without value
+8MK2: Explicit Non-Specific Tag
+9DXL: Spec Example 9.6. Stream [1.3]
+9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
+9TFX: Spec Example 7.6. Double Quoted Lines [1.3]
+B3HG: Spec Example 8.9. Folded Scalar [1.3]
+C2DT: Spec Example 7.18. Flow Mapping Adjacent Values
+DFF7: Spec Example 7.16. Flow Mapping Entries
+E76Z: Aliases in Implicit Block Mapping
+EX5H: Multiline Scalar at Top Level [1.3]
+EXG3: Three dashes and content without space [1.3]
+FBC9: Allowed characters in plain scalars
+FH7J: Tags on Empty Scalars
+FRK4: Spec Example 7.3. Completely Empty Flow Nodes
+J3BT: Spec Example 5.12. Tabs and Spaces
+JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3]
+JTV5: Block Mapping with Multiline Scalars
+K54U: Tab after document header
+KK5P: Various combinations of explicit block mappings
+KSS4: Scalars on --- line
+KZN9: Spec Example 7.21. Single Pair Implicit Entries
+LE5A: Spec Example 7.24. Flow Nodes
+M7A3: Spec Example 9.3. Bare Documents
+M9B4: Spec Example 8.7. Literal Scalar
+NAT4: Various empty or newline only quoted strings
+NHX8: Empty Lines at End of Document
+PUW8: Document start on last line
+PW8X: Anchors on Empty Scalars
+Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3]
+S3PD: Spec Example 8.18. Implicit Block Mapping Entries
+S4JQ: Spec Example 6.28. Non-Specific Tags
+T26H: Spec Example 8.8. Literal Content [1.3]
+T4YY: Spec Example 7.9. Single Quoted Lines [1.3]
+T5N4: Spec Example 8.7. Literal Scalar [1.3]
+UT92: Spec Example 9.4. Explicit Documents
+W42U: Spec Example 8.15. Block Sequence Entry Types
+W4TN: Spec Example 9.5. Directives Documents
+W5VH: Allowed characters in alias
+WZ62: Spec Example 7.2. Empty Content
+X38W: Aliases in Flow Objects
+XLQ9: Multiline scalar that looks like a YAML directive
+Y2GN: Anchor with colon in the middle
+ZWK4: Key with anchor after missing explicit mapping value
diff --git a/blacklist/libyaml-parser b/blacklist/libyaml-parser
new file mode 100644
index 0000000..94029f6
--- /dev/null
+++ b/blacklist/libyaml-parser
@@ -0,0 +1,34 @@
+2JQS: Block Mapping with Missing Keys
+2LFX: Spec Example 6.13. Reserved Directives [1.3]
+2SXE: Anchors With Colon in Name
+4ABK: Spec Example 7.17. Flow Mapping Separate Values
+4MUZ: Flow mapping colon on line after key
+5MUD: Colon and adjacent value on next line
+6BCT: Spec Example 6.3. Separation Spaces
+6LVF: Spec Example 6.13. Reserved Directives
+6M2F: Aliases in Explicit Block Mapping
+7Z25: Bare document after document end marker
+8XYN: Anchor with unicode character
+9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
+9SA2: Multiline double quoted flow mapping key
+A2M4: Spec Example 6.2. Indentation Indicators
+BEC7: Spec Example 6.14. “YAML” directive
+DBG4: Spec Example 7.10. Plain Characters
+DK3J: Zero indented block scalar with line that looks like a comment
+FP8R: Zero indented block scalar
+FRK4: Spec Example 7.3. Completely Empty Flow Nodes
+HWV9: Document-end marker
+K3WX: Colon and adjacent value after comment on next line
+KZN9: Spec Example 7.21. Single Pair Implicit Entries
+M7A3: Spec Example 9.3. Bare Documents
+NHX8: Empty Lines at End of Document
+NJ66: Multiline plain flow mapping key
+Q5MG: Tab at beginning of line followed by a flow mapping
+QT73: Comment and document-end marker
+R4YG: Spec Example 8.2. Block Indentation Indicator
+S3PD: Spec Example 8.18. Implicit Block Mapping Entries
+UT92: Spec Example 9.4. Explicit Documents
+W4TN: Spec Example 9.5. Directives Documents
+W5VH: Allowed characters in alias
+WZ62: Spec Example 7.2. Empty Content
+Y2GN: Anchor with colon in the middle
diff --git a/blacklist/libyaml-parser-error b/blacklist/libyaml-parser-error
new file mode 100644
index 0000000..aa83f1a
--- /dev/null
+++ b/blacklist/libyaml-parser-error
@@ -0,0 +1,10 @@
+9C9N: Wrong indented flow sequence
+9HCY: Need document footer before directives
+9JBA: Invalid comment after end of flow sequence
+CVW2: Invalid comment after comma
+EB22: Missing document-end marker before directive
+QB6E: Wrong indented multiline quoted scalar
+RHX7: YAML directive without document end marker
+S98Z: Block scalar with more spaces than first content line
+SU5Z: Comment without whitespace after doublequoted scalar
+X4QW: Comment without whitespace after block scalar indicator
diff --git a/test/libyaml-emitter.t b/test/libyaml-emitter.t
new file mode 100644
index 0000000..3f3d8be
--- /dev/null
+++ b/test/libyaml-emitter.t
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=1090,2034
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+source "$root"/test/test-runner.bash
+
+check-test() {
+  id=$1
+  t=data/$id
+
+  grep "$id" "$root/blacklist/libyaml-emitter" >/dev/null && return 1
+  [[ -e $t/error ]] && return 1
+
+  return 0
+}
+
+run-test() {
+  dir=$1
+  ok=true
+
+  want=$dir/out.yaml
+  [[ -e $want ]] || want="$dir/in.yaml"
+
+  libyaml/tests/run-emitter-test-suite "$dir/test.event" > /tmp/test.out || {
+    (
+      cat "$dir/test.event"
+      cat "$want"
+    ) | sed 's/^/# /'
+  }
+
+  output="$(${DIFF:-diff} -u "$want" /tmp/test.out)" || ok=false
+}
+
+run-tests "$@"
diff --git a/test/libyaml-parser-error.t b/test/libyaml-parser-error.t
new file mode 100644
index 0000000..a51578e
--- /dev/null
+++ b/test/libyaml-parser-error.t
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=1090,2034
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+source "$root"/test/test-runner.bash
+
+check-test() {
+  id=$1
+  t=data/$id
+
+  grep "$id" "$root/blacklist/libyaml-parser-error" >/dev/null && return 1
+  [[ -e $t/error ]] || return 1
+
+  return 0
+}
+
+run-test() {
+  dir=$1
+  ok=false
+
+  libyaml/tests/run-parser-test-suite "$dir/in.yaml" > /tmp/test.out 2>&1 || ok=true
+
+  $ok || output=$(< /tmp/test.out)
+}
+
+run-tests "$@"
diff --git a/test/libyaml-parser.t b/test/libyaml-parser.t
new file mode 100644
index 0000000..5e9cb94
--- /dev/null
+++ b/test/libyaml-parser.t
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=1090,2034
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+source "$root"/test/test-runner.bash
+
+check-test() {
+  id=$1
+  t=data/$id
+
+  grep "$id" "$root/blacklist/libyaml-parser" >/dev/null && return 1
+  [[ -e $t/error ]] && return 1
+
+  return 0
+}
+
+run-test() {
+  dir=$1
+  ok=true
+
+  libyaml/tests/run-parser-test-suite "$dir/in.yaml" > /tmp/test.out || {
+    (
+      cat "$dir/in.yaml"
+      cat "$dir/test.event"
+    ) | sed 's/^/# /'
+  }
+
+  output=$(${DIFF:-diff} -u "$dir/test.event" /tmp/test.out) || ok=false
+}
+
+run-tests "$@"
diff --git a/test/test-runner.bash b/test/test-runner.bash
new file mode 100644
index 0000000..8360308
--- /dev/null
+++ b/test/test-runner.bash
@@ -0,0 +1,35 @@
+# shellcheck disable=2001,2154,2207
+
+set -e -u -o pipefail
+
+run-tests() (
+  root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+  cd "$root"
+
+  if [[ $# -gt 0 ]]; then
+    ids=("$@")
+  else
+    ids=($(cd data; printf "%s\n" * | grep '[0-9]'))
+  fi
+
+  count=0
+  for id in "${ids[@]}"; do
+    check-test "$id" || continue
+
+    dir=data/$id
+    label="$id: $(< "$dir/===")"
+    [[ -e $dir/in.yaml ]] || continue
+
+    run-test "$dir"
+
+    if $ok; then
+      echo "ok $((++count)) $label"
+    else
+      echo "not ok $((++count)) $label"
+      echo "$output" | sed 's/^/# /'
+    fi
+  done
+
+  echo "1..$count"
+)