From 551356de1bc8ef431fcd1b32624bec0fa1612b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Mon, 1 Jun 2020 10:31:58 -0700 Subject: Support a default environment The env files were renamed to be more obvious. Now pin-* files are symlinks to env-* files. If a pinning can't be found using the current mainline commit, we use env/default default which is a symlink to the most recent. This will probably work most of the time. A big warning box will be printed showing what happened and where to get more info. To try this out: git checkout ingy.test-suite-support export LIBYAML_TEST_SUITE_RUN_BRANCH=run-test-suite-2 ln -fs .makefile GNUmakefile make distclean make test-suite --- Makefile | 6 +- env/blacklist-001 | 130 ------------------- env/default | 1 + env/env-2020-05-24-test-suite-9fe5172ca31 | 138 +++++++++++++++++++++ ...-0.2.4-72e2f75277ff88ad1fb8a81cf422df06bf371578 | 1 + env/pin-0032321756cee86a67171de425267c1d0d406092 | 6 - env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400 | 1 - env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095 | 1 - env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b | 1 - ...master-9deee01508bb3b8ad25822f53d91d9f54eae9a0b | 1 + env/pin-sample | 25 ---- lookup | 62 ++++++++- 12 files changed, 205 insertions(+), 168 deletions(-) delete mode 100644 env/blacklist-001 create mode 120000 env/default create mode 100644 env/env-2020-05-24-test-suite-9fe5172ca31 create mode 120000 env/pin-0.2.4-72e2f75277ff88ad1fb8a81cf422df06bf371578 delete mode 100644 env/pin-0032321756cee86a67171de425267c1d0d406092 delete mode 120000 env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400 delete mode 120000 env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095 delete mode 120000 env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b create mode 120000 env/pin-master-9deee01508bb3b8ad25822f53d91d9f54eae9a0b delete mode 100644 env/pin-sample diff --git a/Makefile b/Makefile index d338e8c..d6b01a2 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,11 @@ test: data [[ "$(debug)" ]] && export LIBYAML_TEST_SUITE_DEBUG=1 export LIBYAML_TEST_SUITE_ENV=$$(./lookup env) [[ $$LIBYAML_TEST_SUITE_ENV ]] || exit 1 - prove -v test/ + set +ex + (set -x; prove -v test/) + if [[ $$LIBYAML_TEST_SUITE_ENV == env/default ]]; then + ./lookup default-warning + fi test-all: prove -v test/test-all.sh diff --git a/env/blacklist-001 b/env/blacklist-001 deleted file mode 100644 index e39d924..0000000 --- a/env/blacklist-001 +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash - -whitelist all - -blacklist parser "\ -2JQS: Block Mapping with Missing Keys -2LFX: Spec Example 6.13. Reserved Directives [1.3] -2SXE: Anchors With Colon in Name -4ABK: Spec Example 7.17. Flow Mapping Separate Values -4MUZ: Flow mapping colon on line after key -5MUD: Colon and adjacent value on next line -6BCT: Spec Example 6.3. Separation Spaces -6LVF: Spec Example 6.13. Reserved Directives -6M2F: Aliases in Explicit Block Mapping -7Z25: Bare document after document end marker -8XYN: Anchor with unicode character -9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3 -9SA2: Multiline double quoted flow mapping key -A2M4: Spec Example 6.2. Indentation Indicators -BEC7: Spec Example 6.14. “YAML” directive -DBG4: Spec Example 7.10. Plain Characters -DK3J: Zero indented block scalar with line that looks like a comment -FP8R: Zero indented block scalar -FRK4: Spec Example 7.3. Completely Empty Flow Nodes -HWV9: Document-end marker -K3WX: Colon and adjacent value after comment on next line -KZN9: Spec Example 7.21. Single Pair Implicit Entries -M7A3: Spec Example 9.3. Bare Documents -NHX8: Empty Lines at End of Document -NJ66: Multiline plain flow mapping key -Q5MG: Tab at beginning of line followed by a flow mapping -QT73: Comment and document-end marker -R4YG: Spec Example 8.2. Block Indentation Indicator -S3PD: Spec Example 8.18. Implicit Block Mapping Entries -UT92: Spec Example 9.4. Explicit Documents -W4TN: Spec Example 9.5. Directives Documents -W5VH: Allowed characters in alias -WZ62: Spec Example 7.2. Empty Content -Y2GN: Anchor with colon in the middle -" - -blacklist parser-error "\ -9C9N: Wrong indented flow sequence -9HCY: Need document footer before directives -9JBA: Invalid comment after end of flow sequence -CVW2: Invalid comment after comma -EB22: Missing document-end marker before directive -QB6E: Wrong indented multiline quoted scalar -RHX7: YAML directive without document end marker -S98Z: Block scalar with more spaces than first content line -SU5Z: Comment without whitespace after doublequoted scalar -X4QW: Comment without whitespace after block scalar indicator -" - -blacklist emitter "\ -26DV: Whitespace around colon in mappings -2EBW: Allowed characters in keys -2JQS: Block Mapping with Missing Keys -2LFX: Spec Example 6.13. Reserved Directives [1.3] -2SXE: Anchors With Colon in Name -2XXW: Spec Example 2.25. Unordered Sets -3MYT: Plain Scalar looking like key, comment, anchor and tag -4ABK: Spec Example 7.17. Flow Mapping Separate Values -4MUZ: Flow mapping colon on line after key -4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3] -52DL: Explicit Non-Specific Tag [1.3] -565N: Construct Binary -5TYM: Spec Example 6.21. Local Tag Prefix -5WE3: Spec Example 8.17. Explicit Block Mapping Entries -6CK3: Spec Example 6.26. Tag Shorthands -6FWR: Block Scalar Keep -6KGN: Anchor for empty node -6M2F: Aliases in Explicit Block Mapping -6PBE: Zero-indented sequences in explicit mapping keys -6SLA: Allowed characters in quoted mapping key -6WLZ: Spec Example 6.18. Primary Tag Handle [1.3] -6WPF: Spec Example 6.8. Flow Folding [1.3] -6XDY: Two document start markers -6ZKB: Spec Example 9.6. Stream -7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines -7W2P: Block Mapping with Missing Values -7Z25: Bare document after document end marker -8KB6: Multiline plain flow mapping key without value -8XYN: Anchor with unicode character -9BXH: Multiline doublequoted flow mapping key without value -8MK2: Explicit Non-Specific Tag -9DXL: Spec Example 9.6. Stream [1.3] -9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3 -9TFX: Spec Example 7.6. Double Quoted Lines [1.3] -B3HG: Spec Example 8.9. Folded Scalar [1.3] -C2DT: Spec Example 7.18. Flow Mapping Adjacent Values -DFF7: Spec Example 7.16. Flow Mapping Entries -E76Z: Aliases in Implicit Block Mapping -EX5H: Multiline Scalar at Top Level [1.3] -EXG3: Three dashes and content without space [1.3] -F8F9: Spec Example 8.5. Chomping Trailing Lines -FBC9: Allowed characters in plain scalars -FH7J: Tags on Empty Scalars -FRK4: Spec Example 7.3. Completely Empty Flow Nodes -J3BT: Spec Example 5.12. Tabs and Spaces -JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3] -JTV5: Block Mapping with Multiline Scalars -K54U: Tab after document header -K858: Spec Example 8.6. Empty Scalar Chomping -KK5P: Various combinations of explicit block mappings -KSS4: Scalars on --- line -KZN9: Spec Example 7.21. Single Pair Implicit Entries -LE5A: Spec Example 7.24. Flow Nodes -M7A3: Spec Example 9.3. Bare Documents -M9B4: Spec Example 8.7. Literal Scalar -NAT4: Various empty or newline only quoted strings -NHX8: Empty Lines at End of Document -PUW8: Document start on last line -PW8X: Anchors on Empty Scalars -Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3] -S3PD: Spec Example 8.18. Implicit Block Mapping Entries -S4JQ: Spec Example 6.28. Non-Specific Tags -T26H: Spec Example 8.8. Literal Content [1.3] -T4YY: Spec Example 7.9. Single Quoted Lines [1.3] -T5N4: Spec Example 8.7. Literal Scalar [1.3] -UT92: Spec Example 9.4. Explicit Documents -W42U: Spec Example 8.15. Block Sequence Entry Types -W4TN: Spec Example 9.5. Directives Documents -W5VH: Allowed characters in alias -WZ62: Spec Example 7.2. Empty Content -X38W: Aliases in Flow Objects -XLQ9: Multiline scalar that looks like a YAML directive -Y2GN: Anchor with colon in the middle -ZWK4: Key with anchor after missing explicit mapping value -" diff --git a/env/default b/env/default new file mode 120000 index 0000000..033d08c --- /dev/null +++ b/env/default @@ -0,0 +1 @@ +env-2020-05-24-test-suite-9fe5172ca31 \ No newline at end of file diff --git a/env/env-2020-05-24-test-suite-9fe5172ca31 b/env/env-2020-05-24-test-suite-9fe5172ca31 new file mode 100644 index 0000000..37a6ace --- /dev/null +++ b/env/env-2020-05-24-test-suite-9fe5172ca31 @@ -0,0 +1,138 @@ +#!/bin/bash + +LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite +LIBYAML_TEST_SUITE_DATA_COMMIT=9fe5172ca313349ad5b8200f8168de0b0a48ea5d + +whitelist all + +blacklist emitter "\ +26DV: Whitespace around colon in mappings +2EBW: Allowed characters in keys +2JQS: Block Mapping with Missing Keys +2LFX: Spec Example 6.13. Reserved Directives [1.3] +2SXE: Anchors With Colon in Name +2XXW: Spec Example 2.25. Unordered Sets +3MYT: Plain Scalar looking like key, comment, anchor and tag +4ABK: Spec Example 7.17. Flow Mapping Separate Values +4MUZ: Flow mapping colon on line after key +4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3] +52DL: Explicit Non-Specific Tag [1.3] +565N: Construct Binary +5T43: Colon at the beginning of adjacent flow scalar +5TYM: Spec Example 6.21. Local Tag Prefix +5WE3: Spec Example 8.17. Explicit Block Mapping Entries +6CK3: Spec Example 6.26. Tag Shorthands +6FWR: Block Scalar Keep +6KGN: Anchor for empty node +6M2F: Aliases in Explicit Block Mapping +6PBE: Zero-indented sequences in explicit mapping keys +6SLA: Allowed characters in quoted mapping key +6WLZ: Spec Example 6.18. Primary Tag Handle [1.3] +6WPF: Spec Example 6.8. Flow Folding [1.3] +6XDY: Two document start markers +6ZKB: Spec Example 9.6. Stream +7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines +7W2P: Block Mapping with Missing Values +7Z25: Bare document after document end marker +8KB6: Multiline plain flow mapping key without value +8XYN: Anchor with unicode character +9BXH: Multiline doublequoted flow mapping key without value +8MK2: Explicit Non-Specific Tag +9DXL: Spec Example 9.6. Stream [1.3] +9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3 +9TFX: Spec Example 7.6. Double Quoted Lines [1.3] +B3HG: Spec Example 8.9. Folded Scalar [1.3] +C2DT: Spec Example 7.18. Flow Mapping Adjacent Values +DFF7: Spec Example 7.16. Flow Mapping Entries +E76Z: Aliases in Implicit Block Mapping +EX5H: Multiline Scalar at Top Level [1.3] +EXG3: Three dashes and content without space [1.3] +F8F9: Spec Example 8.5. Chomping Trailing Lines +FBC9: Allowed characters in plain scalars +FH7J: Tags on Empty Scalars +FRK4: Spec Example 7.3. Completely Empty Flow Nodes +J3BT: Spec Example 5.12. Tabs and Spaces +JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3] +JTV5: Block Mapping with Multiline Scalars +K54U: Tab after document header +K858: Spec Example 8.6. Empty Scalar Chomping +KK5P: Various combinations of explicit block mappings +KSS4: Scalars on --- line +KZN9: Spec Example 7.21. Single Pair Implicit Entries +LE5A: Spec Example 7.24. Flow Nodes +M7A3: Spec Example 9.3. Bare Documents +M9B4: Spec Example 8.7. Literal Scalar +NAT4: Various empty or newline only quoted strings +NHX8: Empty Lines at End of Document +PUW8: Document start on last line +PW8X: Anchors on Empty Scalars +Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3] +S3PD: Spec Example 8.18. Implicit Block Mapping Entries +S4JQ: Spec Example 6.28. Non-Specific Tags +T26H: Spec Example 8.8. Literal Content [1.3] +T4YY: Spec Example 7.9. Single Quoted Lines [1.3] +T5N4: Spec Example 8.7. Literal Scalar [1.3] +UT92: Spec Example 9.4. Explicit Documents +W42U: Spec Example 8.15. Block Sequence Entry Types +W4TN: Spec Example 9.5. Directives Documents +W5VH: Allowed characters in alias +WZ62: Spec Example 7.2. Empty Content +X38W: Aliases in Flow Objects +XLQ9: Multiline scalar that looks like a YAML directive +Y2GN: Anchor with colon in the middle +ZWK4: Key with anchor after missing explicit mapping value +" + +blacklist parser "\ +2JQS: Block Mapping with Missing Keys +2LFX: Spec Example 6.13. Reserved Directives [1.3] +2SXE: Anchors With Colon in Name +4ABK: Spec Example 7.17. Flow Mapping Separate Values +4MUZ: Flow mapping colon on line after key +5MUD: Colon and adjacent value on next line +5T43: Colon at the beginning of adjacent flow scalar +6BCT: Spec Example 6.3. Separation Spaces +6LVF: Spec Example 6.13. Reserved Directives +6M2F: Aliases in Explicit Block Mapping +7Z25: Bare document after document end marker +8XYN: Anchor with unicode character +9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3 +9SA2: Multiline double quoted flow mapping key +A2M4: Spec Example 6.2. Indentation Indicators +BEC7: Spec Example 6.14. “YAML” directive +DBG4: Spec Example 7.10. Plain Characters +DK3J: Zero indented block scalar with line that looks like a comment +FP8R: Zero indented block scalar +FRK4: Spec Example 7.3. Completely Empty Flow Nodes +HWV9: Document-end marker +JR7V: Question marks in scalars +K3WX: Colon and adjacent value after comment on next line +KZN9: Spec Example 7.21. Single Pair Implicit Entries +M7A3: Spec Example 9.3. Bare Documents +NHX8: Empty Lines at End of Document +NJ66: Multiline plain flow mapping key +Q5MG: Tab at beginning of line followed by a flow mapping +QT73: Comment and document-end marker +R4YG: Spec Example 8.2. Block Indentation Indicator +S3PD: Spec Example 8.18. Implicit Block Mapping Entries +UT92: Spec Example 9.4. Explicit Documents +W4TN: Spec Example 9.5. Directives Documents +W5VH: Allowed characters in alias +WZ62: Spec Example 7.2. Empty Content +Y2GN: Anchor with colon in the middle +" + +blacklist parser-error "\ +9C9N: Wrong indented flow sequence +9HCY: Need document footer before directives +9JBA: Invalid comment after end of flow sequence +CVW2: Invalid comment after comma +EB22: Missing document-end marker before directive +QB6E: Wrong indented multiline quoted scalar +RHX7: YAML directive without document end marker +S98Z: Block scalar with more spaces than first content line +5T43: Colon at the beginning of adjacent flow scalar +SU5Z: Comment without whitespace after doublequoted scalar +U99R: Invalid comma in tag +X4QW: Comment without whitespace after block scalar indicator +" diff --git a/env/pin-0.2.4-72e2f75277ff88ad1fb8a81cf422df06bf371578 b/env/pin-0.2.4-72e2f75277ff88ad1fb8a81cf422df06bf371578 new file mode 120000 index 0000000..033d08c --- /dev/null +++ b/env/pin-0.2.4-72e2f75277ff88ad1fb8a81cf422df06bf371578 @@ -0,0 +1 @@ +env-2020-05-24-test-suite-9fe5172ca31 \ No newline at end of file diff --git a/env/pin-0032321756cee86a67171de425267c1d0d406092 b/env/pin-0032321756cee86a67171de425267c1d0d406092 deleted file mode 100644 index 246762c..0000000 --- a/env/pin-0032321756cee86a67171de425267c1d0d406092 +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite -LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17 - -source "$ENV/blacklist-001" diff --git a/env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400 b/env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400 deleted file mode 120000 index 3ff882c..0000000 --- a/env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400 +++ /dev/null @@ -1 +0,0 @@ -pin-0032321756cee86a67171de425267c1d0d406092 \ No newline at end of file diff --git a/env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095 b/env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095 deleted file mode 120000 index 3ff882c..0000000 --- a/env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095 +++ /dev/null @@ -1 +0,0 @@ -pin-0032321756cee86a67171de425267c1d0d406092 \ No newline at end of file diff --git a/env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b b/env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b deleted file mode 120000 index 3ff882c..0000000 --- a/env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b +++ /dev/null @@ -1 +0,0 @@ -pin-0032321756cee86a67171de425267c1d0d406092 \ No newline at end of file diff --git a/env/pin-master-9deee01508bb3b8ad25822f53d91d9f54eae9a0b b/env/pin-master-9deee01508bb3b8ad25822f53d91d9f54eae9a0b new file mode 120000 index 0000000..033d08c --- /dev/null +++ b/env/pin-master-9deee01508bb3b8ad25822f53d91d9f54eae9a0b @@ -0,0 +1 @@ +env-2020-05-24-test-suite-9fe5172ca31 \ No newline at end of file diff --git a/env/pin-sample b/env/pin-sample deleted file mode 100644 index 7ae7cbc..0000000 --- a/env/pin-sample +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# This pinning file can be used for development (before committing to master) -# like this: -# -# make test-suite LIBYAML_TEST_SUITE_ENV=pin - -# yaml-test-suite info: -LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite -LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17 - -# Start with a base blacklist (optional): -source "$ENV/blacklist-001" - -# Add to blacklists: -blacklist parser XXX1 XXX2 -blacklist parser-error XXX3 XXX4 -blacklist emitter XXX5 XXX6 - -# Whitelist (un-blacklist) certain tests: -whitelist parser AAA1 AAA2 -whitelist parser-error AAA3 AAA4 -whitelist emitter AAA5 AAA6 - -whitelist parser 2JQS diff --git a/lookup b/lookup index df0d5bb..cd15707 100755 --- a/lookup +++ b/lookup @@ -22,11 +22,17 @@ main() { base=$(cd ../.. && pwd) head=$(get-head-commit) + if [[ $lookup == default-warning ]]; then + default-warning + exit + fi + if [[ ${LIBYAML_TEST_SUITE_ENV-} ]]; then lookup-env else [[ $env ]] || lookup-log [[ $env ]] || lookup-local + [[ $env ]] || lookup-default fi if [[ $lookup == data ]]; then @@ -103,15 +109,25 @@ lookup-log() { lookup-local() { debug "Looking for local env for '$head'" - local e=env/pin-$head + e=$(echo env/pin-*-$head) if [[ -e $e ]]; then env=$e - source "./env/pin-$head" + source "$env" found fi } +lookup-default() { + debug "Using default environment 'env/default'" + + [[ $lookup == env ]] && default-warning + + env=env/default + source "$env" + found +} + found() { debug "Found '$env'" @@ -119,14 +135,54 @@ found() { commit=$LIBYAML_TEST_SUITE_DATA_COMMIT } +default-warning() ( + main_branch=$(cd "$base" && git rev-parse --abbrev-ref HEAD) + test_branch=$(git rev-parse --abbrev-ref HEAD) + + default=env/default + while [[ -h $default ]]; do + default=$(readlink "$default") + done + + message-in-a-box " +WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING + +Using the default test-suite env: + + env/$default + +No test-suite environment info was found for the HEAD of '$main_branch': + + $head + +This info is kept in tests/run-test-suite/env/ + +For more information, see: + +* https://github.com/yaml/libyaml/blob/$test_branch/ReadMe.md +* https://github.com/yaml/libyaml/tree/$test_branch/env +" +) + whitelist() (:) blacklist() (:) die() { echo "Died: $*" >&2; exit 1; } warn() ( echo "$*" >&2 ) debug() ( - if [[ ${LIBYAML_TEST_SUITE_DEBUG-} ]]; then + if [[ ${LIBYAML_TEST_SUITE_DEBUG-} && $lookup != default-warning ]]; then warn "[DEBUG] $*" fi ) +message-in-a-box() ( + len=${COLUMNS:-$(tput cols)} + len=${len:-80} + line=$(eval "printf '*%.0s' {1..$len}") + + ( + echo "$line" + echo "* ${1//$'\n'/$'\n'* }" + echo "$line" + ) >&2 +) main "$@" -- cgit v1.2.1