From 2e18b77f5b6926e6616ce22d2d3e6d511de69c9b Mon Sep 17 00:00:00 2001 From: "Arnold D. Robbins" Date: Thu, 16 Mar 2023 18:25:43 +0200 Subject: Start revamp of CSV handling. --- test/ChangeLog | 7 +++++++ test/Gentests | 11 +++++++++++ test/Makefile.am | 5 ++++- test/Makefile.in | 7 +++++-- test/Maketests | 2 +- test/badargs.ok | 1 + test/csv1.awk | 6 +++--- 7 files changed, 32 insertions(+), 7 deletions(-) (limited to 'test') diff --git a/test/ChangeLog b/test/ChangeLog index e206fa9b..f247c540 100644 --- a/test/ChangeLog +++ b/test/ChangeLog @@ -1,3 +1,10 @@ +2023-03-16 Arnold D. Robbins + + * Gentests: Handle NEED_CSV. + * Makefile.am (NEED_CSV): New list of tests that need --csv. + * badargs.ok: Update after code changes. + * csv1.awk: Adjust after code changes. + 2023-03-09 Arnold D. Robbins * badargs.ok: Update after code changes. diff --git a/test/Gentests b/test/Gentests index 42a81cff..b3a8f787 100755 --- a/test/Gentests +++ b/test/Gentests @@ -108,6 +108,13 @@ BEGIN { next } +/^NEED_CSV *=/,/[^\\]$/ { + gsub(/(^NEED_CSV *=|\\$)/,"") + for (i = 1; i <= NF; i++) + csv[$i] + next +} + /^GENTESTS_UNUSED *=/,/[^\\]$/ { gsub(/(^GENTESTS_UNUSED *=|\\$)/,"") for (i = 1; i <= NF; i++) @@ -229,6 +236,10 @@ function generate(x, s, i, locale_string) s = s " --re-interval" delete re_interval[x] } + if (x in csv) { + s = s " --csv" + delete csv[x] + } if (x".in" in files) { s = s " < \"$(srcdir)\"/$@.in" delete files[x".in"] diff --git a/test/Makefile.am b/test/Makefile.am index 6d000178..7bb4c983 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1604,9 +1604,12 @@ NEED_SANDBOX = sandbox1 # List of tests that need --traditional NEED_TRADITIONAL = litoct tradanch rscompat -# Lists of tests that need the PMA allocator and a backing file +# List of tests that need the PMA allocator and a backing file NEED_PMA = pma +# List of tests that need --csv +NEED_CSV = csv1 + # Lists of tests that run a shell script RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01 diff --git a/test/Makefile.in b/test/Makefile.in index 1cd775d1..28c5ebc5 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -1868,9 +1868,12 @@ NEED_SANDBOX = sandbox1 # List of tests that need --traditional NEED_TRADITIONAL = litoct tradanch rscompat -# Lists of tests that need the PMA allocator and a backing file +# List of tests that need the PMA allocator and a backing file NEED_PMA = pma +# List of tests that need --csv +NEED_CSV = csv1 + # Lists of tests that run a shell script RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01 @@ -4484,7 +4487,7 @@ crlf: csv1: @echo $@ - @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ + @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ dbugeval2: diff --git a/test/Maketests b/test/Maketests index 628ff3fa..d284aab6 100644 --- a/test/Maketests +++ b/test/Maketests @@ -1414,7 +1414,7 @@ crlf: csv1: @echo $@ - @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ + @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ dbugeval2: diff --git a/test/badargs.ok b/test/badargs.ok index d2c67cac..1d79bc78 100644 --- a/test/badargs.ok +++ b/test/badargs.ok @@ -17,6 +17,7 @@ Short options: GNU long options: (extensions) -h --help -i includefile --include=includefile -I --trace + -k --csv -l library --load=library -L[fatal|invalid|no-ext] --lint[=fatal|invalid|no-ext] -M --bignum diff --git a/test/csv1.awk b/test/csv1.awk index 12bbf1e5..4896ef7c 100644 --- a/test/csv1.awk +++ b/test/csv1.awk @@ -1,6 +1,6 @@ -BEGIN { - FS = "," -} +# BEGIN { +# FS = "," +# } { printf(" \t%s\t", $0) -- cgit v1.2.1 From 428e9bffb9a3410452d76b142b24d6e8d48397ab Mon Sep 17 00:00:00 2001 From: "Arnold D. Robbins" Date: Fri, 17 Mar 2023 15:22:54 +0200 Subject: Update split() for CSV data. --- test/ChangeLog | 5 +++++ test/Makefile.am | 7 ++++--- test/Makefile.in | 12 +++++++++--- test/Maketests | 5 +++++ test/csv2.awk | 34 ++++++++++++++++++++++++++++++++++ test/csv2.ok | 0 6 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 test/csv2.awk create mode 100644 test/csv2.ok (limited to 'test') diff --git a/test/ChangeLog b/test/ChangeLog index f247c540..989d105b 100644 --- a/test/ChangeLog +++ b/test/ChangeLog @@ -1,3 +1,8 @@ +2023-03-17 Arnold D. Robbins + + * Makefile.am (EXTRA_DIST, NEED_CSV, GAWK_EXT_TESTS): New test, csv2. + * csv2.awk, csv2.ok: New files. + 2023-03-16 Arnold D. Robbins * Gentests: Handle NEED_CSV. diff --git a/test/Makefile.am b/test/Makefile.am index 7bb4c983..e0be391c 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -225,7 +225,8 @@ EXTRA_DIST = \ crlf.ok \ csv1.awk \ csv1.in \ - csv1.ok \ + csv2.awk \ + csv2.ok \ datanonl.awk \ datanonl.in \ datanonl.ok \ @@ -1506,7 +1507,7 @@ GAWK_EXT_TESTS = \ aadelete1 aadelete2 aarray1 aasort aasorti argtest arraysort \ arraysort2 arraytype asortbool backw badargs beginfile1 beginfile2 \ binmode1 charasbytes clos1way clos1way2 clos1way3 clos1way4 \ - clos1way5 clos1way6 colonwarn commas crlf csv1 dbugeval dbugeval2 \ + clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 dbugeval dbugeval2 \ dbugeval3 dbugeval4 dbugtypedre1 dbugtypedre2 delsub \ devfd devfd1 devfd2 dfacheck1 dumpvars \ elemnew1 elemnew2 elemnew3 errno exit fieldwdth forcenum fpat1 fpat2 \ @@ -1608,7 +1609,7 @@ NEED_TRADITIONAL = litoct tradanch rscompat NEED_PMA = pma # List of tests that need --csv -NEED_CSV = csv1 +NEED_CSV = csv1 csv2 # Lists of tests that run a shell script RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01 diff --git a/test/Makefile.in b/test/Makefile.in index 28c5ebc5..ce2c7744 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -489,7 +489,8 @@ EXTRA_DIST = \ crlf.ok \ csv1.awk \ csv1.in \ - csv1.ok \ + csv2.awk \ + csv2.ok \ datanonl.awk \ datanonl.in \ datanonl.ok \ @@ -1770,7 +1771,7 @@ GAWK_EXT_TESTS = \ aadelete1 aadelete2 aarray1 aasort aasorti argtest arraysort \ arraysort2 arraytype asortbool backw badargs beginfile1 beginfile2 \ binmode1 charasbytes clos1way clos1way2 clos1way3 clos1way4 \ - clos1way5 clos1way6 colonwarn commas crlf csv1 dbugeval dbugeval2 \ + clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 dbugeval dbugeval2 \ dbugeval3 dbugeval4 dbugtypedre1 dbugtypedre2 delsub \ devfd devfd1 devfd2 dfacheck1 dumpvars \ elemnew1 elemnew2 elemnew3 errno exit fieldwdth forcenum fpat1 fpat2 \ @@ -1872,7 +1873,7 @@ NEED_TRADITIONAL = litoct tradanch rscompat NEED_PMA = pma # List of tests that need --csv -NEED_CSV = csv1 +NEED_CSV = csv1 csv2 # Lists of tests that run a shell script RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01 @@ -4490,6 +4491,11 @@ csv1: @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ +csv2: + @echo $@ + @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ + @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ + dbugeval2: @echo $@ @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --debug < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ diff --git a/test/Maketests b/test/Maketests index d284aab6..ed1a050e 100644 --- a/test/Maketests +++ b/test/Maketests @@ -1417,6 +1417,11 @@ csv1: @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ +csv2: + @echo $@ + @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ + @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ + dbugeval2: @echo $@ @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --debug < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ diff --git a/test/csv2.awk b/test/csv2.awk new file mode 100644 index 00000000..93cbca65 --- /dev/null +++ b/test/csv2.awk @@ -0,0 +1,34 @@ +BEGIN { + text[1] = "a"; expected[1] = 1 + text[2] = " a"; expected[2] = 1 + text[3] = ",a"; expected[3] = 2 + text[4] = " , a"; expected[4] = 2 + text[5] = "a,b"; expected[5] = 2 + text[6] = "a,b,c"; expected[6] = 3 + text[7] = "\"\""; expected[7] = 1 + text[8] = "\"abc\""; expected[8] = 1 + text[9] = "\"a\"\"b\""; expected[9] = 1 + text[10] = "\"a\",\"b\""; expected[10] = 2 + text[11] = "a\"\"b"; expected[11] = 1 + text[12] = "\"a,b\""; expected[12] = 1 + text[13] = "\"\"\"\""; expected[13] = 1 + text[14] = "\"\"\"\"\"\""; expected[14] = 1 + text[15] = "\"\"\"x\"\"\""; expected[15] = 1 + text[16] = ",,\"\""; expected[16] = 3 + text[17] = "a\"\"b"; expected[17] = 1 + text[18] = "a\"b"; expected[18] = 1 + text[19] = "a''b"; expected[19] = 1 + text[20] = "\"abc"; expected[20] = 1 + text[21] = ",,"; expected[21] = 3 + text[22] = "a,"; expected[22] = 2 + text[23] = "\"\","; expected[23] = 2 + text[24] = ","; expected[24] = 2 + text[25] = "\"abc\",def"; expected[25] = 2 + + for (i = 1; i <= length(text); i++) { + n = split(text[i], array) + if (n != expected[i]) + printf("text[%d] = <%s>, expected %d, got %d\n", + i, text[i], expected[i], n) + } +} diff --git a/test/csv2.ok b/test/csv2.ok new file mode 100644 index 00000000..e69de29b -- cgit v1.2.1 From 927143821e5124bb30cae744e6767b4d75463d18 Mon Sep 17 00:00:00 2001 From: "Arnold D. Robbins" Date: Tue, 21 Mar 2023 22:03:35 +0200 Subject: Add initial csvscan() routine. --- test/ChangeLog | 5 +++++ test/Makefile.am | 9 +++++++-- test/Makefile.in | 14 ++++++++++++-- test/Maketests | 5 +++++ 4 files changed, 29 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/ChangeLog b/test/ChangeLog index 989d105b..8e0791a3 100644 --- a/test/ChangeLog +++ b/test/ChangeLog @@ -1,3 +1,8 @@ +2023-03-21 Arnold D. Robbins + + * Makefile.am (EXTRA_DIST, NEED_CSV, GAWK_EXT_TESTS): New test, csv3. + * csv3.awk, csv3.in, csv3.ok: New files. + 2023-03-17 Arnold D. Robbins * Makefile.am (EXTRA_DIST, NEED_CSV, GAWK_EXT_TESTS): New test, csv2. diff --git a/test/Makefile.am b/test/Makefile.am index e0be391c..d8f5ea40 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -225,8 +225,12 @@ EXTRA_DIST = \ crlf.ok \ csv1.awk \ csv1.in \ + csv1.ok \ csv2.awk \ csv2.ok \ + csv3.awk \ + csv3.in \ + csv3.ok \ datanonl.awk \ datanonl.in \ datanonl.ok \ @@ -1507,7 +1511,8 @@ GAWK_EXT_TESTS = \ aadelete1 aadelete2 aarray1 aasort aasorti argtest arraysort \ arraysort2 arraytype asortbool backw badargs beginfile1 beginfile2 \ binmode1 charasbytes clos1way clos1way2 clos1way3 clos1way4 \ - clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 dbugeval dbugeval2 \ + clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 csv3 \ + dbugeval dbugeval2 \ dbugeval3 dbugeval4 dbugtypedre1 dbugtypedre2 delsub \ devfd devfd1 devfd2 dfacheck1 dumpvars \ elemnew1 elemnew2 elemnew3 errno exit fieldwdth forcenum fpat1 fpat2 \ @@ -1609,7 +1614,7 @@ NEED_TRADITIONAL = litoct tradanch rscompat NEED_PMA = pma # List of tests that need --csv -NEED_CSV = csv1 csv2 +NEED_CSV = csv1 csv2 csv3 # Lists of tests that run a shell script RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01 diff --git a/test/Makefile.in b/test/Makefile.in index ce2c7744..22f67f0a 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -489,8 +489,12 @@ EXTRA_DIST = \ crlf.ok \ csv1.awk \ csv1.in \ + csv1.ok \ csv2.awk \ csv2.ok \ + csv3.awk \ + csv3.in \ + csv3.ok \ datanonl.awk \ datanonl.in \ datanonl.ok \ @@ -1771,7 +1775,8 @@ GAWK_EXT_TESTS = \ aadelete1 aadelete2 aarray1 aasort aasorti argtest arraysort \ arraysort2 arraytype asortbool backw badargs beginfile1 beginfile2 \ binmode1 charasbytes clos1way clos1way2 clos1way3 clos1way4 \ - clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 dbugeval dbugeval2 \ + clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 csv3 \ + dbugeval dbugeval2 \ dbugeval3 dbugeval4 dbugtypedre1 dbugtypedre2 delsub \ devfd devfd1 devfd2 dfacheck1 dumpvars \ elemnew1 elemnew2 elemnew3 errno exit fieldwdth forcenum fpat1 fpat2 \ @@ -1873,7 +1878,7 @@ NEED_TRADITIONAL = litoct tradanch rscompat NEED_PMA = pma # List of tests that need --csv -NEED_CSV = csv1 csv2 +NEED_CSV = csv1 csv2 csv3 # Lists of tests that run a shell script RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01 @@ -4496,6 +4501,11 @@ csv2: @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ +csv3: + @echo $@ + @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ + @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ + dbugeval2: @echo $@ @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --debug < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ diff --git a/test/Maketests b/test/Maketests index ed1a050e..6dae6a56 100644 --- a/test/Maketests +++ b/test/Maketests @@ -1422,6 +1422,11 @@ csv2: @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ +csv3: + @echo $@ + @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ + @-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@ + dbugeval2: @echo $@ @-AWKPATH="$(srcdir)" $(AWK) -f $@.awk --debug < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@ -- cgit v1.2.1 From 0089ada17feeb26a58b96c434cb82a58a83c5a67 Mon Sep 17 00:00:00 2001 From: "Arnold D. Robbins" Date: Thu, 23 Mar 2023 17:02:43 +0200 Subject: Adjust csv1 test. --- test/ChangeLog | 5 +++++ test/csv1.in | 4 ---- test/csv1.ok | 4 ---- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'test') diff --git a/test/ChangeLog b/test/ChangeLog index 8e0791a3..4b15eaa8 100644 --- a/test/ChangeLog +++ b/test/ChangeLog @@ -1,3 +1,8 @@ +2023-03-23 Arnold D. Robbins + + * csv1.in, csv1.ok: Adjusted after code change and to match + RFC 4180 format. + 2023-03-21 Arnold D. Robbins * Makefile.am (EXTRA_DIST, NEED_CSV, GAWK_EXT_TESTS): New test, csv3. diff --git a/test/csv1.in b/test/csv1.in index 620b2ab3..abdcf27e 100644 --- a/test/csv1.in +++ b/test/csv1.in @@ -8,16 +8,12 @@ a,b,c "abc" "a""b" "a","b" -a""b "a,b" """" """""" """x""" ,,"" -a""b -a"b a''b -"abc ,, a, "", diff --git a/test/csv1.ok b/test/csv1.ok index 178daed1..e25c3cb8 100644 --- a/test/csv1.ok +++ b/test/csv1.ok @@ -8,16 +8,12 @@ "abc" [abc] "a""b" [a"b] "a","b" [a][b] - a""b [a""b] "a,b" [a,b] """" ["] """""" [""] """x""" ["x"] ,,"" [][][] - a""b [a""b] - a"b [a"b] a''b [a''b] - "abc [abc] ,, [][][] a, [a][] "", [][] -- cgit v1.2.1 From 7af995289b05c5e5843facc4ba3a21672c2028ad Mon Sep 17 00:00:00 2001 From: "Arnold D. Robbins" Date: Thu, 23 Mar 2023 20:41:26 +0200 Subject: Add test files. --- test/csv3.awk | 6 ++++++ test/csv3.in | 13 +++++++++++++ test/csv3.ok | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 test/csv3.awk create mode 100644 test/csv3.in create mode 100644 test/csv3.ok (limited to 'test') diff --git a/test/csv3.awk b/test/csv3.awk new file mode 100644 index 00000000..ac4e182f --- /dev/null +++ b/test/csv3.awk @@ -0,0 +1,6 @@ +{ + print "Record:", NR + for (i = 1; i <= NF; i++) { + printf("\t$%d = <%s>\n", i, $i) + } +} diff --git a/test/csv3.in b/test/csv3.in new file mode 100644 index 00000000..d0f45223 --- /dev/null +++ b/test/csv3.in @@ -0,0 +1,13 @@ +Forename,Lastname,Address,State,Zipcode +John,Doe,"120 jefferson st. +Riverside", NJ, 08075 +Jack,McGinnis,"220 hobo Av. +Philadelphia", PA,09119 +"John ""Da Man""",Repici,"120 Jefferson St. +Riverside", NJ,08075 +Stephen,Tyler,"7452 Terrace ""At the Plaza"" road +SomeTown",SD, 91234 +,Blankman," +SomeTown", SD, 00298 +"Joan ""the bone"", Anne",Jet,"9th, at Terrace plc +Desert City",CO,00123 diff --git a/test/csv3.ok b/test/csv3.ok new file mode 100644 index 00000000..eadb878a --- /dev/null +++ b/test/csv3.ok @@ -0,0 +1,48 @@ +Record: 1 + $1 = + $2 = + $3 =
+ $4 = + $5 = +Record: 2 + $1 = + $2 = + $3 = <120 jefferson st. +Riverside> + $4 = < NJ> + $5 = < 08075> +Record: 3 + $1 = + $2 = + $3 = <220 hobo Av. +Philadelphia> + $4 = < PA> + $5 = <09119> +Record: 4 + $1 = + $2 = + $3 = <120 Jefferson St. +Riverside> + $4 = < NJ> + $5 = <08075> +Record: 5 + $1 = + $2 = + $3 = <7452 Terrace "At the Plaza" road +SomeTown> + $4 = + $5 = < 91234> +Record: 6 + $1 = <> + $2 = + $3 = < +SomeTown> + $4 = < SD> + $5 = < 00298> +Record: 7 + $1 = + $2 = + $3 = <9th, at Terrace plc +Desert City> + $4 = + $5 = <00123> -- cgit v1.2.1