Merge branch 'feature/csv-revamp'

14 files changed, 182 insertions, 17 deletions

diff --git a/test/ChangeLog b/test/ChangeLog
index 8853d068..db6720ea 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -4,6 +4,28 @@
 	* unicode1.awk, unicode1.ok: New files.
 	* funstack.ok: Updated after code change.
 
+2023-03-23         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* csv1.in, csv1.ok: Adjusted after code change and to match
+	RFC 4180 format.
+
+2023-03-21         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* Makefile.am (EXTRA_DIST, NEED_CSV, GAWK_EXT_TESTS): New test, csv3.
+	* csv3.awk, csv3.in, csv3.ok: New files.
+
+2023-03-17         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* Makefile.am (EXTRA_DIST, NEED_CSV, GAWK_EXT_TESTS): New test, csv2.
+	* csv2.awk, csv2.ok: New files.
+
+2023-03-16         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* Gentests: Handle NEED_CSV.
+	* Makefile.am (NEED_CSV): New list of tests that need --csv.
+	* badargs.ok: Update after code changes.
+	* csv1.awk: Adjust after code changes.
+
 2023-03-09         Arnold D. Robbins     <arnold@skeeve.com>
 
 	* badargs.ok: Update after code changes.
diff --git a/test/Gentests b/test/Gentests
index 42a81cff..b3a8f787 100755
--- a/test/Gentests
+++ b/test/Gentests
@@ -108,6 +108,13 @@ BEGIN {
 	next
 }
 
+/^NEED_CSV *=/,/[^\\]$/ {
+	gsub(/(^NEED_CSV *=|\\$)/,"")
+	for (i = 1; i <= NF; i++)
+		csv[$i]
+	next
+}
+
 /^GENTESTS_UNUSED *=/,/[^\\]$/ {
 	gsub(/(^GENTESTS_UNUSED *=|\\$)/,"")
 	for (i = 1; i <= NF; i++)
@@ -229,6 +236,10 @@ function generate(x,	s, i, locale_string)
 		s = s " --re-interval"
 		delete re_interval[x]
 	}
+	if (x in csv) {
+		s = s " --csv"
+		delete csv[x]
+	}
 	if (x".in" in files) {
 		s = s " < \"$(srcdir)\"/$@.in"
 		delete files[x".in"]
diff --git a/test/Makefile.am b/test/Makefile.am
index 00034a15..026db35f 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -226,6 +226,11 @@ EXTRA_DIST = \
 	csv1.awk \
 	csv1.in \
 	csv1.ok \
+	csv2.awk \
+	csv2.ok \
+	csv3.awk \
+	csv3.in \
+	csv3.ok \
 	datanonl.awk \
 	datanonl.in \
 	datanonl.ok \
@@ -1508,7 +1513,8 @@ GAWK_EXT_TESTS = \
 	aadelete1 aadelete2 aarray1 aasort aasorti argtest arraysort \
 	arraysort2 arraytype asortbool backw badargs beginfile1 beginfile2 \
 	binmode1 charasbytes clos1way clos1way2 clos1way3 clos1way4 \
-	clos1way5 clos1way6 colonwarn commas crlf csv1 dbugeval dbugeval2 \
+	clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 csv3 \
+	dbugeval dbugeval2 \
 	dbugeval3 dbugeval4 dbugtypedre1 dbugtypedre2 delsub \
 	devfd devfd1 devfd2 dfacheck1 dumpvars \
 	elemnew1 elemnew2 elemnew3 errno exit fieldwdth forcenum fpat1 fpat2 \
@@ -1606,9 +1612,12 @@ NEED_SANDBOX = sandbox1
 # List of tests that need --traditional
 NEED_TRADITIONAL = litoct tradanch rscompat
 
-# Lists of tests that need the PMA allocator and a backing file
+# List of tests that need the PMA allocator and a backing file
 NEED_PMA = pma
 
+# List of tests that need --csv
+NEED_CSV = csv1 csv2 csv3
+
 # Lists of tests that run a shell script
 RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01
 
diff --git a/test/Makefile.in b/test/Makefile.in
index d2343a7d..4ef4c5df 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -490,6 +490,11 @@ EXTRA_DIST = \
 	csv1.awk \
 	csv1.in \
 	csv1.ok \
+	csv2.awk \
+	csv2.ok \
+	csv3.awk \
+	csv3.in \
+	csv3.ok \
 	datanonl.awk \
 	datanonl.in \
 	datanonl.ok \
@@ -1772,7 +1777,8 @@ GAWK_EXT_TESTS = \
 	aadelete1 aadelete2 aarray1 aasort aasorti argtest arraysort \
 	arraysort2 arraytype asortbool backw badargs beginfile1 beginfile2 \
 	binmode1 charasbytes clos1way clos1way2 clos1way3 clos1way4 \
-	clos1way5 clos1way6 colonwarn commas crlf csv1 dbugeval dbugeval2 \
+	clos1way5 clos1way6 colonwarn commas crlf csv1 csv2 csv3 \
+	dbugeval dbugeval2 \
 	dbugeval3 dbugeval4 dbugtypedre1 dbugtypedre2 delsub \
 	devfd devfd1 devfd2 dfacheck1 dumpvars \
 	elemnew1 elemnew2 elemnew3 errno exit fieldwdth forcenum fpat1 fpat2 \
@@ -1870,9 +1876,12 @@ NEED_SANDBOX = sandbox1
 # List of tests that need --traditional
 NEED_TRADITIONAL = litoct tradanch rscompat
 
-# Lists of tests that need the PMA allocator and a backing file
+# List of tests that need the PMA allocator and a backing file
 NEED_PMA = pma
 
+# List of tests that need --csv
+NEED_CSV = csv1 csv2 csv3
+
 # Lists of tests that run a shell script
 RUN_SHELL = exit fflush localenl modifiers next randtest rtlen rtlen01
 
@@ -4486,7 +4495,17 @@ crlf:
 
 csv1:
 	@echo $@
-	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
+	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
+	@-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@
+
+csv2:
+	@echo $@
+	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  --csv >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
+	@-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@
+
+csv3:
+	@echo $@
+	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
 	@-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@
 
 dbugeval2:
diff --git a/test/Maketests b/test/Maketests
index 09d9c02c..01db7413 100644
--- a/test/Maketests
+++ b/test/Maketests
@@ -1414,7 +1414,17 @@ crlf:
 
 csv1:
 	@echo $@
-	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
+	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
+	@-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@
+
+csv2:
+	@echo $@
+	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  --csv >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
+	@-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@
+
+csv3:
+	@echo $@
+	@-AWKPATH="$(srcdir)" $(AWK) -f $@.awk  --csv < "$(srcdir)"/$@.in >_$@ 2>&1 || echo EXIT CODE: $$? >>_$@
 	@-$(CMP) "$(srcdir)"/$@.ok _$@ && rm -f _$@
 
 dbugeval2:
diff --git a/test/badargs.ok b/test/badargs.ok
index d2c67cac..1d79bc78 100644
--- a/test/badargs.ok
+++ b/test/badargs.ok
@@ -17,6 +17,7 @@ Short options:		GNU long options: (extensions)
 	-h			--help
 	-i includefile		--include=includefile
 	-I			--trace
+	-k			--csv
 	-l library		--load=library
 	-L[fatal|invalid|no-ext]	--lint[=fatal|invalid|no-ext]
 	-M			--bignum
diff --git a/test/csv1.awk b/test/csv1.awk
index 12bbf1e5..4896ef7c 100644
--- a/test/csv1.awk
+++ b/test/csv1.awk
@@ -1,6 +1,6 @@
-BEGIN {
-	FS = ","
-}
+# BEGIN {
+# 	FS = ","
+# }
 
 {
 	printf(" \t%s\t", $0)
diff --git a/test/csv1.in b/test/csv1.in
index 620b2ab3..abdcf27e 100644
--- a/test/csv1.in
+++ b/test/csv1.in
@@ -8,16 +8,12 @@ a,b,c
 "abc"
 "a""b"
 "a","b"
-a""b
 "a,b"
 """"
 """"""
 """x"""
 ,,""
-a""b
-a"b
 a''b
-"abc
 ,,
 a,
 "",
diff --git a/test/csv1.ok b/test/csv1.ok
index 178daed1..e25c3cb8 100644
--- a/test/csv1.ok
+++ b/test/csv1.ok
@@ -8,16 +8,12 @@
  	"abc"	[abc]
  	"a""b"	[a"b]
  	"a","b"	[a][b]
- 	a""b	[a""b]
  	"a,b"	[a,b]
  	""""	["]
  	""""""	[""]
  	"""x"""	["x"]
  	,,""	[][][]
- 	a""b	[a""b]
- 	a"b	[a"b]
  	a''b	[a''b]
- 	"abc	[abc]
  	,,	[][][]
  	a,	[a][]
  	"",	[][]
diff --git a/test/csv2.awk b/test/csv2.awk
new file mode 100644
index 00000000..93cbca65
--- /dev/null
+++ b/test/csv2.awk
@@ -0,0 +1,34 @@
+BEGIN {
+	text[1] = "a";			expected[1] = 1
+	text[2] = "  a";		expected[2] = 1
+	text[3] = ",a";			expected[3] = 2
+	text[4] = " , a";		expected[4] = 2
+	text[5] = "a,b";		expected[5] = 2
+	text[6] = "a,b,c";		expected[6] = 3
+	text[7] = "\"\"";		expected[7] = 1
+	text[8] = "\"abc\"";		expected[8] = 1
+	text[9] = "\"a\"\"b\"";		expected[9] = 1
+	text[10] = "\"a\",\"b\"";	expected[10] = 2
+	text[11] = "a\"\"b";		expected[11] = 1
+	text[12] = "\"a,b\"";		expected[12] = 1
+	text[13] = "\"\"\"\"";		expected[13] = 1
+	text[14] = "\"\"\"\"\"\"";	expected[14] = 1
+	text[15] = "\"\"\"x\"\"\"";	expected[15] = 1
+	text[16] = ",,\"\"";		expected[16] = 3
+	text[17] = "a\"\"b";		expected[17] = 1
+	text[18] = "a\"b";		expected[18] = 1
+	text[19] = "a''b";		expected[19] = 1
+	text[20] = "\"abc";		expected[20] = 1
+	text[21] = ",,";		expected[21] = 3
+	text[22] = "a,";		expected[22] = 2
+	text[23] = "\"\",";		expected[23] = 2
+	text[24] = ",";			expected[24] = 2
+	text[25] = "\"abc\",def";	expected[25] = 2
+
+	for (i = 1; i <= length(text); i++) {
+		n = split(text[i], array)
+		if (n != expected[i])
+			printf("text[%d] = <%s>, expected %d, got %d\n",
+			       i, text[i], expected[i], n)
+	}
+}
diff --git a/test/csv2.ok b/test/csv2.ok
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/csv2.ok
diff --git a/test/csv3.awk b/test/csv3.awk
new file mode 100644
index 00000000..ac4e182f
--- /dev/null
+++ b/test/csv3.awk
@@ -0,0 +1,6 @@
+{
+	print "Record:", NR
+	for (i = 1; i <= NF; i++) {
+		printf("\t$%d = <%s>\n", i, $i)
+	}
+}
diff --git a/test/csv3.in b/test/csv3.in
new file mode 100644
index 00000000..d0f45223
--- /dev/null
+++ b/test/csv3.in
@@ -0,0 +1,13 @@
+Forename,Lastname,Address,State,Zipcode
+John,Doe,"120 jefferson st.
+Riverside", NJ, 08075
+Jack,McGinnis,"220 hobo Av.
+Philadelphia", PA,09119
+"John ""Da Man""",Repici,"120 Jefferson St.
+Riverside", NJ,08075
+Stephen,Tyler,"7452 Terrace ""At the Plaza"" road
+SomeTown",SD, 91234
+,Blankman,"
+SomeTown", SD, 00298
+"Joan ""the bone"", Anne",Jet,"9th, at Terrace plc
+Desert City",CO,00123
diff --git a/test/csv3.ok b/test/csv3.ok
new file mode 100644
index 00000000..eadb878a
--- /dev/null
+++ b/test/csv3.ok
@@ -0,0 +1,48 @@
+Record: 1
+	$1 = <Forename>
+	$2 = <Lastname>
+	$3 = <Address>
+	$4 = <State>
+	$5 = <Zipcode>
+Record: 2
+	$1 = <John>
+	$2 = <Doe>
+	$3 = <120 jefferson st.
+Riverside>
+	$4 = < NJ>
+	$5 = < 08075>
+Record: 3
+	$1 = <Jack>
+	$2 = <McGinnis>
+	$3 = <220 hobo Av.
+Philadelphia>
+	$4 = < PA>
+	$5 = <09119>
+Record: 4
+	$1 = <John "Da Man">
+	$2 = <Repici>
+	$3 = <120 Jefferson St.
+Riverside>
+	$4 = < NJ>
+	$5 = <08075>
+Record: 5
+	$1 = <Stephen>
+	$2 = <Tyler>
+	$3 = <7452 Terrace "At the Plaza" road
+SomeTown>
+	$4 = <SD>
+	$5 = < 91234>
+Record: 6
+	$1 = <>
+	$2 = <Blankman>
+	$3 = <
+SomeTown>
+	$4 = < SD>
+	$5 = < 00298>
+Record: 7
+	$1 = <Joan "the bone", Anne>
+	$2 = <Jet>
+	$3 = <9th, at Terrace plc
+Desert City>
+	$4 = <CO>
+	$5 = <00123>