PR/573: Nick Terrell: Add zstd support

46 files changed, 100 insertions, 2 deletions

diff --git a/magic/Magdir/compress b/magic/Magdir/compress
index 910545a7..3297d5dc 100644
--- a/magic/Magdir/compress
+++ b/magic/Magdir/compress
@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# $File: compress,v 1.65 2015/12/04 20:48:03 christos Exp $
+# $File: compress,v 1.66 2016/09/16 12:12:05 christos Exp $
 # compress:  file(1) magic for pure-compression formats (no archives)
 #
 # compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc.
@@ -220,6 +220,56 @@
 0	lelong		0x184c2102	LZ4 compressed data (v0.1-v0.9)
 !:mime	application/x-lz4
 
+# Zstandard/LZ4 skippable frames
+# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
+0         lelong&0xFFFFFFF0  0x184D2A50
+>(4.l+8)  indirect
+
+# Zstandard Dictionary ID subroutine
+0     name        zstd-dictionary-id
+# Single Segment = True
+>0    byte        &0x20   \b, Dictionary ID:
+>>0   byte&0x03   0       None
+>>0   byte&0x03   1
+>>>1  byte        x       %u
+>>0   byte&0x03   2
+>>>1  leshort     x       %u
+>>0   byte&0x03   3
+>>>1  lelong      x       %u
+# Single Segment = False
+>0    byte        ^0x20   \b, Dictionary ID:
+>>0   byte&0x03   0       None
+>>0   byte&0x03   1
+>>>2  byte        x       %u
+>>0   byte&0x03   2
+>>>2  leshort     x       %u
+>>0   byte&0x03   3
+>>>2  lelong      x       %u
+
+# Zstandard compressed data
+# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
+0     lelong       0xFD2FB522  Zstandard compressed data (v0.2)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB523  Zstandard compressed data (v0.3)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB524  Zstandard compressed data (v0.4)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB525  Zstandard compressed data (v0.5)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB526  Zstandard compressed data (v0.6)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB527  Zstandard compressed data (v0.7)
+!:mime  application/x-zstd
+>4    use          zstd-dictionary-id
+0     lelong       0xFD2FB528  Zstandard compressed data (v0.8+)
+!:mime  application/x-zstd
+>4    use          zstd-dictionary-id
+
+# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
+0  lelong    0xEC30A437  Zstandard dictionary
+!:mime  application/x-zstd-dictionary
+>4 lelong    x           (ID %u)
+
 # AFX compressed files (Wolfram Kleff)
 2	string		-afx-		AFX compressed file data
 
diff --git a/src/compress.c b/src/compress.c
index cbcad5e6..63f3106f 100644
--- a/src/compress.c
+++ b/src/compress.c
@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: compress.c,v 1.98 2016/06/28 16:38:26 christos Exp $")
+FILE_RCSID("@(#)$File: compress.c,v 1.99 2016/09/16 12:12:05 christos Exp $")
 #endif
 
 #include "magic.h"
@@ -132,6 +132,9 @@ static const char *lrzip_args[] = {
 static const char *lz4_args[] = {
 	"lz4", "-cd", NULL
 };
+static const char *zstd_args[] = {
+	"zstd", "-cd", NULL
+};
 
 private const struct {
 	const void *magic;
@@ -154,6 +157,7 @@ private const struct {
  	{ "\3757zXZ\0",	6, xz_args },		/* XZ Utils */
  	{ "LRZI",	4, lrzip_args },	/* LRZIP */
  	{ "\004\"M\030",4, lz4_args },		/* LZ4 */
+ 	{ "\x28\xB5\x2F\xFD", 4, zstd_args },	/* zstd */
 #ifdef ZLIBSUPPORT
 	{ zlibcmp,	0, zlib_args },		/* zlib */
 #endif
diff --git a/tests/zstd-3-skippable-frames.result b/tests/zstd-3-skippable-frames.result
new file mode 100644
index 00000000..468070c3
--- /dev/null
+++ b/tests/zstd-3-skippable-frames.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 1
\ No newline at end of file
diff --git a/tests/zstd-dictionary-0.result b/tests/zstd-dictionary-0.result
new file mode 100644
index 00000000..e1b9c620
--- /dev/null
+++ b/tests/zstd-dictionary-0.result
@@ -0,0 +1 @@
+Zstandard dictionary (ID 0)
\ No newline at end of file
diff --git a/tests/zstd-dictionary-1.result b/tests/zstd-dictionary-1.result
new file mode 100644
index 00000000..dd9f5f1b
--- /dev/null
+++ b/tests/zstd-dictionary-1.result
@@ -0,0 +1 @@
+Zstandard dictionary (ID 1)
\ No newline at end of file
diff --git a/tests/zstd-dictionary-2.result b/tests/zstd-dictionary-2.result
new file mode 100644
index 00000000..e1ac5b3a
--- /dev/null
+++ b/tests/zstd-dictionary-2.result
@@ -0,0 +1 @@
+Zstandard dictionary (ID 285212672)
\ No newline at end of file
diff --git a/tests/zstd-skippable-frame-0.result b/tests/zstd-skippable-frame-0.result
new file mode 100644
index 00000000..a4e3e482
--- /dev/null
+++ b/tests/zstd-skippable-frame-0.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.2)
\ No newline at end of file
diff --git a/tests/zstd-skippable-frame-4.result b/tests/zstd-skippable-frame-4.result
new file mode 100644
index 00000000..cecb5e71
--- /dev/null
+++ b/tests/zstd-skippable-frame-4.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.3)
\ No newline at end of file
diff --git a/tests/zstd-skippable-frame-8.result b/tests/zstd-skippable-frame-8.result
new file mode 100644
index 00000000..528f701d
--- /dev/null
+++ b/tests/zstd-skippable-frame-8.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.4)
\ No newline at end of file
diff --git a/tests/zstd-skippable-frame-C.result b/tests/zstd-skippable-frame-C.result
new file mode 100644
index 00000000..468070c3
--- /dev/null
+++ b/tests/zstd-skippable-frame-C.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 1
\ No newline at end of file
diff --git a/tests/zstd-v0.2-FF.result b/tests/zstd-v0.2-FF.result
new file mode 100644
index 00000000..a4e3e482
--- /dev/null
+++ b/tests/zstd-v0.2-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.2)
\ No newline at end of file
diff --git a/tests/zstd-v0.2-FF.testfile b/tests/zstd-v0.2-FF.testfile
new file mode 100644
index 00000000..6fe4f273
--- /dev/null
+++ b/tests/zstd-v0.2-FF.testfile
@@ -0,0 +1 @@
+"µ/ýÿ

\ No newline at end of file
diff --git a/tests/zstd-v0.3-FF.result b/tests/zstd-v0.3-FF.result
new file mode 100644
index 00000000..cecb5e71
--- /dev/null
+++ b/tests/zstd-v0.3-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.3)
\ No newline at end of file
diff --git a/tests/zstd-v0.3-FF.testfile b/tests/zstd-v0.3-FF.testfile
new file mode 100644
index 00000000..dc504691
--- /dev/null
+++ b/tests/zstd-v0.3-FF.testfile
@@ -0,0 +1 @@
+#µ/ýÿ

\ No newline at end of file
diff --git a/tests/zstd-v0.4-FF.result b/tests/zstd-v0.4-FF.result
new file mode 100644
index 00000000..528f701d
--- /dev/null
+++ b/tests/zstd-v0.4-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.4)
\ No newline at end of file
diff --git a/tests/zstd-v0.4-FF.testfile b/tests/zstd-v0.4-FF.testfile
new file mode 100644
index 00000000..f2768a21
--- /dev/null
+++ b/tests/zstd-v0.4-FF.testfile
@@ -0,0 +1 @@
+$µ/ýÿ

\ No newline at end of file
diff --git a/tests/zstd-v0.5-FF.result b/tests/zstd-v0.5-FF.result
new file mode 100644
index 00000000..815c80d3
--- /dev/null
+++ b/tests/zstd-v0.5-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.5)
\ No newline at end of file
diff --git a/tests/zstd-v0.5-FF.testfile b/tests/zstd-v0.5-FF.testfile
new file mode 100644
index 00000000..a25f337b
--- /dev/null
+++ b/tests/zstd-v0.5-FF.testfile
@@ -0,0 +1 @@
+%µ/ýÿ

\ No newline at end of file
diff --git a/tests/zstd-v0.6-FF.result b/tests/zstd-v0.6-FF.result
new file mode 100644
index 00000000..b28af6b2
--- /dev/null
+++ b/tests/zstd-v0.6-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.6)
\ No newline at end of file
diff --git a/tests/zstd-v0.6-FF.testfile b/tests/zstd-v0.6-FF.testfile
new file mode 100644
index 00000000..1c8ca598
--- /dev/null
+++ b/tests/zstd-v0.6-FF.testfile
@@ -0,0 +1 @@
+&µ/ýÿ

\ No newline at end of file
diff --git a/tests/zstd-v0.7-00.result b/tests/zstd-v0.7-00.result
new file mode 100644
index 00000000..72e0e27a
--- /dev/null
+++ b/tests/zstd-v0.7-00.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.7), Dictionary ID: None
\ No newline at end of file
diff --git a/tests/zstd-v0.7-21.result b/tests/zstd-v0.7-21.result
new file mode 100644
index 00000000..a4d43c81
--- /dev/null
+++ b/tests/zstd-v0.7-21.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.7), Dictionary ID: 1
\ No newline at end of file
diff --git a/tests/zstd-v0.7-21.testfile b/tests/zstd-v0.7-21.testfile
new file mode 100644
index 00000000..b40294ea
--- /dev/null
+++ b/tests/zstd-v0.7-21.testfile
@@ -0,0 +1 @@
+'µ/ý!

\ No newline at end of file
diff --git a/tests/zstd-v0.7-22.result b/tests/zstd-v0.7-22.result
new file mode 100644
index 00000000..b4336c9a
--- /dev/null
+++ b/tests/zstd-v0.7-22.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.7), Dictionary ID: 513
\ No newline at end of file
diff --git a/tests/zstd-v0.7-22.testfile b/tests/zstd-v0.7-22.testfile
new file mode 100644
index 00000000..8b72d683
--- /dev/null
+++ b/tests/zstd-v0.7-22.testfile
@@ -0,0 +1 @@
+'µ/ý"

\ No newline at end of file
diff --git a/tests/zstd-v0.8-00.result b/tests/zstd-v0.8-00.result
new file mode 100644
index 00000000..67edebd8
--- /dev/null
+++ b/tests/zstd-v0.8-00.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: None
\ No newline at end of file
diff --git a/tests/zstd-v0.8-01.result b/tests/zstd-v0.8-01.result
new file mode 100644
index 00000000..a1239a71
--- /dev/null
+++ b/tests/zstd-v0.8-01.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 2
\ No newline at end of file
diff --git a/tests/zstd-v0.8-01.testfile b/tests/zstd-v0.8-01.testfile
new file mode 100644
index 00000000..88735e47
--- /dev/null
+++ b/tests/zstd-v0.8-01.testfile
@@ -0,0 +1 @@
+(µ/ý


\ No newline at end of file
diff --git a/tests/zstd-v0.8-02.result b/tests/zstd-v0.8-02.result
new file mode 100644
index 00000000..82beab8a
--- /dev/null
+++ b/tests/zstd-v0.8-02.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 770
\ No newline at end of file
diff --git a/tests/zstd-v0.8-02.testfile b/tests/zstd-v0.8-02.testfile
new file mode 100644
index 00000000..db554336
--- /dev/null
+++ b/tests/zstd-v0.8-02.testfile
@@ -0,0 +1 @@
+(µ/ý

\ No newline at end of file
diff --git a/tests/zstd-v0.8-03.result b/tests/zstd-v0.8-03.result
new file mode 100644
index 00000000..6aaaa197
--- /dev/null
+++ b/tests/zstd-v0.8-03.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 84148994
\ No newline at end of file
diff --git a/tests/zstd-v0.8-03.testfile b/tests/zstd-v0.8-03.testfile
new file mode 100644
index 00000000..506b344a
--- /dev/null
+++ b/tests/zstd-v0.8-03.testfile
@@ -0,0 +1 @@
+(µ/ý

\ No newline at end of file
diff --git a/tests/zstd-v0.8-16.result b/tests/zstd-v0.8-16.result
new file mode 100644
index 00000000..82beab8a
--- /dev/null
+++ b/tests/zstd-v0.8-16.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 770
\ No newline at end of file
diff --git a/tests/zstd-v0.8-16.testfile b/tests/zstd-v0.8-16.testfile
new file mode 100644
index 00000000..3f87f79d
--- /dev/null
+++ b/tests/zstd-v0.8-16.testfile
@@ -0,0 +1 @@
+(µ/ý

\ No newline at end of file
diff --git a/tests/zstd-v0.8-20.result b/tests/zstd-v0.8-20.result
new file mode 100644
index 00000000..67edebd8
--- /dev/null
+++ b/tests/zstd-v0.8-20.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: None
\ No newline at end of file
diff --git a/tests/zstd-v0.8-20.testfile b/tests/zstd-v0.8-20.testfile
new file mode 100644
index 00000000..76fdbb8a
--- /dev/null
+++ b/tests/zstd-v0.8-20.testfile
@@ -0,0 +1 @@
+(µ/ý 

\ No newline at end of file
diff --git a/tests/zstd-v0.8-21.result b/tests/zstd-v0.8-21.result
new file mode 100644
index 00000000..468070c3
--- /dev/null
+++ b/tests/zstd-v0.8-21.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 1
\ No newline at end of file
diff --git a/tests/zstd-v0.8-21.testfile b/tests/zstd-v0.8-21.testfile
new file mode 100644
index 00000000..9ebeff48
--- /dev/null
+++ b/tests/zstd-v0.8-21.testfile
@@ -0,0 +1 @@
+(µ/ý!

\ No newline at end of file
diff --git a/tests/zstd-v0.8-22.result b/tests/zstd-v0.8-22.result
new file mode 100644
index 00000000..9869d24f
--- /dev/null
+++ b/tests/zstd-v0.8-22.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 513
\ No newline at end of file
diff --git a/tests/zstd-v0.8-22.testfile b/tests/zstd-v0.8-22.testfile
new file mode 100644
index 00000000..f2e55bf7
--- /dev/null
+++ b/tests/zstd-v0.8-22.testfile
@@ -0,0 +1 @@
+(µ/ý"

\ No newline at end of file
diff --git a/tests/zstd-v0.8-23.result b/tests/zstd-v0.8-23.result
new file mode 100644
index 00000000..d38420fe
--- /dev/null
+++ b/tests/zstd-v0.8-23.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 67305985
\ No newline at end of file
diff --git a/tests/zstd-v0.8-23.testfile b/tests/zstd-v0.8-23.testfile
new file mode 100644
index 00000000..f66a18fc
--- /dev/null
+++ b/tests/zstd-v0.8-23.testfile
@@ -0,0 +1 @@
+(µ/ý#

\ No newline at end of file
diff --git a/tests/zstd-v0.8-F4.result b/tests/zstd-v0.8-F4.result
new file mode 100644
index 00000000..67edebd8
--- /dev/null
+++ b/tests/zstd-v0.8-F4.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: None
\ No newline at end of file
diff --git a/tests/zstd-v0.8-F4.testfile b/tests/zstd-v0.8-F4.testfile
new file mode 100644
index 00000000..a4e4240e
--- /dev/null
+++ b/tests/zstd-v0.8-F4.testfile
@@ -0,0 +1 @@
+(µ/ýô

\ No newline at end of file
diff --git a/tests/zstd-v0.8-FF.result b/tests/zstd-v0.8-FF.result
new file mode 100644
index 00000000..d38420fe
--- /dev/null
+++ b/tests/zstd-v0.8-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 67305985
\ No newline at end of file
diff --git a/tests/zstd-v0.8-FF.testfile b/tests/zstd-v0.8-FF.testfile
new file mode 100644
index 00000000..bc639113
--- /dev/null
+++ b/tests/zstd-v0.8-FF.testfile
@@ -0,0 +1 @@
+(µ/ýÿ

\ No newline at end of file