Fix major bug when compressing with dictionary (and add more tests)

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c
index ec0c3295b0a82652f9f7d53ac4286bfb520582d9..628edc75b83df41ceb4868068829b5202ff6d678 100644 (file)
--- a/src/lib/comp/zstd/zstd.c
+++ b/src/lib/comp/zstd/zstd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 Jonathan Dieter <jdieter@gmail.com>
+ * Copyright 2018, 2021 Jonathan Dieter <jdieter@gmail.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -155,8 +155,23 @@ static bool end_cchunk(zckCtx *zck, zckComp *comp, char **dst, size_t *dst_size,
                                   comp->dc_data_size, comp->level);
     }
 #else
-    *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data,
-                               comp->dc_data_size);
+    if(!use_dict && comp->dict_size > 0) {
+        size_t retval = ZSTD_CCtx_loadDictionary(comp->cctx, NULL, 0);
+        if(ZSTD_isError(retval)) {
+            set_fatal_error(zck, "Unable to add zdict to compression context");
+            return false;
+        }
+        *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data,
+                                   comp->dc_data_size);
+        retval = ZSTD_CCtx_loadDictionary(comp->cctx, comp->dict, comp->dict_size);
+        if(ZSTD_isError(retval)) {
+            set_fatal_error(zck, "Unable to add zdict to compression context");
+            return false;
+        }
+    } else {
+        *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data,
+                                   comp->dc_data_size);
+    }
 #endif //OLD_ZSTD
 
     free(comp->dc_data);

diff --git a/test/meson.build b/test/meson.build
index 3fc59b6b5e4825e80fc21e35d8f518d412a03c86..41861269aa930bd3ced121ce61e1822aaf8affaa 100644 (file)
--- a/test/meson.build
+++ b/test/meson.build
@@ -152,7 +152,7 @@ test(
     ]
 )
 test(
-    'decompress auto-chunked file - nocomp',
+    'decompress previously generated auto-chunked file - nocomp',
     shacheck,
     args: [
         unzck,
@@ -162,7 +162,7 @@ test(
     ]
 )
 test(
-    'decompress auto-chunked file - no dict',
+    'decompress previously generated auto-chunked file - no dict',
     shacheck,
     args: [
         unzck,
@@ -172,7 +172,7 @@ test(
     ]
 )
 test(
-    'decompress auto-chunked file - dict',
+    'decompress previously generated auto-chunked file - dict',
     shacheck,
     args: [
         unzck,
@@ -182,7 +182,7 @@ test(
     ]
 )
 test(
-    'decompress manual file - no dict',
+    'decompress previously generated manual file - no dict',
     shacheck,
     args: [
         unzck,
@@ -192,7 +192,7 @@ test(
     ]
 )
 test(
-    'decompress manual file - dict',
+    'decompress previously generated manual file - dict',
     shacheck,
     args: [
         unzck,
@@ -202,7 +202,7 @@ test(
     ]
 )
 test(
-    'decompress dict from auto-chunked file',
+    'decompress dict from previously generated auto-chunked file',
     shacheck,
     args: [
         unzck,
@@ -238,8 +238,20 @@ if build_machine.endian() != 'big'
             join_paths(file_path, 'LICENSE.fodt')
         ]
     )
+    test(
+        'decompress auto-chunked file - no dict',
+        shacheck,
+        args: [
+            unzck,
+            'LICENSE.nodict.fodt',
+            '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e',
+            'LICENSE.nodict.fodt.zck'
+        ],
+        is_parallel: false
+    )
+
 
-    check_sha = '4a495d993483f5f7915f0267c5ce6fc94a263b9dd58cf0bbf33aa0dbcdbb6669'
+    check_sha = '11d08d01c7877d51b84a0f97ebf651ca3304de10cdf207223df2dbc3295fa532'
     if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
         check_sha = 'b86795ca14eb04b382d1c7f94501aa5d1a2ddb05a5351c0235d00edb954e9b66'
     endif
@@ -261,6 +273,17 @@ if build_machine.endian() != 'big'
             join_paths(file_path, 'LICENSE.fodt')
         ]
     )
+    test(
+        'decompress auto-chunked file - dict',
+        shacheck,
+        args: [
+            unzck,
+            'LICENSE.dict.fodt',
+            '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e',
+            'LICENSE.dict.fodt.zck'
+        ],
+        is_parallel: false
+    )
 
     check_sha = 'db42e903a2cf3b859835298272ff45bff6d2435d32fe7fda2bfe2815ab161994'
     if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
@@ -291,8 +314,19 @@ if build_machine.endian() != 'big'
             join_paths(file_path, 'LICENSE.fodt')
         ]
     )
+    test(
+        'decompress manual file - no dict',
+        shacheck,
+        args: [
+            unzck,
+            'LICENSE.manual.nodict.fodt',
+            '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e',
+            'LICENSE.manual.nodict.fodt.zck'
+        ],
+        is_parallel: false
+    )
 
-    check_sha = '4d165a1412013ca38ab2195166e0b91d989137b08065ea23cd49779b7ef6fd09'
+    check_sha = 'ee8df850bc4797e6696b966a74a459129b0f6d23dfb720d1cc8cba6f2b209dd2'
     if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
         check_sha = '5fc2449aeb51f6a898529fa4d53b04de6df6a352826f819dc2b39b153ddaa788'
     endif
@@ -322,6 +356,17 @@ if build_machine.endian() != 'big'
             join_paths(file_path, 'LICENSE.fodt')
         ]
     )
+    test(
+        'decompress manual file - dict',
+        shacheck,
+        args: [
+            unzck,
+            'LICENSE.manual.dict.fodt',
+            '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e',
+            'LICENSE.manual.dict.fodt.zck'
+        ],
+        is_parallel: false
+    )
 endif
 
 run_target('test-abi',