Make builds consistent across architectures for zstd-1.5.0

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

diff --git a/meson.build b/meson.build
index 223de097bc8658d97a8e1cfe3900bc146d336654..61bf40da9f2b2eb437773dc8b6086b380110788d 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -23,6 +23,9 @@ else
     zstd_dep = dependency('libzstd', required : get_option('with-zstd') == 'enabled')
     if zstd_dep.found()
         add_project_arguments('-DZCHUNK_ZSTD', language : 'c')
+        if zstd_dep.version().version_compare('<=1.4.9')
+            add_project_arguments('-DOLD_ZSTD', language : 'c')
+        endif
     endif
 endif
 

diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c
index feffc6cea2ed63e78d2ee129d6dfc12c03e5eeb1..ec0c3295b0a82652f9f7d53ac4286bfb520582d9 100644 (file)
--- a/src/lib/comp/zstd/zstd.c
+++ b/src/lib/comp/zstd/zstd.c
@@ -37,16 +37,39 @@ static bool init(zckCtx *zck, zckComp *comp) {
     VALIDATE_BOOL(zck);
     ALLOCD_BOOL(zck, comp);
 
+    size_t retval = 0;
+
     comp->cctx = ZSTD_createCCtx();
+#ifndef OLD_ZSTD
+    retval = ZSTD_CCtx_setParameter(comp->cctx, ZSTD_c_compressionLevel, comp->level);
+    if(ZSTD_isError(retval)) {
+        set_fatal_error(zck, "Unable to set compression level to %i", comp->level);
+        return false;
+    }
+    // This seems to be the only way to make the compression deterministic across
+    // architectures with zstd 1.5.0
+    retval = ZSTD_CCtx_setParameter(comp->cctx, ZSTD_c_strategy, ZSTD_btopt);
+    if(ZSTD_isError(retval)) {
+        set_fatal_error(zck, "Unable to set compression strategy");
+        return false;
+    }
+#endif //OLD_ZSTD
     comp->dctx = ZSTD_createDCtx();
     if(comp->dict && comp->dict_size > 0) {
+#ifdef OLD_ZSTD
         comp->cdict_ctx = ZSTD_createCDict(comp->dict, comp->dict_size,
                                            comp->level);
         if(comp->cdict_ctx == NULL) {
-            set_fatal_error(zck,
-                            "Unable to create zstd compression dict context");
+            set_fatal_error(zck, "Unable to create zstd compression dict context");
+            return false;
+        }
+#else
+        retval = ZSTD_CCtx_loadDictionary(comp->cctx, comp->dict, comp->dict_size);
+        if(ZSTD_isError(retval)) {
+            set_fatal_error(zck, "Unable to add zdict to compression context");
             return false;
         }
+#endif //OLD_ZSTD
         comp->ddict_ctx = ZSTD_createDDict(comp->dict, comp->dict_size);
         if(comp->ddict_ctx == NULL) {
             set_fatal_error(zck,
@@ -115,7 +138,7 @@ static bool end_cchunk(zckCtx *zck, zckComp *comp, char **dst, size_t *dst_size,
     }
 
     *dst = zmalloc(max_size);
-
+#ifdef OLD_ZSTD
     /* Currently, compression isn't deterministic when using contexts in
      * zstd 1.3.5, so this works around it */
     if(use_dict && comp->cdict_ctx) {
@@ -131,6 +154,11 @@ static bool end_cchunk(zckCtx *zck, zckComp *comp, char **dst, size_t *dst_size,
         *dst_size = ZSTD_compress(*dst, max_size, comp->dc_data,
                                   comp->dc_data_size, comp->level);
     }
+#else
+    *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data,
+                               comp->dc_data_size);
+#endif //OLD_ZSTD
+
     free(comp->dc_data);
     comp->dc_data = NULL;
     comp->dc_data_loc = 0;

diff --git a/test/meson.build b/test/meson.build
index d552e2d40dbdca16247ef49b69e4227b6e591b07..7665c1e9589e34f080e06fb9d4dadecb60253560 100644 (file)
--- a/test/meson.build
+++ b/test/meson.build
@@ -214,7 +214,7 @@ test(
 )
 
 if build_machine.endian() != 'big'
-    check_sha = '3d2fa50ede0d2d70c9683e5a175819f2cb597bc4b2a6fe3cea5f2b8b2f78d2cc'
+    check_sha = '4f07f865bb15624cf854aa369e14a3538ad9e9bf98e233036d37d2568e60b7cc'
     if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
         check_sha = 'eff3098803ba80f0c446d49f48188f89167d7f29cdc8a98c19f0ecfb4e2ee3c9'
     endif
@@ -240,7 +240,7 @@ if build_machine.endian() != 'big'
         ]
     )
 
-    check_sha = 'a939da7db34d3e1cbb9970c092a8902a3dc3fc2dda8469c24dfda4aebbd6e164'
+    check_sha = '4a495d993483f5f7915f0267c5ce6fc94a263b9dd58cf0bbf33aa0dbcdbb6669'
     if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
         check_sha = 'b86795ca14eb04b382d1c7f94501aa5d1a2ddb05a5351c0235d00edb954e9b66'
     endif
@@ -264,7 +264,7 @@ if build_machine.endian() != 'big'
     )
 endif
 
-check_sha = 'c715ef49e4582ef155e185eb665551fe065bafb21c07a69cc71d90075da034ff'
+check_sha = 'db42e903a2cf3b859835298272ff45bff6d2435d32fe7fda2bfe2815ab161994'
 if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
     check_sha = 'c46929367cd3d05daaca3b44657726791b428fb2198f5e7e5367b5cc781307aa'
 endif
@@ -294,7 +294,7 @@ test(
     ]
 )
 
-check_sha = 'acda73e0092dd33de24e51db09f8611dd7330837c393cabdaaee7ea0f79e6309'
+check_sha = '4d165a1412013ca38ab2195166e0b91d989137b08065ea23cd49779b7ef6fd09'
 if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9')
     check_sha = '5fc2449aeb51f6a898529fa4d53b04de6df6a352826f819dc2b39b153ddaa788'
 endif