Work around non-deterministic compression when using contexts in

zstd-1.3.5

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c
index d4a9b36d6da36dd864089f2c0adc3d56065afcd5..1a9d14f6d5a66ce0ee0c280c4fb7d5bfe0bbc04a 100644 (file)
--- a/src/lib/comp/comp.c
+++ b/src/lib/comp/comp.c
@@ -256,9 +256,6 @@ bool comp_init(zckCtx *zck) {
                 return false;
         }
     }
-    free(zck->comp.dict);
-    zck->comp.dict = NULL;
-    zck->comp.dict_size = 0;
     zck->comp.started = true;
     return true;
 }
@@ -289,6 +286,11 @@ bool comp_close(zckCtx *zck) {
         zck->comp.data_loc = 0;
         zck->comp.data_idx = NULL;
     }
+    if(zck->comp.dict)
+        free(zck->comp.dict);
+    zck->comp.dict = NULL;
+    zck->comp.dict_size = 0;
+
     return comp_reset(zck);
 }
 

diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c
index 94b4a55dd8ae61131999134c975ebd05c512fec3..21188e35f64573ba67f5d08038f46b46972909e8 100644 (file)
--- a/src/lib/comp/zstd/zstd.c
+++ b/src/lib/comp/zstd/zstd.c
@@ -57,6 +57,26 @@ static bool init(zckCtx *zck, zckComp *comp) {
     return true;
 }
 
+static bool close(zckCtx *zck, zckComp *comp) {
+    if(comp->cdict_ctx) {
+        ZSTD_freeCDict(comp->cdict_ctx);
+        comp->cdict_ctx = NULL;
+    }
+    if(comp->ddict_ctx) {
+        ZSTD_freeDDict(comp->ddict_ctx);
+        comp->ddict_ctx = NULL;
+    }
+    if(comp->cctx) {
+        ZSTD_freeCCtx(comp->cctx);
+        comp->cctx = NULL;
+    }
+    if(comp->dctx) {
+        ZSTD_freeDCtx(comp->dctx);
+        comp->dctx = NULL;
+    }
+    return true;
+}
+
 /* The zstd compression format doesn't allow streaming compression with a dict
  * unless you statically link to it.  If we have a dict, we do pseudo-streaming
  * compression where we buffer the data until the chunk ends. */
@@ -96,13 +116,20 @@ static bool end_cchunk(zckCtx *zck, zckComp *comp, char **dst, size_t *dst_size,
         return false;
     }
 
+    /* Currently, compression isn't deterministic when using contexts in
+     * zstd 1.3.5, so this works around it */
     if(use_dict && comp->cdict_ctx) {
-        *dst_size = ZSTD_compress_usingCDict(comp->cctx, *dst, max_size,
-                                             comp->dc_data, comp->dc_data_size,
-                                             comp->cdict_ctx);
+        if(comp->cctx)
+            ZSTD_freeCCtx(comp->cctx);
+        comp->cctx = ZSTD_createCCtx();
+
+        *dst_size = ZSTD_compress_usingDict(comp->cctx, *dst, max_size,
+                                            comp->dc_data, comp->dc_data_size,
+                                            comp->dict, comp->dict_size,
+                                            comp->level);
     } else {
-        *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, comp->dc_data,
-                                      comp->dc_data_size, comp->level);
+        *dst_size = ZSTD_compress(*dst, max_size, comp->dc_data,
+                                  comp->dc_data_size, comp->level);
     }
     free(comp->dc_data);
     comp->dc_data = NULL;
@@ -167,26 +194,6 @@ decomp_error_1:
     return false;
 }
 
-static bool close(zckCtx *zck, zckComp *comp) {
-    if(comp->cdict_ctx) {
-        ZSTD_freeCDict(comp->cdict_ctx);
-        comp->cdict_ctx = NULL;
-    }
-    if(comp->ddict_ctx) {
-        ZSTD_freeDDict(comp->ddict_ctx);
-        comp->ddict_ctx = NULL;
-    }
-    if(comp->cctx) {
-        ZSTD_freeCCtx(comp->cctx);
-        comp->cctx = NULL;
-    }
-    if(comp->dctx) {
-        ZSTD_freeDCtx(comp->dctx);
-        comp->dctx = NULL;
-    }
-    return true;
-}
-
 static bool set_parameter(zckCtx *zck, zckComp *comp, int option,
                           const void *value) {
     if(option == ZCK_ZSTD_COMP_LEVEL) {