From 813a1f66083eec4ed925c74a43198f57d973c53a Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Tue, 1 Jun 2021 20:35:20 +0100 Subject: [PATCH] Fix major bug when compressing with dictionary (and add more tests) Signed-off-by: Jonathan Dieter --- src/lib/comp/zstd/zstd.c | 21 ++++++++++++-- test/meson.build | 61 ++++++++++++++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c index ec0c329..628edc7 100644 --- a/src/lib/comp/zstd/zstd.c +++ b/src/lib/comp/zstd/zstd.c @@ -1,5 +1,5 @@ /* - * Copyright 2018 Jonathan Dieter + * Copyright 2018, 2021 Jonathan Dieter * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -155,8 +155,23 @@ static bool end_cchunk(zckCtx *zck, zckComp *comp, char **dst, size_t *dst_size, comp->dc_data_size, comp->level); } #else - *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data, - comp->dc_data_size); + if(!use_dict && comp->dict_size > 0) { + size_t retval = ZSTD_CCtx_loadDictionary(comp->cctx, NULL, 0); + if(ZSTD_isError(retval)) { + set_fatal_error(zck, "Unable to add zdict to compression context"); + return false; + } + *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data, + comp->dc_data_size); + retval = ZSTD_CCtx_loadDictionary(comp->cctx, comp->dict, comp->dict_size); + if(ZSTD_isError(retval)) { + set_fatal_error(zck, "Unable to add zdict to compression context"); + return false; + } + } else { + *dst_size = ZSTD_compress2(comp->cctx, *dst, max_size, comp->dc_data, + comp->dc_data_size); + } #endif //OLD_ZSTD free(comp->dc_data); diff --git a/test/meson.build b/test/meson.build index 3fc59b6..4186126 100644 --- a/test/meson.build +++ b/test/meson.build @@ -152,7 +152,7 @@ test( ] ) test( - 'decompress auto-chunked file - nocomp', + 'decompress previously generated auto-chunked file - nocomp', shacheck, args: [ unzck, @@ -162,7 +162,7 @@ test( ] ) test( - 'decompress auto-chunked file - no dict', + 'decompress previously generated auto-chunked file - no dict', shacheck, args: [ unzck, @@ -172,7 +172,7 @@ test( ] ) test( - 'decompress auto-chunked file - dict', + 'decompress previously generated auto-chunked file - dict', shacheck, args: [ unzck, @@ -182,7 +182,7 @@ test( ] ) test( - 'decompress manual file - no dict', + 'decompress previously generated manual file - no dict', shacheck, args: [ unzck, @@ -192,7 +192,7 @@ test( ] ) test( - 'decompress manual file - dict', + 'decompress previously generated manual file - dict', shacheck, args: [ unzck, @@ -202,7 +202,7 @@ test( ] ) test( - 'decompress dict from auto-chunked file', + 'decompress dict from previously generated auto-chunked file', shacheck, args: [ unzck, @@ -238,8 +238,20 @@ if build_machine.endian() != 'big' join_paths(file_path, 'LICENSE.fodt') ] ) + test( + 'decompress auto-chunked file - no dict', + shacheck, + args: [ + unzck, + 'LICENSE.nodict.fodt', + '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e', + 'LICENSE.nodict.fodt.zck' + ], + is_parallel: false + ) + - check_sha = '4a495d993483f5f7915f0267c5ce6fc94a263b9dd58cf0bbf33aa0dbcdbb6669' + check_sha = '11d08d01c7877d51b84a0f97ebf651ca3304de10cdf207223df2dbc3295fa532' if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9') check_sha = 'b86795ca14eb04b382d1c7f94501aa5d1a2ddb05a5351c0235d00edb954e9b66' endif @@ -261,6 +273,17 @@ if build_machine.endian() != 'big' join_paths(file_path, 'LICENSE.fodt') ] ) + test( + 'decompress auto-chunked file - dict', + shacheck, + args: [ + unzck, + 'LICENSE.dict.fodt', + '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e', + 'LICENSE.dict.fodt.zck' + ], + is_parallel: false + ) check_sha = 'db42e903a2cf3b859835298272ff45bff6d2435d32fe7fda2bfe2815ab161994' if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9') @@ -291,8 +314,19 @@ if build_machine.endian() != 'big' join_paths(file_path, 'LICENSE.fodt') ] ) + test( + 'decompress manual file - no dict', + shacheck, + args: [ + unzck, + 'LICENSE.manual.nodict.fodt', + '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e', + 'LICENSE.manual.nodict.fodt.zck' + ], + is_parallel: false + ) - check_sha = '4d165a1412013ca38ab2195166e0b91d989137b08065ea23cd49779b7ef6fd09' + check_sha = 'ee8df850bc4797e6696b966a74a459129b0f6d23dfb720d1cc8cba6f2b209dd2' if zstd_dep.found() and zstd_dep.version().version_compare('<=1.4.9') check_sha = '5fc2449aeb51f6a898529fa4d53b04de6df6a352826f819dc2b39b153ddaa788' endif @@ -322,6 +356,17 @@ if build_machine.endian() != 'big' join_paths(file_path, 'LICENSE.fodt') ] ) + test( + 'decompress manual file - dict', + shacheck, + args: [ + unzck, + 'LICENSE.manual.dict.fodt', + '394ed6c2fc4ac47e5ee111a46f2a35b8010a56c7747748216f52105e868d5a3e', + 'LICENSE.manual.dict.fodt.zck' + ], + is_parallel: false + ) endif run_target('test-abi', -- 2.30.2