From: Jonathan Dieter Date: Sat, 24 Mar 2018 14:54:19 +0000 (+0200) Subject: Change compression so it's streamed. Decompression is still on a chunk-by- X-Git-Tag: archive/raspbian/1.1.9+ds1-1+rpi1~1^2~340 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=c3e5d271cca94b5e739c5c58dec2708659b9442d;p=zchunk.git Change compression so it's streamed. Decompression is still on a chunk-by- chunk basis. Signed-off-by: Jonathan Dieter --- diff --git a/include/zck.h b/include/zck.h index 193e4e6..59d833d 100644 --- a/include/zck.h +++ b/include/zck.h @@ -118,6 +118,8 @@ int zck_comp_init(zckCtx *zck); int zck_comp_close(zckCtx *zck); /* Compress data src of size src_size, and write to chunk */ int zck_compress(zckCtx *zck, const char *src, const size_t src_size); +/* Finish compressing chunk */ +int zck_end_chunk(zckCtx *zck); /* Decompress data src of size src_size, and write to dst, while setting * dst_size */ int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c index 11c7506..59e1bd5 100644 --- a/src/lib/comp/comp.c +++ b/src/lib/comp/comp.c @@ -37,6 +37,10 @@ #endif #define BLK_SIZE 32768 +#define VALIDATE(f) if(!f) { \ + zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n"); \ + return False; \ + } static char unknown[] = "Unknown(\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; @@ -46,6 +50,8 @@ const static char *COMP_NAME[] = { }; int zck_comp_init(zckCtx *zck) { + VALIDATE(zck); + zckComp *comp = &(zck->comp); char *dst = NULL; size_t dst_size = 0; @@ -66,12 +72,22 @@ int zck_comp_init(zckCtx *zck) { if(!zck->comp.compress(comp, zck->comp.dict, zck->comp.dict_size, &dst, &dst_size, 0)) return False; - if(!zck_write(zck->temp_fd, dst, dst_size)) { free(dst); return False; } zck_index_add_to_chunk(zck, dst, dst_size, zck->comp.dict_size); + free(dst); + dst = NULL; + dst_size = 0; + + if(!zck->comp.end_chunk(comp, &dst, &dst_size, 0)) + return False; + if(!zck_write(zck->temp_fd, dst, dst_size)) { + free(dst); + return False; + } + zck_index_add_to_chunk(zck, dst, dst_size, 0); zck_index_finish_chunk(zck); free(dst); } @@ -82,9 +98,7 @@ int zck_comp_init(zckCtx *zck) { } int zck_compress(zckCtx *zck, const char *src, const size_t src_size) { - zckComp *comp = &(zck->comp); - char *dst; - size_t dst_size; + VALIDATE(zck); if(!zck->comp.started) { zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n"); @@ -94,21 +108,61 @@ int zck_compress(zckCtx *zck, const char *src, const size_t src_size) { if(src_size == 0) return True; - if(!zck->comp.compress(comp, src, src_size, &dst, &dst_size, 1)) + char *dst = NULL; + size_t dst_size = 0; + if(!zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1)) return False; + if(dst_size > 0 && !zck_write(zck->temp_fd, dst, dst_size)) { + free(dst); + return False; + } + if(!zck_index_add_to_chunk(zck, dst, dst_size, src_size)) { + free(dst); + return False; + } + free(dst); + return True; +} - if(!zck_write(zck->temp_fd, dst, dst_size)) { +int zck_end_chunk(zckCtx *zck) { + VALIDATE(zck); + + if(!zck->comp.started) { + zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n"); + return False; + } + + /* No point in compressing empty data */ + if(zck->comp.data_size == 0) { + if(!zck_index_finish_chunk(zck)) + return False; + return True; + } + + char *dst = NULL; + size_t dst_size = 0; + if(!zck->comp.end_chunk(&(zck->comp), &dst, &dst_size, 1)) + return False; + if(dst_size > 0 && !zck_write(zck->temp_fd, dst, dst_size)) { + free(dst); + return False; + } + if(!zck_index_add_to_chunk(zck, dst, dst_size, 0)) { + free(dst); + return False; + } + if(!zck_index_finish_chunk(zck)) { free(dst); return False; } - zck_index_add_to_chunk(zck, dst, dst_size, src_size); - zck_index_finish_chunk(zck); free(dst); return True; } int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, char **dst, size_t dst_size) { + VALIDATE(zck); + zckComp *comp = &(zck->comp); *dst = NULL; @@ -127,14 +181,17 @@ int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, } int zck_comp_close(zckCtx *zck) { - if(zck->comp.cctx) { - zck->comp.started = 0; - return zck->comp.close(&(zck->comp)); - } - return True; + VALIDATE(zck); + + zck->comp.started = 0; + if(zck->comp.close == NULL) + return True; + return zck->comp.close(&(zck->comp)); } int zck_set_compression_type(zckCtx *zck, int type) { + VALIDATE(zck); + zckComp *comp = &(zck->comp); /* Cannot change compression type after compression has started */ @@ -163,6 +220,8 @@ int zck_set_compression_type(zckCtx *zck, int type) { } int zck_set_comp_parameter(zckCtx *zck, int option, void *value) { + VALIDATE(zck); + /* Cannot change compression parameters after compression has started */ if(zck && zck->comp.started) { zck_log(ZCK_LOG_ERROR, diff --git a/src/lib/comp/nocomp/nocomp.c b/src/lib/comp/nocomp/nocomp.c index 11cb77c..0b67206 100644 --- a/src/lib/comp/nocomp/nocomp.c +++ b/src/lib/comp/nocomp/nocomp.c @@ -35,8 +35,16 @@ static int zck_nocomp_init(zckComp *comp) { return True; } +static int zck_nocomp_end_chunk(zckComp *comp, char **dst, size_t *dst_size, + int use_dict) { + *dst = NULL; + *dst_size = 0; + + return True; +} + static int zck_nocomp_comp(zckComp *comp, const char *src, const size_t src_size, - char **dst, size_t *dst_size, int use_dict) { + char **dst, size_t *dst_size, int use_dict) { *dst = zmalloc(src_size); if(dst == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size); @@ -49,8 +57,9 @@ static int zck_nocomp_comp(zckComp *comp, const char *src, const size_t src_size return True; } -static int zck_nocomp_decomp(zckComp *comp, const char *src, const size_t src_size, - char **dst, size_t dst_size, int use_dict) { +static int zck_nocomp_decomp(zckComp *comp, const char *src, + const size_t src_size, char **dst, size_t dst_size, + int use_dict) { *dst = zmalloc(src_size); if(dst == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size); @@ -82,6 +91,7 @@ int zck_nocomp_setup(zckComp *comp) { comp->init = zck_nocomp_init; comp->set_parameter = zck_nocomp_set_parameter; comp->compress = zck_nocomp_comp; + comp->end_chunk = zck_nocomp_end_chunk; comp->decompress = zck_nocomp_decomp; comp->close = zck_nocomp_close; comp->type = ZCK_COMP_NONE; diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c index 924f6d6..51a9ed2 100644 --- a/src/lib/comp/zstd/zstd.c +++ b/src/lib/comp/zstd/zstd.c @@ -26,12 +26,18 @@ #include #include +#include #include #include #include "zck_private.h" -static int zck_zstd_init(zckComp *comp) { +#define VALIDATE(f) if(!f) { \ + zck_log(ZCK_LOG_ERROR, "zckComp not initialized\n"); \ + return False; \ + } + +static int init(zckComp *comp) { comp->cctx = ZSTD_createCCtx(); comp->dctx = ZSTD_createDCtx(); if(comp->dict && comp->dict_size > 0) { @@ -52,10 +58,31 @@ static int zck_zstd_init(zckComp *comp) { return True; } -static int zck_zstd_compress(zckComp *comp, const char *src, - const size_t src_size, char **dst, - size_t *dst_size, int use_dict) { - size_t max_size = ZSTD_compressBound(src_size); +/* The zstd compression format doesn't allow streaming compression with a dict + * unless you statically link to it. If we have a dict, we do pseudo-streaming + * compression where we buffer the data until the chunk ends. */ +static int compress(zckComp *comp, const char *src, const size_t src_size, + char **dst, size_t *dst_size, int use_dict) { + if(comp->data == NULL) + comp->data = zmalloc(src_size); + else + comp->data = realloc(comp->data, comp->data_size + src_size); + if(comp->data == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + comp->data_size + src_size); + return False; + } + memcpy(comp->data + comp->data_size, src, src_size); + *dst = NULL; + *dst_size = 0; + comp->data_size += src_size; + return True; +} + +static int end_chunk(zckComp *comp, char **dst, size_t *dst_size, + int use_dict) { + VALIDATE(comp); + size_t max_size = ZSTD_compressBound(comp->data_size); if(ZSTD_isError(max_size)) { zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n", ZSTD_getErrorName(max_size)); @@ -69,12 +96,16 @@ static int zck_zstd_compress(zckComp *comp, const char *src, } if(use_dict && comp->cdict_ctx) { - *dst_size = ZSTD_compress_usingCDict(comp->cctx, *dst, max_size, src, - src_size, comp->cdict_ctx); + *dst_size = ZSTD_compress_usingCDict(comp->cctx, *dst, max_size, + comp->data, comp->data_size, + comp->cdict_ctx); } else { - *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, src, src_size, - comp->level); + *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, comp->data, + comp->data_size, comp->level); } + free(comp->data); + comp->data = NULL; + comp->data_size = 0; if(ZSTD_isError(*dst_size)) { zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n", ZSTD_getErrorName(*dst_size)); @@ -83,9 +114,10 @@ static int zck_zstd_compress(zckComp *comp, const char *src, return True; } -static int zck_zstd_decompress(zckComp *comp, const char *src, - const size_t src_size, char **dst, - size_t dst_size, int use_dict) { +static int decompress(zckComp *comp, const char *src, const size_t src_size, + char **dst, size_t dst_size, int use_dict) { + VALIDATE(comp); + *dst = zmalloc(dst_size); if(dst == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", dst_size); @@ -109,7 +141,7 @@ static int zck_zstd_decompress(zckComp *comp, const char *src, return True; } -static int zck_zstd_close(zckComp *comp) { +static int close(zckComp *comp) { if(comp->cdict_ctx) { ZSTD_freeCDict(comp->cdict_ctx); comp->cdict_ctx = NULL; @@ -126,10 +158,15 @@ static int zck_zstd_close(zckComp *comp) { ZSTD_freeDCtx(comp->dctx); comp->dctx = NULL; } + if(comp->data) { + free(comp->data); + comp->data = NULL; + } + comp->data_size = 0; return True; } -static int zck_zstd_set_parameter(zckComp *comp, int option, void *value) { +static int set_parameter(zckComp *comp, int option, void *value) { if(option == ZCK_ZCK_COMP_LEVEL) { if(*(int*)value >= 0 && *(int*)value <= ZSTD_maxCLevel()) { comp->level = *(int*)value; @@ -140,18 +177,19 @@ static int zck_zstd_set_parameter(zckComp *comp, int option, void *value) { return False; } -static int zck_zstd_set_default_parameters(zckComp *comp) { +static int set_default_parameters(zckComp *comp) { /* Set default compression level to 16 */ int level=16; - return zck_zstd_set_parameter(comp, ZCK_ZCK_COMP_LEVEL, &level); + return set_parameter(comp, ZCK_ZCK_COMP_LEVEL, &level); } int zck_zstd_setup(zckComp *comp) { - comp->init = zck_zstd_init; - comp->set_parameter = zck_zstd_set_parameter; - comp->compress = zck_zstd_compress; - comp->decompress = zck_zstd_decompress; - comp->close = zck_zstd_close; + comp->init = init; + comp->set_parameter = set_parameter; + comp->compress = compress; + comp->end_chunk = end_chunk; + comp->decompress = decompress; + comp->close = close; comp->type = ZCK_COMP_ZSTD; - return zck_zstd_set_default_parameters(comp); + return set_default_parameters(comp); } diff --git a/src/lib/index/index_create.c b/src/lib/index/index_create.c index 9902b38..d0eeeae 100644 --- a/src/lib/index/index_create.c +++ b/src/lib/index/index_create.c @@ -199,19 +199,19 @@ int zck_index_add_to_chunk(zckCtx *zck, char *data, size_t comp_size, size_t orig_size) { VALIDATE(zck); - if(comp_size == 0) - return True; - if(zck->work_index_item == NULL && !zck_index_create_chunk(zck)) return False; + zck->work_index_item->length += orig_size; + if(comp_size == 0) + return True; + if(!zck_hash_update(&(zck->full_hash), data, comp_size)) return False; if(!zck_hash_update(&(zck->work_index_hash), data, comp_size)) return False; zck->work_index_item->comp_length += comp_size; - zck->work_index_item->length += orig_size; return True; } diff --git a/src/lib/zck.c b/src/lib/zck.c index d236b1b..63c713c 100644 --- a/src/lib/zck.c +++ b/src/lib/zck.c @@ -262,12 +262,16 @@ int zck_import_dict(zckCtx *zck, char *data, size_t size) { "Attempting to initialize an empty compression dictionary\n"); return False; } + zck_log(ZCK_LOG_DEBUG, "Closing compression\n"); if(!zck_comp_close(zck)) return False; + zck_log(ZCK_LOG_DEBUG, "setting dict 1\n"); if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT, data)) return False; + zck_log(ZCK_LOG_DEBUG, "setting dict 2\n"); if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT_SIZE, &size)) return False; + zck_log(ZCK_LOG_DEBUG, "Initializing\n"); if(!zck_comp_init(zck)) return False; return True; @@ -325,11 +329,13 @@ int zck_validate_file(zckCtx *zck) { "Unable to calculate %s checksum for full file\n"); return False; } + zck_log(ZCK_LOG_DEBUG, "Checking data checksum\n"); if(memcmp(digest, zck->full_hash_digest, zck->hash_type.digest_size) != 0) { free(digest); - zck_log(ZCK_LOG_ERROR, "Final file failed checksum\n"); + zck_log(ZCK_LOG_ERROR, "Data checksum failed!\n"); return False; } + zck_log(ZCK_LOG_DEBUG, "Data checksum valid\n"); free(digest); return True; } diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h index 80f8c89..5aa5764 100644 --- a/src/lib/zck_private.h +++ b/src/lib/zck_private.h @@ -15,6 +15,8 @@ struct zckComp; typedef int (*finit)(struct zckComp *comp); typedef int (*fparam)(struct zckComp *comp, int option, void *value); +typedef int (*fcompend)(struct zckComp *comp, char **dst, size_t *dst_size, + int use_dict); typedef int (*fcomp)(struct zckComp *comp, const char *src, const size_t src_size, char **dst, size_t *dst_size, int use_dict); @@ -71,9 +73,14 @@ typedef struct zckComp { void *ddict_ctx; void *dict; size_t dict_size; + + char *data; + size_t data_size; + finit init; fparam set_parameter; fcomp compress; + fcompend end_chunk; fdecomp decompress; fcclose close; } zckComp; diff --git a/src/zck.c b/src/zck.c index e2cfc13..23d84ce 100644 --- a/src/zck.c +++ b/src/zck.c @@ -43,7 +43,6 @@ int main (int argc, char *argv[]) { char *out_name; char *dict = NULL; size_t dict_size = 0; - zckCtx *zck = zck_create(); zck_set_log_level(ZCK_LOG_DEBUG); @@ -86,6 +85,10 @@ int main (int argc, char *argv[]) { exit(1); } free(out_name); + + zckCtx *zck = zck_create(); + if(zck == NULL) + exit(1); if(!zck_init_write(zck, dst_fd)) exit(1); @@ -161,6 +164,8 @@ int main (int argc, char *argv[]) { printf("Compressing %li bytes\n", next-found); if(!zck_compress(zck, found, next-found)) exit(1); + if(!zck_end_chunk(zck)) + exit(1); found = next; search = next + 1; if(search > data + in_size) @@ -169,6 +174,8 @@ int main (int argc, char *argv[]) { printf("Completing %li bytes\n", data+in_size-found); if(!zck_compress(zck, found, data+in_size-found)) exit(1); + if(!zck_end_chunk(zck)) + exit(1); search = NULL; } } @@ -201,6 +208,8 @@ int main (int argc, char *argv[]) { printf("Completing %li bytes\n", cur_loc-start); if(!zck_compress(zck, start, cur_loc-start)) exit(1); + if(!zck_end_chunk(zck)) + exit(1); start = cur_loc; } }