Store uncompressed size in index

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

diff --git a/include/zck.h b/include/zck.h
index 04758f6d3bcb6718fd8d1214817173c3421e67f2..193e4e69cf2182776dfd2071d8a468460baa3340 100644 (file)
--- a/include/zck.h
+++ b/include/zck.h
@@ -33,6 +33,7 @@ typedef struct zckIndexItem {
     int finished;
     size_t start;
     size_t comp_length;
+    size_t length;
     struct zckIndexItem *next;
 } zckIndexItem;
 
@@ -120,7 +121,7 @@ int zck_compress(zckCtx *zck, const char *src, const size_t src_size);
 /* Decompress data src of size src_size, and write to dst, while setting
  * dst_size */
 int zck_decompress(zckCtx *zck, const char *src, const size_t src_size,
-                   char **dst, size_t *dst_size);
+                   char **dst, size_t dst_size);
 
 /*******************************************************************
  * Creating a zchunk file

diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c
index 03ca0b1b612e60e549c5e4930ea724e10815add0..831555165dff3168b3b79a190a7b52b0a042f866 100644 (file)
--- a/src/lib/comp/comp.c
+++ b/src/lib/comp/comp.c
@@ -71,7 +71,7 @@ int zck_comp_init(zckCtx *zck) {
             free(dst);
             return False;
         }
-        zck_index_add_chunk(zck, dst, dst_size);
+        zck_index_add_chunk(zck, dst, dst_size, zck->comp.dict_size);
         free(dst);
     }
     zck->comp.dict = NULL;
@@ -100,15 +100,15 @@ int zck_compress(zckCtx *zck, const char *src, const size_t src_size) {
         free(dst);
         return False;
     }
-    zck_index_add_chunk(zck, dst, dst_size);
+    zck_index_add_chunk(zck, dst, dst_size, src_size);
     free(dst);
     return True;
 }
 
-int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, char **dst, size_t *dst_size) {
+int zck_decompress(zckCtx *zck, const char *src, const size_t src_size,
+                   char **dst, size_t dst_size) {
     zckComp *comp = &(zck->comp);
     *dst = NULL;
-    *dst_size = 0;
 
     if(!zck->comp.started) {
         zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n");

diff --git a/src/lib/comp/nocomp/nocomp.c b/src/lib/comp/nocomp/nocomp.c
index abeade78467b7029414b1a51f6ed37b1390f74f8..11cb77c983d163c655b64fd87b76ef7c3024a0b5 100644 (file)
--- a/src/lib/comp/nocomp/nocomp.c
+++ b/src/lib/comp/nocomp/nocomp.c
@@ -35,8 +35,7 @@ static int zck_nocomp_init(zckComp *comp) {
     return True;
 }
 
-/* Nocomp just copies data for both compression and decompression */
-static int zck_nocomp_de_comp(zckComp *comp, const char *src, const size_t src_size,
+static int zck_nocomp_comp(zckComp *comp, const char *src, const size_t src_size,
                         char **dst, size_t *dst_size, int use_dict) {
     *dst = zmalloc(src_size);
     if(dst == NULL) {
@@ -50,6 +49,19 @@ static int zck_nocomp_de_comp(zckComp *comp, const char *src, const size_t src_s
     return True;
 }
 
+static int zck_nocomp_decomp(zckComp *comp, const char *src, const size_t src_size,
+                        char **dst, size_t dst_size, int use_dict) {
+    *dst = zmalloc(src_size);
+    if(dst == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size);
+        return False;
+    }
+
+    memcpy(*dst, src, src_size);
+
+    return True;
+}
+
 static int zck_nocomp_close(zckComp *comp) {
     return True;
 }
@@ -69,8 +81,8 @@ static int zck_nocomp_set_default_parameters(zckComp *comp) {
 int zck_nocomp_setup(zckComp *comp) {
     comp->init = zck_nocomp_init;
     comp->set_parameter = zck_nocomp_set_parameter;
-    comp->compress = zck_nocomp_de_comp;
-    comp->decompress = zck_nocomp_de_comp;
+    comp->compress = zck_nocomp_comp;
+    comp->decompress = zck_nocomp_decomp;
     comp->close = zck_nocomp_close;
     comp->type = ZCK_COMP_NONE;
     return zck_nocomp_set_default_parameters(comp);

diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c
index fc7dd062e5f7fbfc7aa09390dec7a04caa20b573..924f6d69af6213202fb365f34b0534ec699b42f1 100644 (file)
--- a/src/lib/comp/zstd/zstd.c
+++ b/src/lib/comp/zstd/zstd.c
@@ -85,30 +85,20 @@ static int zck_zstd_compress(zckComp *comp, const char *src,
 
 static int zck_zstd_decompress(zckComp *comp, const char *src,
                                const size_t src_size, char **dst,
-                               size_t *dst_size, int use_dict) {
-    unsigned long long frame_size;
-    size_t retval;
-
-    frame_size = ZSTD_getFrameContentSize(src, src_size);
-    if(frame_size == ZSTD_CONTENTSIZE_UNKNOWN ||
-       frame_size == ZSTD_CONTENTSIZE_ERROR) {
-        zck_log(ZCK_LOG_ERROR, "Unknown content size\n");
-        return False;
-    }
-    *dst_size = (size_t)frame_size;
-    *dst = zmalloc(*dst_size);
+                               size_t dst_size, int use_dict) {
+    *dst = zmalloc(dst_size);
     if(dst == NULL) {
-        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", *dst_size);
+        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", dst_size);
         return False;
     }
 
-    if(use_dict && comp->ddict_ctx) {
-        retval = ZSTD_decompress_usingDDict(comp->dctx, *dst, *dst_size, src,
+    size_t retval;
+    if(use_dict && comp->ddict_ctx)
+        retval = ZSTD_decompress_usingDDict(comp->dctx, *dst, dst_size, src,
                                             src_size, comp->ddict_ctx);
-    } else {
-        retval = ZSTD_decompressDCtx(comp->dctx, *dst, *dst_size, src,
+    else
+        retval = ZSTD_decompressDCtx(comp->dctx, *dst, dst_size, src,
                                      src_size);
-    }
     if(ZSTD_isError(retval)) {
         free(*dst);
         *dst = NULL;

diff --git a/src/lib/dl/range.c b/src/lib/dl/range.c
index 57a99b0726c22e3510c756756f4537e365567fa5..718c3977c891fb696b8e6b2f9cd0c8a3022cb5ad 100644 (file)
--- a/src/lib/dl/range.c
+++ b/src/lib/dl/range.c
@@ -74,7 +74,7 @@ zckRangeItem *zck_range_insert_new(zckRangeItem *prev, zckRangeItem *next, uint6
     }
     if(add_index)
         if(!zck_index_new_chunk(&(info->index), idx->digest, idx->digest_size,
-                                end-start+1, False)) {
+                                end-start+1, end-start+1, False)) {
             free(new);
             return NULL;
         }

diff --git a/src/lib/index/index_create.c b/src/lib/index/index_create.c
index f387ef00574e534a197d152ae8b6b728f2bad1a2..b50e9d592c0c39f9ab2fdd2a4cd67db281b67b5e 100644 (file)
--- a/src/lib/index/index_create.c
+++ b/src/lib/index/index_create.c
@@ -50,7 +50,7 @@ int zck_index_finalize(zckCtx *zck) {
     if(zck->index.first) {
         zckIndexItem *tmp = zck->index.first;
         while(tmp) {
-            index_malloc += zck->index.digest_size + MAX_COMP_SIZE;
+            index_malloc += zck->index.digest_size + MAX_COMP_SIZE*2;
             tmp = tmp->next;
         }
     }
@@ -64,10 +64,15 @@ int zck_index_finalize(zckCtx *zck) {
     if(zck->index.first) {
         zckIndexItem *tmp = zck->index.first;
         while(tmp) {
+            /* Write digest */
             memcpy(index+index_size, tmp->digest, zck->index.digest_size);
             index_size += zck->index.digest_size;
+            /* Write compressed size */
             zck_compint_from_size(index+index_size, tmp->comp_length,
                                   &index_size);
+            /* Write uncompressed size */
+            zck_compint_from_size(index+index_size, tmp->length, &index_size);
+
             tmp = tmp->next;
         }
     }
@@ -117,7 +122,7 @@ int zck_index_finalize(zckCtx *zck) {
 }
 
 int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
-                        size_t length, int finished) {
+                        size_t comp_size, size_t orig_size, int finished) {
     if(index == NULL) {
         zck_log(ZCK_LOG_ERROR, "Invalid index\n");
         return False;
@@ -141,7 +146,8 @@ int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
         memcpy(idx->digest, digest, digest_size);
     idx->digest_size = digest_size;
     idx->start = index->length;
-    idx->comp_length = length;
+    idx->comp_length = comp_size;
+    idx->length = orig_size;
     idx->finished = finished;
     if(index->first == NULL) {
         index->first = idx;
@@ -152,11 +158,12 @@ int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
         tmp->next = idx;
     }
     index->count += 1;
-    index->length += length;
+    index->length += comp_size;
     return True;
 }
 
-int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) {
+int zck_index_add_chunk(zckCtx *zck, char *data, size_t comp_size,
+                        size_t orig_size) {
     zckHash hash;
 
     if(zck == NULL) {
@@ -164,16 +171,16 @@ int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) {
         return False;
     }
 
-    if(size == 0) {
+    if(comp_size == 0) {
         if(!zck_index_new_chunk(&(zck->index), NULL, zck->index.digest_size,
-                                size, True))
+                                comp_size, orig_size, True))
             return False;
     } else {
-        if(!zck_hash_update(&(zck->full_hash), data, size))
+        if(!zck_hash_update(&(zck->full_hash), data, comp_size))
             return False;
         if(!zck_hash_init(&hash, &(zck->chunk_hash_type)))
             return False;
-        if(!zck_hash_update(&hash, data, size))
+        if(!zck_hash_update(&hash, data, comp_size))
             return False;
 
         char *digest = zck_hash_finalize(&hash);
@@ -184,7 +191,7 @@ int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) {
             return False;
         }
         if(!zck_index_new_chunk(&(zck->index), digest, zck->index.digest_size,
-                                size, True))
+                                comp_size, orig_size, True))
             return False;
         free(digest);
     }

diff --git a/src/lib/index/index_read.c b/src/lib/index/index_read.c
index 7d9651e4ef177ff3d112825f06d4615edf7c5d69..ca2d3c14e0fe2b97968b23bee901d8e029d2cd51 100644 (file)
--- a/src/lib/index/index_read.c
+++ b/src/lib/index/index_read.c
@@ -117,9 +117,15 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) {
             return False;
         new->start = idx_loc;
         new->comp_length = chunk_length;
-        new->finished = False;
 
-        idx_loc += chunk_length;
+        /* Read and store uncompressed entry length */
+        chunk_length = 0;
+        if(!zck_compint_to_size(&chunk_length, data+length, &length))
+            return False;
+        new->length = chunk_length;
+
+        new->finished = False;
+        idx_loc += new->comp_length;
         zck->index.length = idx_loc;
 
         if(prev)

diff --git a/src/lib/zck.c b/src/lib/zck.c
index fa35b1c9c916912ec19a276b0c1dbe80aba470c9..99bd04504887d8550b35a61ddc09380c0f92d089 100644 (file)
--- a/src/lib/zck.c
+++ b/src/lib/zck.c
@@ -344,6 +344,7 @@ int zck_decompress_to_file(zckCtx *zck, int src_fd, int dst_fd) {
     /* Check if zck file is empty */
     for(int count=0; idx; count++) {
         size_t csize = idx->comp_length;
+        size_t size = idx->length;
         char *cdata;
 
         if(csize == 0)
@@ -366,9 +367,8 @@ int zck_decompress_to_file(zckCtx *zck, int src_fd, int dst_fd) {
             return False;
         }
 
-        size_t size = 0;
         char *data = NULL;
-        if(!zck_decompress(zck, cdata, csize, &data, &size)) {
+        if(!zck_decompress(zck, cdata, csize, &data, size)) {
             free(cdata);
             zck_log(ZCK_LOG_ERROR, "Unable to decompress chunk %i\n", count);
             return False;

diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h
index 24f481afdffb8cab34bbe86163444182fc56d9a9..75465f5359cc7b98d64f65cbabf4275b6d933df5 100644 (file)
--- a/src/lib/zck_private.h
+++ b/src/lib/zck_private.h
@@ -19,7 +19,7 @@ typedef int (*fcomp)(struct zckComp *comp, const char *src,
                      const size_t src_size, char **dst, size_t *dst_size,
                      int use_dict);
 typedef int (*fdecomp)(struct zckComp *comp, const char *src,
-                       const size_t src_size, char **dst, size_t *dst_size,
+                       const size_t src_size, char **dst, size_t dst_size,
                        int use_dict);
 typedef int (*fcclose)(struct zckComp *comp);
 
@@ -100,14 +100,6 @@ const char *zck_hash_name_from_type(int hash_type);
 int zck_get_tmp_fd();
 int zck_validate_file(zckCtx *zck);
 
-/* comp/comp.c */
-int zck_comp_init(zckCtx *zck);
-int zck_compress(zckCtx *zck, const char *src, const size_t src_size);
-int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, char **dst, size_t *dst_size);
-int zck_comp_close(zckCtx *zck);
-int zck_set_compression_type(zckCtx *zck, int type);
-int zck_set_comp_parameter(zckCtx *zck, int option, void *value);
-
 /* hash/hash.h */
 int zck_hash_setup(zckHashType *ht, int h);
 int zck_hash_init(zckHash *hash, zckHashType *hash_type);
@@ -120,8 +112,9 @@ const char *zck_hash_get_printable(const char *digest, zckHashType *type);
 int zck_index_read(zckCtx *zck, char *data, size_t size);
 int zck_index_finalize(zckCtx *zck);
 int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
-                        size_t length, int finished);
-int zck_index_add_chunk(zckCtx *zck, char *data, size_t size);
+                        size_t comp_size, size_t orig_size, int finished);
+int zck_index_add_chunk(zckCtx *zck, char *data, size_t comp_size,
+                        size_t orig_size);
 void zck_index_clean(zckIndex *index);
 void zck_index_free(zckCtx *zck);
 int zck_write_index(zckCtx *zck);

diff --git a/src/zck_read_header.c b/src/zck_read_header.c
index 9d01da054f4d46809d3054f7372fd5c97f3b2d66..81fb67b11d869d1aca29e179da56e68eb5a10aca 100644 (file)
--- a/src/zck_read_header.c
+++ b/src/zck_read_header.c
@@ -75,7 +75,7 @@ int main (int argc, char *argv[]) {
     while(idx) {
         for(int i=0; i<zck_get_chunk_digest_size(zck); i++)
             printf("%02x", (unsigned char)idx->digest[i]);
-        printf(" %12lu %12lu %12lu\n", idx->start + zck_get_header_length(zck), idx->comp_length, idx->comp_length);
+        printf(" %12lu %12lu %12lu\n", idx->start + zck_get_header_length(zck), idx->comp_length, idx->length);
         idx = idx->next;
     }