#define BLK_SIZE 32768
#define VALIDATE(f) if(!f) { \
- zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n"); \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckCtx not initialized\n"); \
return False; \
}
+#define VALIDATE_READ(f) VALIDATE(f); \
+ if(f->mode != ZCK_MODE_READ) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckCtx not opened for reading\n"); \
+ return False; \
+ }
+
+#define VALIDATE_WRITE(f) VALIDATE(f); \
+ if(f->mode != ZCK_MODE_WRITE) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckCtx not opened for writing\n"); \
+ return False; \
+ }
+
+#define VALIDATE_SIZE(f) if(!f) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckComp not initialized\n"); \
+ return -1; \
+ }
+
+#define VALIDATE_READ_SIZE(f) VALIDATE_SIZE(f); \
+ if(f->mode != ZCK_MODE_READ) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckComp not opened for reading\n"); \
+ return -1; \
+ }
+
+#define VALIDATE_WRITE_SIZE(f) VALIDATE_SIZE(f); \
+ if(f->mode != ZCK_MODE_WRITE) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckComp not opened for writing\n"); \
+ return -1; \
+ }
+
static char unknown[] = "Unknown(\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
const static char *COMP_NAME[] = {
"no",
+ "Unknown (1)",
"zstd"
};
VALIDATE(zck);
zckComp *comp = &(zck->comp);
- char *dst = NULL;
- size_t dst_size = 0;
if(zck->comp.started) {
zck_log(ZCK_LOG_ERROR, "Compression already initialized\n");
zck_log(ZCK_LOG_ERROR, "Invalid dictionary configuration\n");
return False;
}
+ zck_log(ZCK_LOG_DEBUG, "Initializing %s compression\n",
+ zck_comp_name_from_type(comp->type));
if(!zck->comp.init(&(zck->comp)))
return False;
-
if(zck->temp_fd) {
if(zck->comp.dict) {
- if(!zck->comp.compress(comp, zck->comp.dict, zck->comp.dict_size, &dst,
- &dst_size, 0))
+ char *dst = NULL;
+ size_t dst_size = 0;
+
+ if(zck->comp.compress(comp, zck->comp.dict, zck->comp.dict_size,
+ &dst, &dst_size, 0) < 0)
return False;
if(!write_data(zck->temp_fd, dst, dst_size)) {
free(dst);
dst = NULL;
dst_size = 0;
- if(!zck->comp.end_chunk(comp, &dst, &dst_size, 0))
+ if(!zck->comp.end_cchunk(comp, &dst, &dst_size, 0))
return False;
if(!write_data(zck->temp_fd, dst, dst_size)) {
free(dst);
return True;
}
-int zck_comp_close(zckCtx *zck) {
+int zck_comp_reset(zckCtx *zck) {
VALIDATE(zck);
zck->comp.started = 0;
+ if(zck->comp.dc_data) {
+ free(zck->comp.dc_data);
+ zck->comp.dc_data = NULL;
+ zck->comp.dc_data_loc = 0;
+ zck->comp.dc_data_size = 0;
+ }
if(zck->comp.close == NULL)
return True;
return zck->comp.close(&(zck->comp));
}
+int zck_comp_close(zckCtx *zck) {
+ VALIDATE(zck);
+ zck_log(ZCK_LOG_DEBUG, "Closing compression\n");
+ if(zck->comp.data) {
+ free(zck->comp.data);
+ zck->comp.data = NULL;
+ zck->comp.data_size = 0;
+ zck->comp.data_loc = 0;
+ zck->comp.data_idx = NULL;
+ }
+ return zck_comp_reset(zck);
+}
+
int zck_set_compression_type(zckCtx *zck, int type) {
VALIDATE(zck);
}
const char *zck_comp_name_from_type(int comp_type) {
- if(comp_type > 1) {
+ if(comp_type > 2) {
snprintf(unknown+8, 21, "%i)", comp_type);
return unknown;
}
return COMP_NAME[comp_type];
}
-int zck_write(zckCtx *zck, const char *src, const size_t src_size) {
- VALIDATE(zck);
+size_t zck_comp_read_from_dc(zckComp *comp, char *dst, size_t dst_size) {
+ VALIDATE_SIZE(comp);
+ VALIDATE_SIZE(dst);
+
+ size_t dl_size = dst_size;
+ if(dl_size > comp->dc_data_size - comp->dc_data_loc)
+ dl_size = comp->dc_data_size - comp->dc_data_loc;
+ memcpy(dst, comp->dc_data+comp->dc_data_loc, dl_size);
+ comp->dc_data_loc += dl_size;
+ if(dl_size > 0)
+ zck_log(ZCK_LOG_DEBUG, "Reading %lu bytes from decompressed buffer\n",
+ dl_size);
+ return dl_size;
+}
- if(!zck->comp.started && !zck_comp_init(zck))
+int zck_comp_add_to_dc(zckComp *comp, const char *src, size_t src_size) {
+ VALIDATE(comp);
+ VALIDATE(src);
+
+ /* Get rid of any already read data and allocate space for new data */
+ char *temp = zmalloc(comp->dc_data_size - comp->dc_data_loc + src_size);
+ if(temp == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to reallocate %lu bytes\n",
+ comp->dc_data_size - comp->dc_data_loc + src_size);
return False;
+ }
+ if(comp->dc_data_loc != 0)
+ zck_log(ZCK_LOG_DEBUG, "Freeing %lu bytes from decompressed buffer\n",
+ comp->dc_data_loc);
+ zck_log(ZCK_LOG_DEBUG, "Adding %lu bytes to decompressed buffer\n",
+ src_size);
+ memcpy(temp, comp->dc_data + comp->dc_data_loc,
+ comp->dc_data_size - comp->dc_data_loc);
+ free(comp->dc_data);
+ comp->dc_data_size -= comp->dc_data_loc;
+ comp->dc_data_loc = 0;
+ comp->dc_data = temp;
+
+ /* Copy new uncompressed data into comp */
+ memcpy(comp->dc_data + comp->dc_data_size, src, src_size);
+ comp->dc_data_size += src_size;
+ return True;
+}
+
+int zck_comp_add_to_data(zckComp *comp, const char *src, size_t src_size) {
+ VALIDATE(comp);
+ VALIDATE(src);
+ comp->data = realloc(comp->data, comp->data_size + src_size);
+ if(comp->data == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to reallocate %lu bytes\n",
+ comp->data_size + src_size);
+ return False;
+ }
+ zck_log(ZCK_LOG_DEBUG, "Adding %lu bytes to compressed buffer\n",
+ src_size);
+ memcpy(comp->data + comp->data_size, src, src_size);
+ comp->data_size += src_size;
+ comp->data_loc += src_size;
+ return True;
+}
+
+ssize_t zck_write(zckCtx *zck, const char *src, const size_t src_size) {
+ VALIDATE_WRITE_SIZE(zck);
+
+ if(!zck->comp.started && !zck_comp_init(zck))
+ return -1;
if(src_size == 0)
- return True;
+ return 0;
char *dst = NULL;
size_t dst_size = 0;
- if(!zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1))
- return False;
+ if(zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1) < 0)
+ return -1;
if(dst_size > 0 && !write_data(zck->temp_fd, dst, dst_size)) {
free(dst);
- return False;
+ return -1;
}
if(!zck_index_add_to_chunk(zck, dst, dst_size, src_size)) {
free(dst);
- return False;
+ return -1;
}
free(dst);
- return True;
+ return src_size;
}
-int zck_end_chunk(zckCtx *zck) {
- VALIDATE(zck);
+ssize_t zck_end_chunk(zckCtx *zck) {
+ VALIDATE_WRITE_SIZE(zck);
if(!zck->comp.started && !zck_comp_init(zck))
- return False;
+ return -1;
/* No point in compressing empty data */
- if(zck->comp.data_size == 0)
- return True;
+ if(zck->comp.dc_data_size == 0)
+ return 0;
+ size_t data_size = zck->comp.dc_data_size;
char *dst = NULL;
size_t dst_size = 0;
- if(!zck->comp.end_chunk(&(zck->comp), &dst, &dst_size, 1))
- return False;
+ if(!zck->comp.end_cchunk(&(zck->comp), &dst, &dst_size, 1))
+ return -1;
if(dst_size > 0 && !write_data(zck->temp_fd, dst, dst_size)) {
free(dst);
- return False;
+ return -1;
}
if(!zck_index_add_to_chunk(zck, dst, dst_size, 0)) {
free(dst);
- return False;
+ return -1;
}
if(!zck_index_finish_chunk(zck)) {
free(dst);
- return False;
+ return -1;
}
free(dst);
- return True;
+ return data_size;
}
-int zck_read(zckCtx *zck, const char *src, const size_t src_size, char **dst,
- size_t dst_size) {
- VALIDATE(zck);
+ssize_t comp_end_dchunk(zckCtx *zck, int use_dict, size_t fd_size) {
+ ssize_t rb = zck->comp.end_dchunk(&(zck->comp), use_dict, fd_size);
+ if(!zck_validate_current_chunk(zck))
+ return -1;
+ zck->comp.data_loc = 0;
+ zck->comp.data_idx = zck->comp.data_idx->next;
+ zck_hash_init(&(zck->check_chunk_hash), &(zck->chunk_hash_type));
+ return rb;
+}
- zckComp *comp = &(zck->comp);
- *dst = NULL;
+ssize_t comp_read(zckCtx *zck, char *dst, size_t dst_size, int use_dict) {
+ VALIDATE_READ_SIZE(zck);
if(!zck->comp.started) {
zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n");
- return False;
+ return -1;
}
- if(src_size == 0)
- return True;
+ if(dst_size == 0)
+ return 0;
- if(!zck->comp.decompress(comp, src, src_size, dst, dst_size, 1))
+ size_t dc = 0;
+ char *src = zmalloc(dst_size - dc);
+ if(src == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", dst_size-dc);
return False;
+ }
+ int finished_rd = False;
+ int finished_dc = False;
+ zck_log(ZCK_LOG_DEBUG, "Trying to read %lu bytes\n", dst_size);
+ while(dc < dst_size) {
+ /* Get bytes from decompressed buffer */
+ ssize_t rb = zck_comp_read_from_dc(&(zck->comp), dst+dc, dst_size-dc);
+ if(rb < 0)
+ goto zck_read_error;
+ dc += rb;
+ if(dc == dst_size)
+ break;
+ if(rb > 0)
+ continue;
+ if(finished_dc || zck->comp.data_eof)
+ break;
+
+ /* Decompress compressed buffer into decompressed buffer */
+ size_t dc_data_size = zck->comp.dc_data_size;
+ size_t dc_data_loc = zck->comp.dc_data_loc;
+ if(!zck->comp.decompress(&(zck->comp), use_dict))
+ goto zck_read_error;
+
+ /* Check whether we decompressed more data */
+ if(zck->comp.dc_data_size != dc_data_size ||
+ zck->comp.dc_data_loc != dc_data_loc)
+ continue;
+
+ /* End decompression chunk if we're on a chunk boundary */
+ if(zck->comp.data_idx == NULL) {
+ zck->comp.data_idx = zck->index.first;
+ zck_hash_init(&(zck->check_chunk_hash), &(zck->chunk_hash_type));
+ zck->comp.data_loc = 0;
+ }
+ if(zck->comp.data_loc == zck->comp.data_idx->comp_length) {
+ if(comp_end_dchunk(zck, use_dict, zck->comp.data_idx->length) < 0)
+ return -1;
+ if(zck->comp.data_idx == NULL) {
+ if(!zck_validate_file(zck))
+ goto zck_hash_error;
+ zck->comp.data_eof = True;
+ }
+ continue;
+ }
- return True;
+ /* If we finished reading and we've reached here, we're done
+ * decompressing */
+ if(finished_rd) {
+ finished_dc = True;
+ continue;
+ }
+
+ /* Make sure we don't read beyond current chunk length */
+ size_t rs = dst_size;
+ if(zck->comp.data_loc + rs > zck->comp.data_idx->comp_length)
+ rs = zck->comp.data_idx->comp_length - zck->comp.data_loc;
+
+ /* Decompressed buffer is empty, so read data from file and fill
+ * compressed buffer */
+ rb = read_data(zck->fd, src, rs);
+ if(rb < 0)
+ goto zck_read_error;
+ if(rb < rs) {
+ zck_log(ZCK_LOG_DEBUG, "EOF\n");
+ finished_rd = True;
+ }
+ zck_hash_update(&(zck->check_full_hash), src, rb);
+ zck_hash_update(&(zck->check_chunk_hash), src, rb);
+ if(!zck_comp_add_to_data(&(zck->comp), src, rb))
+ goto zck_read_error;
+ }
+ free(src);
+ return dc;
+zck_read_error:
+ free(src);
+ return -1;
+zck_hash_error:
+ free(src);
+ return -2;
+}
+
+ssize_t zck_read(zckCtx *zck, char *dst, size_t dst_size) {
+ VALIDATE_READ_SIZE(zck);
+
+ return comp_read(zck, dst, dst_size, 1);
}
#include "zck_private.h"
-static int zck_nocomp_init(zckComp *comp) {
+static int init(zckComp *comp) {
return True;
}
-static int zck_nocomp_end_chunk(zckComp *comp, char **dst, size_t *dst_size,
- int use_dict) {
+static int end_cchunk(zckComp *comp, char **dst, size_t *dst_size,
+ int use_dict) {
*dst = NULL;
*dst_size = 0;
return True;
}
-static int zck_nocomp_comp(zckComp *comp, const char *src, const size_t src_size,
- char **dst, size_t *dst_size, int use_dict) {
+static ssize_t compress(zckComp *comp, const char *src, const size_t src_size,
+ char **dst, size_t *dst_size, int use_dict) {
*dst = zmalloc(src_size);
if(dst == NULL) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size);
- return False;
+ return -1;
}
memcpy(*dst, src, src_size);
*dst_size = src_size;
- return True;
+ return *dst_size;
}
-static int zck_nocomp_decomp(zckComp *comp, const char *src,
- const size_t src_size, char **dst, size_t dst_size,
- int use_dict) {
- *dst = zmalloc(src_size);
- if(dst == NULL) {
- zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size);
+
+static int decompress(zckComp *comp, const int use_dict) {
+ char *src = comp->data;
+ char src_size = comp->data_size;
+ comp->data = NULL;
+ comp->data_size = 0;
+ if(!zck_comp_add_to_dc(comp, src, src_size)) {
+ free(src);
return False;
}
+ free(src);
+ return True;
+}
- memcpy(*dst, src, src_size);
-
+static int end_dchunk(zckComp *comp, const int use_dict, const size_t fd_size) {
return True;
}
-static int zck_nocomp_close(zckComp *comp) {
+static int close(zckComp *comp) {
return True;
}
/* Nocomp doesn't support any parameters, so return error if setting a parameter
* was attempted */
-static int zck_nocomp_set_parameter(zckComp *comp, int option, void *value) {
+static int set_parameter(zckComp *comp, int option, void *value) {
zck_log(ZCK_LOG_ERROR, "Invalid compression parameter for ZCK_COMP_NONE\n");
return False;
}
/* No default parameters to set when there's no compression */
-static int zck_nocomp_set_default_parameters(zckComp *comp) {
+static int set_default_parameters(zckComp *comp) {
return True;
}
int zck_nocomp_setup(zckComp *comp) {
- comp->init = zck_nocomp_init;
- comp->set_parameter = zck_nocomp_set_parameter;
- comp->compress = zck_nocomp_comp;
- comp->end_chunk = zck_nocomp_end_chunk;
- comp->decompress = zck_nocomp_decomp;
- comp->close = zck_nocomp_close;
+ comp->init = init;
+ comp->set_parameter = set_parameter;
+ comp->compress = compress;
+ comp->end_cchunk = end_cchunk;
+ comp->decompress = decompress;
+ comp->end_dchunk = end_dchunk;
+ comp->close = close;
comp->type = ZCK_COMP_NONE;
- return zck_nocomp_set_default_parameters(comp);
+ return set_default_parameters(comp);
}
/* The zstd compression format doesn't allow streaming compression with a dict
* unless you statically link to it. If we have a dict, we do pseudo-streaming
* compression where we buffer the data until the chunk ends. */
-static int compress(zckComp *comp, const char *src, const size_t src_size,
- char **dst, size_t *dst_size, int use_dict) {
- if(comp->data == NULL)
- comp->data = zmalloc(src_size);
- else
- comp->data = realloc(comp->data, comp->data_size + src_size);
- if(comp->data == NULL) {
+static ssize_t compress(zckComp *comp, const char *src, const size_t src_size,
+ char **dst, size_t *dst_size, int use_dict) {
+ comp->dc_data = realloc(comp->dc_data, comp->dc_data_size + src_size);
+ if(comp->dc_data == NULL) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- comp->data_size + src_size);
- return False;
+ comp->dc_data_size + src_size);
+ return -1;
}
- memcpy(comp->data + comp->data_size, src, src_size);
+ memcpy(comp->dc_data + comp->dc_data_size, src, src_size);
*dst = NULL;
*dst_size = 0;
- comp->data_size += src_size;
- return True;
+ comp->dc_data_size += src_size;
+ return 0;
}
-static int end_chunk(zckComp *comp, char **dst, size_t *dst_size,
- int use_dict) {
+static int end_cchunk(zckComp *comp, char **dst, size_t *dst_size,
+ int use_dict) {
VALIDATE(comp);
- size_t max_size = ZSTD_compressBound(comp->data_size);
+ size_t max_size = ZSTD_compressBound(comp->dc_data_size);
if(ZSTD_isError(max_size)) {
zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n",
ZSTD_getErrorName(max_size));
if(use_dict && comp->cdict_ctx) {
*dst_size = ZSTD_compress_usingCDict(comp->cctx, *dst, max_size,
- comp->data, comp->data_size,
+ comp->dc_data, comp->dc_data_size,
comp->cdict_ctx);
} else {
- *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, comp->data,
- comp->data_size, comp->level);
+ *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, comp->dc_data,
+ comp->dc_data_size, comp->level);
}
- free(comp->data);
- comp->data = NULL;
- comp->data_size = 0;
+ free(comp->dc_data);
+ comp->dc_data = NULL;
+ comp->dc_data_size = 0;
+ comp->dc_data_loc = 0;
if(ZSTD_isError(*dst_size)) {
zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n",
ZSTD_getErrorName(*dst_size));
return True;
}
-static int decompress(zckComp *comp, const char *src, const size_t src_size,
- char **dst, size_t dst_size, int use_dict) {
+static int decompress(zckComp *comp, const int use_dict) {
VALIDATE(comp);
- *dst = zmalloc(dst_size);
- if(dst == NULL) {
- zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", dst_size);
- return False;
- }
+ return True;
+}
+
+static int end_dchunk(zckComp *comp, const int use_dict, const size_t fd_size) {
+ char *src = comp->data;
+ size_t src_size = comp->data_size;
+ comp->data = NULL;
+ comp->data_size = 0;
+
+ char *dst = zmalloc(fd_size);
+ if(dst == NULL)
+ goto decomp_error_1;
size_t retval;
- if(use_dict && comp->ddict_ctx)
- retval = ZSTD_decompress_usingDDict(comp->dctx, *dst, dst_size, src,
+ zck_log(ZCK_LOG_DEBUG, "Decompressing %lu bytes to %lu bytes\n", src_size,
+ fd_size);
+ if(use_dict && comp->ddict_ctx) {
+ zck_log(ZCK_LOG_DEBUG, "Running decompression using dict\n");
+ retval = ZSTD_decompress_usingDDict(comp->dctx, dst, fd_size, src,
src_size, comp->ddict_ctx);
- else
- retval = ZSTD_decompressDCtx(comp->dctx, *dst, dst_size, src,
- src_size);
+ } else {
+ zck_log(ZCK_LOG_DEBUG, "Running decompression\n");
+ retval = ZSTD_decompressDCtx(comp->dctx, dst, fd_size, src, src_size);
+ }
+
if(ZSTD_isError(retval)) {
- free(*dst);
- *dst = NULL;
zck_log(ZCK_LOG_ERROR, "zstd decompression error: %s\n",
ZSTD_getErrorName(retval));
- return False;
+ goto decomp_error_2;
}
+ if(!zck_comp_add_to_dc(comp, dst, fd_size))
+ goto decomp_error_2;
+ free(dst);
+ free(src);
return True;
+decomp_error_2:
+ free(dst);
+decomp_error_1:
+ free(src);
+ return False;
}
static int close(zckComp *comp) {
ZSTD_freeDCtx(comp->dctx);
comp->dctx = NULL;
}
- if(comp->data) {
- free(comp->data);
- comp->data = NULL;
- }
- comp->data_size = 0;
return True;
}
comp->init = init;
comp->set_parameter = set_parameter;
comp->compress = compress;
- comp->end_chunk = end_chunk;
+ comp->end_cchunk = end_cchunk;
comp->decompress = decompress;
+ comp->end_dchunk = end_dchunk;
comp->close = close;
comp->type = ZCK_COMP_ZSTD;
return set_default_parameters(comp);
return False;
if(!found && !zck_range_add(info, tgt_idx, tgt))
return False;
-
tgt_idx = tgt_idx->next;
}
return True;
/* Download first hundred bytes and read magic and hash type */
if(!zck_dl_bytes(dl, url, 100, start, &buffer_len))
return False;
- if(!zck_read_initial(zck, dl->dst_fd))
+ if(!zck_read_initial(zck))
return False;
start = tell_data(dl->dst_fd);
start, &buffer_len))
return False;
/* Read and store the index hash */
- if(!zck_read_index_hash(zck, dl->dst_fd))
+ if(!zck_read_index_hash(zck))
return False;
start += zck->hash_type.digest_size;
char *digest = zck_get_index_digest(zck);
zck_log(ZCK_LOG_DEBUG, "\n");
/* Read and store compression type and index size */
- if(!zck_read_ct_is(zck, dl->dst_fd))
+ if(!zck_read_ct_is(zck))
return False;
start = tell_data(dl->dst_fd);
zck_log(ZCK_LOG_DEBUG, "Index size: %llu\n", zck->index_size);
if(!zck_dl_bytes(dl, url, zck->index_size, start,
&buffer_len))
return False;
- if(!zck_read_index(zck, dl->dst_fd))
+ if(!zck_read_index(zck))
return False;
/* Write zeros to rest of file */
#include "zck_private.h"
+#define VALIDATE(f) if(!f) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckCtx not initialized\n"); \
+ return False; \
+ }
+
+#define VALIDATE_READ(f) VALIDATE(f); \
+ if(f->mode != ZCK_MODE_READ) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckCtx not opened for reading\n"); \
+ return False; \
+ }
+
+#define VALIDATE_WRITE(f) VALIDATE(f); \
+ if(f->mode != ZCK_MODE_WRITE) { \
+ zck_log(ZCK_LOG_ERROR, \
+ "zckCtx not opened for writing\n"); \
+ return False; \
+ }
+
+int zck_read_initial(zckCtx *zck) {
+ VALIDATE_READ(zck);
-int zck_read_initial(zckCtx *zck, int src_fd) {
char *header = zmalloc(5 + MAX_COMP_SIZE);
size_t length = 0;
if(header == NULL) {
}
zck_log(ZCK_LOG_DEBUG, "Reading magic and hash type\n");
- if(!read_data(src_fd, header, 5 + MAX_COMP_SIZE)) {
+ if(!read_data(zck->fd, header, 5 + MAX_COMP_SIZE)) {
free(header);
return False;
}
return False;
if(!zck_hash_setup(&(zck->hash_type), hash_type))
return False;
- if(!seek_data(src_fd, length, SEEK_SET))
+ if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type)))
+ return False;
+ if(!seek_data(zck->fd, length, SEEK_SET))
return False;
zck->header_string = header;
zck->header_size = length;
return True;
}
-int zck_read_index_hash(zckCtx *zck, int src_fd) {
+int zck_read_index_hash(zckCtx *zck) {
+ VALIDATE_READ(zck);
+
if(zck->header_string == NULL) {
zck_log(ZCK_LOG_ERROR,
"Reading index hash before initial bytes are read\n");
return False;
}
zck_log(ZCK_LOG_DEBUG, "Reading index hash\n");
- if(!read_data(src_fd, digest, zck->hash_type.digest_size)) {
+ if(!read_data(zck->fd, digest, zck->hash_type.digest_size)) {
free(digest);
free(header);
return False;
return True;
}
-int zck_read_ct_is(zckCtx *zck, int src_fd) {
+int zck_read_ct_is(zckCtx *zck) {
+ VALIDATE_READ(zck);
+
if(zck->header_string == NULL) {
zck_log(ZCK_LOG_ERROR,
"Reading compression type before hash type is read\n");
return False;
}
zck_log(ZCK_LOG_DEBUG, "Reading compression type and index size\n");
- if(!read_data(src_fd, header + length, MAX_COMP_SIZE*2))
+ if(!read_data(zck->fd, header + length, MAX_COMP_SIZE*2))
return False;
int tmp = 0;
return False;
zck->index_size = tmp;
- if(!seek_data(src_fd, length, SEEK_SET))
+ if(!seek_data(zck->fd, length, SEEK_SET))
return False;
zck->header_string = header;
zck->header_size = length;
return True;
}
-int zck_read_index(zckCtx *zck, int src_fd) {
+int zck_read_index(zckCtx *zck) {
+ VALIDATE_READ(zck);
+
char *index = zmalloc(zck->index_size);
if(!index) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
return False;
}
zck_log(ZCK_LOG_DEBUG, "Reading index\n");
- if(!read_data(src_fd, index, zck->index_size)) {
+ if(!read_data(zck->fd, index, zck->index_size)) {
free(index);
return False;
}
return True;
}
-int zck_read_header(zckCtx *zck, int src_fd) {
- if(zck == NULL) {
- zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n");
+int zck_read_header(zckCtx *zck) {
+ VALIDATE_READ(zck);
+
+ if(!zck_read_initial(zck))
return False;
- }
- zck->fd = src_fd;
- if(!zck_read_initial(zck, src_fd))
+ if(!zck_read_index_hash(zck))
return False;
- if(!zck_read_index_hash(zck, src_fd))
+ if(!zck_read_ct_is(zck))
return False;
- if(!zck_read_ct_is(zck, src_fd))
+ if(!zck_read_index(zck))
return False;
- if(!zck_read_index(zck, src_fd))
+ if(!zck_import_dict(zck))
return False;
return True;
}
}
int zck_write_header(zckCtx *zck) {
+ VALIDATE_WRITE(zck);
+
if(!write_data(zck->fd, zck->header_string, zck->header_size))
return False;
return True;
#include "zck_private.h"
-int read_data(int fd, char *data, size_t length) {
+ssize_t read_data(int fd, char *data, size_t length) {
if(length == 0)
- return True;
+ return 0;
if(data == NULL) {
zck_log(ZCK_LOG_ERROR, "Unable to read to NULL data pointer\n");
- return False;
+ return -1;
}
ssize_t read_bytes = read(fd, data, length);
if(read_bytes == -1) {
zck_log(ZCK_LOG_ERROR, "Error reading data: %s\n", strerror(errno));
- return False;
- } else if(read_bytes != length) {
- zck_log(ZCK_LOG_ERROR, "Short read\n");
- return False;
+ return -1;
}
- return True;
+ return read_bytes;
}
int write_data(int fd, const char *data, size_t length) {
return True;
}
-size_t tell_data(int fd) {
- return lseek(fd, 0, SEEK_CUR);
+ssize_t tell_data(int fd) {
+ ssize_t loc = lseek(fd, 0, SEEK_CUR);
+ return loc;
}
int chunks_from_temp(zckCtx *zck) {
return False; \
}
-int zck_write_file(zckCtx *zck) {
+int zck_close(zckCtx *zck) {
VALIDATE(zck);
- zck_index_finalize(zck);
- zck_log(ZCK_LOG_DEBUG, "Writing header\n");
- if(!zck_write_header(zck))
- return False;
- zck_log(ZCK_LOG_DEBUG, "Writing index\n");
- if(!zck_write_index(zck))
- return False;
- zck_log(ZCK_LOG_DEBUG, "Writing chunks\n");
- if(!chunks_from_temp(zck))
- return False;
- zck_log(ZCK_LOG_DEBUG, "Finished writing file, cleaning up\n");
- zck_index_free(zck);
- zck_comp_close(zck);
- if(zck->temp_fd) {
- close(zck->temp_fd);
- zck->temp_fd = 0;
- }
+ if(zck->mode == ZCK_MODE_WRITE) {
+ zck_index_finalize(zck);
+ zck_log(ZCK_LOG_DEBUG, "Writing header\n");
+ if(!zck_write_header(zck))
+ return False;
+ zck_log(ZCK_LOG_DEBUG, "Writing index\n");
+ if(!zck_write_index(zck))
+ return False;
+ zck_log(ZCK_LOG_DEBUG, "Writing chunks\n");
+ if(!chunks_from_temp(zck))
+ return False;
+ zck_log(ZCK_LOG_DEBUG, "Finished writing file, cleaning up\n");
+ zck_index_free(zck);
+ zck_comp_close(zck);
+ if(zck->temp_fd) {
+ close(zck->temp_fd);
+ zck->temp_fd = 0;
+ }
+ }
return True;
}
zck_comp_close(zck);
zck_hash_close(&(zck->full_hash));
zck_hash_close(&(zck->check_full_hash));
+ zck_hash_close(&(zck->check_chunk_hash));
zck_clear_work_index(zck);
if(zck->full_hash_digest) {
free(zck->full_hash_digest);
return zck;
}
+zckCtx *zck_init_adv_read (int src_fd) {
+ zckCtx *zck = zck_create();
+ if(zck == NULL)
+ return NULL;
+
+ zck->mode = ZCK_MODE_READ;
+ zck->fd = src_fd;
+ return zck;
+}
+
+zckCtx *zck_init_read (int src_fd) {
+ zckCtx *zck = zck_init_adv_read(src_fd);
+ if(zck == NULL)
+ return NULL;
+
+ if(!zck_read_header(zck))
+ return False;
+
+ return zck;
+}
+
+zckCtx *zck_init_write (int dst_fd) {
+ zckCtx *zck = zck_create();
+ if(zck == NULL)
+ return NULL;
+
+ zck->mode = ZCK_MODE_WRITE;
+ zck->temp_fd = zck_get_tmp_fd();
+ if(zck->temp_fd < 0)
+ goto iw_error;
+
+ /* Set defaults */
+#ifdef ZCHUNK_ZSTD
+ if(!zck_set_compression_type(zck, ZCK_COMP_ZSTD))
+ goto iw_error;
+#else
+ if(!zck_set_compression_type(zck, ZCK_COMP_NONE))
+ goto iw_error;
+#endif
+ if(!zck_set_full_hash_type(zck, ZCK_HASH_SHA256))
+ goto iw_error;
+ if(!zck_set_chunk_hash_type(zck, ZCK_HASH_SHA1))
+ goto iw_error;
+ zck->fd = dst_fd;
+
+ return zck;
+iw_error:
+ free(zck);
+ return NULL;
+}
+
int zck_set_full_hash_type(zckCtx *zck, int hash_type) {
VALIDATE(zck);
zck_log(ZCK_LOG_INFO, "Setting full hash to %s\n",
return zck->header_size + zck->index_size;
}
+ssize_t zck_get_data_length(zckCtx *zck) {
+ zckIndexItem *idx = zck->index.first;
+ while(idx->next != NULL)
+ idx = idx->next;
+ return idx->start + idx->comp_length;
+}
+
int zck_get_tmp_fd() {
int temp_fd;
char *fname = NULL;
return temp_fd;
}
-int zck_init_write (zckCtx *zck, int dst_fd) {
+int zck_import_dict(zckCtx *zck) {
VALIDATE(zck);
- zck_clear(zck);
- memset(zck, 0, sizeof(zckCtx));
- zck->temp_fd = zck_get_tmp_fd();
- if(zck->temp_fd < 0)
- return False;
-
- /* Set defaults */
-#ifdef ZCHUNK_ZSTD
- zck_set_compression_type(zck, ZCK_COMP_ZSTD);
-#else
- zck_set_compression_type(zck, ZCK_COMP_NONE);
-#endif
- if(!zck_set_full_hash_type(zck, ZCK_HASH_SHA256))
- return False;
- if(!zck_set_chunk_hash_type(zck, ZCK_HASH_SHA1))
- return False;
- zck->fd = dst_fd;
+ size_t size = zck->index.first->length;
- return True;
-}
+ /* No dict */
+ if(size == 0)
+ return True;
-int zck_import_dict(zckCtx *zck, char *data, size_t size) {
- VALIDATE(zck);
- if(data == NULL || size == 0) {
- zck_log(ZCK_LOG_ERROR,
- "Attempting to initialize an empty compression dictionary\n");
+ char *data = zmalloc(size);
+ if(data == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", size);
return False;
}
- zck_log(ZCK_LOG_DEBUG, "Closing compression\n");
- if(!zck_comp_close(zck))
+ if(comp_read(zck, data, size, 0) != size) {
+ zck_log(ZCK_LOG_ERROR, "Error reading compressed dict\n");
+ return False;
+ }
+ zck_log(ZCK_LOG_DEBUG, "Resetting compression\n");
+ if(!zck_comp_reset(zck))
return False;
zck_log(ZCK_LOG_DEBUG, "setting dict 1\n");
if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT, data))
zck_log(ZCK_LOG_DEBUG, "setting dict 2\n");
if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT_SIZE, &size))
return False;
- zck_log(ZCK_LOG_DEBUG, "Initializing\n");
if(!zck_comp_init(zck))
return False;
+ free(data);
+
return True;
}
return True;
}
-int zck_validate_file(zckCtx *zck) {
+int zck_validate_current_chunk(zckCtx *zck) {
VALIDATE(zck);
- char *digest = zck_hash_finalize(&(zck->check_full_hash));
+ char *digest = zck_hash_finalize(&(zck->check_chunk_hash));
if(digest == NULL) {
zck_log(ZCK_LOG_ERROR,
- "Unable to calculate %s checksum for full file\n");
+ "Unable to calculate %s checksum for chunk\n");
return False;
}
- zck_log(ZCK_LOG_DEBUG, "Checking data checksum\n");
- if(memcmp(digest, zck->full_hash_digest, zck->hash_type.digest_size) != 0) {
+ zck_log(ZCK_LOG_DEBUG, "Checking chunk checksum\n");
+ zck_log(ZCK_LOG_DEBUG, "Expected chunk checksum: ");
+ for(int i=0; i<zck->chunk_hash_type.digest_size; i++)
+ zck_log(ZCK_LOG_DEBUG, "%02x",
+ (unsigned char)zck->comp.data_idx->digest[i]);
+ zck_log(ZCK_LOG_DEBUG, "\n");
+ zck_log(ZCK_LOG_DEBUG, "Calculated chunk checksum: ");
+ for(int i=0; i<zck->chunk_hash_type.digest_size; i++)
+ zck_log(ZCK_LOG_DEBUG, "%02x", (unsigned char)digest[i]);
+ zck_log(ZCK_LOG_DEBUG, "\n");
+ if(memcmp(digest, zck->comp.data_idx->digest,
+ zck->chunk_hash_type.digest_size) != 0) {
free(digest);
- zck_log(ZCK_LOG_ERROR, "Data checksum failed!\n");
+ zck_log(ZCK_LOG_ERROR, "Chunk checksum failed!\n");
return False;
}
- zck_log(ZCK_LOG_DEBUG, "Data checksum valid\n");
+ zck_log(ZCK_LOG_DEBUG, "Chunk checksum valid\n");
free(digest);
return True;
}
-int zck_decompress_to_file(zckCtx *zck, int src_fd, int dst_fd) {
+int zck_validate_file(zckCtx *zck) {
VALIDATE(zck);
- if(!zck_read_header(zck, src_fd))
- return False;
- if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type)))
- return False;
-
- int start = lseek(src_fd, 0, SEEK_CUR);
- if(start == -1) {
- zck_log(ZCK_LOG_ERROR, "Unable to set starting point source file: %s\n",
- strerror(errno));
+ char *digest = zck_hash_finalize(&(zck->check_full_hash));
+ if(digest == NULL) {
+ zck_log(ZCK_LOG_ERROR,
+ "Unable to calculate %s checksum for full file\n");
return False;
}
-
- zckIndex *index = zck_get_index(zck);
- zckIndexItem *idx = index->first;
-
- /* Check if zck file is empty */
- for(int count=0; idx; count++) {
- size_t csize = idx->comp_length;
- size_t size = idx->length;
- char *cdata;
-
- if(csize == 0) {
- idx = idx->next;
- continue;
- }
-
- cdata = zmalloc(csize);
- if(cdata == NULL) {
- zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", csize);
- return False;
- }
- if(!seek_data(src_fd, start + idx->start, SEEK_SET))
- return False;
- if(!read_data(src_fd, cdata, csize)) {
- free(cdata);
- zck_log(ZCK_LOG_ERROR, "Error reading chunk %i\n", count);
- return False;
- }
- if(!zck_validate_chunk(zck, cdata, csize, idx, count)) {
- free(cdata);
- return False;
- }
-
- char *data = NULL;
- if(!zck_read(zck, cdata, csize, &data, size)) {
- free(cdata);
- zck_log(ZCK_LOG_ERROR, "Unable to decompress chunk %i\n", count);
- return False;
- }
- free(cdata);
- if(count == 0) {
- if(!zck_import_dict(zck, data, size)) {
- free(data);
- return False;
- }
- } else {
- if(!write_data(dst_fd, data, size)) {
- free(data);
- zck_log(ZCK_LOG_ERROR, "Unable to write chunk %i\n", count);
- return False;
- }
- }
- free(data);
- idx = idx->next;
- }
- if(!zck_validate_file(zck))
+ zck_log(ZCK_LOG_DEBUG, "Checking data checksum\n");
+ zck_log(ZCK_LOG_INFO, "Excpected data checksum: ");
+ for(int i=0; i<zck->hash_type.digest_size; i++)
+ zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)zck->full_hash_digest[i]);
+ zck_log(ZCK_LOG_INFO, "\n");
+ zck_log(ZCK_LOG_INFO, "Calculated data checksum: ");
+ for(int i=0; i<zck->hash_type.digest_size; i++)
+ zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)digest[i]);
+ zck_log(ZCK_LOG_INFO, "\n");
+ if(memcmp(digest, zck->full_hash_digest, zck->hash_type.digest_size) != 0) {
+ free(digest);
+ zck_log(ZCK_LOG_ERROR, "Data checksum failed!\n");
return False;
+ }
+ zck_log(ZCK_LOG_DEBUG, "Data checksum valid\n");
+ free(digest);
return True;
}
/* Maximum string length for a compressed size_t */
#define MAX_COMP_SIZE (((sizeof(size_t) * 8) / 7) + 1)
+#define ZCK_MODE_READ 0
+#define ZCK_MODE_WRITE 1
+
#define zmalloc(x) calloc(1, x)
struct zckComp;
typedef int (*finit)(struct zckComp *comp);
typedef int (*fparam)(struct zckComp *comp, int option, void *value);
-typedef int (*fcompend)(struct zckComp *comp, char **dst, size_t *dst_size,
- int use_dict);
-typedef int (*fcomp)(struct zckComp *comp, const char *src,
- const size_t src_size, char **dst, size_t *dst_size,
- int use_dict);
-typedef int (*fdecomp)(struct zckComp *comp, const char *src,
- const size_t src_size, char **dst, size_t dst_size,
- int use_dict);
+typedef int (*fccompend)(struct zckComp *comp, char **dst, size_t *dst_size,
+ int use_dict);
+typedef ssize_t (*fcomp)(struct zckComp *comp, const char *src,
+ const size_t src_size, char **dst, size_t *dst_size,
+ int use_dict);
+typedef int (*fdecomp)(struct zckComp *comp, const int use_dict);
+typedef int (*fdcompend)(struct zckComp *comp, const int use_dict,
+ const size_t fd_size);
typedef int (*fcclose)(struct zckComp *comp);
typedef enum log_type log_type;
char *data;
size_t data_size;
+ size_t data_loc;
+ zckIndexItem *data_idx;
+ int data_eof;
+ char *dc_data;
+ size_t dc_data_size;
+ size_t dc_data_loc;
finit init;
fparam set_parameter;
fcomp compress;
- fcompend end_chunk;
+ fccompend end_cchunk;
fdecomp decompress;
+ fdcompend end_dchunk;
fcclose close;
} zckComp;
typedef struct zckCtx {
int temp_fd;
int fd;
+ int mode;
char *full_hash_digest;
char *header_string;
zckIndex index;
zckIndexItem *work_index_item;
zckHash work_index_hash;
+
char *index_digest;
zckHash full_hash;
zckHash check_full_hash;
+ zckHash check_chunk_hash;
zckComp comp;
zckHashType hash_type;
zckHashType chunk_hash_type;
+
+ char *data;
+ size_t data_size;
} zckCtx;
const char *zck_hash_name_from_type(int hash_type);
int zck_get_tmp_fd();
+int zck_import_dict(zckCtx *zck);
int zck_validate_file(zckCtx *zck);
+int zck_validate_current_chunk(zckCtx *zck);
void zck_clear_work_index(zckCtx *zck);
/* hash/hash.h */
void zck_index_free(zckCtx *zck);
void zck_index_free_item(zckIndexItem **item);
int zck_write_index(zckCtx *zck);
+zckIndexItem *zck_get_index_of_loc(zckIndex *index, size_t loc);
/* io.c */
int seek_data(int fd, off_t offset, int whence);
-size_t tell_data(int fd);
-int read_data(int fd, char *data, size_t length);
+ssize_t tell_data(int fd);
+ssize_t read_data(int fd, char *data, size_t length);
int write_data(int fd, const char *data, size_t length);
int write_comp_size(int fd, size_t val);
int read_comp_size(int fd, size_t *val, size_t *length);
int chunks_from_temp(zckCtx *zck);
/* header.c */
-int zck_read_initial(zckCtx *zck, int src_fd);
-int zck_read_index_hash(zckCtx *zck, int src_fd);
-int zck_read_ct_is(zckCtx *zck, int src_fd);
-int zck_read_index(zckCtx *zck, int src_fd);
+int zck_read_initial(zckCtx *zck);
+int zck_read_index_hash(zckCtx *zck);
+int zck_read_ct_is(zckCtx *zck);
+int zck_read_index(zckCtx *zck);
+int zck_read_header(zckCtx *zck);
int zck_header_create(zckCtx *zck);
-int zck_read_header(zckCtx *zck, int src_fd);
int zck_write_header(zckCtx *zck);
+/* comp/comp.c */
+int zck_comp_add_to_dc(zckComp *comp, const char *src, size_t src_size);
+int zck_comp_add_to_data(zckComp *comp, const char *src, size_t src_size);
+size_t zck_comp_read_from_dc(zckComp *comp, char *dst, size_t dst_size);
+ssize_t comp_read(zckCtx *zck, char *dst, size_t dst_size, int use_dict);
+
/* dl/range.c */
char *zck_range_get_char(zckRangeItem **range, int max_ranges);
int zck_range_add(zckRange *info, zckIndexItem *idx, zckCtx *zck);
int zck_compint_to_size(size_t *val, const char *compint, size_t *length);
-
/* log.c */
void zck_log(log_type lt, const char *format, ...);
#endif
projects / zchunk.git / commitdiff