typedef struct zckCtx zckCtx;
typedef struct zckHash zckHash;
+typedef struct zckChunk zckChunk;
+typedef struct zckIndex zckIndex;
+typedef struct zckRange zckRange;
+typedef struct zckDL zckDL;
-/* Contains an index item pointing to a chunk */
-typedef struct zckIndexItem {
- char *digest;
- int digest_size;
- int valid;
- size_t start;
- size_t comp_length;
- size_t length;
- struct zckIndexItem *next;
-} zckIndexItem;
-
-/* Contains everything about an index and a pointer to the first index item */
-typedef struct zckIndex {
- size_t count;
- size_t length;
- int hash_type;
- size_t digest_size;
- zckIndexItem *first;
-} zckIndex;
-
-/* Contains a single range */
-typedef struct zckRangeItem {
- size_t start;
- size_t end;
- struct zckRangeItem *next;
- struct zckRangeItem *prev;
-} zckRangeItem;
-
-/* Contains a series of ranges, information about them, a link to the first
- * range item, and an index describing what information is in the ranges */
-typedef struct zckRange {
- unsigned int count;
- zckRangeItem *first;
- zckIndex index;
-} zckRange;
+typedef size_t (*zck_wcb)(void *ptr, size_t l, size_t c, void *dl_v);
/*******************************************************************
* Reading a zchunk file
/*******************************************************************
* Indexes
*******************************************************************/
-/* Get index count */
-ssize_t zck_get_index_count(zckCtx *zck)
+/* Get chunk count */
+ssize_t zck_get_chunk_count(zckCtx *zck)
+ __attribute__ ((warn_unused_result));
+/* Get first chunk */
+zckChunk *zck_get_first_chunk(zckCtx *zck)
+ __attribute__ ((warn_unused_result));
+/* Get next chunk */
+zckChunk *zck_get_next_chunk(zckChunk *idx)
+ __attribute__ ((warn_unused_result));
+/* Get chunk starting location */
+ssize_t zck_get_chunk_start(zckChunk *idx)
+ __attribute__ ((warn_unused_result));
+/* Get uncompressed chunk size */
+ssize_t zck_get_chunk_size(zckChunk *idx)
__attribute__ ((warn_unused_result));
-/* Get index */
-zckIndex *zck_get_index(zckCtx *zck)
+/* Get compressed chunk size */
+ssize_t zck_get_chunk_comp_size(zckChunk *idx)
+ __attribute__ ((warn_unused_result));
+/* Get validity of current chunk - 1 = valid, 0 = missing, -1 = invalid */
+int zck_get_chunk_valid(zckChunk *idx)
__attribute__ ((warn_unused_result));
/* Get chunk digest */
-char *zck_get_chunk_digest(zckIndexItem *item)
+char *zck_get_chunk_digest(zckChunk *item)
+ __attribute__ ((warn_unused_result));
+/* Find out if two chunk digests are the same */
+int zck_compare_chunk_digest(zckChunk *a, zckChunk *b)
__attribute__ ((warn_unused_result));
-
/*******************************************************************
* Advanced hash functions
* Downloading (should this go in a separate header and library?)
*******************************************************************/
-typedef size_t (*zck_wcb)(void *ptr, size_t l, size_t c, void *dl_v);
-typedef struct zckDLPriv zckDLPriv;
-
-/* Contains a zchunk download context */
-typedef struct zckDL {
- size_t dl;
- size_t ul;
- zck_wcb write_cb;
- void *wdata;
- zck_wcb header_cb;
- void *hdrdata;
- zckRange *range;
- zckDLPriv *priv;
- struct zckCtx *zck;
-} zckDL;
-
/*******************************************************************
* Ranges
*******************************************************************/
* is */
int zck_get_min_download_size()
__attribute__ ((warn_unused_result));
+/* Get the number of separate range items in the range */
+int zck_get_range_count(zckRange *range)
+ __attribute__ ((warn_unused_result));
/*******************************************************************
* Downloading
void zck_dl_reset(zckDL *dl);
/* Free zchunk download context */
void zck_dl_free(zckDL **dl);
-/* Write callback. You *must* pass this and your initialized zchunk download
- * context to the downloader when downloading a zchunk file. If you have your
- * own callback, set dl->write_cb to your callback and dl->wdata to your
- * callback data. */
-size_t zck_write_chunk_cb(void *ptr, size_t l, size_t c, void *dl_v);
-size_t zck_write_zck_header_cb(void *ptr, size_t l, size_t c, void *dl_v);
-size_t zck_header_cb(char *b, size_t l, size_t c, void *dl_v);
+/* Get zchunk context from download context */
+zckCtx *zck_dl_get_zck(zckDL *dl)
+ __attribute__ ((warn_unused_result));
/* Clear regex used for extracting download ranges from multipart download */
void zck_dl_clear_regex(zckDL *dl);
/* Download and process the header from url */
/* Get number of bytes uploaded using download context */
size_t zck_dl_get_bytes_uploaded(zckDL *dl)
__attribute__ ((warn_unused_result));
+/* Set download ranges for zchunk download context */
+int zck_dl_set_range(zckDL *dl, zckRange *range)
+ __attribute__ ((warn_unused_result));
+/* Get download ranges from zchunk download context */
+int zck_dl_set_range(zckDL *dl, zckRange *range)
+ __attribute__ ((warn_unused_result));
+
+/* Set header callback function */
+int zck_dl_set_header_cb(zckDL *dl, zck_wcb func)
+ __attribute__ ((warn_unused_result));
+/* Set header userdata */
+int zck_dl_set_header_data(zckDL *dl, void *data)
+ __attribute__ ((warn_unused_result));
+/* Set write callback function */
+int zck_dl_set_write_cb(zckDL *dl, zck_wcb func)
+ __attribute__ ((warn_unused_result));
+/* Set write userdata */
+int zck_dl_set_write_data(zckDL *dl, void *data)
+ __attribute__ ((warn_unused_result));
+
+/* Write callback. You *must* pass this and your initialized zchunk download
+ * context to the downloader when downloading a zchunk file. If you have your
+ * own callback, set dl->write_cb to your callback and dl->wdata to your
+ * callback data. */
+size_t zck_write_chunk_cb(void *ptr, size_t l, size_t c, void *dl_v);
+size_t zck_write_zck_header_cb(void *ptr, size_t l, size_t c, void *dl_v);
+size_t zck_header_cb(char *b, size_t l, size_t c, void *dl_v);
#endif
/* Free zckDL header regex used for downloading ranges */
static void clear_dl_regex(zckDL *dl) {
- if(dl == NULL || dl->priv == NULL)
+ if(dl == NULL || dl == NULL)
return;
- if(dl->priv->hdr_regex) {
- regfree(dl->priv->hdr_regex);
- free(dl->priv->hdr_regex);
- dl->priv->hdr_regex = NULL;
+ if(dl->hdr_regex) {
+ regfree(dl->hdr_regex);
+ free(dl->hdr_regex);
+ dl->hdr_regex = NULL;
}
- if(dl->priv->dl_regex) {
- regfree(dl->priv->dl_regex);
- free(dl->priv->dl_regex);
- dl->priv->dl_regex = NULL;
+ if(dl->dl_regex) {
+ regfree(dl->dl_regex);
+ free(dl->dl_regex);
+ dl->dl_regex = NULL;
}
- if(dl->priv->end_regex) {
- regfree(dl->priv->end_regex);
- free(dl->priv->end_regex);
- dl->priv->end_regex = NULL;
+ if(dl->end_regex) {
+ regfree(dl->end_regex);
+ free(dl->end_regex);
+ dl->end_regex = NULL;
}
}
/* Write zeros to tgt->fd in location of tgt_idx */
-static int zero_chunk(zckCtx *tgt, zckIndexItem *tgt_idx) {
+static int zero_chunk(zckCtx *tgt, zckChunk *tgt_idx) {
char buf[BUF_SIZE] = {0};
size_t to_read = tgt_idx->comp_length;
if(!seek_data(tgt->fd, tgt->data_offset + tgt_idx->start, SEEK_SET))
/* Check whether last downloaded chunk is valid and zero it out if it isn't */
static int set_chunk_valid(zckDL *dl) {
VALIDATE(dl);
- VALIDATE(dl->priv);
+ VALIDATE(dl);
- int retval = validate_chunk(dl->zck, dl->priv->tgt_check, ZCK_LOG_WARNING,
- dl->priv->tgt_number);
+ int retval = validate_chunk(dl->zck, dl->tgt_check, ZCK_LOG_WARNING,
+ dl->tgt_number);
if(retval < 1) {
- if(!zero_chunk(dl->zck, dl->priv->tgt_check))
+ if(!zero_chunk(dl->zck, dl->tgt_check))
return False;
- dl->priv->tgt_check->valid = -1;
+ dl->tgt_check->valid = -1;
if(retval == 0)
return False;
} else {
- dl->priv->tgt_check->valid = 1;
+ dl->tgt_check->valid = 1;
}
- dl->priv->tgt_check = NULL;
+ dl->tgt_check = NULL;
return True;
}
/* Write length or to end of current chunk, whichever comes first */
static int dl_write(zckDL *dl, const char *at, size_t length) {
VALIDATE(dl);
- VALIDATE(dl->priv);
+ VALIDATE(dl);
int wb = 0;
- if(dl->priv->write_in_chunk > 0) {
- if(dl->priv->write_in_chunk < length)
- wb = dl->priv->write_in_chunk;
+ if(dl->write_in_chunk > 0) {
+ if(dl->write_in_chunk < length)
+ wb = dl->write_in_chunk;
else
wb = length;
if(!write_data(dl->zck->fd, at, wb))
return -1;
- dl->priv->write_in_chunk -= wb;
+ dl->write_in_chunk -= wb;
if(!hash_update(&(dl->zck->check_chunk_hash), at, wb))
return -1;
zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes\n", wb);
- dl->priv->dl_chunk_data += wb;
+ dl->dl_chunk_data += wb;
}
return wb;
}
/* Copy chunk identified by src_idx into location specified by tgt_idx */
static int write_and_verify_chunk(zckCtx *src, zckCtx *tgt,
- zckIndexItem *src_idx,
- zckIndexItem *tgt_idx) {
+ zckChunk *src_idx,
+ zckChunk *tgt_idx) {
static char buf[BUF_SIZE] = {0};
size_t to_read = src_idx->comp_length;
/* Split current read into the appropriate chunks and write appropriately */
int dl_write_range(zckDL *dl, const char *at, size_t length) {
VALIDATE(dl);
- VALIDATE(dl->priv);
+ VALIDATE(dl);
if(dl->range == NULL) {
zck_log(ZCK_LOG_ERROR, "zckDL range not initialized\n");
return 0;
int wb = dl_write(dl, at, length);
if(wb < 0)
return 0;
- if(dl->priv->write_in_chunk == 0) {
+ if(dl->write_in_chunk == 0) {
/* Check whether we just finished downloading a chunk and verify it */
- if(dl->priv->tgt_check && !set_chunk_valid(dl))
+ if(dl->tgt_check && !set_chunk_valid(dl))
return False;
- for(zckIndexItem *idx = dl->range->index.first; idx; idx = idx->next) {
- if(dl->priv->dl_chunk_data == idx->start) {
+ for(zckChunk *idx = dl->range->index.first; idx; idx = idx->next) {
+ if(dl->dl_chunk_data == idx->start) {
int count = 0;
- for(zckIndexItem *tgt_idx = dl->zck->index.first; tgt_idx;
+ for(zckChunk *tgt_idx = dl->zck->index.first; tgt_idx;
tgt_idx = tgt_idx->next, count++) {
if(tgt_idx->valid == 1)
continue;
if(idx->comp_length == tgt_idx->comp_length &&
memcmp(idx->digest, tgt_idx->digest,
idx->digest_size) == 0) {
- dl->priv->tgt_check = tgt_idx;
- dl->priv->tgt_number = count;
+ dl->tgt_check = tgt_idx;
+ dl->tgt_number = count;
if(!hash_init(&(dl->zck->check_chunk_hash),
&(dl->zck->chunk_hash_type)))
return 0;
- dl->priv->write_in_chunk = idx->comp_length;
+ dl->write_in_chunk = idx->comp_length;
if(!seek_data(dl->zck->fd,
dl->zck->data_offset + tgt_idx->start,
SEEK_SET))
}
int wb2 = 0;
/* We've still got data, call recursively */
- if(dl->priv->write_in_chunk > 0 && wb < length) {
+ if(dl->write_in_chunk > 0 && wb < length) {
wb2 = dl_write_range(dl, at+wb, length-wb);
if(wb2 == 0)
return 0;
}
int PUBLIC zck_copy_chunks(zckCtx *src, zckCtx *tgt) {
- zckIndex *tgt_info = zck_get_index(tgt);
- zckIndex *src_info = zck_get_index(src);
- zckIndexItem *tgt_idx = tgt_info->first;
- zckIndexItem *src_idx = src_info->first;
+ zckIndex *tgt_info = &(tgt->index);
+ zckIndex *src_info = &(src->index);
+ zckChunk *tgt_idx = tgt_info->first;
+ zckChunk *src_idx = src_info->first;
while(tgt_idx) {
/* No need to copy already valid chunk */
if(tgt_idx->valid == 1) {
sizeof(zckDL));
return NULL;
}
- dl->priv = zmalloc(sizeof(zckDLPriv));
- if(!dl->priv) {
- zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes for dl->priv\n",
- sizeof(zckDL));
- return NULL;
- }
- dl->priv->mp = zmalloc(sizeof(zckMP));
- if(!dl->priv->mp) {
+ dl->mp = zmalloc(sizeof(zckMP));
+ if(!dl->mp) {
zck_log(ZCK_LOG_ERROR,
- "Unable to allocate %lu bytes for dl->priv->mp\n",
+ "Unable to allocate %lu bytes for dl->mp\n",
sizeof(zckMP));
return NULL;
}
void PUBLIC zck_dl_reset(zckDL *dl) {
if(!dl)
return;
- if(dl->priv) {
- if(dl->priv->mp) {
- if(dl->priv->mp->buffer)
- free(dl->priv->mp->buffer);
- memset(dl->priv->mp, 0, sizeof(zckMP));
- }
- dl->priv->dl_chunk_data = 0;
- clear_dl_regex(dl);
- if(dl->priv->boundary)
- free(dl->priv->boundary);
- dl->priv->boundary = NULL;
- }
- if(dl->range)
- zck_range_free(&(dl->range));
+ reset_mp(dl->mp);
+ dl->dl_chunk_data = 0;
+ clear_dl_regex(dl);
+ if(dl->boundary)
+ free(dl->boundary);
+ dl->boundary = NULL;
zckCtx *zck = dl->zck;
size_t db = dl->dl;
size_t ub = dl->ul;
- zckDLPriv *priv = dl->priv;
+ zckMP *mp = dl->mp;
memset(dl, 0, sizeof(zckDL));
- dl->priv = priv;
dl->zck = zck;
dl->dl = db;
dl->ul = ub;
+ dl->mp = mp;
}
/* Free zckDL and set pointer to NULL */
void PUBLIC zck_dl_free(zckDL **dl) {
zck_dl_reset(*dl);
- if((*dl)->priv->mp)
- free((*dl)->priv->mp);
- if((*dl)->priv)
- free((*dl)->priv);
+ if((*dl)->mp)
+ free((*dl)->mp);
free(*dl);
*dl = NULL;
}
+zckCtx PUBLIC *zck_dl_get_zck(zckDL *dl) {
+ if(dl == NULL)
+ return NULL;
+ return dl->zck;
+}
+
+int PUBLIC zck_dl_set_range(zckDL *dl, zckRange *range) {
+ if(dl == NULL)
+ return False;
+ dl->range = range;
+ return True;
+}
+
+zckRange PUBLIC *zck_dl_get_range(zckDL *dl) {
+ if(dl == NULL)
+ return NULL;
+ return dl->range;
+}
+
+int PUBLIC zck_dl_set_header_cb(zckDL *dl, zck_wcb func) {
+ if(dl == NULL)
+ return False;
+ dl->header_cb = func;
+ return True;
+}
+
+int PUBLIC zck_dl_set_header_data(zckDL *dl, void *data) {
+ if(dl == NULL)
+ return False;
+ dl->header_data = data;
+ return True;
+}
+
+int PUBLIC zck_dl_set_write_cb(zckDL *dl, zck_wcb func) {
+ if(dl == NULL)
+ return False;
+ dl->write_cb = func;
+ return True;
+}
+
+int PUBLIC zck_dl_set_write_data(zckDL *dl, void *data) {
+ if(dl == NULL)
+ return False;
+ dl->write_data = data;
+ return True;
+}
+
/*******************************************************************
* Callbacks
*******************************************************************/
zck_log(ZCK_LOG_DEBUG, "No boundary detected");
if(dl->header_cb)
- return dl->header_cb(b, l, c, dl->hdrdata);
+ return dl->header_cb(b, l, c, dl->header_data);
return c*l;
}
zck_log(ZCK_LOG_DEBUG, "Downloading %lu bytes to position %lu\n", l*c, loc);
wb = write(dl->zck->fd, ptr, l*c);
if(dl->write_cb)
- return dl->write_cb(ptr, l, c, dl->wdata);
+ return dl->write_cb(ptr, l, c, dl->write_data);
return wb;
}
zckDL *dl = (zckDL*)dl_v;
size_t wb = 0;
dl->dl += l*c;
- if(dl->priv->boundary != NULL) {
+ if(dl->boundary != NULL) {
int retval = multipart_extract(dl, ptr, l*c);
if(retval == 0)
wb = 0;
wb = l*c;
}
if(dl->write_cb)
- return dl->write_cb(ptr, l, c, dl->wdata);
+ return dl->write_cb(ptr, l, c, dl->write_data);
return wb;
}
char *next = "\r\n--%s\r\ncontent-type:.*\r\n" \
"content-range: *bytes *([0-9]+) *- *([0-9]+) */.*\r\n\r";
char *end = "\r\n--%s--\r\n\r";
- char *regex_n = add_boundary_to_regex(next, dl->priv->boundary);
+ char *regex_n = add_boundary_to_regex(next, dl->boundary);
if(regex_n == NULL)
return False;
- char *regex_e = add_boundary_to_regex(end, dl->priv->boundary);
+ char *regex_e = add_boundary_to_regex(end, dl->boundary);
if(regex_n == NULL)
return False;
- dl->priv->dl_regex = zmalloc(sizeof(regex_t));
- if(!create_regex(dl->priv->dl_regex, regex_n)) {
+ dl->dl_regex = zmalloc(sizeof(regex_t));
+ if(!create_regex(dl->dl_regex, regex_n)) {
free(regex_n);
return False;
}
free(regex_n);
- dl->priv->end_regex = zmalloc(sizeof(regex_t));
- if(!create_regex(dl->priv->end_regex, regex_e)) {
+ dl->end_regex = zmalloc(sizeof(regex_t));
+ if(!create_regex(dl->end_regex, regex_e)) {
free(regex_e);
return False;
}
return True;
}
-static void reset_mp(zckMP *mp) {
+void reset_mp(zckMP *mp) {
+ if(mp == NULL)
+ return;
if(mp->buffer)
free(mp->buffer);
memset(mp, 0, sizeof(zckMP));
size_t multipart_extract(zckDL *dl, char *b, size_t l) {
VALIDATE(dl);
- if(dl->priv == NULL || dl->priv->mp == NULL)
+ if(dl == NULL || dl->mp == NULL)
return 0;
- zckMP *mp = dl->priv->mp;
+ zckMP *mp = dl->mp;
char *buf = b;
int alloc_buf = False;
}
/* If regex hasn't been created, create it */
- if(dl->priv->dl_regex == NULL && !gen_regex(dl))
+ if(dl->dl_regex == NULL && !gen_regex(dl))
return 0;
char *header_start = buf;
/* Run regex against download range string */
regmatch_t match[4] = {{0}};
- if(regexec(dl->priv->dl_regex, i, 3, match, 0) != 0) {
- if(regexec(dl->priv->end_regex, i, 3, match, 0) != 0)
+ if(regexec(dl->dl_regex, i, 3, match, 0) != 0) {
+ if(regexec(dl->end_regex, i, 3, match, 0) != 0)
zck_log(ZCK_LOG_ERROR, "Unable to find multipart download range\n");
goto end;
}
size_t multipart_get_boundary(zckDL *dl, char *b, size_t size) {
VALIDATE(dl);
- if(dl->priv == NULL)
+ if(dl == NULL)
return 0;
/* Create regex to find boundary */
- if(dl->priv->hdr_regex == NULL) {
+ if(dl->hdr_regex == NULL) {
char *regex = "boundary *= *([0-9a-fA-F]+)";
- dl->priv->hdr_regex = zmalloc(sizeof(regex_t));
- if(!create_regex(dl->priv->hdr_regex, regex))
+ dl->hdr_regex = zmalloc(sizeof(regex_t));
+ if(!create_regex(dl->hdr_regex, regex))
return 0;
}
/* Check whether this header contains the boundary and set it if it does */
regmatch_t match[2] = {{0}};
- if(regexec(dl->priv->hdr_regex, buf, 2, match, 0) == 0) {
- reset_mp(dl->priv->mp);
+ if(regexec(dl->hdr_regex, buf, 2, match, 0) == 0) {
+ reset_mp(dl->mp);
char *boundary = zmalloc(match[1].rm_eo - match[1].rm_so + 1);
memcpy(boundary, buf + match[1].rm_so, match[1].rm_eo - match[1].rm_so);
zck_log(ZCK_LOG_DEBUG, "Multipart boundary: %s\n", boundary);
- dl->priv->boundary = boundary;
+ dl->boundary = boundary;
}
if(buf)
free(buf);
static zckRangeItem *range_insert_new(zckRangeItem *prev, zckRangeItem *next,
uint64_t start, uint64_t end,
- zckRange *info, zckIndexItem *idx,
+ zckRange *info, zckChunk *idx,
int add_index) {
zckRangeItem *new = zmalloc(sizeof(zckRangeItem));
if(!new) {
}
if(add_index)
if(!index_new_chunk(&(info->index), idx->digest, idx->digest_size,
- end-start+1, end-start+1, False)) {
+ end-start+1, end-start+1, False, NULL)) {
free(new);
return NULL;
}
}
}
-static int range_add(zckRange *info, zckIndexItem *idx, zckCtx *zck) {
+static int range_add(zckRange *info, zckChunk *idx, zckCtx *zck) {
if(info == NULL || idx == NULL) {
- zck_log(ZCK_LOG_ERROR, "zckRange or zckIndexItem not allocated\n");
+ zck_log(ZCK_LOG_ERROR, "zckRange or zckChunk not allocated\n");
return False;
}
size_t header_len = 0;
sizeof(zckRange));
return NULL;
}
- zckIndexItem *idx = zck->index.first;
+ zckChunk *idx = zck->index.first;
while(idx) {
if(idx->valid) {
idx = idx->next;
ri.end = end;
return zck_get_range_char(&range);
}
+
+int PUBLIC zck_get_range_count(zckRange *range) {
+ if(range == NULL)
+ return -1;
+ return range->count;
+}
static int validate_checksums(zckCtx *zck, zck_log_type bad_checksums) {
VALIDATE_READ(zck);
- char buf[BUF_SIZE];
+ char buf[BUF_SIZE] = {0};
if(zck->data_offset == 0) {
zck_log(ZCK_LOG_ERROR, "Header hasn't been read yet\n");
return 0;
}
- hash_close(&(zck->check_full_hash));
if(!hash_init(&(zck->check_full_hash), &(zck->hash_type)))
return 0;
/* Check each chunk checksum */
int all_good = True;
int count = 0;
- for(zckIndexItem *idx = zck->index.first; idx; idx = idx->next, count++) {
+ for(zckChunk *idx = zck->index.first; idx; idx = idx->next, count++) {
if(idx == zck->index.first && idx->length == 0) {
idx->valid = 1;
continue;
/* If data checksum failed, invalidate *all* chunks */
if(valid_file == -1)
- for(zckIndexItem *idx = zck->index.first; idx; idx = idx->next)
+ for(zckChunk *idx = zck->index.first; idx; idx = idx->next)
idx->valid = -1;
}
}
int hash_init(zckHash *hash, zckHashType *hash_type) {
+ hash_close(hash);
if(hash && hash_type) {
if(hash_type->type == ZCK_HASH_SHA1) {
zck_log(ZCK_LOG_DDEBUG, "Initializing SHA-1 hash\n");
return (char *)digest;
}
zck_log(ZCK_LOG_ERROR, "Unsupported hash type: %i\n", hash->type);
+ hash_close(hash);
return NULL;
}
zck_log(ZCK_LOG_ERROR, "Hash hasn't been initialized\n");
+ hash_close(hash);
return NULL;
}
zck_hash_name_from_type(hash_type));
return False;
}
- hash_close(&(zck->full_hash));
if(!hash_init(&(zck->full_hash), &(zck->hash_type))) {
zck_log(ZCK_LOG_ERROR, "Unable initialize full hash\n");
return False;
}
/* Validate chunk, returning -1 if checksum fails, 1 if good, 0 if error */
-int validate_chunk(zckCtx *zck, zckIndexItem *idx,
+int validate_chunk(zckCtx *zck, zckChunk *idx,
zck_log_type bad_checksum, int chunk_number) {
VALIDATE(zck);
if(idx == NULL) {
/* Returns 1 if data hash matches, -1 if it doesn't and 0 if error */
int PUBLIC zck_validate_data_checksum(zckCtx *zck) {
- hash_close(&(zck->check_full_hash));
if(!seek_data(zck->fd, zck->data_offset, SEEK_SET))
- return -1;
+ return 0;
if(!hash_init(&(zck->check_full_hash), &(zck->hash_type)))
- return -1;
+ return 0;
char buf[BUF_SIZE] = {0};
- zckIndexItem *idx = zck->index.first;
+ zckChunk *idx = zck->index.first;
zck_log(ZCK_LOG_DEBUG, "Checking full hash\n");
while(idx) {
size_t to_read = idx->comp_length;
if(rb > to_read)
rb = to_read;
if(!read_data(zck->fd, buf, rb))
- return -1;
+ return 0;
if(!hash_update(&(zck->check_full_hash), buf, rb))
- return -1;
+ return 0;
to_read -= rb;
}
idx = idx->next;
return get_digest_string(zck->full_hash_digest, zck->hash_type.digest_size);
}
-char PUBLIC *zck_get_chunk_digest(zckIndexItem *item) {
+char PUBLIC *zck_get_chunk_digest(zckChunk *item) {
if(item == NULL)
return NULL;
return get_digest_string(item->digest, item->digest_size);
}
ssize_t PUBLIC zck_get_data_length(zckCtx *zck) {
- zckIndexItem *idx = zck->index.first;
+ zckChunk *idx = zck->index.first;
while(idx->next != NULL)
idx = idx->next;
return idx->start + idx->comp_length;
#include "zck_private.h"
-static void index_free_item(zckIndexItem **item) {
+static void index_free_item(zckChunk **item) {
if(*item == NULL)
return;
return;
if(index->first) {
- zckIndexItem *next;
- zckIndexItem *tmp=index->first;
+ zckChunk *next;
+ zckChunk *tmp=index->first;
while(tmp != NULL) {
next = tmp->next;
index_free_item(&tmp);
VALIDATE(zck);
clear_work_index(zck);
- zck->work_index_item = zmalloc(sizeof(zckIndexItem));
+ zck->work_index_item = zmalloc(sizeof(zckChunk));
if(zck->work_index_item == NULL) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- sizeof(zckIndexItem));
+ sizeof(zckChunk));
return False;
}
if(!hash_init(&(zck->work_index_hash), &(zck->chunk_hash_type)))
return True;
}
-static int finish_chunk(zckIndex *index, zckIndexItem *item, char *digest,
- int valid) {
+static int finish_chunk(zckIndex *index, zckChunk *item, char *digest,
+ int valid, zckCtx *zck) {
VALIDATE(index);
VALIDATE(item);
}
item->start = index->length;
item->valid = valid;
+ item->zck = zck;
if(index->first == NULL) {
index->first = item;
} else {
- zckIndexItem *tmp = index->first;
+ zckChunk *tmp = index->first;
while(tmp->next)
tmp = tmp->next;
tmp->next = item;
/* Add digest size + MAX_COMP_SIZE bytes for length of each entry in
* index */
if(zck->index.first) {
- zckIndexItem *tmp = zck->index.first;
+ zckChunk *tmp = zck->index.first;
while(tmp) {
index_malloc += zck->index.digest_size + MAX_COMP_SIZE*2;
tmp = tmp->next;
compint_from_size(index+index_size, zck->index.hash_type, &index_size);
compint_from_size(index+index_size, zck->index.count, &index_size);
if(zck->index.first) {
- zckIndexItem *tmp = zck->index.first;
+ zckChunk *tmp = zck->index.first;
while(tmp) {
/* Write digest */
memcpy(index+index_size, tmp->digest, zck->index.digest_size);
}
int index_new_chunk(zckIndex *index, char *digest, int digest_size,
- size_t comp_size, size_t orig_size, int finished) {
+ size_t comp_size, size_t orig_size, int finished,
+ zckCtx *zck) {
if(index == NULL) {
zck_log(ZCK_LOG_ERROR, "Invalid index\n");
return False;
zck_log(ZCK_LOG_ERROR, "Digest size 0 too small\n");
return False;
}
- zckIndexItem *idx = zmalloc(sizeof(zckIndexItem));
+ zckChunk *idx = zmalloc(sizeof(zckChunk));
if(idx == NULL) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- sizeof(zckIndexItem));
+ sizeof(zckChunk));
return False;
}
index->digest_size = digest_size;
idx->comp_length = comp_size;
idx->length = orig_size;
- return finish_chunk(index, idx, digest, finished);
+ return finish_chunk(index, idx, digest, finished, zck);
}
int index_add_to_chunk(zckCtx *zck, char *data, size_t comp_size,
return False;
}
}
- if(!finish_chunk(&(zck->index), zck->work_index_item, digest, True))
+ if(!finish_chunk(&(zck->index), zck->work_index_item, digest, True, zck))
return False;
free(digest);
return False;
zck->index.count = index_count;
- zckIndexItem *prev = zck->index.first;
+ zckChunk *prev = zck->index.first;
size_t idx_loc = 0;
while(length < size) {
if(length + zck->index.digest_size > max_length) {
return False;
}
- zckIndexItem *new = zmalloc(sizeof(zckIndexItem));
+ zckChunk *new = zmalloc(sizeof(zckChunk));
if(!new) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- sizeof(zckIndexItem));
+ sizeof(zckChunk));
return False;
}
if(!compint_to_size(&chunk_length, data+length, &length, max_length))
return False;
new->length = chunk_length;
-
+ new->zck = zck;
new->valid = 0;
idx_loc += new->comp_length;
zck->index.length = idx_loc;
return True;
}
-ssize_t PUBLIC zck_get_index_count(zckCtx *zck) {
+ssize_t PUBLIC zck_get_chunk_count(zckCtx *zck) {
if(zck == NULL)
return -1;
return zck->index.count;
}
-zckIndex PUBLIC *zck_get_index(zckCtx *zck) {
+zckChunk PUBLIC *zck_get_first_chunk(zckCtx *zck) {
if(zck == NULL)
return NULL;
- return &(zck->index);
+ return zck->index.first;
+}
+
+zckChunk PUBLIC *zck_get_next_chunk(zckChunk *idx) {
+ if(idx == NULL)
+ return NULL;
+ return idx->next;
+}
+
+ssize_t PUBLIC zck_get_chunk_start(zckChunk *idx) {
+ if(idx == NULL)
+ return -1;
+ if(idx->zck)
+ return idx->start + zck_get_header_length(idx->zck);
+ else
+ return idx->start;
+}
+
+ssize_t PUBLIC zck_get_chunk_size(zckChunk *idx) {
+ if(idx == NULL)
+ return -1;
+ return idx->length;
+}
+
+ssize_t PUBLIC zck_get_chunk_comp_size(zckChunk *idx) {
+ if(idx == NULL)
+ return -1;
+ return idx->comp_length;
+}
+
+int PUBLIC zck_get_chunk_valid(zckChunk *idx) {
+ if(idx == NULL)
+ return -1;
+ return idx->valid;
+}
+
+int PUBLIC zck_compare_chunk_digest(zckChunk *a, zckChunk *b) {
+ if(a == NULL || b == NULL)
+ return False;
+ if(a->digest_size != b->digest_size)
+ return False;
+ if(memcmp(a->digest, b->digest, a->digest_size) != 0)
+ return False;
+ return True;
}
int PUBLIC zck_missing_chunks(zckCtx *zck) {
return -1;
}
int missing = 0;
- for(zckIndexItem *idx = zck->index.first; idx; idx=idx->next)
+ for(zckChunk *idx = zck->index.first; idx; idx=idx->next)
if(idx->valid == 0)
missing++;
return missing;
return -1;
}
int failed = 0;
- for(zckIndexItem *idx = zck->index.first; idx; idx=idx->next)
+ for(zckChunk *idx = zck->index.first; idx; idx=idx->next)
if(idx->valid == -1)
failed++;
return failed;
if(!zck)
return;
- for(zckIndexItem *idx = zck->index.first; idx; idx=idx->next)
+ for(zckChunk *idx = zck->index.first; idx; idx=idx->next)
if(idx->valid == -1)
idx->valid = 0;
return;
size_t buffer_len;
} zckMP;
-typedef struct zckDLPriv {
+typedef struct zckDL {
+ struct zckCtx *zck;
+ size_t dl;
+ size_t ul;
+ zckRange *range;
zckMP *mp;
char *boundary;
int parser_started;
regex_t *dl_regex;
regex_t *end_regex;
regex_t *hdr_regex;
- zckIndexItem *tgt_check;
+ zckChunk *tgt_check;
int tgt_number;
-} zckDLPriv;
+
+ /* Callbacks */
+ zck_wcb write_cb;
+ void *write_data;
+ zck_wcb header_cb;
+ void *header_data;
+} zckDL;
+
+/* Contains an index item pointing to a chunk */
+typedef struct zckChunk {
+ char *digest;
+ int digest_size;
+ int valid;
+ size_t start;
+ size_t comp_length;
+ size_t length;
+ struct zckChunk *next;
+ zckCtx *zck;
+} zckChunk;
+
+/* Contains everything about an index and a pointer to the first index item */
+typedef struct zckIndex {
+ size_t count;
+ size_t length;
+ int hash_type;
+ size_t digest_size;
+ zckChunk *first;
+} zckIndex;
+
+/* Contains a single range */
+typedef struct zckRangeItem {
+ size_t start;
+ size_t end;
+ struct zckRangeItem *next;
+ struct zckRangeItem *prev;
+} zckRangeItem;
+
+/* Contains a series of ranges, information about them, a link to the first
+ * range item, and an index describing what information is in the ranges */
+typedef struct zckRange {
+ unsigned int count;
+ zckRangeItem *first;
+ zckIndex index;
+} zckRange;
typedef struct zckComp {
int started;
char *data;
size_t data_size;
size_t data_loc;
- zckIndexItem *data_idx;
+ zckChunk *data_idx;
int data_eof;
char *dc_data;
size_t dc_data_size;
ssize_t prep_hdr_size;
zckIndex index;
- zckIndexItem *work_index_item;
+ zckChunk *work_index_item;
zckHash work_index_hash;
size_t stream;
int has_streams;
char *hash_finalize(zckHash *hash)
__attribute__ ((warn_unused_result));
void hash_close(zckHash *hash);
-int validate_chunk(zckCtx *zck, zckIndexItem *idx, zck_log_type bad_checksum,
+int validate_chunk(zckCtx *zck, zckChunk *idx, zck_log_type bad_checksum,
int chunk_number)
__attribute__ ((warn_unused_result));
int validate_file(zckCtx *zck, zck_log_type bad_checksums)
int index_create(zckCtx *zck)
__attribute__ ((warn_unused_result));
int index_new_chunk(zckIndex *index, char *digest, int digest_size,
- size_t comp_size, size_t orig_size, int valid)
+ size_t comp_size, size_t orig_size, int valid,
+ zckCtx *zck)
__attribute__ ((warn_unused_result));
int index_add_to_chunk(zckCtx *zck, char *data, size_t comp_size,
size_t orig_size)
__attribute__ ((warn_unused_result));
size_t multipart_get_boundary(zckDL *dl, char *b, size_t size)
__attribute__ ((warn_unused_result));
+void reset_mp(zckMP *mp);
/* dl/dl.c */
int dl_write_range(zckDL *dl, const char *at, size_t length)
if(!zck_close(zck))
exit(1);
if(arguments.log_level <= ZCK_LOG_INFO) {
- zckIndex *idx = zck_get_index(zck);
printf("Wrote %lu bytes in %lu chunks\n",
(unsigned long)(zck_get_data_length(zck) +
zck_get_header_length(zck)),
- (unsigned long)idx->count);
+ (long)zck_get_chunk_count(zck));
}
zck_free(&zck);
printf(" %s: %s\n", arguments.args[1], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_src)));
exit(1);
}
- zckIndex *tgt_info = zck_get_index(zck_tgt);
- if(tgt_info == NULL)
+ zckChunk *tgt_idx = zck_get_first_chunk(zck_tgt);
+ zckChunk *src_idx = zck_get_first_chunk(zck_src);
+ if(tgt_idx == NULL || src_idx == NULL)
exit(1);
- zckIndex *src_info = zck_get_index(zck_src);
- if(src_info == NULL)
- exit(1);
- zckIndexItem *tgt_idx = tgt_info->first;
- zckIndexItem *src_idx = src_info->first;
- if(memcmp(tgt_idx->digest, src_idx->digest, tgt_idx->digest_size) != 0)
+
+ if(!zck_compare_chunk_digest(tgt_idx, src_idx))
printf("WARNING: Dicts don't match\n");
ssize_t dl_size = zck_get_header_length(zck_tgt);
if(dl_size < 0)
exit(1);
ssize_t total_size = zck_get_header_length(zck_tgt);
ssize_t matched_chunks = 0;
- while(tgt_idx) {
+ for(tgt_idx = zck_get_first_chunk(zck_tgt); tgt_idx;
+ tgt_idx = zck_get_next_chunk(tgt_idx)) {
int found = False;
- src_idx = src_info->first;
-
- while(src_idx) {
- if(memcmp(tgt_idx->digest, src_idx->digest, tgt_idx->digest_size) == 0) {
+ for(src_idx = zck_get_first_chunk(zck_src); src_idx;
+ src_idx = zck_get_next_chunk(src_idx)) {
+ if(zck_compare_chunk_digest(tgt_idx, src_idx)) {
found = True;
break;
}
- src_idx = src_idx->next;
}
if(!found) {
- dl_size += tgt_idx->comp_length;
+ dl_size += zck_get_chunk_comp_size(tgt_idx);
} else {
matched_chunks += 1;
}
- total_size += tgt_idx->comp_length;
- tgt_idx = tgt_idx->next;
+ total_size += zck_get_chunk_comp_size(tgt_idx);
}
printf("Would download %li of %li bytes\n", (long)dl_size,
(long)total_size);
printf("Matched %li of %lu chunks\n", (long)matched_chunks,
- (long unsigned)zck_get_index_count(zck_tgt));
+ (long unsigned)zck_get_chunk_count(zck_tgt));
zck_free(&zck_tgt);
zck_free(&zck_src);
}
/* Return 0 on error, -1 on 200 response (if dl_ctx->fail_no_ranges),
* and 1 on complete success */
int dl_range(dlCtx *dl_ctx, char *url, char *range, int is_chunk) {
- if(dl_ctx == NULL || dl_ctx->dl == NULL || dl_ctx->dl->priv == NULL) {
+ if(dl_ctx == NULL || dl_ctx->dl == NULL) {
free(range);
printf("Struct not defined\n");
return 0;
if(start + bytes > *buffer_len) {
zckDL *dl = dl_ctx->dl;
- int fd = zck_get_fd(dl->zck);
+ int fd = zck_get_fd(zck_dl_get_zck(dl));
if(lseek(fd, *buffer_len, SEEK_SET) == -1) {
printf("Seek to download location failed: %s\n",
if(retval < 1)
return retval;
- if(!zck_read_lead(dl->zck))
+ zckCtx *zck = zck_dl_get_zck(dl);
+ if(zck == NULL)
return 0;
- start = zck_get_lead_length(dl->zck);
- if(!dl_bytes(&dl_ctx, url, zck_get_header_length(dl->zck) - start,
+
+ if(!zck_read_lead(zck))
+ return 0;
+ start = zck_get_lead_length(zck);
+ if(!dl_bytes(&dl_ctx, url, zck_get_header_length(zck) - start,
start, &buffer_len, log_level))
return 0;
- if(!zck_read_header(dl->zck))
+ if(!zck_read_header(zck))
return 0;
return 1;
}
goto out;
}
lseek(dst_fd, 0, SEEK_SET);
- if(!zck_read_lead(dl->zck) || !zck_read_header(dl->zck)) {
+ if(!zck_read_lead(zck_tgt) || !zck_read_header(zck_tgt)) {
printf("Error reading zchunk file\n");
exit_val = 10;
goto out;
while(zck_missing_chunks(zck_tgt) > 0) {
dl_ctx.range_fail = 0;
zck_dl_reset(dl);
- dl->range = zck_get_dl_range(zck_tgt, dl_ctx.max_ranges);
- if(dl->range == NULL) {
+ zckRange *range = zck_get_dl_range(zck_tgt, dl_ctx.max_ranges);
+ if(range == NULL || !zck_dl_set_range(dl, range)) {
exit_val = 10;
goto out;
}
while(range_attempt[ra_index] > 1 &&
- range_attempt[ra_index+1] > dl->range->count)
+ range_attempt[ra_index+1] > zck_get_range_count(range))
ra_index++;
- char *range_string = zck_get_range_char(dl->range);
+ char *range_string = zck_get_range_char(range);
if(range_string == NULL) {
exit_val = 10;
goto out;
}
printf("Tried downloading too many ranges, reducing to %i\n", dl_ctx.max_ranges);
}
- zck_range_free(&(dl->range));
+ if(!zck_dl_set_range(dl, NULL)) {
+ exit_val = 10;
+ goto out;
+ }
+ zck_range_free(&range);
if(!retval) {
+ exit_val = 1;
goto out;
}
}
goto out;
}
- switch(zck_validate_data_checksum(dl->zck)) {
+ switch(zck_validate_data_checksum(zck_tgt)) {
case -1:
exit_val = 1;
break;
digest = zck_get_data_digest(zck);
printf("Data checksum: %s\n", digest);
free(digest);
- printf("Index count: %lu\n", (long unsigned)zck_get_index_count(zck));
+ printf("Chunk count: %lu\n", (long unsigned)zck_get_chunk_count(zck));
printf("Chunk checksum type: %s\n", zck_hash_name_from_type(zck_get_chunk_hash_type(zck)));
}
if(arguments.log_level <= ZCK_LOG_INFO) {
- zckIndex *idxi = zck_get_index(zck);
- if(idxi == NULL)
- exit(1);
- for(zckIndexItem *idx = idxi->first; idx; idx=idx->next) {
- char *digest = zck_get_chunk_digest(idx);
+ for(zckChunk *chk = zck_get_first_chunk(zck); chk;
+ chk=zck_get_next_chunk(chk)) {
+ char *digest = zck_get_chunk_digest(chk);
if(digest == NULL)
exit(1);
printf("%s %12lu %12lu %12lu", digest,
- (long unsigned)(idx->start + zck_get_header_length(zck)),
- (long unsigned)idx->comp_length, (long unsigned)idx->length);
+ (long unsigned)zck_get_chunk_start(chk),
+ (long unsigned)zck_get_chunk_comp_size(chk),
+ (long unsigned)zck_get_chunk_size(chk));
if(arguments.verify) {
- if(idx->valid)
+ if(zck_get_chunk_valid(chk) == 1)
printf(" +");
else
printf(" !");
projects / zchunk.git / commitdiff