From c9355bf6bc5955c1999aac50849d9a4017eb5f37 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Mon, 4 Jun 2018 10:47:51 +0300 Subject: [PATCH] API revamp to give control over the actual download to the program Signed-off-by: Jonathan Dieter --- include/zck.h | 116 +++++++++------- src/lib/dl/dl.c | 315 +++++++++++------------------------------- src/lib/dl/range.c | 123 +++++++++-------- src/lib/zck_private.h | 12 +- 4 files changed, 214 insertions(+), 352 deletions(-) diff --git a/include/zck.h b/include/zck.h index 14b0d91..c1b45d7 100644 --- a/include/zck.h +++ b/include/zck.h @@ -67,6 +67,21 @@ typedef struct zckIndex { zckIndexItem *first; } zckIndex; +/* Contains a single range */ +typedef struct zckRangeItem { + size_t start; + size_t end; + struct zckRangeItem *next; + struct zckRangeItem *prev; +} zckRangeItem; + +/* Contains a series of ranges, information about them, a link to the first + * range item, and an index describing what information is in the ranges */ +typedef struct zckRange { + unsigned int count; + zckRangeItem *first; + zckIndex index; +} zckRange; /******************************************************************* * Reading a zchunk file @@ -126,8 +141,24 @@ int zck_set_ioption(zckCtx *zck, zck_ioption option, ssize_t value) void zck_set_log_level(zck_log_type ll); /* Validate the chunk and data checksums for the current file. * Returns -1 for error, 0 for invalid checksum and 1 for valid checksum */ -int zck_validate_checksums(zckCtx *zck); +int zck_validate_checksums(zckCtx *zck) + __attribute__ ((warn_unused_result)); +/* Validate just the data checksum for the current file */ +int zck_validate_data_checksum(zckCtx *zck) + __attribute__ ((warn_unused_result)); +/* Get a zckRange of ranges that need to still be downloaded. + * max_ranges is the maximum number of ranges supported in a single request + * by the server. If the server supports unlimited ranges, set to -1 + * Returns NULL if there's an error */ +zckRange *zck_get_dl_range(zckCtx *zck, int max_ranges) + __attribute__ ((warn_unused_result)); +/* Get a string representation of a zckRange */ +char *zck_get_range_char(zckRange *range) + __attribute__ ((warn_unused_result)); +/* Get file descriptor attached to zchunk context */ +int zck_get_fd(zckCtx *zck) + __attribute__ ((warn_unused_result)); /******************************************************************* * The functions should be all you need to read and write a zchunk @@ -142,7 +173,10 @@ int zck_validate_checksums(zckCtx *zck); /* Initialize zchunk context */ zckCtx *zck_create() __attribute__ ((warn_unused_result)); -/* Get header length (header + index) */ +/* Get lead length */ +ssize_t zck_get_lead_length(zckCtx *zck) + __attribute__ ((warn_unused_result)); +/* Get header length (lead + preface + index + sigs) */ ssize_t zck_get_header_length(zckCtx *zck) __attribute__ ((warn_unused_result)); /* Get data length */ @@ -185,6 +219,9 @@ int zck_comp_reset(zckCtx *zck) /* Initialize zchunk for reading using advanced options */ zckCtx *zck_init_adv_read (int src_fd) __attribute__ ((warn_unused_result)); +/* Read zchunk lead */ +int zck_read_lead(zckCtx *zck) + __attribute__ ((warn_unused_result)); /* Read zchunk header */ int zck_read_header(zckCtx *zck) __attribute__ ((warn_unused_result)); @@ -211,51 +248,35 @@ char *zck_get_chunk_digest(zckIndexItem *item) int zck_get_full_hash_type(zckCtx *zck) __attribute__ ((warn_unused_result)); /* Get digest size of overall hash type */ -int zck_get_full_digest_size(zckCtx *zck) +ssize_t zck_get_full_digest_size(zckCtx *zck) __attribute__ ((warn_unused_result)); /* Get chunk hash type */ int zck_get_chunk_hash_type(zckCtx *zck) __attribute__ ((warn_unused_result)); /* Get digest size of chunk hash type */ -int zck_get_chunk_digest_size(zckCtx *zck) +ssize_t zck_get_chunk_digest_size(zckCtx *zck) __attribute__ ((warn_unused_result)); /* Get name of hash type */ const char *zck_hash_name_from_type(int hash_type) __attribute__ ((warn_unused_result)); -/* Check data hash */ -int zck_hash_check_data(zckCtx *zck, int dst_fd) - __attribute__ ((warn_unused_result)); + /******************************************************************* * Downloading (should this go in a separate header and library?) *******************************************************************/ -/* Contains a single range */ -typedef struct zckRangeItem { - size_t start; - size_t end; - struct zckRangeItem *next; - struct zckRangeItem *prev; -} zckRangeItem; - -/* Contains a series of ranges, information about them, a link to the first - * range item, and an index describing what information is in the ranges */ -typedef struct zckRange { - unsigned int count; - unsigned int segments; - unsigned int max_ranges; - zckRangeItem *first; - zckIndex index; -} zckRange; +typedef size_t (*zck_wcb)(void *ptr, size_t l, size_t c, void *dl_v); typedef struct zckDLPriv zckDLPriv; /* Contains a zchunk download context */ typedef struct zckDL { size_t dl; size_t ul; - int dst_fd; - char *boundary; + zck_wcb write_cb; + void *wdata; + zck_wcb header_cb; + void *hdrdata; zckRange info; zckDLPriv *priv; struct zckCtx *zck; @@ -264,41 +285,36 @@ typedef struct zckDL { /******************************************************************* * Ranges *******************************************************************/ -/* Update info with the maximum number of ranges in a single request */ -int zck_range_calc_segments(zckRange *info, unsigned int max_ranges) - __attribute__ ((warn_unused_result)); /* Get any matching chunks from src and put them in the right place in tgt */ -int zck_dl_copy_src_chunks(zckRange *info, zckCtx *src, zckCtx *tgt) - __attribute__ ((warn_unused_result)); -/* Get index of chunks not available in src, and put them in info */ -int zck_range_get_need_dl(zckRange *info, zckCtx *zck_src, zckCtx *zck_tgt) - __attribute__ ((warn_unused_result)); -/* Get array of range request strings. ra must be allocated to size - * info->segments, and the strings must be freed by the caller after use */ -int zck_range_get_array(zckRange *info, char **ra) +int zck_copy_chunks(zckCtx *src, zckCtx *tgt) __attribute__ ((warn_unused_result)); /* Free any resources in zckRange */ void zck_range_close(zckRange *info); /* Get range string from start and end location */ char *zck_get_range(size_t start, size_t end) __attribute__ ((warn_unused_result)); - +/* Get the minimum size needed to download in order to know how large the header + * is */ +int get_min_download_size() + __attribute__ ((warn_unused_result)); /******************************************************************* * Downloading *******************************************************************/ -/* Initialize curl stuff, should be run at beginning of any program using any - * following functions */ -void zck_dl_global_init(); -/* Clean up curl stuff, should be run at end of any program using any following - * functions */ -void zck_dl_global_cleanup(); - /* Initialize zchunk download context */ -zckDL *zck_dl_init() +zckDL *zck_dl_init(zckCtx *zck) __attribute__ ((warn_unused_result)); +/* Reset zchunk download context for reuse */ +void zck_dl_reset(zckDL *dl); /* Free zchunk download context */ void zck_dl_free(zckDL **dl); +/* Write callback. You *must* pass this and your initialized zchunk download + * context to the downloader when downloading a zchunk file. If you have your + * own callback, set dl->write_cb to your callback and dl->wdata to your + * callback data. */ +size_t zck_write_chunk_cb(void *ptr, size_t l, size_t c, void *dl_v); +size_t zck_write_zck_header_cb(void *ptr, size_t l, size_t c, void *dl_v); +size_t zck_header_cb(char *b, size_t l, size_t c, void *dl_v); /* Clear regex used for extracting download ranges from multipart download */ void zck_dl_clear_regex(zckDL *dl); /* Download and process the header from url */ @@ -310,10 +326,8 @@ size_t zck_dl_get_bytes_downloaded(zckDL *dl) /* Get number of bytes uploaded using download context */ size_t zck_dl_get_bytes_uploaded(zckDL *dl) __attribute__ ((warn_unused_result)); + /* Download ranges specified in dl->info from url */ -int zck_dl_range(zckDL *dl, char *url) - __attribute__ ((warn_unused_result)); -/* Return string with range request from start to end (inclusive) */ -char *zck_dl_get_range(unsigned int start, unsigned int end) - __attribute__ ((warn_unused_result)); +/*int zck_dl_range(zckDL *dl, char *url) + __attribute__ ((warn_unused_result));*/ #endif diff --git a/src/lib/dl/dl.c b/src/lib/dl/dl.c index 02015e1..b3923e6 100644 --- a/src/lib/dl/dl.c +++ b/src/lib/dl/dl.c @@ -41,10 +41,15 @@ } /* Free zckDL regex's used for downloading ranges */ -void zck_dl_free_dl_regex(zckDL *dl) { +static void free_dl_regex(zckDL *dl) { if(dl == NULL || dl->priv == NULL) return; + if(dl->priv->hdr_regex) { + regfree(dl->priv->hdr_regex); + free(dl->priv->hdr_regex); + dl->priv->hdr_regex = NULL; + } if(dl->priv->dl_regex) { regfree(dl->priv->dl_regex); free(dl->priv->dl_regex); @@ -58,7 +63,7 @@ void zck_dl_free_dl_regex(zckDL *dl) { } /* Write zeros to tgt->fd in location of tgt_idx */ -int zck_dl_write_zero(zckCtx *tgt, zckIndexItem *tgt_idx) { +static int zero_chunk(zckCtx *tgt, zckIndexItem *tgt_idx) { char buf[BUF_SIZE] = {0}; size_t to_read = tgt_idx->comp_length; if(!seek_data(tgt->fd, tgt->data_offset + tgt_idx->start, SEEK_SET)) @@ -74,23 +79,16 @@ int zck_dl_write_zero(zckCtx *tgt, zckIndexItem *tgt_idx) { return True; } -int zck_dl_write(zckDL *dl, const char *at, size_t length) { - VALIDATE(dl); - VALIDATE(dl->priv); - if(dl->priv->write_in_chunk < length) - length = dl->priv->write_in_chunk; - if(!write_data(dl->dst_fd, at, length)) - return -1; - dl->priv->write_in_chunk -= length; - return length; -} - -int zck_dl_md_write(zckDL *dl, const char *at, size_t length) { +static int dl_write(zckDL *dl, const char *at, size_t length) { VALIDATE(dl); VALIDATE(dl->priv); int wb = 0; if(dl->priv->write_in_chunk > 0) { - wb = zck_dl_write(dl, at, length); + if(dl->priv->write_in_chunk < length) + wb = dl->priv->write_in_chunk; + if(!write_data(dl->zck->fd, at, wb)) + return -1; + dl->priv->write_in_chunk -= wb; if(!zck_hash_update(dl->priv->chunk_hash, at, wb)) return 0; if(wb < 0) @@ -101,7 +99,7 @@ int zck_dl_md_write(zckDL *dl, const char *at, size_t length) { return wb; } -int zck_dl_write_chunk(zckDL *dl) { +static int validate_chunk(zckDL *dl) { VALIDATE(dl); VALIDATE(dl->priv); if(dl->priv->chunk_hash == NULL) { @@ -114,7 +112,7 @@ int zck_dl_write_chunk(zckDL *dl) { dl->priv->tgt_check->digest_size) != 0) { zck_log(ZCK_LOG_WARNING, "Downloaded chunk failed hash check\n"); - if(!zck_dl_write_zero(dl->zck, dl->priv->tgt_check)) + if(!zero_chunk(dl->zck, dl->priv->tgt_check)) return False; } else { dl->priv->tgt_check->valid = True; @@ -136,10 +134,10 @@ int zck_dl_write_range(zckDL *dl, const char *at, size_t length) { zck_log(ZCK_LOG_ERROR, "zckCtx index not initialized\n"); return 0; } - int wb = zck_dl_md_write(dl, at, length); + int wb = dl_write(dl, at, length); if(dl->priv->write_in_chunk == 0) { /* Check whether we just finished downloading a chunk and verify it */ - if(dl->priv->tgt_check && !zck_dl_write_chunk(dl)) + if(dl->priv->tgt_check && !validate_chunk(dl)) return False; zckIndexItem *idx = dl->info.index.first; while(idx) { @@ -157,7 +155,7 @@ int zck_dl_write_range(zckDL *dl, const char *at, size_t length) { &(dl->zck->chunk_hash_type))) return 0; dl->priv->write_in_chunk = idx->comp_length; - if(!seek_data(dl->dst_fd, + if(!seek_data(dl->zck->fd, dl->zck->data_offset + tgt_idx->start, SEEK_SET)) return 0; @@ -181,42 +179,44 @@ int zck_dl_write_range(zckDL *dl, const char *at, size_t length) { return wb + wb2; } -char *zck_dl_get_range_char(unsigned int start, unsigned int end) { - zckRangeItem r = {0}; - zckRangeItem *range = &r; - - r.start = start; - r.end = end; - char *range_header = zck_range_get_char(&range, 2); - return range_header; +size_t PUBLIC zck_write_zck_header_cb(void *ptr, size_t l, size_t c, void *dl_v) { + if(dl_v == NULL) + return 0; + zckDL *dl = (zckDL*)dl_v; + size_t wb = 0; + dl->dl += l*c; + wb = write(dl->zck->fd, ptr, l*c); + if(dl->write_cb) + dl->write_cb(ptr, l, c, dl->wdata); + return wb; } -static size_t dl_write_data(void *ptr, size_t l, size_t c, void *dl_v) { +size_t PUBLIC zck_write_chunk_cb(void *ptr, size_t l, size_t c, void *dl_v) { if(dl_v == NULL) return 0; zckDL *dl = (zckDL*)dl_v; size_t wb = 0; dl->dl += l*c; - if(dl->boundary != NULL) { + if(dl->priv->boundary != NULL) { int retval = zck_multipart_extract(dl, ptr, l*c); if(retval == 0) wb = 0; else wb = l*c; - } else if(dl->priv->is_chunk) { + } else { int retval = zck_dl_write_range(dl, ptr, l*c); if(retval == 0) wb = 0; else wb = l*c; - } else { - wb = write(dl->dst_fd, ptr, l*c); } + if(dl->write_cb) + dl->write_cb(ptr, l, c, dl->wdata); return wb; } -int zck_dl_write_and_verify(zckRange *info, zckCtx *src, zckCtx *tgt, - zckIndexItem *src_idx, zckIndexItem *tgt_idx) { +int write_and_verify_chunk(zckCtx *src, zckCtx *tgt, zckIndexItem *src_idx, + zckIndexItem *tgt_idx) { static char buf[BUF_SIZE] = {0}; size_t to_read = src_idx->comp_length; @@ -248,31 +248,34 @@ int zck_dl_write_and_verify(zckRange *info, zckCtx *src, zckCtx *tgt, pdigest = get_digest_string(digest, src_idx->digest_size); zck_log(ZCK_LOG_WARNING, "Target hash: %s\n", pdigest); free(pdigest); - if(!zck_dl_write_zero(tgt, tgt_idx)) - return False; - if(!zck_range_add(info, tgt_idx, tgt)) + if(!zero_chunk(tgt, tgt_idx)) return False; } else { tgt_idx->valid = True; - zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes at %lu\n", + zck_log(ZCK_LOG_DEBUG, "Wrote %lu bytes at %lu\n", tgt_idx->comp_length, tgt_idx->start); } free(digest); return True; } -int PUBLIC zck_dl_copy_src_chunks(zckRange *info, zckCtx *src, zckCtx *tgt) { +int PUBLIC zck_copy_chunks(zckCtx *src, zckCtx *tgt) { zckIndex *tgt_info = zck_get_index(tgt); zckIndex *src_info = zck_get_index(src); zckIndexItem *tgt_idx = tgt_info->first; zckIndexItem *src_idx = src_info->first; while(tgt_idx) { + /* No need to copy already valid chunk */ + if(tgt_idx->valid) + continue; + int found = False; src_idx = src_info->first; while(src_idx) { if(tgt_idx->comp_length == src_idx->comp_length && tgt_idx->length == src_idx->length && + tgt_idx->digest_size == src_idx->digest_size && memcmp(tgt_idx->digest, src_idx->digest, tgt_idx->digest_size) == 0) { found = True; @@ -281,122 +284,30 @@ int PUBLIC zck_dl_copy_src_chunks(zckRange *info, zckCtx *src, zckCtx *tgt) { src_idx = src_idx->next; } /* Write out found chunk, then verify that it's valid */ - if(found && !zck_dl_write_and_verify(info, src, tgt, src_idx, tgt_idx)) - return False; - if(!found && !zck_range_add(info, tgt_idx, tgt)) - return False; + if(found) + write_and_verify_chunk(src, tgt, src_idx, tgt_idx); tgt_idx = tgt_idx->next; } return True; } -static size_t get_header(char *b, size_t l, size_t c, void *dl_v) { +size_t PUBLIC zck_header_cb(char *b, size_t l, size_t c, void *dl_v) { if(dl_v == NULL) return 0; zckDL *dl = (zckDL*)dl_v; - return zck_multipart_get_boundary(dl, b, c*l); -} - -int zck_dl_range_chk_chunk(zckDL *dl, char *url, int is_chunk) { - if(dl == NULL || dl->priv == NULL || dl->info.first == NULL) { - zck_log(ZCK_LOG_ERROR, "Struct not defined\n"); - return False; - } - if(dl->priv->parser_started) { - zck_log(ZCK_LOG_ERROR, "Multipart parser already started\n"); - return False; - } - if(dl->info.segments == 0) - dl->info.segments = 1; - dl->priv->is_chunk = is_chunk; - - char **ra = calloc(sizeof(char*), dl->info.segments); - if(!zck_range_get_array(&(dl->info), ra)) { - free(ra); - return False; - } - CURLcode res; - - for(int i=0; iinfo.segments; i++) { - if(dl->priv->dl_regex != NULL) - zck_dl_free_dl_regex(dl); - if(dl->boundary != NULL) - free(dl->boundary); - - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_URL, url); - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_HEADERFUNCTION, get_header); - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_HEADERDATA, dl); - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_WRITEFUNCTION, dl_write_data); - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_WRITEDATA, dl); - curl_easy_setopt(dl->priv->curl_ctx, CURLOPT_RANGE, ra[i]); - res = curl_easy_perform(dl->priv->curl_ctx); - free(ra[i]); - - if(res != CURLE_OK) { - zck_log(ZCK_LOG_ERROR, "Download failed: %s\n", - curl_easy_strerror(res)); - return False; - } - long code; - curl_easy_getinfo (dl->priv->curl_ctx, CURLINFO_RESPONSE_CODE, &code); - if (code != 206 && code != 200) { - zck_log(ZCK_LOG_ERROR, "HTTP Error: %li when download %s\n", code, - url); - return False; - } - zck_dl_clear_regex(dl); - } - free(ra); - return True; -} - -int PUBLIC zck_dl_range(zckDL *dl, char *url) { - return zck_dl_range_chk_chunk(dl, url, 1); -} + size_t retval = zck_multipart_get_boundary(dl, b, c*l); -int zck_dl_bytes(zckDL *dl, char *url, size_t bytes, size_t start, - size_t *buffer_len) { - if(dl == NULL) { - zck_log(ZCK_LOG_ERROR, "zckDL not initialized\n"); - return False; - } - if(start + bytes > *buffer_len) { - zckIndexItem idx = {0}; - - zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n"); - if(lseek(dl->dst_fd, 0, SEEK_END) == -1) { - zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n", - strerror(errno)); - return False; - } - zck_log(ZCK_LOG_DEBUG, "Downloading %lu bytes at position %lu\n", start+bytes-*buffer_len, *buffer_len); - idx.start = *buffer_len; - idx.comp_length = start+bytes-*buffer_len; - zck_range_close(&(dl->info)); - if(!zck_range_add(&(dl->info), &idx, NULL)) - return False; - if(!zck_dl_range_chk_chunk(dl, url, 0)) - return False; - zck_range_close(&(dl->info)); - *buffer_len = start+bytes; - zck_log(ZCK_LOG_DEBUG, "Seeking to position %lu\n", start); - if(lseek(dl->dst_fd, start, SEEK_SET) == -1) { - zck_log(ZCK_LOG_ERROR, - "Seek to byte %lu of temporary file failed: %s\n", start, - strerror(errno)); - return False; - } - } - return True; + if(dl->header_cb) + dl->header_cb(b, l, c, dl->hdrdata); + return c*l; } int zck_zero_bytes(zckDL *dl, size_t bytes, size_t start, size_t *buffer_len) { char buf[BUF_SIZE] = {0}; if(start + bytes > *buffer_len) { zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n"); - if(lseek(dl->dst_fd, 0, SEEK_END) == -1) { + if(lseek(dl->zck->fd, 0, SEEK_END) == -1) { zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n", strerror(errno)); return False; @@ -406,14 +317,14 @@ int zck_zero_bytes(zckDL *dl, size_t bytes, size_t start, size_t *buffer_len) { size_t wb = BUF_SIZE; if(write + wb > start + bytes) wb = (start + bytes) - write; - if(!write_data(dl->dst_fd, buf, wb)) + if(!write_data(dl->zck->fd, buf, wb)) return False; write += wb; } zck_log(ZCK_LOG_DEBUG, "Wrote %lu zeros at position %lu\n", start+bytes-*buffer_len, *buffer_len); *buffer_len = start+bytes; zck_log(ZCK_LOG_DEBUG, "Seeking to position %lu\n", start); - if(lseek(dl->dst_fd, start, SEEK_SET) == -1) { + if(lseek(dl->zck->fd, start, SEEK_SET) == -1) { zck_log(ZCK_LOG_ERROR, "Seek to byte %lu of temporary file failed: %s\n", start, strerror(errno)); @@ -423,72 +334,6 @@ int zck_zero_bytes(zckDL *dl, size_t bytes, size_t start, size_t *buffer_len) { return True; } -/* Download header */ -int PUBLIC zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) { - if(zck == NULL) { - zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n"); - return False; - } - if(dl == NULL) { - zck_log(ZCK_LOG_ERROR, "zckDL not initialized\n"); - return False; - } - size_t buffer_len = 0; - size_t start = 0; - zck->fd = dl->dst_fd; - - /* Download first hundred bytes and read magic and hash type */ - if(!zck_dl_bytes(dl, url, 200, start, &buffer_len)) - return False; - if(!read_lead_1(zck)) - return False; - start = tell_data(dl->dst_fd); - - if(!zck_dl_bytes(dl, url, zck->lead_size + zck->hash_type.digest_size, - start, &buffer_len)) - return False; - if(!read_lead_2(zck)) - return False; - zck_log(ZCK_LOG_DEBUG, "Header hash: (%s)", - zck_hash_name_from_type(zck_get_full_hash_type(zck))); - char *digest = zck_get_header_digest(zck); - zck_log(ZCK_LOG_DEBUG, "%s\n", digest); - free(digest); - start = tell_data(dl->dst_fd); - - /* If we haven't downloaded enough for the index hash plus a few others, do - * it now */ - if(!zck_dl_bytes(dl, url, zck->lead_size + zck->header_length, - start, &buffer_len)) - return False; - - /* Verify header checksum */ - if(!validate_header(zck)) - return False; - zck_hash_close(&(zck->check_full_hash)); - - /* Read the header */ - if(!read_preface(zck)) - return False; - start += zck->preface_size; - zck_log(ZCK_LOG_DEBUG, "Index size: %llu\n", zck->index_size); - - /* Read the index */ - if(!read_index(zck)) - return False; - - /* Read signatures */ - if(!read_sig(zck)) - return False; - - /* Write zeros to rest of file */ - zckIndex *info = &(dl->info.index); - info->hash_type = zck->index.hash_type; - zck_log(ZCK_LOG_DEBUG, "Writing zeros to rest of file: %llu\n", zck->index.length + zck->index_size + start); - if(!zck_zero_bytes(dl, zck->index.length, zck->data_offset, &buffer_len)) - return False; - return True; -} size_t PUBLIC zck_dl_get_bytes_downloaded(zckDL *dl) { VALIDATE(dl); @@ -500,20 +345,12 @@ size_t PUBLIC zck_dl_get_bytes_uploaded(zckDL *dl) { return dl->ul; } -void PUBLIC zck_dl_global_init() { - curl_global_init(CURL_GLOBAL_ALL); -} - -void PUBLIC zck_dl_global_cleanup() { - curl_global_cleanup(); -} - /* Free zckDL header regex used for downloading ranges */ void zck_dl_clear_regex(zckDL *dl) { if(dl == NULL || dl->priv == NULL) return; - zck_dl_free_dl_regex(dl); + free_dl_regex(dl); if(dl->priv->hdr_regex) { regfree(dl->priv->hdr_regex); free(dl->priv->hdr_regex); @@ -522,7 +359,7 @@ void zck_dl_clear_regex(zckDL *dl) { } /* Initialize zckDL. When finished, zckDL *must* be freed by zck_dl_free() */ -zckDL PUBLIC *zck_dl_init() { +zckDL PUBLIC *zck_dl_init(zckCtx *zck) { zckDL *dl = zmalloc(sizeof(zckDL)); if(!dl) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes for zckDL\n", @@ -542,33 +379,35 @@ zckDL PUBLIC *zck_dl_init() { sizeof(zckMP)); return NULL; } - dl->priv->curl_ctx = curl_easy_init(); - if(!dl->priv->curl_ctx) { - zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes for dl->curl_ctx\n", - sizeof(CURL)); - return NULL; - } + dl->zck = zck; return dl; } /* Free zckDL and set pointer to NULL */ -void PUBLIC zck_dl_free(zckDL **dl) { - if(!*dl) +void PUBLIC zck_dl_reset(zckDL *dl) { + if(!dl) return; - if((*dl)->priv) { - if((*dl)->priv->mp) { - if((*dl)->priv->mp->buffer) - free((*dl)->priv->mp->buffer); - free((*dl)->priv->mp); + if(dl->priv) { + if(dl->priv->mp) { + if(dl->priv->mp->buffer) + free(dl->priv->mp->buffer); + free(dl->priv->mp); } - zck_dl_clear_regex(*dl); - curl_easy_cleanup((*dl)->priv->curl_ctx); - free((*dl)->priv); + zck_dl_clear_regex(dl); + if(dl->priv->boundary) + free(dl->priv->boundary); + free(dl->priv); } - if((*dl)->info.first) - zck_range_close(&((*dl)->info)); - if((*dl)->boundary) - free((*dl)->boundary); + zckCtx *zck = dl->zck; + if(dl->info.first) + zck_range_close(&(dl->info)); + memset(dl, 0, sizeof(zckDL)); + dl->zck = zck; +} + +/* Free zckDL and set pointer to NULL */ +void PUBLIC zck_dl_free(zckDL **dl) { + zck_dl_reset(*dl); free(*dl); *dl = NULL; } diff --git a/src/lib/dl/range.c b/src/lib/dl/range.c index 6580be1..4afc137 100644 --- a/src/lib/dl/range.c +++ b/src/lib/dl/range.c @@ -147,40 +147,7 @@ int zck_range_add(zckRange *info, zckIndexItem *idx, zckCtx *zck) { return True; } -int PUBLIC zck_range_calc_segments(zckRange *info, unsigned int max_ranges) { - if(max_ranges == 0) - return False; - info->segments = (info->count + max_ranges - 1) / max_ranges; - info->max_ranges = max_ranges; - return True; -} - -int zck_range_get_need_dl(zckRange *info, zckCtx *zck_src, zckCtx *zck_tgt) { - zckIndex *tgt_info = zck_get_index(zck_tgt); - zckIndex *src_info = zck_get_index(zck_src); - zckIndexItem *tgt_idx = tgt_info->first; - zckIndexItem *src_idx = src_info->first; - while(tgt_idx) { - int found = False; - src_idx = src_info->first; - - while(src_idx) { - if(memcmp(tgt_idx->digest, src_idx->digest, tgt_idx->digest_size) == 0) { - found = True; - break; - } - src_idx = src_idx->next; - } - if(!found) - if(!zck_range_add(info, tgt_idx, zck_tgt)) - return False; - - tgt_idx = tgt_idx->next; - } - return True; -} - -char *zck_range_get_char(zckRangeItem **range, int max_ranges) { +char PUBLIC *zck_get_range_char(zckRange *range) { int buf_size=32768; char *output=malloc(buf_size); if(!output) { @@ -190,10 +157,11 @@ char *zck_range_get_char(zckRangeItem **range, int max_ranges) { int loc = 0; int count = 0; - while(*range) { + zckRangeItem *ri = range->first; + while(ri) { int length = snprintf(output+loc, buf_size-loc, "%lu-%lu,", - (long unsigned)(*range)->start, - (long unsigned)(*range)->end); + (long unsigned)ri->start, + (long unsigned)ri->end); if(length < 0) { zck_log(ZCK_LOG_ERROR, "Unable to get range: %s\n", strerror(errno)); @@ -212,9 +180,7 @@ char *zck_range_get_char(zckRangeItem **range, int max_ranges) { } loc += length; count++; - *range = (*range)->next; - if(count == max_ranges) - break; + ri = ri->next; } output[loc-1]='\0'; // Remove final comma output = realloc(output, loc); @@ -227,28 +193,69 @@ char *zck_range_get_char(zckRangeItem **range, int max_ranges) { return output; } -char PUBLIC *zck_get_range(size_t start, size_t end) { - zckRangeItem range = {0}; - range.start = start; - range.end = end; - zckRangeItem *r = ⦥ - return zck_range_get_char(&r, 1); -} +zckRange PUBLIC *zck_get_dl_range(zckCtx *zck, int max_ranges) { + zckRange *range = zmalloc(sizeof(zckRange)); + if(range == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(zckRange)); + return NULL; + } + zckIndexItem **range_idx = &(range->index.first); + zckIndexItem *idx = zck->index.first; + size_t start = 0; + while(idx) { + if(idx->valid) + continue; + if(!zck_range_add(range, idx, zck)) { + zck_range_close(range); + return NULL; + } + *range_idx = copy_index_item(idx); + if(*range_idx == NULL) { + zck_range_close(range); + return NULL; + } + (*range_idx)->start = start; + range_idx = &((*range_idx)->next); + start += idx->comp_length; -int zck_range_get_array(zckRange *info, char **ra) { - if(!info) { - zck_log(ZCK_LOG_ERROR, "zckRange not allocated\n"); - return False; + if(max_ranges >= 0 && range->count >= max_ranges) + break; + idx = idx->next; } - zckRangeItem *tmp = info->first; - for(int i=0; isegments; i++) { - ra[i] = zck_range_get_char(&tmp, info->max_ranges); - if(ra[i] == NULL) - return False; + return range; +} + +int zck_range_get_need_dl(zckRange *info, zckCtx *zck_src, zckCtx *zck_tgt) { + zckIndex *tgt_info = zck_get_index(zck_tgt); + zckIndex *src_info = zck_get_index(zck_src); + zckIndexItem *tgt_idx = tgt_info->first; + zckIndexItem *src_idx = src_info->first; + while(tgt_idx) { + int found = False; + src_idx = src_info->first; + + while(src_idx) { + if(memcmp(tgt_idx->digest, src_idx->digest, tgt_idx->digest_size) == 0) { + found = True; + break; + } + src_idx = src_idx->next; + } + if(!found) + if(!zck_range_add(info, tgt_idx, zck_tgt)) + return False; + + tgt_idx = tgt_idx->next; } return True; } -int zck_range_get_max(zckRange *info, char *url) { - return True; +char PUBLIC *zck_get_range(size_t start, size_t end) { + zckRange range = {0}; + zckRangeItem ri = {0}; + range.first = &ri; + ri.start = start; + ri.end = end; + return zck_get_range_char(&range); } diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h index a29a9be..f73e0fd 100644 --- a/src/lib/zck_private.h +++ b/src/lib/zck_private.h @@ -32,6 +32,7 @@ typedef int (*fcclose)(struct zckComp *comp); typedef enum zck_log_type zck_log_type; + typedef struct zckHashType { int type; int digest_size; @@ -52,8 +53,8 @@ typedef struct zckMP { } zckMP; typedef struct zckDLPriv { - CURL *curl_ctx; zckMP *mp; + char *boundary; int parser_started; int is_chunk; size_t write_in_chunk; @@ -188,6 +189,9 @@ int set_full_hash_type(zckCtx *zck, int hash_type) __attribute__ ((warn_unused_result)); int set_chunk_hash_type(zckCtx *zck, int hash_type) __attribute__ ((warn_unused_result)); +int get_max_hash_size() + __attribute__ ((warn_unused_result)); + /* index/index.c */ int zck_index_read(zckCtx *zck, char *data, size_t size, size_t max_length) @@ -209,6 +213,8 @@ int zck_write_index(zckCtx *zck) __attribute__ ((warn_unused_result)); zckIndexItem *zck_get_index_of_loc(zckIndex *index, size_t loc) __attribute__ ((warn_unused_result)); +zckIndexItem *copy_index_item(zckIndexItem *src) + __attribute__ ((warn_unused_result)); /* io.c */ int seek_data(int fd, off_t offset, int whence) @@ -227,10 +233,6 @@ int chunks_from_temp(zckCtx *zck) __attribute__ ((warn_unused_result)); /* header.c */ -int read_lead_1(zckCtx *zck) - __attribute__ ((warn_unused_result)); -int read_lead_2(zckCtx *zck) - __attribute__ ((warn_unused_result)); int validate_header(zckCtx *zck) __attribute__ ((warn_unused_result)); int read_preface(zckCtx *zck) -- 2.30.2