static char args_doc[] = "<file>";
static struct argp_option options[] = {
- {"verbose", 'v', 0, 0, "Increase verbosity"},
- {"quiet", 'q', 0, 0,
+ {"verbose", 'v', 0, 0, "Increase verbosity"},
+ {"quiet", 'q', 0, 0,
"Only show warnings (can be specified twice to only show errors)"},
- {"source", 's', "FILE", 0, "File to use as delta source"},
- {"version", 'V', 0, 0, "Show program version"},
+ {"source", 's', "FILE", 0, "File to use as delta source"},
+ {"fail-no-ranges", 1000, 0, 0,
+ "If server doesn't support ranges, fail instead of downloading full file"},
+ {"version", 'V', 0, 0, "Show program version"},
{ 0 }
};
+static int range_attempt[] = {
+ 255,
+ 127,
+ 7,
+ 2,
+ 1
+};
+
struct arguments {
char *args[1];
zck_log_type log_level;
char *source;
+ int fail_no_ranges;
};
static error_t parse_opt (int key, char *arg, struct argp_state *state) {
case 'V':
version();
break;
-
+ case 1000:
+ arguments->fail_no_ranges = 1;
+ break;
case ARGP_KEY_ARG:
if (state->arg_num >= 1) {
argp_usage (state);
static struct argp argp = {options, parse_opt, args_doc, doc};
-int dl_range(CURL *curl, zckDL *dl, char *url, char *range, int is_chunk) {
- if(dl == NULL || dl->priv == NULL) {
+typedef struct dlCtx {
+ CURL *curl;
+ zckDL *dl;
+ int fail_no_ranges;
+ int range_fail;
+ int max_ranges;
+} dlCtx;
+
+size_t dl_header_cb(char *b, size_t l, size_t c, void *dl_v) {
+ dlCtx *dl_ctx = (dlCtx*)dl_v;
+ if(dl_ctx->fail_no_ranges) {
+ long code = -1;
+ curl_easy_getinfo(dl_ctx->curl, CURLINFO_RESPONSE_CODE, &code);
+ if(code == 200) {
+ dl_ctx->range_fail = 1;
+ return 0;
+ }
+ }
+ return zck_header_cb(b, l, c, dl_ctx->dl);
+}
+
+/* Return -1 on error, 0 on 200 response (if is_chunk), and 1 on complete
+ * success */
+int dl_range(dlCtx *dl_ctx, char *url, char *range, int is_chunk) {
+ if(dl_ctx == NULL || dl_ctx->dl == NULL || dl_ctx->dl->priv == NULL) {
+ free(range);
printf("Struct not defined\n");
return 0;
}
+ CURL *curl = dl_ctx->curl;
CURLcode res;
- zck_dl_reset(dl);
-
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
- curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, zck_header_cb);
- curl_easy_setopt(curl, CURLOPT_HEADERDATA, dl);
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, dl_header_cb);
+ curl_easy_setopt(curl, CURLOPT_HEADERDATA, dl_ctx);
if(is_chunk)
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, zck_write_chunk_cb);
else
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, zck_write_zck_header_cb);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, dl);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, dl_ctx->dl);
curl_easy_setopt(curl, CURLOPT_RANGE, range);
res = curl_easy_perform(curl);
+ free(range);
+
+ if(dl_ctx->range_fail)
+ return -1;
if(res != CURLE_OK) {
printf("Download failed: %s\n", curl_easy_strerror(res));
- return False;
+ return 0;
}
long code;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &code);
if (code != 206 && code != 200) {
- printf("HTTP Error: %li when download %s\n", code,
+ printf("HTTP Error: %li when downloading %s\n", code,
url);
- return False;
+ return 0;
}
- return True;
+
+ return 1;
}
-int dl_byte_range(CURL *curl, zckDL *dl, char *url, int start, int end) {
- char *range = zck_get_range(start, end);
- return dl_range(curl, dl, url, range, 0);
+
+int dl_byte_range(dlCtx *dl_ctx, char *url, int start, int end) {
+ char *range = NULL;
+ zck_dl_reset(dl_ctx->dl);
+ if(start > -1 && end > -1)
+ range = zck_get_range(start, end);
+ return dl_range(dl_ctx, url, range, 0);
}
-int dl_bytes(CURL *curl, zckDL *dl, char *url, size_t bytes, size_t start, size_t *buffer_len, int log_level) {
+int dl_bytes(dlCtx *dl_ctx, char *url, size_t bytes, size_t start,
+ size_t *buffer_len, int log_level) {
if(start + bytes > *buffer_len) {
+ zckDL *dl = dl_ctx->dl;
+
int fd = zck_get_fd(dl->zck);
- if(lseek(fd, 0, SEEK_END) == -1) {
- printf("Seek to end of temporary file failed: %s\n",
+ if(lseek(fd, *buffer_len, SEEK_SET) == -1) {
+ printf("Seek to download location failed: %s\n",
strerror(errno));
return 0;
}
if(*buffer_len >= start + bytes)
return 1;
- if(!dl_byte_range(curl, dl, url, *buffer_len,
- (start + bytes - *buffer_len) - 1)) {
- printf("Error downloading bytes\n");
- return 0;
- }
+
+ int retval = dl_byte_range(dl_ctx, url, *buffer_len,
+ (start + bytes) - 1);
+ if(retval < 1)
+ return retval;
+
if(log_level <= ZCK_LOG_DEBUG)
printf("Downloading %lu bytes at position %lu\n",
start+bytes-*buffer_len, *buffer_len);
strerror(errno));
return 0;
}
+ printf("Seeking to location %lu\n", start);
}
return 1;
}
-int dl_header(CURL *curl, zckDL *dl, char *url, int log_level) {
+int dl_header(CURL *curl, zckDL *dl, char *url, int fail_no_ranges,
+ int log_level) {
size_t buffer_len = 0;
size_t start = 0;
- /* Download first two hundred bytes and read magic and hash type */
- if(!dl_bytes(curl, dl, url, get_min_download_size(), start, &buffer_len, log_level))
- return 0;
+ dlCtx dl_ctx = {0};
+ dl_ctx.fail_no_ranges = 1;
+ dl_ctx.dl = dl;
+ dl_ctx.curl = curl;
+ dl_ctx.max_ranges = 1;
+
+ /* Download minimum download size and read magic and hash type */
+ int retval = dl_bytes(&dl_ctx, url, zck_get_min_download_size(), start,
+ &buffer_len, log_level);
+ if(retval < 1)
+ return retval;
+
if(!zck_read_lead(dl->zck))
return 0;
start = zck_get_lead_length(dl->zck);
printf("Now we need %lu bytes\n", zck_get_header_length(dl->zck) - start);
-
- if(!dl_bytes(curl, dl, url, zck_get_header_length(dl->zck) - start,
+ if(!dl_bytes(&dl_ctx, url, zck_get_header_length(dl->zck) - start,
start, &buffer_len, log_level))
return 0;
if(!zck_read_header(dl->zck))
zck_set_log_level(arguments.log_level);
- int src_fd = open(arguments.source, O_RDONLY);
- if(src_fd < 0) {
- printf("Unable to open %s\n", arguments.source);
- perror("");
- exit(1);
+ zckCtx *zck_src = NULL;
+ if(arguments.source) {
+ int src_fd = open(arguments.source, O_RDONLY);
+ if(src_fd < 0) {
+ printf("Unable to open %s\n", arguments.source);
+ perror("");
+ exit(10);
+ }
+ zck_src = zck_init_read(src_fd);
+ if(zck_src == NULL) {
+ printf("Unable to open %s\n", arguments.source);
+ exit(10);
+ }
}
- zckCtx *zck_src = zck_init_read(src_fd);
- if(zck_src == NULL) {
- printf("Unable to open %s\n", arguments.source);
- exit(1);
+
+ CURL *curl_ctx = curl_easy_init();
+ if(!curl_ctx) {
+ printf("Unable to allocate %lu bytes for curl context\n",
+ sizeof(CURL));
+ exit(10);
}
char *outname_full = calloc(1, strlen(arguments.args[0])+1);
if(dst_fd < 0) {
printf("Unable to open %s: %s\n", outname, strerror(errno));
free(outname_full);
- exit(1);
+ exit(10);
}
- free(outname_full);
zckCtx *zck_tgt = zck_init_adv_read(dst_fd);
if(zck_tgt == NULL)
- exit(1);
-
- CURL *curl_ctx = curl_easy_init();
- if(!curl_ctx) {
- printf("Unable to allocate %lu bytes for curl context\n",
- sizeof(CURL));
- exit(1);
- }
+ exit(10);
zckDL *dl = zck_dl_init(zck_tgt);
if(dl == NULL)
- exit(1);
- dl->zck = zck_tgt;
+ exit(10);
- if(!dl_header(curl_ctx, dl, arguments.args[0], arguments.log_level))
- exit(1);
-
- if(!zck_copy_chunks(zck_src, zck_tgt))
- exit(1);
+ int exit_val = 0;
+ int retval = dl_header(curl_ctx, dl, arguments.args[0],
+ arguments.fail_no_ranges, arguments.log_level);
+ if(!retval) {
+ exit_val = 10;
+ goto out;
+ }
+ /* The server doesn't support ranges */
+ if(retval == -1) {
+ if(arguments.fail_no_ranges) {
+ printf("Server doesn't support ranges and --fail-no-ranges was "
+ "set\n");
+ exit_val = 2;
+ goto out;
+ }
+ /* Download the full file */
+ lseek(dst_fd, 0, SEEK_SET);
+ ftruncate(dst_fd, 0);
+ dlCtx dl_ctx = {0};
+ dl_ctx.dl = dl;
+ dl_ctx.curl = curl_ctx;
+ dl_ctx.max_ranges = 0;
+ if(!dl_byte_range(&dl_ctx, arguments.args[0], -1, -1)) {
+ exit_val = 10;
+ goto out;
+ }
+ lseek(dst_fd, 0, SEEK_SET);
+ if(!zck_read_lead(dl->zck) || !zck_read_header(dl->zck)) {
+ printf("Error reading zchunk file\n");
+ exit_val = 10;
+ goto out;
+ }
+ } else {
+ /* If file is already fully downloaded, let's get out of here! */
+ if(zck_validate_checksums(zck_tgt)) {
+ printf("Downloaded %lu bytes\n",
+ (long unsigned)zck_dl_get_bytes_downloaded(dl));
+ ftruncate(dst_fd, zck_get_length(zck_tgt));
+ exit_val = 0;
+ //goto out;
+ }
+ if(zck_src && !zck_copy_chunks(zck_src, zck_tgt)) {
+ exit_val = 10;
+ goto out;
+ }
+ dlCtx dl_ctx = {0};
+ dl_ctx.dl = dl;
+ dl_ctx.curl = curl_ctx;
+ dl_ctx.max_ranges = range_attempt[0];
+ dl_ctx.fail_no_ranges = 1;
+ int ra_index = 0;
+ printf("Missing chunks: %i\n", zck_missing_chunks(zck_tgt));
+ while(zck_missing_chunks(zck_tgt) > 0) {
+ dl_ctx.range_fail = 0;
+ zck_dl_reset(dl);
+ dl->range = zck_get_dl_range(zck_tgt, dl_ctx.max_ranges);
+ if(dl->range == NULL) {
+ exit_val = 10;
+ goto out;
+ }
+ while(range_attempt[ra_index] > 1 &&
+ range_attempt[ra_index+1] > dl->range->count)
+ ra_index++;
+ char *range_string = zck_get_range_char(dl->range);
+ if(range_string == NULL) {
+ exit_val = 10;
+ goto out;
+ }
+ int retval = dl_range(&dl_ctx, arguments.args[0], range_string, 1);
+ if(retval == -1) {
+ if(dl_ctx.max_ranges > 1) {
+ ra_index += 1;
+ dl_ctx.max_ranges = range_attempt[ra_index];
+ }
+ printf("Tried downloading too many ranges, reducing to %i\n", dl_ctx.max_ranges);
+ }
+ zck_range_free(&(dl->range));
+ if(!retval) {
+ goto out;
+ }
+ }
+ }
printf("Downloaded %lu bytes\n",
(long unsigned)zck_dl_get_bytes_downloaded(dl));
- int exit_val = 0;
+ ftruncate(dst_fd, zck_get_length(zck_tgt));
+
switch(zck_validate_data_checksum(dl->zck)) {
case -1:
exit_val = 1;
default:
break;
}
+out:
+ free(outname_full);
zck_dl_free(&dl);
zck_free(&zck_tgt);
zck_free(&zck_src);
projects / zchunk.git / commitdiff