typedef enum log_type { ZCK_LOG_DEBUG, ZCK_LOG_INFO, ZCK_LOG_WARNING,
ZCK_LOG_ERROR } log_type;
+typedef struct zckIndex {
+ char *digest;
+ int digest_size;
+ int finished;
+ uint64_t start;
+ size_t length;
+ struct zckIndex *next;
+} zckIndex;
+
+typedef struct zckIndexInfo {
+ uint64_t count;
+ size_t length;
+ uint8_t hash_type;
+ uint8_t digest_size;
+ zckIndex *first;
+} zckIndexInfo;
typedef struct zckRange {
uint64_t start;
unsigned int segments;
unsigned int max_ranges;
zckRange *first;
+ zckIndexInfo index;
} zckRangeInfo;
-typedef struct zckIndex {
- char *digest;
- uint64_t start;
- size_t length;
- struct zckIndex *next;
-} zckIndex;
-
-typedef struct zckHashType zckHashType;
-
-typedef struct zckIndexInfo {
- uint64_t count;
- size_t length;
- zckHashType *hash_type;
- zckIndex *first;
-} zckIndexInfo;
-
typedef struct zckDLPriv zckDLPriv;
+typedef struct zckCtx zckCtx;
+typedef struct zckHash zckHash;
typedef struct zckDL {
size_t dl;
size_t ul;
+ size_t write_in_chunk;
+ size_t dl_chunk_data;
int dst_fd;
char *boundary;
- zckIndexInfo index;
zckRangeInfo info;
zckDLPriv *priv;
+ struct zckCtx *zck;
+ zckIndex *tgt_check;
+ zckHash *chunk_hash;
} zckDL;
-typedef struct zckCtx zckCtx;
zckCtx *zck_create();
void zck_free(zckCtx *zck);
int zck_decompress_to_file (zckCtx *zck, int src_fd, int dst_fd);
int zck_set_full_hash_type(zckCtx *zck, uint8_t hash_type);
int zck_set_chunk_hash_type(zckCtx *zck, uint8_t hash_type);
+int64_t zck_get_predata_length(zckCtx *zck);
char *zck_get_index_digest(zckCtx *zck);
char *zck_get_full_digest(zckCtx *zck);
int zck_get_full_digest_size(zckCtx *zck);
const char *zck_comp_name_from_type(uint8_t comp_type);
int zck_range_calc_segments(zckRangeInfo *info, unsigned int max_ranges);
int zck_range_get_need_dl(zckRangeInfo *info, zckCtx *zck_src, zckCtx *zck_tgt);
+int zck_dl_copy_src_chunks(zckRangeInfo *info, zckCtx *src, zckCtx *tgt);
int zck_range_get_array(zckRangeInfo *info, char **ra);
void zck_range_close(zckRangeInfo *info);
void zck_set_log_level(log_type ll);
size_t zck_dl_get_bytes_uploaded(zckDL *dl);
int zck_dl_range(zckDL *dl, char *url);
char *zck_dl_get_range(unsigned int start, unsigned int end);
+int zck_hash_check_full_file(zckCtx *zck, int dst_fd);
#endif
return False; \
}
+int zck_dl_write_zero(zckCtx *tgt, zckIndex *tgt_idx) {
+ char buf[BUF_SIZE] = {0};
+ size_t tgt_data_offset = tgt->preindex_size + tgt->comp_index_size;
+ size_t to_read = tgt_idx->length;
+ if(!zck_seek(tgt->fd, tgt_data_offset + tgt_idx->start, SEEK_SET))
+ return False;
+ while(to_read > 0) {
+ int rb = BUF_SIZE;
+ if(rb > to_read)
+ rb = to_read;
+ if(!zck_write(tgt->fd, buf, rb))
+ return False;
+ to_read -= rb;
+ }
+ return True;
+}
+
+int zck_dl_write(zckDL *dl, const char *at, size_t length) {
+ if(dl->write_in_chunk < length)
+ length = dl->write_in_chunk;
+ if(!zck_write(dl->dst_fd, at, length))
+ return -1;
+ dl->write_in_chunk -= length;
+ return length;
+}
+
+int zck_dl_md_write(zckDL *dl, const char *at, size_t length) {
+ int wb = 0;
+ if(dl->write_in_chunk > 0) {
+ wb = zck_dl_write(dl, at, length);
+ if(!zck_hash_update(dl->chunk_hash, at, wb))
+ return 0;
+ if(wb < 0)
+ return 0;
+ zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes\n", wb);
+ dl->dl_chunk_data += wb;
+ }
+ return wb;
+}
+
+int zck_dl_write_chunk(zckDL *dl) {
+ if(dl->chunk_hash == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Chunk hash not initialized\n");
+ return False;
+ }
+ char *digest = zck_hash_finalize(dl->chunk_hash);
+ free(dl->chunk_hash);
+ if(memcmp(digest, dl->tgt_check->digest, dl->tgt_check->digest_size) != 0) {
+ zck_log(ZCK_LOG_WARNING,
+ "Downloaded chunk failed hash check\n");
+ if(!zck_dl_write_zero(dl->zck, dl->tgt_check))
+ return False;
+ } else {
+ dl->tgt_check->finished = True;
+ }
+ dl->tgt_check = NULL;
+ dl->chunk_hash = NULL;
+ free(digest);
+ return True;
+}
+
int zck_dl_multidata_cb(zckDL *dl, const char *at, size_t length) {
- if(dl == NULL) {
- zck_log(ZCK_LOG_ERROR, "zckDL not initialized");
+ if(dl == NULL || dl->info.index.first == NULL) {
+ zck_log(ZCK_LOG_ERROR, "zckDL index not initialized\n");
return 0;
}
- zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes\n", length);
- size_t wb = write(dl->dst_fd, at, length);
- return wb;
+ if(dl->zck == NULL || dl->zck->index.first == NULL) {
+ zck_log(ZCK_LOG_ERROR, "zckCtx index not initialized\n");
+ return 0;
+ }
+ int wb = zck_dl_md_write(dl, at, length);
+ if(dl->write_in_chunk == 0) {
+ /* Check whether we just finished downloading a chunk and verify it */
+ if(dl->tgt_check && !zck_dl_write_chunk(dl))
+ return False;
+ zckIndex *idx = dl->info.index.first;
+ while(idx) {
+ if(dl->dl_chunk_data == idx->start) {
+ zckIndex *tgt_idx = dl->zck->index.first;
+ while(tgt_idx) {
+ if(tgt_idx->finished)
+ tgt_idx = tgt_idx->next;
+ if(idx->length == tgt_idx->length &&
+ memcmp(idx->digest, tgt_idx->digest,
+ idx->digest_size) == 0) {
+ dl->tgt_check = tgt_idx;
+ dl->chunk_hash = zmalloc(sizeof(zckHash));
+ if(!zck_hash_init(dl->chunk_hash,
+ &(dl->zck->chunk_hash_type)))
+ return 0;
+ dl->write_in_chunk = idx->length;
+ size_t offset = dl->zck->preindex_size +
+ dl->zck->comp_index_size;
+ if(!zck_seek(dl->dst_fd, offset + tgt_idx->start,
+ SEEK_SET))
+ return 0;
+ idx = NULL;
+ tgt_idx = NULL;
+ } else {
+ tgt_idx = tgt_idx->next;
+ }
+ }
+ }
+ if(idx)
+ idx = idx->next;
+ }
+ }
+ int wb2 = 0;
+ if(dl->write_in_chunk > 0 && wb < length) {
+ wb2 = zck_dl_multidata_cb(dl, at+wb, length-wb);
+ if(wb2 == 0)
+ return 0;
+ }
+ return wb + wb2;
}
zckDL *zck_dl_init() {
return wb;
}
+int zck_dl_write_and_verify(zckRangeInfo *info, zckCtx *src, zckCtx *tgt,
+ zckIndex *src_idx, zckIndex *tgt_idx) {
+ static char buf[BUF_SIZE] = {0};
+
+ size_t src_data_offset = src->preindex_size + src->comp_index_size;
+ size_t tgt_data_offset = tgt->preindex_size + tgt->comp_index_size;
+ size_t to_read = src_idx->length;
+ if(!zck_seek(src->fd, src_data_offset + src_idx->start, SEEK_SET))
+ return False;
+ if(!zck_seek(tgt->fd, tgt_data_offset + tgt_idx->start, SEEK_SET))
+ return False;
+ zckHash check_hash = {0};
+ if(!zck_hash_init(&check_hash, &(src->chunk_hash_type)))
+ return False;
+ while(to_read > 0) {
+ int rb = BUF_SIZE;
+ if(rb > to_read)
+ rb = to_read;
+ if(!zck_read(src->fd, buf, rb))
+ return False;
+ if(!zck_hash_update(&check_hash, buf, rb))
+ return False;
+ if(!zck_write(tgt->fd, buf, rb))
+ return False;
+ to_read -= rb;
+ }
+ char *digest = zck_hash_finalize(&check_hash);
+ /* If chunk is invalid, overwrite with zeros and add to download range */
+ if(memcmp(digest, src_idx->digest, src_idx->digest_size) != 0) {
+ zck_log(ZCK_LOG_WARNING, "Source hash: %s\n",
+ zck_hash_get_printable(src_idx->digest, &(src->chunk_hash_type)));
+ zck_log(ZCK_LOG_WARNING, "Target hash: %s\n",
+ zck_hash_get_printable(digest, &(src->chunk_hash_type)));
+ if(!zck_dl_write_zero(tgt, tgt_idx))
+ return False;
+ if(!zck_range_add(info, tgt_idx, tgt))
+ return False;
+ } else {
+ tgt_idx->finished = True;
+ zck_log(ZCK_LOG_DEBUG, "Writing %lu bytes at %lu\n", tgt_idx->length,
+ tgt_idx->start);
+ }
+ free(digest);
+ return True;
+}
+
+int zck_dl_copy_src_chunks(zckRangeInfo *info, zckCtx *src, zckCtx *tgt) {
+ zckIndexInfo *tgt_info = zck_get_index(tgt);
+ zckIndexInfo *src_info = zck_get_index(src);
+ zckIndex *tgt_idx = tgt_info->first;
+ zckIndex *src_idx = src_info->first;
+ while(tgt_idx) {
+ int found = False;
+ src_idx = src_info->first;
+
+ while(src_idx) {
+ if(tgt_idx->length == src_idx->length &&
+ memcmp(tgt_idx->digest, src_idx->digest,
+ zck_get_chunk_digest_size(tgt)) == 0) {
+ found = True;
+ break;
+ }
+ src_idx = src_idx->next;
+ }
+ /* Write out found chunk, then verify that it's valid */
+ if(found && !zck_dl_write_and_verify(info, src, tgt, src_idx, tgt_idx))
+ return False;
+ if(!found && !zck_range_add(info, tgt_idx, tgt))
+ return False;
+
+ tgt_idx = tgt_idx->next;
+ }
+ return True;
+}
+
int zck_dl_range(zckDL *dl, char *url) {
if(dl == NULL || dl->priv == NULL || dl->info.first == NULL) {
zck_log(ZCK_LOG_ERROR, "Struct not defined\n");
return False;
}
if(start + bytes > *buffer_len) {
+ zckIndex idx = {0};
+
zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n");
if(lseek(dl->dst_fd, 0, SEEK_END) == -1) {
zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n",
return False;
}
zck_log(ZCK_LOG_DEBUG, "Downloading %lu bytes at position %lu\n", start+bytes-*buffer_len, *buffer_len);
+ idx.start = *buffer_len;
+ idx.length = start+bytes-*buffer_len;
zck_range_close(&(dl->info));
- zck_range_add(&(dl->info), *buffer_len, start+bytes-1);
+ zck_range_add(&(dl->info), &idx, NULL);
if(!zck_dl_range(dl, url))
return False;
zck_range_close(&(dl->info));
return True;
}
+int zck_zero_bytes(zckDL *dl, size_t bytes, size_t start, size_t *buffer_len) {
+ char buf[BUF_SIZE] = {0};
+ if(start + bytes > *buffer_len) {
+ zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n");
+ if(lseek(dl->dst_fd, 0, SEEK_END) == -1) {
+ zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n",
+ strerror(errno));
+ return False;
+ }
+ size_t write = *buffer_len;
+ while(write < start + bytes) {
+ size_t wb = BUF_SIZE;
+ if(write + wb > start + bytes)
+ wb = (start + bytes) - write;
+ if(!zck_write(dl->dst_fd, buf, wb))
+ return False;
+ write += wb;
+ }
+ zck_log(ZCK_LOG_DEBUG, "Wrote %lu zeros at position %lu\n", start+bytes-*buffer_len, *buffer_len);
+ *buffer_len = start+bytes;
+ zck_log(ZCK_LOG_DEBUG, "Seeking to position %lu\n", start);
+ if(lseek(dl->dst_fd, start, SEEK_SET) == -1) {
+ zck_log(ZCK_LOG_ERROR,
+ "Seek to byte %lu of temporary file failed: %s\n", start,
+ strerror(errno));
+ return False;
+ }
+ }
+ return True;
+}
+
int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) {
size_t buffer_len = 0;
size_t start = 0;
-
+ if(zck == NULL) {
+ zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n");
+ return False;
+ }
+ if(dl == NULL) {
+ zck_log(ZCK_LOG_ERROR, "zckDL not initialized\n");
+ return False;
+ }
+ zck->fd = dl->dst_fd;
if(!zck_dl_bytes(dl, url, 100, start, &buffer_len))
return False;
if(!zck_read_initial(zck, dl->dst_fd))
return False;
if(!zck_read_index(zck, dl->dst_fd))
return False;
+ zckIndexInfo *info = &(dl->info.index);
+ info->hash_type = zck->index.hash_type;
+ zck_log(ZCK_LOG_DEBUG, "Writing zeros to rest of file: %llu\n", zck->index.length + zck->comp_index_size + start);
+ if(!zck_zero_bytes(dl, zck->index.length, zck->comp_index_size + start, &buffer_len))
+ return False;
return True;
}
next = next->next;
free(tmp);
}
+ zck_index_clean(&(info->index));
memset(info, 0, sizeof(zckRangeInfo));
}
-zckRange *zck_range_insert_new(zckRange *prev, zckRange *next, uint64_t start, uint64_t end) {
+zckRange *zck_range_insert_new(zckRange *prev, zckRange *next, uint64_t start,
+ uint64_t end, zckRangeInfo *info,
+ zckIndex *idx, int add_index) {
zckRange *new = zmalloc(sizeof(zckRange));
if(!new) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
new->next = next;
next->prev = new;
}
+ if(add_index)
+ if(!zck_index_new_chunk(&(info->index), idx->digest, idx->digest_size,
+ end-start+1, False)) {
+ free(new);
+ return NULL;
+ }
return new;
}
}
}
-int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end) {
- if(!info) {
- zck_log(ZCK_LOG_ERROR, "zckRangeInfo not allocated\n");
+int zck_range_add(zckRangeInfo *info, zckIndex *idx, zckCtx *zck) {
+ if(info == NULL || idx == NULL) {
+ zck_log(ZCK_LOG_ERROR, "zckRangeInfo or zckIndex not allocated\n");
return False;
}
+ size_t predata_len = 0;
+ int add_index = False;
+ if(zck) {
+ predata_len = zck_get_predata_length(zck);
+ add_index = True;
+ }
+
+ size_t start = idx->start + predata_len;
+ size_t end = idx->start + predata_len + idx->length - 1;
zckRange *prev = info->first;
for(zckRange *ptr=info->first; ptr;) {
prev = ptr;
ptr = ptr->next;
continue;
} else if(start < ptr->start) {
- if(zck_range_insert_new(ptr->prev, ptr, start, end) == NULL)
+
+ if(zck_range_insert_new(ptr->prev, ptr, start, end, info, idx, add_index) == NULL)
return False;
if(info->first == ptr) {
info->first = ptr->prev;
}
}
/* We've only reached here if we should be last item */
- zckRange *new = zck_range_insert_new(prev, NULL, start, end);
+ zckRange *new = zck_range_insert_new(prev, NULL, start, end, info, idx, add_index);
if(new == NULL)
return False;
if(info->first == NULL)
src_idx = src_idx->next;
}
if(!found)
- if(!zck_range_add(info, tgt_idx->start, tgt_idx->start+tgt_idx->length-1))
+ if(!zck_range_add(info, tgt_idx, zck_tgt))
return False;
tgt_idx = tgt_idx->next;
#include "sha2/sha2.h"
static char unknown[] = "Unknown(\0\0\0\0\0";
+static char hash_text[BUF_SIZE] = {0};
const static char *HASH_NAME[] = {
"SHA-1",
return;
}
+/* Returns 1 if full file hash matches, 0 if it doesn't and -1 if failure */
+int zck_hash_check_full_file(zckCtx *zck, int dst_fd) {
+ if(!zck_seek(dst_fd, zck->preindex_size + zck->comp_index_size, SEEK_SET))
+ return -1;
+ if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type)))
+ return -1;
+ char buf[BUF_SIZE] = {0};
+ zckIndex *idx = zck->index.first;
+ zck_log(ZCK_LOG_INFO, "Checking full hash\n");
+ while(idx) {
+ size_t to_read = idx->length;
+ while(to_read > 0) {
+ size_t rb = BUF_SIZE;
+ if(rb > to_read)
+ rb = to_read;
+ if(!zck_read(dst_fd, buf, rb))
+ return -1;
+ zck_hash_update(&(zck->check_full_hash), buf, rb);
+ to_read -= rb;
+ }
+ idx = idx->next;
+ }
+ return zck_validate_file(zck);
+ return True;
+}
+
char *zck_hash_finalize(zckHash *hash) {
if(hash && hash->ctx && hash->type) {
if(hash->type->type == ZCK_HASH_SHA1) {
return HASH_NAME[hash_type];
}
-
+const char *zck_hash_get_printable(const char *digest, zckHashType *type) {
+ if(digest == NULL || type == NULL) {
+ zck_log(ZCK_LOG_ERROR,
+ "digest or zckHashType haven't been initialized\n");
+ return False;
+ }
+ for(int i=0; i<type->digest_size; i++) {
+ if(snprintf(hash_text + (i*2), 3, "%02x", (unsigned char)digest[i]) < 0) {
+ zck_log(ZCK_LOG_ERROR, "Unable to generate printable hash\n");
+ return NULL;
+ }
+ }
+ return hash_text;
+}
#include "zck_private.h"
-int zck_index_free(zckCtx *zck) {
- if(zck->index.first) {
+void zck_index_clean(zckIndexInfo *index) {
+ if(index == NULL)
+ return;
+
+ if(index->first) {
zckIndex *next;
- zckIndex *tmp=zck->index.first;
+ zckIndex *tmp=index->first;
while(tmp != NULL) {
next = tmp->next;
if(tmp->digest)
tmp = next;
}
}
- if(zck->index.hash_type)
- free(zck->index.hash_type);
- memset(&(zck->index), 0, sizeof(zckIndexInfo));
+ memset(index, 0, sizeof(zckIndexInfo));
+}
+
+void zck_index_free(zckCtx *zck) {
+ zck_index_clean(&(zck->index));
if(zck->full_hash_digest) {
free(zck->full_hash_digest);
zck->full_hash_digest = NULL;
free(zck->index_digest);
zck->index_digest = NULL;
}
- return True;
}
if(zck->index.first) {
zckIndex *tmp = zck->index.first;
while(tmp) {
- index_size += zck->index.hash_type->digest_size + sizeof(uint64_t);
+ index_size += zck->index.digest_size + sizeof(uint64_t);
tmp = tmp->next;
}
}
/* Write index */
index = zmalloc(index_size);
index_loc = index;
- memcpy(index_loc, &(zck->index.hash_type->type), 1);
+ memcpy(index_loc, &(zck->index.hash_type), 1);
index_loc += 1;
index_count = htole64(zck->index.count);
memcpy(index_loc, &index_count, sizeof(uint64_t));
zckIndex *tmp = zck->index.first;
while(tmp) {
uint64_t end = htole64(tmp->start + tmp->length);
- memcpy(index_loc, tmp->digest, zck->index.hash_type->digest_size);
- index_loc += zck->index.hash_type->digest_size;
+ memcpy(index_loc, tmp->digest, zck->index.digest_size);
+ index_loc += zck->index.hash_type;
memcpy(index_loc, &end, sizeof(uint64_t));
index_loc += sizeof(uint64_t);
tmp = tmp->next;
return True;
}
-int zck_index_new_chunk(zckIndexInfo *index, char *digest, size_t length) {
+int zck_index_new_chunk(zckIndexInfo *index, char *digest, int digest_size,
+ size_t length, int finished) {
if(index == NULL) {
- zck_log(ZCK_LOG_ERROR, "Invalid index");
+ zck_log(ZCK_LOG_ERROR, "Invalid index\n");
+ return False;
+ }
+ if(digest_size == 0) {
+ zck_log(ZCK_LOG_ERROR, "Digest size 0 too small\n");
return False;
}
zckIndex *idx = zmalloc(sizeof(zckIndex));
sizeof(zckIndex));
return False;
}
- if(digest == NULL || length == 0) {
- idx->digest = zmalloc(index->hash_type->digest_size);
- if(idx->digest == NULL) {
- zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- index->hash_type->digest_size);
- return False;
- }
- } else {
- idx->digest = digest;
+ idx->digest = zmalloc(digest_size);
+ if(idx->digest == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", digest_size);
+ return False;
}
+ if(digest)
+ memcpy(idx->digest, digest, digest_size);
+ idx->digest_size = digest_size;
idx->start = index->length;
idx->length = length;
+ idx->finished = finished;
if(index->first == NULL) {
index->first = idx;
} else {
return True;
}
-int zck_index_add_dl_chunk(zckDL *dl, char *digest, size_t size) {
- if(dl == NULL) {
- zck_log(ZCK_LOG_ERROR, "Invalid dl context");
- return False;
- }
- zckIndex *new_index = zmalloc(sizeof(zckIndex));
- if(new_index == NULL) {
- zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- sizeof(zckIndex));
- return False;
- }
- dl->index.count = dl->index.count + 1;
- return True;
-}
-
int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) {
zckHash hash;
}
if(size == 0) {
- if(!zck_index_new_chunk(&(zck->index), NULL, size))
+ if(!zck_index_new_chunk(&(zck->index), NULL, zck->index.digest_size,
+ size, True))
return False;
} else {
if(!zck_hash_update(&(zck->full_hash), data, size))
return False;
- if(!zck_hash_init(&hash, zck->index.hash_type))
+ if(!zck_hash_init(&hash, &(zck->chunk_hash_type)))
return False;
if(!zck_hash_update(&hash, data, size))
return False;
if(digest == NULL) {
zck_log(ZCK_LOG_ERROR,
"Unable to calculate %s checksum for new chunk\n",
- zck_hash_name_from_type(zck->index.hash_type->type));
+ zck_hash_name_from_type(zck->index.hash_type));
return False;
}
- if(!zck_index_new_chunk(&(zck->index), digest, size))
+ if(!zck_index_new_chunk(&(zck->index), digest, zck->index.digest_size,
+ size, True))
return False;
}
return True;
char *digest;
uint64_t index_size;
uint64_t index_count;
- uint8_t hash_type;
char *dst = NULL;
size_t dst_size = 0;
char *cur_loc;
zck->hash_type.digest_size);
return False;
}
+ uint8_t hash_type;
memcpy(&hash_type, dst, 1);
if(!zck_set_chunk_hash_type(zck, hash_type)) {
if(dst)
}
if((dst_size - (zck->hash_type.digest_size + sizeof(uint64_t)+ 1)) %
- (zck->index.hash_type->digest_size + sizeof(uint64_t)) != 0) {
+ (zck->index.digest_size + sizeof(uint64_t)) != 0) {
zck_log(ZCK_LOG_ERROR, "Index size is invalid\n");
if(dst)
free(dst);
}
uint64_t end = 0;
- new->digest = zmalloc(zck->index.hash_type->digest_size);
+ new->digest = zmalloc(zck->index.digest_size);
if(!new->digest) {
zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
- zck->index.hash_type->digest_size);
+ zck->index.digest_size);
return False;
}
- memcpy(new->digest, cur_loc, zck->index.hash_type->digest_size);
- cur_loc += zck->index.hash_type->digest_size;
+ memcpy(new->digest, cur_loc, zck->index.digest_size);
+ new->digest_size = zck->index.digest_size;
+ cur_loc += zck->index.digest_size;
memcpy(&end, cur_loc, sizeof(uint64_t));
new->start = prev_loc;
new->length = le64toh(end) - prev_loc;
+ new->finished = False;
prev_loc = le64toh(end);
zck->index.length += new->length;
cur_loc += sizeof(uint64_t);
#include <stdlib.h>
#include <stdint.h>
+#include <string.h>
#include <unistd.h>
+#include <errno.h>
#include <zck.h>
#include "zck_private.h"
-#define BLK_SIZE 32768
-
int zck_read(int fd, char *data, size_t length) {
if(length == 0)
return True;
-
- if(read(fd, data, length) != length) {
+ if(data == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to read to NULL data pointer\n");
+ return False;
+ }
+ ssize_t read_bytes = read(fd, data, length);
+ if(read_bytes == -1) {
+ zck_log(ZCK_LOG_ERROR, "Error reading data: %s\n", strerror(errno));
+ return False;
+ } else if(read_bytes != length) {
zck_log(ZCK_LOG_ERROR, "Short read\n");
return False;
}
-
return True;
}
int zck_write(int fd, const char *data, size_t length) {
if(length == 0)
return True;
+ if(data == NULL) {
+ zck_log(ZCK_LOG_ERROR, "Unable to write from NULL data pointer\n");
+ return False;
+ }
+ ssize_t write_bytes = write(fd, data, length);
+ if(write_bytes == -1) {
+ zck_log(ZCK_LOG_ERROR, "Error write data: %s\n", strerror(errno));
+ return False;
+ } else if(write_bytes != length) {
+ zck_log(ZCK_LOG_ERROR, "Short write\n");
+ return False;
+ }
+ return True;
+}
+
+int zck_seek(int fd, off_t offset, int whence) {
+ if(lseek(fd, offset, whence) == -1) {
+ char *wh_str = NULL;
- if(write(fd, data, length) != length)
+ if(whence == SEEK_CUR) {
+ wh_str = "from current position";
+ } else if(whence == SEEK_END) {
+ wh_str = "from end of file";
+ } else if(whence == SEEK_SET) {
+ wh_str = "from beginning of file";
+ } else {
+ wh_str = "using unknown measurement";
+ }
+ zck_log(ZCK_LOG_ERROR, "Unable to seek to %lu %s: %s\n", offset, wh_str,
+ strerror(errno));
return False;
+ }
return True;
}
int zck_chunks_from_temp(zckCtx *zck) {
int read_count;
- char *data = zmalloc(BLK_SIZE);
+ char *data = zmalloc(BUF_SIZE);
if(data == NULL)
return False;
if(lseek(zck->temp_fd, 0, SEEK_SET) == -1)
return False;
- while((read_count = read(zck->temp_fd, data, BLK_SIZE)) > 0) {
+ while((read_count = read(zck->temp_fd, data, BUF_SIZE)) > 0) {
if(read_count == -1 || !zck_write(zck->fd, data, read_count)) {
free(data);
return False;
#include <stdint.h>
#include <stddef.h>
+#define BUF_SIZE 32768
+
#define zmalloc(x) calloc(1, x)
struct zckComp;
int digest_size;
} zckHashType;
-typedef struct {
+typedef struct zckHash {
zckHashType *type;
void *ctx;
} zckHash;
int temp_fd;
int fd;
+ size_t preindex_size;
char *full_hash_digest;
char *comp_index;
size_t comp_index_size;
zckHash check_full_hash;
zckComp comp;
zckHashType hash_type;
+ zckHashType chunk_hash_type;
} zckCtx;
const char *zck_hash_name_from_type(uint8_t hash_type);
int zck_get_tmp_fd();
+int zck_validate_file(zckCtx *zck);
/* comp/comp.c */
int zck_comp_init(zckCtx *zck);
int zck_hash_update(zckHash *hash, const char *message, const size_t size);
char *zck_hash_finalize(zckHash *hash);
void zck_hash_close(zckHash *hash);
+const char *zck_hash_get_printable(const char *digest, zckHashType *type);
/* index/index.c */
int zck_index_read(zckCtx *zck, char *data, size_t size);
int zck_index_finalize(zckCtx *zck);
+int zck_index_new_chunk(zckIndexInfo *index, char *digest, int digest_size,
+ size_t length, int finished);
int zck_index_add_chunk(zckCtx *zck, char *data, size_t size);
-int zck_index_free(zckCtx *zck);
+void zck_index_clean(zckIndexInfo *index);
+void zck_index_free(zckCtx *zck);
int zck_write_index(zckCtx *zck);
/* io.c */
+int zck_seek(int fd, off_t offset, int whence);
int zck_read(int fd, char *data, size_t length);
int zck_write(int fd, const char *data, size_t length);
int zck_chunks_from_temp(zckCtx *zck);
/* dl/range.c */
char *zck_range_get_char(zckRange **range, int max_ranges);
-int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end);
+int zck_range_add(zckRangeInfo *info, zckIndex *idx, zckCtx *zck);
/* log.c */
void zck_log(log_type lt, const char *format, ...);
printf("Unable to open %s\n", argv[1]);
exit(1);
}
- close(src_fd);
zckDL *dl = zck_dl_init();
if(dl == NULL)
exit(1);
+ dl->zck = zck_tgt;
dl->dst_fd = zck_get_tmp_fd();
if(dl->dst_fd < 0)
exit(1);
zck_range_close(&(dl->info));
- if(!zck_range_get_need_dl(&(dl->info), zck_src, zck_tgt))
+ if(!zck_dl_copy_src_chunks(&(dl->info), zck_src, zck_tgt))
exit(1);
int max_ranges = 256;
if(!zck_range_calc_segments(&(dl->info), max_ranges))
free(outname_full);*/
printf("Downloaded %lu bytes\n", zck_dl_get_bytes_downloaded(dl));
+ switch(zck_hash_check_full_file(dl->zck, dl->dst_fd)) {
+ case -1:
+ exit(1);
+ break;
+ case 0:
+ exit(1);
+ break;
+ default:
+ break;
+ }
zck_dl_free(dl);
zck_free(zck_tgt);
zck_free(zck_src);
zck_dl_global_cleanup();
exit(0);
-
- /*
- if(zck_get_chunk_hash_type(zck_tgt) != zck_get_chunk_hash_type(zck_src)) {
- printf("ERROR: Chunk hash types don't match:\n");
- printf(" %s: %s\n", argv[1], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_tgt)));
- printf(" %s: %s\n", argv[2], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_src)));
- return 1;
- }
- zckIndex *tgt_idx = zck_get_index(zck_tgt);
- zckIndex *src_idx = zck_get_index(zck_src);
- if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) != 0)
- printf("WARNING: Dicts don't match\n");
- int dl_size = 0;
- int total_size = 0;
- int matched_chunks = 0;
- while(tgt_idx) {
- int found = False;
- src_idx = zck_get_index(zck_src);
-
- while(src_idx) {
- if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) == 0) {
- found = True;
- break;
- }
- src_idx = src_idx->next;
- }
- if(!found) {
- dl_size += tgt_idx->length;
- } else {
- matched_chunks += 1;
- }
- total_size += tgt_idx->length;
- tgt_idx = tgt_idx->next;
- }
- printf("Would download %i of %i bytes\n", dl_size, total_size);
- printf("Matched %i of %lu chunks\n", matched_chunks, zck_get_index_count(zck_tgt));
-
- zckRangeInfo info = {0};
- if(!zck_range_get_need_dl(&info, zck_src, zck_tgt))
- exit(1);
- zckRange *tmp = info.first;
- while(tmp) {
- printf("Range: %lu - %lu\n", tmp->start, tmp->end);
- tmp = tmp->next;
- }
- printf("Count: %u\n", info.count);
- zck_range_calc_segments(&info, 5);
- char **ra = calloc(sizeof(char*), info.segments);
- if(!ra) {
- printf("Unable to allocate %lu bytes\n", sizeof(char*) * info.segments);
- exit(1);
- }
- if(!zck_range_get_array(&info, ra))
- exit(1);
- for(int i=0; i<info.segments; i++) {
- printf("%s\n", ra[i]);
- free(ra[i]);
- }
- free(ra);
- zck_range_close(&info);
- zck_free(zck_tgt);
- zck_free(zck_src);
- zck_dl_global_cleanup();
- */
}
projects / zchunk.git / commitdiff