Move buzhash into lib/ and automatically chunk by default
authorJonathan Dieter <jdieter@gmail.com>
Wed, 11 Jul 2018 13:35:25 +0000 (14:35 +0100)
committerJonathan Dieter <jdieter@gmail.com>
Wed, 11 Jul 2018 13:35:25 +0000 (14:35 +0100)
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
12 files changed:
src/buzhash/LICENSE [deleted file]
src/buzhash/buzhash.c [deleted file]
src/buzhash/buzhash.h [deleted file]
src/lib/buzhash/LICENSE [new file with mode: 0644]
src/lib/buzhash/buzhash.c [new file with mode: 0644]
src/lib/buzhash/buzhash.h [new file with mode: 0644]
src/lib/buzhash/meson.build [new file with mode: 0644]
src/lib/comp/comp.c
src/lib/meson.build
src/lib/zck.c
src/lib/zck_private.h
src/meson.build

diff --git a/src/buzhash/LICENSE b/src/buzhash/LICENSE
deleted file mode 100644 (file)
index ef6bdaf..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-Copyright (c) 2015, the urlblock developers.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
diff --git a/src/buzhash/buzhash.c b/src/buzhash/buzhash.c
deleted file mode 100644 (file)
index 06ecc9a..0000000
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright (c) 2015, the urlblock developers.
- * Copyright (c) 2018 Jonathan Dieter <jdieter@gmail.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "buzhash.h"
-
-#include <string.h>
-
-#define rol32(v,s) (((v) << (s)) | ((v) >> (32 - (s))))
-
-const uint32_t buzhash_table[] = {
-    0x458be752, 0xc10748cc, 0xfbbcdbb8, 0x6ded5b68,
-    0xb10a82b5, 0x20d75648, 0xdfc5665f, 0xa8428801,
-    0x7ebf5191, 0x841135c7, 0x65cc53b3, 0x280a597c,
-    0x16f60255, 0xc78cbc3e, 0x294415f5, 0xb938d494,
-    0xec85c4e6, 0xb7d33edc, 0xe549b544, 0xfdeda5aa,
-    0x882bf287, 0x3116737c, 0x05569956, 0xe8cc1f68,
-    0x0806ac5e, 0x22a14443, 0x15297e10, 0x50d090e7,
-    0x4ba60f6f, 0xefd9f1a7, 0x5c5c885c, 0x82482f93,
-    0x9bfd7c64, 0x0b3e7276, 0xf2688e77, 0x8fad8abc,
-    0xb0509568, 0xf1ada29f, 0xa53efdfe, 0xcb2b1d00,
-    0xf2a9e986, 0x6463432b, 0x95094051, 0x5a223ad2,
-    0x9be8401b, 0x61e579cb, 0x1a556a14, 0x5840fdc2,
-    0x9261ddf6, 0xcde002bb, 0x52432bb0, 0xbf17373e,
-    0x7b7c222f, 0x2955ed16, 0x9f10ca59, 0xe840c4c9,
-    0xccabd806, 0x14543f34, 0x1462417a, 0x0d4a1f9c,
-    0x087ed925, 0xd7f8f24c, 0x7338c425, 0xcf86c8f5,
-    0xb19165cd, 0x9891c393, 0x325384ac, 0x0308459d,
-    0x86141d7e, 0xc922116a, 0xe2ffa6b6, 0x53f52aed,
-    0x2cd86197, 0xf5b9f498, 0xbf319c8f, 0xe0411fae,
-    0x977eb18c, 0xd8770976, 0x9833466a, 0xc674df7f,
-    0x8c297d45, 0x8ca48d26, 0xc49ed8e2, 0x7344f874,
-    0x556f79c7, 0x6b25eaed, 0xa03e2b42, 0xf68f66a4,
-    0x8e8b09a2, 0xf2e0e62a, 0x0d3a9806, 0x9729e493,
-    0x8c72b0fc, 0x160b94f6, 0x450e4d3d, 0x7a320e85,
-    0xbef8f0e1, 0x21d73653, 0x4e3d977a, 0x1e7b3929,
-    0x1cc6c719, 0xbe478d53, 0x8d752809, 0xe6d8c2c6,
-    0x275f0892, 0xc8acc273, 0x4cc21580, 0xecc4a617,
-    0xf5f7be70, 0xe795248a, 0x375a2fe9, 0x425570b6,
-    0x8898dcf8, 0xdc2d97c4, 0x0106114b, 0x364dc22f,
-    0x1e0cad1f, 0xbe63803c, 0x5f69fac2, 0x4d5afa6f,
-    0x1bc0dfb5, 0xfb273589, 0x0ea47f7b, 0x3c1c2b50,
-    0x21b2a932, 0x6b1223fd, 0x2fe706a8, 0xf9bd6ce2,
-    0xa268e64e, 0xe987f486, 0x3eacf563, 0x1ca2018c,
-    0x65e18228, 0x2207360a, 0x57cf1715, 0x34c37d2b,
-    0x1f8f3cde, 0x93b657cf, 0x31a019fd, 0xe69eb729,
-    0x8bca7b9b, 0x4c9d5bed, 0x277ebeaf, 0xe0d8f8ae,
-    0xd150821c, 0x31381871, 0xafc3f1b0, 0x927db328,
-    0xe95effac, 0x305a47bd, 0x426ba35b, 0x1233af3f,
-    0x686a5b83, 0x50e072e5, 0xd9d3bb2a, 0x8befc475,
-    0x487f0de6, 0xc88dff89, 0xbd664d5e, 0x971b5d18,
-    0x63b14847, 0xd7d3c1ce, 0x7f583cf3, 0x72cbcb09,
-    0xc0d0a81c, 0x7fa3429b, 0xe9158a1b, 0x225ea19a,
-    0xd8ca9ea3, 0xc763b282, 0xbb0c6341, 0x020b8293,
-    0xd4cd299d, 0x58cfa7f8, 0x91b4ee53, 0x37e4d140,
-    0x95ec764c, 0x30f76b06, 0x5ee68d24, 0x679c8661,
-    0xa41979c2, 0xf2b61284, 0x4fac1475, 0x0adb49f9,
-    0x19727a23, 0x15a7e374, 0xc43a18d5, 0x3fb1aa73,
-    0x342fc615, 0x924c0793, 0xbee2d7f0, 0x8a279de9,
-    0x4aa2d70c, 0xe24dd37f, 0xbe862c0b, 0x177c22c2,
-    0x5388e5ee, 0xcd8a7510, 0xf901b4fd, 0xdbc13dbc,
-    0x6c0bae5b, 0x64efe8c7, 0x48b02079, 0x80331a49,
-    0xca3d8ae6, 0xf3546190, 0xfed7108b, 0xc49b941b,
-    0x32baf4a9, 0xeb833a4a, 0x88a3f1a5, 0x3a91ce0a,
-    0x3cc27da1, 0x7112e684, 0x4a3096b1, 0x3794574c,
-    0xa3c8b6f3, 0x1d213941, 0x6e0a2e00, 0x233479f1,
-    0x0f4cd82f, 0x6093edd2, 0x5d7d209e, 0x464fe319,
-    0xd4dcac9e, 0x0db845cb, 0xfb5e4bc3, 0xe0256ce1,
-    0x09fb4ed1, 0x0914be1e, 0xa5bdb2c3, 0xc6eb57bb,
-    0x30320350, 0x3f397e91, 0xa67791bc, 0x86bc0e2c,
-    0xefa0a7e2, 0xe9ff7543, 0xe733612c, 0xd185897b,
-    0x329e5388, 0x91dd236b, 0x2ecb0d93, 0xf4d82a3d,
-    0x35b5c03f, 0xe4e606f0, 0x05b21843, 0x37b45964,
-    0x5eff22f4, 0x6027f4cc, 0x77178b3c, 0xae507131,
-    0x7bf7cabc, 0xf9c18d66, 0x593ade65, 0xd95ddf11,
-};
-
-uint32_t buzhash_setup(buzHash *b, const char *s, size_t n) {
-    b->h = 0;
-    b->window_size = n;
-    for (size_t i = 1; i < n; i++, s++)
-        b->h ^= rol32 (buzhash_table[(size_t) (*s)], n - i);
-    b->h ^= buzhash_table[(size_t) (*s)];
-    return b->h;
-}
-
-uint32_t buzhash_update (buzHash *b, const char *s) {
-    b->h = rol32 (b->h, 1) ^
-           rol32 (buzhash_table[(size_t) *(s-b->window_size)], b->window_size) ^
-           buzhash_table[(size_t) *s];
-    return b->h;
-}
diff --git a/src/buzhash/buzhash.h b/src/buzhash/buzhash.h
deleted file mode 100644 (file)
index 9ea9dd8..0000000
+++ /dev/null
@@ -1,16 +0,0 @@
-#ifndef URLBLOCK_BUZHASH_H
-#define URLBLOCK_BUZHASH_H
-
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdio.h>
-
-typedef struct buzHash {
-    uint32_t h;
-    int window_size;
-} buzHash;
-
-uint32_t buzhash_setup(buzHash *b, const char *s, size_t n);
-uint32_t buzhash_update (buzHash *b, const char *s);
-
-#endif
diff --git a/src/lib/buzhash/LICENSE b/src/lib/buzhash/LICENSE
new file mode 100644 (file)
index 0000000..ef6bdaf
--- /dev/null
@@ -0,0 +1,19 @@
+Copyright (c) 2015, the urlblock developers.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/src/lib/buzhash/buzhash.c b/src/lib/buzhash/buzhash.c
new file mode 100644 (file)
index 0000000..c1fc399
--- /dev/null
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2015, the urlblock developers.
+ * Copyright (c) 2018 Jonathan Dieter <jdieter@gmail.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "buzhash.h"
+
+#include <string.h>
+
+#define rol32(v,s) (((v) << (s)) | ((v) >> (32 - (s))))
+
+const uint32_t buzhash_table[] = {
+    0x458be752, 0xc10748cc, 0xfbbcdbb8, 0x6ded5b68,
+    0xb10a82b5, 0x20d75648, 0xdfc5665f, 0xa8428801,
+    0x7ebf5191, 0x841135c7, 0x65cc53b3, 0x280a597c,
+    0x16f60255, 0xc78cbc3e, 0x294415f5, 0xb938d494,
+    0xec85c4e6, 0xb7d33edc, 0xe549b544, 0xfdeda5aa,
+    0x882bf287, 0x3116737c, 0x05569956, 0xe8cc1f68,
+    0x0806ac5e, 0x22a14443, 0x15297e10, 0x50d090e7,
+    0x4ba60f6f, 0xefd9f1a7, 0x5c5c885c, 0x82482f93,
+    0x9bfd7c64, 0x0b3e7276, 0xf2688e77, 0x8fad8abc,
+    0xb0509568, 0xf1ada29f, 0xa53efdfe, 0xcb2b1d00,
+    0xf2a9e986, 0x6463432b, 0x95094051, 0x5a223ad2,
+    0x9be8401b, 0x61e579cb, 0x1a556a14, 0x5840fdc2,
+    0x9261ddf6, 0xcde002bb, 0x52432bb0, 0xbf17373e,
+    0x7b7c222f, 0x2955ed16, 0x9f10ca59, 0xe840c4c9,
+    0xccabd806, 0x14543f34, 0x1462417a, 0x0d4a1f9c,
+    0x087ed925, 0xd7f8f24c, 0x7338c425, 0xcf86c8f5,
+    0xb19165cd, 0x9891c393, 0x325384ac, 0x0308459d,
+    0x86141d7e, 0xc922116a, 0xe2ffa6b6, 0x53f52aed,
+    0x2cd86197, 0xf5b9f498, 0xbf319c8f, 0xe0411fae,
+    0x977eb18c, 0xd8770976, 0x9833466a, 0xc674df7f,
+    0x8c297d45, 0x8ca48d26, 0xc49ed8e2, 0x7344f874,
+    0x556f79c7, 0x6b25eaed, 0xa03e2b42, 0xf68f66a4,
+    0x8e8b09a2, 0xf2e0e62a, 0x0d3a9806, 0x9729e493,
+    0x8c72b0fc, 0x160b94f6, 0x450e4d3d, 0x7a320e85,
+    0xbef8f0e1, 0x21d73653, 0x4e3d977a, 0x1e7b3929,
+    0x1cc6c719, 0xbe478d53, 0x8d752809, 0xe6d8c2c6,
+    0x275f0892, 0xc8acc273, 0x4cc21580, 0xecc4a617,
+    0xf5f7be70, 0xe795248a, 0x375a2fe9, 0x425570b6,
+    0x8898dcf8, 0xdc2d97c4, 0x0106114b, 0x364dc22f,
+    0x1e0cad1f, 0xbe63803c, 0x5f69fac2, 0x4d5afa6f,
+    0x1bc0dfb5, 0xfb273589, 0x0ea47f7b, 0x3c1c2b50,
+    0x21b2a932, 0x6b1223fd, 0x2fe706a8, 0xf9bd6ce2,
+    0xa268e64e, 0xe987f486, 0x3eacf563, 0x1ca2018c,
+    0x65e18228, 0x2207360a, 0x57cf1715, 0x34c37d2b,
+    0x1f8f3cde, 0x93b657cf, 0x31a019fd, 0xe69eb729,
+    0x8bca7b9b, 0x4c9d5bed, 0x277ebeaf, 0xe0d8f8ae,
+    0xd150821c, 0x31381871, 0xafc3f1b0, 0x927db328,
+    0xe95effac, 0x305a47bd, 0x426ba35b, 0x1233af3f,
+    0x686a5b83, 0x50e072e5, 0xd9d3bb2a, 0x8befc475,
+    0x487f0de6, 0xc88dff89, 0xbd664d5e, 0x971b5d18,
+    0x63b14847, 0xd7d3c1ce, 0x7f583cf3, 0x72cbcb09,
+    0xc0d0a81c, 0x7fa3429b, 0xe9158a1b, 0x225ea19a,
+    0xd8ca9ea3, 0xc763b282, 0xbb0c6341, 0x020b8293,
+    0xd4cd299d, 0x58cfa7f8, 0x91b4ee53, 0x37e4d140,
+    0x95ec764c, 0x30f76b06, 0x5ee68d24, 0x679c8661,
+    0xa41979c2, 0xf2b61284, 0x4fac1475, 0x0adb49f9,
+    0x19727a23, 0x15a7e374, 0xc43a18d5, 0x3fb1aa73,
+    0x342fc615, 0x924c0793, 0xbee2d7f0, 0x8a279de9,
+    0x4aa2d70c, 0xe24dd37f, 0xbe862c0b, 0x177c22c2,
+    0x5388e5ee, 0xcd8a7510, 0xf901b4fd, 0xdbc13dbc,
+    0x6c0bae5b, 0x64efe8c7, 0x48b02079, 0x80331a49,
+    0xca3d8ae6, 0xf3546190, 0xfed7108b, 0xc49b941b,
+    0x32baf4a9, 0xeb833a4a, 0x88a3f1a5, 0x3a91ce0a,
+    0x3cc27da1, 0x7112e684, 0x4a3096b1, 0x3794574c,
+    0xa3c8b6f3, 0x1d213941, 0x6e0a2e00, 0x233479f1,
+    0x0f4cd82f, 0x6093edd2, 0x5d7d209e, 0x464fe319,
+    0xd4dcac9e, 0x0db845cb, 0xfb5e4bc3, 0xe0256ce1,
+    0x09fb4ed1, 0x0914be1e, 0xa5bdb2c3, 0xc6eb57bb,
+    0x30320350, 0x3f397e91, 0xa67791bc, 0x86bc0e2c,
+    0xefa0a7e2, 0xe9ff7543, 0xe733612c, 0xd185897b,
+    0x329e5388, 0x91dd236b, 0x2ecb0d93, 0xf4d82a3d,
+    0x35b5c03f, 0xe4e606f0, 0x05b21843, 0x37b45964,
+    0x5eff22f4, 0x6027f4cc, 0x77178b3c, 0xae507131,
+    0x7bf7cabc, 0xf9c18d66, 0x593ade65, 0xd95ddf11,
+};
+
+uint32_t buzhash_update (buzHash *b, const char *s, size_t window) {
+    if(b->window == NULL || b->window_size != window) {
+        if(b->window)
+            free(b->window);
+        b->window = calloc(1, window);
+        b->window_loc = 0;
+        b->window_fill = 0;
+        b->window_size = window;
+        b->h = 0;
+    }
+    if(b->window_fill < b->window_size) {
+        b->window[b->window_fill] = *s;
+        b->window_fill++;
+        if(b->window_fill < b->window_size) {
+            b->h ^= rol32 (buzhash_table[(size_t) (*s)], window - b->window_fill);
+            return 1;
+        } else {
+            b->h ^= buzhash_table[(size_t) (*s)];
+            return b->h;
+        }
+    }
+    b->h = rol32 (b->h, 1) ^
+           rol32 (buzhash_table[(size_t) b->window[b->window_loc]], b->window_size) ^
+           buzhash_table[(size_t) *s];
+    b->window[b->window_loc++] = *s;
+    b->window_loc %= b->window_size;
+    return b->h;
+}
+
+void buzhash_reset (buzHash *b) {
+    free(b->window);
+    b->window = NULL;
+}
diff --git a/src/lib/buzhash/buzhash.h b/src/lib/buzhash/buzhash.h
new file mode 100644 (file)
index 0000000..06f6917
--- /dev/null
@@ -0,0 +1,19 @@
+#ifndef URLBLOCK_BUZHASH_H
+#define URLBLOCK_BUZHASH_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct buzHash {
+    uint32_t h;
+    int window_size;
+    char *window;
+    int window_loc;
+    int window_fill;
+} buzHash;
+
+uint32_t buzhash_update (buzHash *b, const char *s, size_t window);
+void buzhash_reset (buzHash *b);
+
+#endif
diff --git a/src/lib/buzhash/meson.build b/src/lib/buzhash/meson.build
new file mode 100644 (file)
index 0000000..e3a1917
--- /dev/null
@@ -0,0 +1 @@
+lib_sources += files('buzhash.c')
index 1a3fe9e6f97afbaddc5aff8adf48140f95d5344e..ae4d73af57710cfac1116cee782c670e995c5917 100644 (file)
@@ -28,6 +28,7 @@
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
+#include <math.h>
 #include <zck.h>
 
 #include "zck_private.h"
@@ -165,6 +166,31 @@ static ssize_t comp_end_dchunk(zckCtx *zck, int use_dict, size_t fd_size) {
     return rb;
 }
 
+static ssize_t comp_write(zckCtx *zck, const char *src, const size_t src_size) {
+    VALIDATE_WRITE_SIZE(zck);
+
+    if(!zck->comp.started && !comp_init(zck))
+        return -1;
+
+    if(src_size == 0)
+        return 0;
+
+    char *dst = NULL;
+    size_t dst_size = 0;
+    if(zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1) < 0)
+        return -1;
+    if(dst_size > 0 && !write_data(zck->temp_fd, dst, dst_size)) {
+        free(dst);
+        return -1;
+    }
+    if(!index_add_to_chunk(zck, dst, dst_size, src_size)) {
+        free(dst);
+        return -1;
+    }
+    free(dst);
+    return src_size;
+}
+
 int comp_init(zckCtx *zck) {
     VALIDATE(zck);
 
@@ -461,27 +487,29 @@ const char PUBLIC *zck_comp_name_from_type(int comp_type) {
 }
 
 ssize_t PUBLIC zck_write(zckCtx *zck, const char *src, const size_t src_size) {
-    VALIDATE_WRITE_SIZE(zck);
-
-    if(!zck->comp.started && !comp_init(zck))
-        return -1;
-
-    if(src_size == 0)
-        return 0;
-
-    char *dst = NULL;
-    size_t dst_size = 0;
-    if(zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1) < 0)
-        return -1;
-    if(dst_size > 0 && !write_data(zck->temp_fd, dst, dst_size)) {
-        free(dst);
-        return -1;
+    if(zck->manual_chunk)
+        return comp_write(zck, src, src_size);
+
+    const char *loc = src;
+    size_t loc_size = src_size;
+    for(size_t i=0; i<loc_size; ) {
+        if((buzhash_update(&(zck->buzhash), loc+i, zck->buzhash_width) &
+            zck->buzhash_bitmask) == 0) {
+            if(comp_write(zck, loc, i) != i)
+                return -1;
+            zck_log(ZCK_LOG_DEBUG, "Automatically ending chunk\n");
+            if(zck_end_chunk(zck) < 0)
+                return -1;
+            loc += i;
+            loc_size -= i;
+            i = 0;
+            buzhash_reset(&(zck->buzhash));
+        } else {
+            i++;
+        }
     }
-    if(!index_add_to_chunk(zck, dst, dst_size, src_size)) {
-        free(dst);
+    if(loc_size > 0 && comp_write(zck, loc, loc_size) != loc_size)
         return -1;
-    }
-    free(dst);
     return src_size;
 }
 
@@ -512,6 +540,7 @@ ssize_t PUBLIC zck_end_chunk(zckCtx *zck) {
         free(dst);
         return -1;
     }
+    zck_log(ZCK_LOG_DEBUG, "Finished chunk size: %lu\n", data_size);
     free(dst);
     return data_size;
 }
index df18c4b72c06ee1f4670a85f5f1df998a1164e27..a8647b26d64fdd06b9442393c92313b366242573 100644 (file)
@@ -1,5 +1,6 @@
 lib_sources = files()
 add_global_arguments('-fvisibility=hidden', language : 'c')
+subdir('buzhash')
 subdir('comp')
 subdir('hash')
 subdir('index')
index 225679c61f26c866c48f070fdff5ede10dfb5912..9cadc8e76f7598153725a04557c26eb8eb47031a 100644 (file)
@@ -120,6 +120,14 @@ static char *ascii_checksum_to_bin (char *checksum) {
     return raw_checksum;
 }
 
+static void update_buzhash_bits(zckCtx *zck) {
+    int s=1;
+    for(int i=0; i<zck->buzhash_match_bits; i++)
+        s *= 2;
+    s -= 1;
+    zck->buzhash_bitmask = s;
+}
+
 int get_tmp_fd() {
     int temp_fd;
     char *fname = NULL;
@@ -347,7 +355,8 @@ zckCtx PUBLIC *zck_create() {
     zck->prep_hash_type = -1;
     zck->prep_hdr_size = -1;
     zck->buzhash_width = DEFAULT_BUZHASH_WIDTH;
-    zck->buzhash_match_bits = (2^DEFAULT_BUZHASH_BITS) - 1;
+    zck->buzhash_match_bits = DEFAULT_BUZHASH_BITS;
+    update_buzhash_bits(zck);
     return zck;
 }
 
index c68c5051de66e61a600c03cd1ea712746dd948dc..03061715f5886ba2cb40f0742015a9b7ff1c1a78 100644 (file)
@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include <stddef.h>
 #include <regex.h>
+#include "buzhash/buzhash.h"
 
 #define BUF_SIZE 32768
 /* Maximum string length for a compressed size_t */
@@ -208,8 +209,10 @@ typedef struct zckCtx {
     char *data;
     size_t data_size;
 
+    buzHash buzhash;
     int buzhash_width;
     int buzhash_match_bits;
+    int buzhash_bitmask;
     int manual_chunk;
 } zckCtx;
 
index a572dacf37a11077cf08fa46f408c36a40a85ff7..01978cb6a5d97e70ddfa44c219d68e7de16e2ac9 100644 (file)
@@ -1,7 +1,7 @@
 add_global_arguments('-Wunused-result', language : 'c')
 add_global_arguments('-std=gnu99', language : 'c')
 subdir('lib')
-executable('zck', ['zck.c', 'util_common.c', 'buzhash/buzhash.c'], include_directories: inc, link_with: zcklib, install: true)
+executable('zck', ['zck.c', 'util_common.c'], include_directories: inc, link_with: zcklib, install: true)
 executable('unzck', ['unzck.c', 'util_common.c'], include_directories: inc, link_with: zcklib, install: true)
 executable('zckdl', ['zck_dl.c', 'util_common.c'], include_directories: inc, dependencies: curl_dep, link_with: zcklib, install: true)
 executable('zck_read_header', ['zck_read_header.c', 'util_common.c'], include_directories: inc, link_with: zcklib, install: true)