Excludes: During directory traversal, use QRegularExpression

On Mac, this halves the time spent in csync_excluded_traversal
when using check_csync_excluded_performance. A similar performance
increase is seen on linux.

diff --git a/src/csync/csync_exclude.cpp b/src/csync/csync_exclude.cpp
index dbc54f2bb71e2219874d7c3b080830d2e12d06d7..8b5bd4b88c7dec295369ce9adba98b609bec5bcb 100644 (file)
--- a/src/csync/csync_exclude.cpp
+++ b/src/csync/csync_exclude.cpp
@@ -39,6 +39,8 @@
 
 #include "common/utility.h"
 
+#include <QString>
+
 #ifdef _WIN32
 #include <io.h>
 #else
@@ -234,20 +236,28 @@ static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const ch
     }
     blen = strlen(bname);
 
-    rc = csync_fnmatch("._sync_*.db*", bname, 0);
-    if (rc == 0) {
-        match = CSYNC_FILE_SILENTLY_EXCLUDED;
-        goto out;
-    }
-    rc = csync_fnmatch(".sync_*.db*", bname, 0);
-    if (rc == 0) {
-        match = CSYNC_FILE_SILENTLY_EXCLUDED;
-        goto out;
-    }
-    rc = csync_fnmatch(".csync_journal.db*", bname, 0);
-    if (rc == 0) {
-        match = CSYNC_FILE_SILENTLY_EXCLUDED;
-        goto out;
+    // 9 = strlen(".sync_.db")
+    if (blen >= 9 && bname[0] == '.') {
+        rc = csync_fnmatch("._sync_*.db*", bname, 0);
+        if (rc == 0) {
+            match = CSYNC_FILE_SILENTLY_EXCLUDED;
+            goto out;
+        }
+        rc = csync_fnmatch(".sync_*.db*", bname, 0);
+        if (rc == 0) {
+            match = CSYNC_FILE_SILENTLY_EXCLUDED;
+            goto out;
+        }
+        rc = csync_fnmatch(".csync_journal.db*", bname, 0);
+        if (rc == 0) {
+            match = CSYNC_FILE_SILENTLY_EXCLUDED;
+            goto out;
+        }
+        rc = csync_fnmatch(".owncloudsync.log*", bname, 0);
+        if (rc == 0) {
+            match = CSYNC_FILE_SILENTLY_EXCLUDED;
+            goto out;
+        }
     }
 
     // check the strlen and ignore the file if its name is longer than 254 chars.
@@ -303,12 +313,6 @@ static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const ch
         goto out;
     }
 
-    rc = csync_fnmatch(".owncloudsync.log*", bname, 0);
-    if (rc == 0) {
-        match = CSYNC_FILE_SILENTLY_EXCLUDED;
-        goto out;
-    }
-
     if (!OCC::Utility::shouldUploadConflictFiles()) {
         if (OCC::Utility::isConflictFile(bname)) {
             match = CSYNC_FILE_EXCLUDE_CONFLICT;
@@ -415,8 +419,90 @@ static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const ch
     return match;
 }
 
-CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *path, int filetype) {
-  return _csync_excluded_common(excludes, path, filetype, false);
+/* Only for bnames (not paths) */
+static QString convertToBnameRegexpSyntax(QString exclude)
+{
+    QString s = QRegularExpression::escape(exclude).replace("\\*", ".*").replace("\\?", ".");
+    return s;
+}
+
+void csync_exclude_traversal_prepare(CSYNC *ctx)
+{
+    ctx->parsed_traversal_excludes.prepare(ctx->excludes);
+}
+
+void csync_s::TraversalExcludes::prepare(c_strlist_t *excludes)
+{
+    c_strlist_destroy(list_patterns_fnmatch);
+    list_patterns_fnmatch = nullptr;
+
+    // Start out with regexes that would match nothing
+    QString exclude_only = "a^";
+    QString exclude_and_remove = "a^";
+
+    size_t exclude_count = excludes ? excludes->count : 0;
+    for (unsigned int i = 0; i < exclude_count; i++) {
+        char *exclude = excludes->vector[i];
+        QString *builderToUse = & exclude_only;
+        if (exclude[0] == '\n') continue; // empty line
+        if (exclude[0] == '\r') continue; // empty line
+
+        /* If an exclude entry contains some fnmatch-ish characters, we use the C-style codepath without QRegularEpression */
+        if (strchr(exclude, '/') || strchr(exclude, '[') || strchr(exclude, '{')) {
+            _csync_exclude_add(&list_patterns_fnmatch, exclude);
+            continue;
+        }
+
+        /* Those will attempt to use QRegularExpression */
+        if (exclude[0] == ']'){
+            exclude++;
+            builderToUse = &exclude_and_remove;
+        }
+        if (builderToUse->size() > 0) {
+            builderToUse->append("|");
+        }
+        builderToUse->append(convertToBnameRegexpSyntax(exclude));
+    }
+
+    QString pattern = "^(" + exclude_only + ")$|^(" + exclude_and_remove + ")$";
+    regexp_exclude.setPattern(pattern);
+    QRegularExpression::PatternOptions patternOptions = QRegularExpression::OptimizeOnFirstUsageOption;
+    if (OCC::Utility::fsCasePreserving())
+        patternOptions |= QRegularExpression::CaseInsensitiveOption;
+    regexp_exclude.setPatternOptions(patternOptions);
+    regexp_exclude.optimize();
+}
+
+CSYNC_EXCLUDE_TYPE csync_excluded_traversal(CSYNC *ctx, const char *path, int filetype) {
+    CSYNC_EXCLUDE_TYPE match = CSYNC_NOT_EXCLUDED;
+
+    /* Check only static patterns and only with the reduced list which is empty usually */
+    match = _csync_excluded_common(ctx->parsed_traversal_excludes.list_patterns_fnmatch, path, filetype, false);
+    if (match != CSYNC_NOT_EXCLUDED) {
+        return match;
+    }
+
+    if (ctx->excludes) {
+        /* Now check with our optimized regexps */
+        const char *bname = NULL;
+        /* split up the path */
+        bname = strrchr(path, '/');
+        if (bname) {
+            bname += 1; // don't include the /
+        } else {
+            bname = path;
+        }
+        QString p = QString::fromUtf8(bname);
+        auto m = ctx->parsed_traversal_excludes.regexp_exclude.match(p);
+        if (m.hasMatch()) {
+            if (!m.captured(1).isEmpty()) {
+                match = CSYNC_FILE_EXCLUDE_LIST;
+            } else if (!m.captured(2).isEmpty()) {
+                match = CSYNC_FILE_EXCLUDE_AND_REMOVE;
+            }
+        }
+    }
+    return match;
 }
 
 CSYNC_EXCLUDE_TYPE csync_excluded_no_ctx(c_strlist_t *excludes, const char *path, int filetype) {

diff --git a/src/csync/csync_exclude.h b/src/csync/csync_exclude.h
index 722d27cace71ccf3cdbbed3f69e706fb0f8e0a41..f55e705834d87fdb1f270aaab6f07fee847a4933 100644 (file)
--- a/src/csync/csync_exclude.h
+++ b/src/csync/csync_exclude.h
@@ -51,6 +51,15 @@ int OCSYNC_EXPORT _csync_exclude_add(c_strlist_t **inList, const char *string);
  */
 int OCSYNC_EXPORT csync_exclude_load(const char *fname, c_strlist_t **list);
 
+/**
+ * @brief When all list loads and list are done
+ *
+ * Used to initialize internal data structures that build upon the loaded excludes.
+ *
+ * @param ctx
+ */
+void OCSYNC_EXPORT csync_exclude_traversal_prepare(CSYNC *ctx);
+
 /**
  * @brief Check if the given path should be excluded in a traversal situation.
  *
@@ -66,10 +75,11 @@ int OCSYNC_EXPORT csync_exclude_load(const char *fname, c_strlist_t **list);
  *
  * @return  2 if excluded and needs cleanup, 1 if excluded, 0 if not.
  */
-CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *path, int filetype);
+CSYNC_EXCLUDE_TYPE OCSYNC_EXPORT csync_excluded_traversal(CSYNC *ctx, const char *path, int filetype);
 
 /**
- * @brief csync_excluded_no_ctx
+ * @brief Checks all path components if the whole path should be excluded
+ *
  * @param excludes
  * @param path
  * @param filetype

diff --git a/src/csync/csync_private.h b/src/csync/csync_private.h
index 09d627b2c2e497568a1abe40428258a492f47e96..2c29ece7034111674417244cf9e97ad8273bd9f1 100644 (file)
--- a/src/csync/csync_private.h
+++ b/src/csync/csync_private.h
@@ -48,6 +48,8 @@
 
 #include "csync_macros.h"
 
+#include <QRegularExpression>
+
 /**
  * How deep to scan directories.
  */
@@ -137,10 +139,21 @@ struct OCSYNC_EXPORT csync_s {
       void *checksum_userdata = nullptr;
 
   } callbacks;
-  c_strlist_t *excludes = nullptr;
-  
+
   OCC::SyncJournalDb *statedb;
 
+  c_strlist_t *excludes = nullptr; /* list of individual patterns collected from all exclude files */
+  struct TraversalExcludes {
+      ~TraversalExcludes() {
+          c_strlist_destroy(list_patterns_fnmatch);
+      }
+      void prepare(c_strlist_t *excludes);
+
+      QRegularExpression regexp_exclude;
+      c_strlist_t *list_patterns_fnmatch = nullptr;
+
+  } parsed_traversal_excludes;
+
   struct {
     std::unordered_map<ByteArrayRef, QByteArray, ByteArrayRefHash> folder_renamed_to; // map from->to
     std::unordered_map<ByteArrayRef, QByteArray, ByteArrayRefHash> folder_renamed_from; // map to->from

diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp
index 8f88c324b678881a6097bc34e712753ec179e572..9412d51855769b4acccb17c46b941fea52274a4b 100644 (file)
--- a/src/csync/csync_update.cpp
+++ b/src/csync/csync_update.cpp
@@ -119,7 +119,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f
       excluded =CSYNC_FILE_EXCLUDE_STAT_FAILED;
   } else {
     /* Check if file is excluded */
-    excluded = csync_excluded_traversal(ctx->excludes, fs->path, fs->type);
+    excluded = csync_excluded_traversal(ctx, fs->path, fs->type);
   }
 
   if( excluded == CSYNC_NOT_EXCLUDED ) {
@@ -464,7 +464,7 @@ static bool fill_tree_from_db(CSYNC *ctx, const char *uri)
         /* Check for exclusion from the tree.
          * Note that this is only a safety net in case the ignore list changes
          * without a full remote discovery being triggered. */
-        CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx->excludes, st->path, st->type);
+        CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx, st->path, st->type);
         if (excluded != CSYNC_NOT_EXCLUDED) {
             qDebug(lcUpdate, "%s excluded (%d)", st->path.constData(), excluded);
 

diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp
index e5283f1310516d7883dd25b7b19e5746dde27bc1..fb4ef68968cf8892c90e7b6a0b6186aad2c716bd 100644 (file)
--- a/src/libsync/discoveryphase.cpp
+++ b/src/libsync/discoveryphase.cpp
@@ -20,6 +20,7 @@
 
 #include <csync_private.h>
 #include <csync_rename.h>
+#include <csync_exclude.h>
 
 #include <QLoggingCategory>
 #include <QUrl>
@@ -718,6 +719,8 @@ void DiscoveryJob::start()
     _csync_ctx->callbacks.remote_closedir_hook = remote_vio_closedir_hook;
     _csync_ctx->callbacks.vio_userdata = this;
 
+    csync_exclude_traversal_prepare(_csync_ctx); // Converts the flat exclude list to optimized regexps
+
     csync_set_log_callback(_log_callback);
     csync_set_log_level(_log_level);
     _lastUpdateProgressCallbackCall.invalidate();

diff --git a/src/libsync/excludedfiles.cpp b/src/libsync/excludedfiles.cpp
index 62c4711addcdfce572459441ec9013a54e2bc683..728cbb7fd92f19de18ad0ebac0dd89b17c081923 100644 (file)
--- a/src/libsync/excludedfiles.cpp
+++ b/src/libsync/excludedfiles.cpp
@@ -63,6 +63,7 @@ bool ExcludedFiles::reloadExcludes()
         if (csync_exclude_load(file.toUtf8(), _excludesPtr) < 0)
             success = false;
     }
+    // The csync_exclude_traversal_prepare is called implicitely at sync start.
     return success;
 }
 

diff --git a/test/csync/csync_tests/check_csync_exclude.cpp b/test/csync/csync_tests/check_csync_exclude.cpp
index 3f0180e42c1a78c8a980692a212b1958e2abdbd7..c02b30ff54b5cdaa8a9dba197cb9dd04ca32c177 100644 (file)
--- a/test/csync/csync_tests/check_csync_exclude.cpp
+++ b/test/csync/csync_tests/check_csync_exclude.cpp
@@ -59,6 +59,8 @@ static int setup_init(void **state) {
     rc = _csync_exclude_add(&(csync->excludes), "latex/*/*.tex.tmp");
     assert_int_equal(rc, 0);
 
+    csync_exclude_traversal_prepare(csync);
+
     *state = csync;
     return 0;
 }
@@ -98,6 +100,10 @@ static void check_csync_exclude_load(void **state)
 
     assert_string_equal(csync->excludes->vector[0], "*~");
     assert_int_not_equal(csync->excludes->count, 0);
+
+    assert_true(csync->parsed_traversal_excludes.regexp_exclude.pattern().isEmpty());
+    csync_exclude_traversal_prepare(csync); /* parse into regular expression */
+    assert_false(csync->parsed_traversal_excludes.regexp_exclude.pattern().isEmpty());
 }
 
 static void check_csync_excluded(void **state)
@@ -110,6 +116,9 @@ static void check_csync_excluded(void **state)
     rc = csync_excluded_no_ctx(csync->excludes, "/", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
 
+    rc = csync_excluded_no_ctx(csync->excludes, "A", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
     rc = csync_excluded_no_ctx(csync->excludes, "krawel_krawel", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
     rc = csync_excluded_no_ctx(csync->excludes, ".kde/share/config/kwin.eventsrc", CSYNC_FTW_TYPE_FILE);
@@ -218,6 +227,16 @@ static void check_csync_excluded(void **state)
     rc = csync_excluded_no_ctx(csync->excludes, "file_invalid_char<", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_INVALID_CHAR);
 #endif
+
+    /* ? character */
+    _csync_exclude_add( &(csync->excludes), "bond00?" );
+    csync_exclude_traversal_prepare(csync);
+    rc = csync_excluded_no_ctx(csync->excludes, "bond00", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    rc = csync_excluded_no_ctx(csync->excludes, "bond007", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+    rc = csync_excluded_no_ctx(csync->excludes, "bond0071", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
 }
 
 static void check_csync_excluded_traversal(void **state)
@@ -225,49 +244,188 @@ static void check_csync_excluded_traversal(void **state)
     CSYNC *csync = (CSYNC*)*state;
     int rc;
 
+    rc = csync_excluded_traversal(csync, "", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "/", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    rc = csync_excluded_traversal(csync, "A", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    rc = csync_excluded_traversal(csync,  "krawel_krawel", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    rc = csync_excluded_traversal(csync,  ".kde/share/config/kwin.eventsrc", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    rc = csync_excluded_traversal(csync,  "mozilla/.directory", CSYNC_FTW_TYPE_DIR);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+    /*
+     * Test for patterns in subdirs. '.beagle' is defined as a pattern and has
+     * to be found in top dir as well as in directories underneath.
+     */
+    rc = csync_excluded_traversal(csync,  ".apdisk", CSYNC_FTW_TYPE_DIR);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+    rc = csync_excluded_traversal(csync,  "foo/.apdisk", CSYNC_FTW_TYPE_DIR);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+    rc = csync_excluded_traversal(csync,  "foo/bar/.apdisk", CSYNC_FTW_TYPE_DIR);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+    rc = csync_excluded_traversal(csync, ".java", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    /* csync-journal is ignored in general silently. */
+    rc = csync_excluded_traversal(csync, ".csync_journal.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync, ".csync_journal.db.ctmp", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "subdir/.csync_journal.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "/two/subdir/.csync_journal.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+
+    /* also the new form of the database name */
+    rc = csync_excluded_traversal(csync, "._sync_5bdd60bdfcfa.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "._sync_5bdd60bdfcfa.db.ctmp", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "._sync_5bdd60bdfcfa.db-shm", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "subdir/._sync_5bdd60bdfcfa.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+
+    rc = csync_excluded_traversal(csync,  ".sync_5bdd60bdfcfa.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync,  ".sync_5bdd60bdfcfa.db.ctmp", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync,  ".sync_5bdd60bdfcfa.db-shm", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+    rc = csync_excluded_traversal(csync,  "subdir/.sync_5bdd60bdfcfa.db", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_SILENTLY_EXCLUDED);
+
+
+    /* pattern ]*.directory - ignore and remove */
+    rc = csync_excluded_traversal(csync,  "my.~directory", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_AND_REMOVE);
+
+    rc = csync_excluded_traversal(csync,  "/a_folder/my.~directory", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_AND_REMOVE);
+
+    /* Not excluded because the pattern .netscape/cache requires directory. */
+    rc = csync_excluded_traversal(csync,  ".netscape/cache", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    /* Not excluded  */
+    rc = csync_excluded_traversal(csync,  "unicode/中文.hé", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    /* excluded  */
+    rc = csync_excluded_traversal(csync,  "unicode/пятницы.txt", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+    rc = csync_excluded_traversal(csync,  "unicode/中文.💩", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+    /* path wildcards */
+    rc = csync_excluded_traversal(csync,  "foobar/my_manuscript.out", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+    rc = csync_excluded_traversal(csync,  "latex_tmp/my_manuscript.run.xml", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+    rc = csync_excluded_traversal(csync,  "word_tmp/my_manuscript.run.xml", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    rc = csync_excluded_traversal(csync,  "latex/my_manuscript.tex.tmp", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    rc = csync_excluded_traversal(csync,  "latex/songbook/my_manuscript.tex.tmp", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+#ifdef _WIN32
+    rc = csync_excluded_traversal(csync,  "file_trailing_space ", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_TRAILING_SPACE);
+
+    rc = csync_excluded_traversal(csync,  "file_trailing_dot.", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_INVALID_CHAR);
+
+    rc = csync_excluded_traversal(csync,  "AUX", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_INVALID_CHAR);
+
+    rc = csync_excluded_traversal(csync,  "file_invalid_char<", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_INVALID_CHAR);
+#endif
+
+
+    /* From here the actual traversal tests */
+
     _csync_exclude_add( &(csync->excludes), "/exclude" );
+    csync_exclude_traversal_prepare(csync);
 
     /* Check toplevel dir, the pattern only works for toplevel dir. */
-    rc = csync_excluded_traversal(csync->excludes, "/exclude", CSYNC_FTW_TYPE_DIR);
+    rc = csync_excluded_traversal(csync, "/exclude", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
 
-    rc = csync_excluded_traversal(csync->excludes, "/foo/exclude", CSYNC_FTW_TYPE_DIR);
+    rc = csync_excluded_traversal(csync, "/foo/exclude", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
 
     /* check for a file called exclude. Must still work */
-    rc = csync_excluded_traversal(csync->excludes, "/exclude", CSYNC_FTW_TYPE_FILE);
+    rc = csync_excluded_traversal(csync, "/exclude", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
 
-    rc = csync_excluded_traversal(csync->excludes, "/foo/exclude", CSYNC_FTW_TYPE_FILE);
+    rc = csync_excluded_traversal(csync, "/foo/exclude", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
 
     /* Add an exclude for directories only: excl/ */
     _csync_exclude_add( &(csync->excludes), "excl/" );
-    rc = csync_excluded_traversal(csync->excludes, "/excl", CSYNC_FTW_TYPE_DIR);
+    csync_exclude_traversal_prepare(csync);
+    rc = csync_excluded_traversal(csync, "/excl", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
 
-    rc = csync_excluded_traversal(csync->excludes, "meep/excl", CSYNC_FTW_TYPE_DIR);
+    rc = csync_excluded_traversal(csync, "meep/excl", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
 
-    rc = csync_excluded_traversal(csync->excludes, "meep/excl/file", CSYNC_FTW_TYPE_FILE);
+    rc = csync_excluded_traversal(csync, "meep/excl/file", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED); // because leading dirs aren't checked!
 
-    rc = csync_excluded_traversal(csync->excludes, "/excl", CSYNC_FTW_TYPE_FILE);
+    rc = csync_excluded_traversal(csync, "/excl", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
 
     _csync_exclude_add(&csync->excludes, "/excludepath/withsubdir");
+    csync_exclude_traversal_prepare(csync);
 
-    rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir", CSYNC_FTW_TYPE_DIR);
+    rc = csync_excluded_traversal(csync, "/excludepath/withsubdir", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
 
-    rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir", CSYNC_FTW_TYPE_FILE);
+    rc = csync_excluded_traversal(csync, "/excludepath/withsubdir", CSYNC_FTW_TYPE_FILE);
     assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
 
-    rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir2", CSYNC_FTW_TYPE_DIR);
+    rc = csync_excluded_traversal(csync, "/excludepath/withsubdir2", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
 
-    rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir/foo", CSYNC_FTW_TYPE_DIR);
+    rc = csync_excluded_traversal(csync, "/excludepath/withsubdir/foo", CSYNC_FTW_TYPE_DIR);
     assert_int_equal(rc, CSYNC_NOT_EXCLUDED); // because leading dirs aren't checked!
+
+    /* Check ending of pattern */
+    rc = csync_excluded_traversal(csync, "/exclude", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+    rc = csync_excluded_traversal(csync, "/excludeX", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "exclude", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
+    _csync_exclude_add( &(csync->excludes), "exclude" );
+    csync_exclude_traversal_prepare(csync);
+    rc = csync_excluded_traversal(csync, "exclude", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+
+    /* ? character */
+    _csync_exclude_add( &(csync->excludes), "bond00?" );
+    csync_exclude_traversal_prepare(csync);
+    rc = csync_excluded_traversal(csync, "bond00", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+    rc = csync_excluded_traversal(csync, "bond007", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
+    rc = csync_excluded_traversal(csync, "bond0071", CSYNC_FTW_TYPE_FILE);
+    assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
+
 }
 
 static void check_csync_pathes(void **state)
@@ -338,6 +496,7 @@ static void check_csync_is_windows_reserved_word(void **) {
     assert_true(csync_is_windows_reserved_word("m:"));
 }
 
+/* QT_ENABLE_REGEXP_JIT=0 to get slower results :-) */
 static void check_csync_excluded_performance(void **state)
 {
     CSYNC *csync = (CSYNC*)*state;
@@ -370,8 +529,8 @@ static void check_csync_excluded_performance(void **state)
         gettimeofday(&before, 0);
 
         for (i = 0; i < N; ++i) {
-            totalRc += csync_excluded_traversal(csync->excludes, "/this/is/quite/a/long/path/with/many/components", CSYNC_FTW_TYPE_DIR);
-            totalRc += csync_excluded_traversal(csync->excludes, "/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/29", CSYNC_FTW_TYPE_FILE);
+            totalRc += csync_excluded_traversal(csync, "/this/is/quite/a/long/path/with/many/components", CSYNC_FTW_TYPE_DIR);
+            totalRc += csync_excluded_traversal(csync, "/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/29", CSYNC_FTW_TYPE_FILE);
         }
         assert_int_equal(totalRc, CSYNC_NOT_EXCLUDED); // mainly to avoid optimization