From 3c15e0f7a0ad369a531c78fbd7870453a873d366 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 6 Jun 2023 14:15:47 -0400 Subject: [PATCH] cache negative lookups of global numcopies and mincopies Speeds up eg git-annex sync --content by up to 50%. When it does not need to transfer or drop anything, it now noops a lot more quickly. I didn't see anything else in sync --content noop loop that could really be sped up. It has to cat git objects to keys, stat object files, etc. Sponsored-by: unqueued on Patreon --- Annex.hs | 4 ++-- CHANGELOG | 2 ++ Logs/NumCopies.hs | 8 ++++---- ...4_e961d2b508e3260670bb61a5ae51d109._comment | 12 ++++++++++++ ...5_35d233db5f7410048bd0cf9f5b4fd5cc._comment | 18 ++++++++++++++++++ 5 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 doc/bugs/importtree_spends_hours_reading_cidsdb/comment_14_e961d2b508e3260670bb61a5ae51d109._comment create mode 100644 doc/bugs/importtree_spends_hours_reading_cidsdb/comment_15_35d233db5f7410048bd0cf9f5b4fd5cc._comment diff --git a/Annex.hs b/Annex.hs index 5a9eac3c32..74d9f9891f 100644 --- a/Annex.hs +++ b/Annex.hs @@ -183,8 +183,8 @@ data AnnexState = AnnexState , hashobjecthandle :: Maybe (ResourcePool HashObjectHandle) , checkattrhandle :: Maybe (ResourcePool CheckAttrHandle) , checkignorehandle :: Maybe (ResourcePool CheckIgnoreHandle) - , globalnumcopies :: Maybe NumCopies - , globalmincopies :: Maybe MinCopies + , globalnumcopies :: Maybe (Maybe NumCopies) + , globalmincopies :: Maybe (Maybe MinCopies) , limit :: ExpandableMatcher Annex , timelimit :: Maybe (Duration, POSIXTime) , sizelimit :: Maybe (TVar Integer) diff --git a/CHANGELOG b/CHANGELOG index 8357721888..862e866287 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -79,6 +79,8 @@ git-annex (10.20230408) UNRELEASED; urgency=medium * Large speed up to importing trees from special remotes that contain a lot of files, by only processing changed files. * Some other speedups to importing trees from special remotes. + * Cache negative lookups of global numcopies and mincopies. + Speeds up eg git-annex sync --content by up to 50%. -- Joey Hess Sat, 08 Apr 2023 13:57:18 -0400 diff --git a/Logs/NumCopies.hs b/Logs/NumCopies.hs index 3c358819bb..7579baf3f3 100644 --- a/Logs/NumCopies.hs +++ b/Logs/NumCopies.hs @@ -45,22 +45,22 @@ setGlobalMinCopies new = do {- Value configured in the numcopies log. Cached for speed. -} getGlobalNumCopies :: Annex (Maybe NumCopies) -getGlobalNumCopies = maybe globalNumCopiesLoad (return . Just) +getGlobalNumCopies = maybe globalNumCopiesLoad return =<< Annex.getState Annex.globalnumcopies {- Value configured in the mincopies log. Cached for speed. -} getGlobalMinCopies :: Annex (Maybe MinCopies) -getGlobalMinCopies = maybe globalMinCopiesLoad (return . Just) +getGlobalMinCopies = maybe globalMinCopiesLoad return =<< Annex.getState Annex.globalmincopies globalNumCopiesLoad :: Annex (Maybe NumCopies) globalNumCopiesLoad = do v <- getLog numcopiesLog - Annex.changeState $ \s -> s { Annex.globalnumcopies = v } + Annex.changeState $ \s -> s { Annex.globalnumcopies = Just v } return v globalMinCopiesLoad :: Annex (Maybe MinCopies) globalMinCopiesLoad = do v <- getLog mincopiesLog - Annex.changeState $ \s -> s { Annex.globalmincopies = v } + Annex.changeState $ \s -> s { Annex.globalmincopies = Just v } return v diff --git a/doc/bugs/importtree_spends_hours_reading_cidsdb/comment_14_e961d2b508e3260670bb61a5ae51d109._comment b/doc/bugs/importtree_spends_hours_reading_cidsdb/comment_14_e961d2b508e3260670bb61a5ae51d109._comment new file mode 100644 index 0000000000..42efb8208e --- /dev/null +++ b/doc/bugs/importtree_spends_hours_reading_cidsdb/comment_14_e961d2b508e3260670bb61a5ae51d109._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 14""" + date="2023-06-06T17:11:35Z" + content=""" +There's only one import in the sync, and your output shows it completed +(with error). + +The only other phase of sync that could be run after that and take a lot of +time is content syncing. You would have to have annex.synccontent set +somewhere for sync to do that. Do you? +"""]] diff --git a/doc/bugs/importtree_spends_hours_reading_cidsdb/comment_15_35d233db5f7410048bd0cf9f5b4fd5cc._comment b/doc/bugs/importtree_spends_hours_reading_cidsdb/comment_15_35d233db5f7410048bd0cf9f5b4fd5cc._comment new file mode 100644 index 0000000000..2073dca187 --- /dev/null +++ b/doc/bugs/importtree_spends_hours_reading_cidsdb/comment_15_35d233db5f7410048bd0cf9f5b4fd5cc._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 15""" + date="2023-06-06T17:31:49Z" + content=""" +It would make a lot of sense for --content syncing to be what remains slow. +That has to scan over all the files and when it decides that it does not +need to copy the content anywhere, that's a tight loop with no output. + +In my repo with 10000 files that was set up by the latest test case, +`git-annex sync` takes 13 seconds, and with --content it takes 61 seconds. + +I optimised a numcopies/mincopies lookup away, and that got it +down to 28 seconds. + +The cidsdb does not get accessed by the --content scan +in my testing, although there may be other situations where it does. +"""]] -- 2.30.2