Add git-annex remote refs that are not newer to the merged refs list

Significant startup speed increase by avoiding repeatedly checking if some
remote git-annex branch refs need to be merged when it is not newer.

One way this could happen is when there are 2 remotes that are themselves
connected. The git-annex branch on the first remote gets updated. Then the
second remote pulls from the first, and merges in its git-annex branch.
Then the local repo pulls from the second remote, and merges its git-annex
branch. At this point, a pull from the first remote will get a git-annex
branch that is not newer, but is not on the merged refs list.

In my big repo, git-annex startup time dropped from 4 seconds to 0.1 seconds.
There were 5 to 10 such remote refs out of 18 remotes.

Sponsored-by: Graham Spencer on Patreon

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ab4b43df3cccb695d5dd250375731316c2245565..dce16667d711f68f17a6726e79ce3643d0372c5c 100644 (file)
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -183,23 +183,24 @@ updateTo' pairs = do
        branchref <- getBranch
        ignoredrefs <- getIgnoredRefs
        let unignoredrefs = excludeset ignoredrefs pairs
-       tomerge <- if null unignoredrefs
-               then return []
+       (tomerge, notnewer) <- if null unignoredrefs
+               then return ([], [])
                else do
                        mergedrefs <- getMergedRefs
-                       filterM isnewer (excludeset mergedrefs unignoredrefs)
+                       partitionM isnewer $
+                               excludeset mergedrefs unignoredrefs
        {- In a read-only repository, catching permission denied lets
         - query operations still work, although they will need to do
         - additional work since the refs are not merged. -}
        catchPermissionDenied
                (const (updatefailedperms tomerge))
-               (go branchref tomerge)
+               (go branchref tomerge notnewer)
   where
        excludeset s = filter (\(r, _) -> S.notMember r s)
 
        isnewer (r, _) = inRepo $ Git.Branch.changed fullname r
 
-       go branchref tomerge = do
+       go branchref tomerge notnewer = do
                dirty <- journalDirty gitAnnexJournalDir
                journalcleaned <- if null tomerge
                        {- Even when no refs need to be merged, the index
@@ -229,6 +230,7 @@ updateTo' pairs = do
                journalclean <- if journalcleaned
                        then not <$> privateUUIDsKnown
                        else pure False
+               addMergedRefs notnewer
                return $ UpdateMade
                        { refsWereMerged = not (null tomerge)
                        , journalClean = journalclean 

diff --git a/CHANGELOG b/CHANGELOG
index 9dccfb1730fa69ee4f496657a0e42ef76301829c..d1e28a8fb056c389c7995c6e4b0de2e7d637df44 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,6 +2,8 @@ git-annex (10.20230803) UNRELEASED; urgency=medium
 
   * Fix behavior of onlyingroup.
   * info: Added --dead-repositories option. 
+  * Significant startup speed increase by avoiding repeatedly checking
+    if some remote git-annex branch refs need to be merged.
 
  -- Joey Hess <id@joeyh.name>  Mon, 07 Aug 2023 13:04:13 -0400
 

diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index abe06f335cada4a70525e0aba2372f885a2049e2..6cd2c5e65792aa5748ba239eafa69adb3849af9a 100644 (file)
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -12,6 +12,7 @@ module Utility.Monad (
        getM,
        anyM,
        allM,
+       partitionM,
        untilTrue,
        ifM,
        (<||>),
@@ -45,6 +46,13 @@ allM :: Monad m => (a -> m Bool) -> [a] -> m Bool
 allM _ [] = return True
 allM p (x:xs) = p x <&&> allM p xs
 
+partitionM :: Monad m => (a -> m Bool) -> [a] -> m ([a], [a])
+partitionM _ [] = return ([], [])
+partitionM p (x:xs) = do
+       r <- p x
+       (as, bs) <- partitionM p xs
+       return $ if r then (x:as, bs) else (as, x:bs)
+
 {- Runs an action on values from a list until it succeeds. -}
 untilTrue :: Monad m => [a] -> (a -> m Bool) -> m Bool
 untilTrue = flip anyM