, forcenumcopies :: Maybe NumCopies
, forcemincopies :: Maybe MinCopies
, forcebackend :: Maybe String
+ , rebalance :: Bool
, useragent :: Maybe String
, desktopnotify :: DesktopNotify
, gitcredentialcache :: TMVar CredentialCache
, forcebackend = Nothing
, forcenumcopies = Nothing
, forcemincopies = Nothing
+ , rebalance = False
, useragent = Nothing
, desktopnotify = mempty
, gitcredentialcache = cc
{- git-annex file matching
-
- - Copyright 2012-2023 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2024 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
, ValueToken "metadata" (usev limitMetaData)
, ValueToken "inallgroup" (usev $ limitInAllGroup $ getGroupMap pcd)
, ValueToken "onlyingroup" (usev $ limitOnlyInGroup $ getGroupMap pcd)
+ , ValueToken "balanced" (usev $ limitBalanced (repoUUID pcd) (getGroupMap pcd))
+ , ValueToken "fullybalanced" (usev $ limitFullyBalanced (repoUUID pcd) (getGroupMap pcd))
] ++ commonTokens LimitAnnexFiles
where
preferreddir = maybe "public" fromProposedAccepted $
* The config versioning=true is now reserved for use by versioned special
remotes. External special remotes should not use that config for their
own purposes.
+ * Support "balanced=" and "fullybalanced=" in preferred content expressions.
+ * Added --rebalance option.
-- Joey Hess <id@joeyh.name> Wed, 31 Jul 2024 15:52:03 -0400
<> help "override minimum number of copies"
<> hidden
)
+ , annexFlag (setrebalance True)
+ ( long "rebalance"
+ <> help "move content as needed to improve balance"
+ <> hidden
+ )
, annexOption (setAnnexState . Remote.forceTrust Trusted) $ strOption
( long "trust" <> metavar paramRemote
<> help "deprecated, does not override trust setting"
where
setnumcopies n = setAnnexRead $ \rd -> rd { Annex.forcenumcopies = Just $ configuredNumCopies n }
setmincopies n = setAnnexRead $ \rd -> rd { Annex.forcemincopies = Just $ configuredMinCopies n }
+ setrebalance v = setAnnexRead $ \rd -> rd { Annex.rebalance = v }
setuseragent v = setAnnexRead $ \rd -> rd { Annex.useragent = Just v }
setdesktopnotify v = setAnnexRead $ \rd -> rd { Annex.desktopnotify = Annex.desktopnotify rd <> v }
setgitconfig v = Annex.addGitConfigOverride v
{- user-specified limits on files to act on
-
- - Copyright 2011-2023 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2024 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
import Utility.Glob
import Utility.HumanTime
import Utility.DataUnits
+import Utility.Hash
import qualified Database.Keys
import qualified Utility.RawFilePath as R
import Backend
import qualified Data.Map as M
import qualified System.FilePath.ByteString as P
import System.PosixCompat.Files (accessTime, isSymbolicLink)
+import qualified Data.ByteArray as BA
+import Data.Bits (shiftL)
{- Some limits can look at the current status of files on
- disk, or in the annex. This allows controlling which happens. -}
return $ not (S.null $ present `S.intersection` want)
&& S.null (S.filter (`S.notMember` want) present)
+limitBalanced :: Maybe UUID -> Annex GroupMap -> MkLimit Annex
+limitBalanced mu getgroupmap groupname = do
+ fullybalanced <- limitFullyBalanced mu getgroupmap groupname
+ copies <- limitCopies $ if ':' `elem` groupname
+ then groupname
+ else groupname ++ ":1"
+ let present = limitPresent mu
+ Right $ MatchFiles
+ { matchAction = \a i ->
+ ifM (Annex.getRead Annex.rebalance)
+ ( matchAction fullybalanced a i
+ , matchAction present a i <||>
+ ((not <$> matchAction copies a i)
+ <&&> matchAction fullybalanced a i
+ )
+ )
+ , matchNeedsFileName =
+ matchNeedsFileName present ||
+ matchNeedsFileName fullybalanced ||
+ matchNeedsFileName copies
+ , matchNeedsFileContent =
+ matchNeedsFileContent present ||
+ matchNeedsFileContent fullybalanced ||
+ matchNeedsFileContent copies
+ , matchNeedsKey =
+ matchNeedsKey present ||
+ matchNeedsKey fullybalanced ||
+ matchNeedsKey copies
+ , matchNeedsLocationLog =
+ matchNeedsLocationLog present ||
+ matchNeedsLocationLog fullybalanced ||
+ matchNeedsLocationLog copies
+ , matchDesc = "balanced" =? groupname
+ }
+
+limitFullyBalanced :: Maybe UUID -> Annex GroupMap -> MkLimit Annex
+limitFullyBalanced mu getgroupmap groupname = Right $ MatchFiles
+ { matchAction = const $ checkKey $ \key -> do
+ groupmembers <- fromMaybe S.empty
+ . M.lookup (toGroup groupname)
+ . uuidsByGroup
+ <$> getgroupmap
+ -- TODO free space checking
+ return $ case mu of
+ Just u -> u == pickBalanced key groupmembers
+ Nothing -> False
+ , matchNeedsFileName = False
+ , matchNeedsFileContent = False
+ , matchNeedsKey = True
+ , matchNeedsLocationLog = False
+ , matchDesc = "fullybalanced" =? groupname
+ }
+ where
+
+pickBalanced :: Key -> S.Set UUID -> UUID
+pickBalanced key s =
+ let m = fromIntegral (S.size s)
+ n = keyToInteger key
+ in S.elemAt (fromIntegral (n `mod` m)) s
+
+{- Converts a Key into a stable Integer.
+ -
+ - The SHA2 hash of the key is used to constrain the size of the Integer
+ - and to get an even distribution.
+ -}
+keyToInteger :: Key -> Integer
+keyToInteger key =
+ foldl' (\i b -> (i `shiftL` 8) + fromIntegral b) 0 $
+ BA.unpack (sha2_256s (serializeKey' key))
+
{- Adds a limit to skip files not using a specified key-value backend. -}
addInBackend :: String -> Annex ()
addInBackend = addLimit . limitInBackend
{- Convenience wrapper around cryptonite's hashing.
-
- - Copyright 2013-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2024 Joey Hess <id@joeyh.name>
-
- License: BSD-2-clause
-}
sha2_224_context,
sha2_256,
sha2_256_context,
+ sha2_256s,
sha2_384,
sha2_384_context,
sha2_512,
sha2_256_context :: Context SHA256
sha2_256_context = hashInit
+sha2_256s :: S.ByteString -> Digest SHA256
+sha2_256s = hash
+
sha2_384 :: L.ByteString -> Digest SHA384
sha2_384 = hashlazy
Existing preferred content expressions such as the one for archive group
have this problem.
-So, let's add a new expression: `balanced(group)`
+So, let's add a new expression: `balanced=group`
## implementation
So, we'll want to add some precautions to avoid a lot of data moving around
in such a case:
- ((balanced(backup) and not (copies=backup:1)) or present
+ (balanced=backup and not (copies=backup:1)) or present
So once file lands on a backup drive, it stays there, even if more backup
drives change the balancing.
What if we have 5 backup repos and want each key to be stored in 3 of them?
There's a simple change that can support that:
-`balanced(group:3)`
+`balanced=group:3`
This works the same as before, but rather than just `N mod M`, take
`N+I mod M` where I is [0..2] to get the list of 3 repositories that want a
on some drive there.
This can be implemented by making a group for each datacenter, which all of
-its drives are in, and using `balanced()` to pick the drive that holds the
+its drives are in, and using `balanced` to pick the drive that holds the
copy of the file. The preferred content expression would be eg:
- ((balanced(datacenterA) and not (copies=datacenterA:1)) or present
+ (balanced=datacenterA and not copies=datacenterA:1) or present
In such a situation, to avoid a `N^2` remote interconnect, there might be a
transfer repository in each datacenter, that is in front of its drives. The
It might be sufficient to use `copies=datacenterA:1`, so long as the file
reaching any drive in the datacenter is enough. But may want to add
something analagous to `inallgroup=` that checks if a file is in
-the place that `balanced()` picks for a group. Eg,
+the place that `balanced` picks for a group. Eg,
`balancedgroup=datacenterA` for 1 copy and `balancedgroup=group:datacenterA:2`
for N copies.
In several examples above, we have preferred content expressions in this
form:
- ((balanced(group:N) and not (copies=group:N)) or present
+ (balanced=group:N and not copies=group:N) or present
In order to rebalance, that needs to be changed to:
- balanced(group:N)
+ balanced=group:N
What could be done is make `balanced()` usually expand to the former,
but when --rebalance is used, it only expands to the latter.
-(Might make the fully balanced behavior available as `fullybalanced()` for
+(Might make the fully balanced behavior available as `fullybalanced` for
users who want it, then
-`balanced() == ((fullybalanced(group:N) and not (copies=group:N)) or present`
-usually and when --rebalance is used, `balanced() == fullybalanced(group:N)`
+`balanced=group:N == (fullybalanced=group:N and not copies=group:N) or present`
+usually and when --rebalance is used, `balanced=group:N == fullybalanced=group:N)`
Overrides the mincopies setting.
+* `--rebalance`
+
+ Changes the behavior of the "balanced" preferred content expression
+ to be the same as "fullbalanced". When that expression is used,
+ this can cause a lot of work to be done to rebalance repositories.
+
* `--time-limit=time`
Limits how long a git-annex command runs. The time can be something
says it wants them. (Or, if annex.expireunused is set, it may just delete
them.)
+* `balanced=groupname[:number]`
+
+ Makes content be evenly balanced amoung repositories in the group.
+
+ The number is the number of repositories in the group that will
+ want each file. When not specified, the default is 1.
+
+ For this to work, each repository in the group should have its preferred
+ content set to the same expression. Using `groupwanted` is a good
+ way to do that.
+
+ For example, "balanced=backup:2", when there are 3 members of the backup
+ group, will make each backup repository want 2/3rds of the files.
+
+ The sizes of files are not taken into account, so it's possible for
+ one repository to get larger than usual files and so fill up before
+ the other repositories. But files are only wanted by repositories that
+ have enough free space to hold them. So once a repository is full,
+ the remaining repositories will have any additional files balanced
+ amoung them. In order for this to work, you must use
+ [[git-annex-size]](1) to specify the size of each repository in the
+ group.
+
+ This usually avoids moving files between repositories of the group, even
+ if that means that things are not optimally balanced. Some of the ways
+ that it can get out of balance include adding a new repository to the
+ group, or a file getting copied into more repositories in the group than
+ the specified number. Running git-annex commands with the `--rebalance`
+ option will make this expression instead behave like the `fullybalanced`
+ expression, which will make repositories want to move files around as
+ necessary in order to get fully balanced.
+
+ Note that `not balanced` is a bad thing to put in a preferred content
+ expression for the same reason `not present` is.
+
+* `fullybalanced=groupname`
+
+ This is like `balanced`, but allows moving content between repositories
+ in the group at any time to keep it fully balanced.
+
+ Normally "balanced=groupname:number" is the same as
+ "(fullybalanced=groupname:number and not copies=groupname:number) or present"
+
+ When the `--rebalance` option is used, `balanced` is the same as
+ `fullybalanced`.
+
* `anything`
Always matches.
[[git-annex-wanted]](1)
+[[git-annex-size]](1)
+
<https://git-annex.branchable.com/preferred_content/>
<https://git-annex.branchable.com/preferred_content/standard_groups/>
Here are recent changes to preferred content expressions, and the version
they were added in.
+* "balanced=", "fullybalanced=" 10.20240831
* "securehash" 6.20170228
* "nothing" 6.201600202
* "anything" 5.20150616
## work notes
-* onward to balanced preferred content! But it depends on
- [[track_free_space_in_repos_via_git-annex_branch]] so that will be the
- first task.
+* balanced= and fullybalanced= need to limit the set of repositories to
+ ones with enough free space to contain a key.
+
+* Add `git-annex size` command.
+
+* Implement [[track_free_space_in_repos_via_git-annex_branch]]
+
+## completed items for August's work on balanced preferred content
+
+* Balanced preferred content basic implementation, including --rebalance
+ option.
## completed items for August's work on git-annex proxy support for exporttre
projects / git-annex.git / commitdiff