From e1fc9e204efe11ac973fa1ed0e085de6a2003d5c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 29 Jun 2023 15:34:53 -0400 Subject: [PATCH] added git-annex satisfy This ended up having an interface like sync, rather than like get/copy/drop. That let it be implemented in terms of sync, which took a lot less code. Also, it lets it handle many of the edge cases that sync does, such as getting files that are not visible in a --hide-missing branch, and sending files to exporttree remotes. As well as being easier to implement, `git-annex satisfy myremote` makes sense as it satisfies the preferred content settings of the remote. `git-annex satisfy somefile` does not form a sentence that makes sense. So while -C can be a little bit annoying, it still makes sense to have this syntax. Note that, while I initially thought this would also satisfy numcopies, it does not. Arguably it ought to. But, sync does not send files in order to satisfy numcopies, it only sends files to satisfy preferred content. And it's important that this transfer the same files as sync does, because it will probably be used in a workflow where the user sometimes syncs and sometimes satisfies, and does not expect satisfy to do things that sync would not do. (Also opened a new bug that also affects sync et all, not only this command.) Sponsored-by: Nicholas Golder-Manning on Patreon --- CHANGELOG | 5 +- CmdLine/GitAnnex.hs | 2 + Command/Satisfy.hs | 17 +++ Command/Sync.hs | 103 +++++++++++------- doc/bugs/sync_-C_can_export_other_files.mdwn | 14 +++ doc/git-annex-pull.mdwn | 2 +- doc/git-annex-satisfy.mdwn | 73 +++++-------- doc/git-annex.mdwn | 5 +- .../new_command_for_syncing_content_only.mdwn | 2 + git-annex.cabal | 1 + 10 files changed, 128 insertions(+), 96 deletions(-) create mode 100644 Command/Satisfy.hs create mode 100644 doc/bugs/sync_-C_can_export_other_files.mdwn diff --git a/CHANGELOG b/CHANGELOG index ddc7114f5d..eb9878da7a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,8 @@ git-annex (10.20230627) UNRELEASED; urgency=medium * satify: New command that gets/sends/drops content to satisfy - numcopies and preferred content settings. This is similar to - the --content part of git-annex sync, but with an interface - more like get/move/drop. + preferred content settings. This is like to the --content + part of git-annex sync. * reinject: Added --guesskeys option. * diffdriver: Added --text option for easy diffing of the contents of annexed text files. diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs index 8fcaf956e8..e8f5b703b3 100644 --- a/CmdLine/GitAnnex.hs +++ b/CmdLine/GitAnnex.hs @@ -103,6 +103,7 @@ import qualified Command.Sync import qualified Command.Assist import qualified Command.Pull import qualified Command.Push +import qualified Command.Satisfy import qualified Command.Mirror import qualified Command.AddUrl import qualified Command.ImportFeed @@ -151,6 +152,7 @@ cmds testoptparser testrunner mkbenchmarkgenerator = map addGitAnnexCommonOption , Command.Assist.cmd , Command.Pull.cmd , Command.Push.cmd + , Command.Satisfy.cmd , Command.Mirror.cmd , Command.AddUrl.cmd , Command.ImportFeed.cmd diff --git a/Command/Satisfy.hs b/Command/Satisfy.hs new file mode 100644 index 0000000000..d4fcf3f94f --- /dev/null +++ b/Command/Satisfy.hs @@ -0,0 +1,17 @@ +{- git-annex command + - + - Copyright 2023 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Command.Satisfy (cmd) where + +import Command +import Command.Sync hiding (cmd) + +cmd :: Command +cmd = withAnnexOptions [jobsOption, backendOption] $ + command "satisfy" SectionCommon + "transfer and drop content as configured" + (paramRepeating paramRemote) (seek <--< optParser SatisfyMode) diff --git a/Command/Sync.hs b/Command/Sync.hs index 3aeb20a244..91b6881911 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -95,7 +95,7 @@ cmd = withAnnexOptions [jobsOption, backendOption] $ "synchronize local repository with remotes" (paramRepeating paramRemote) (seek <--< optParser SyncMode) -data OperationMode = SyncMode | PullMode | PushMode | AssistMode +data OperationMode = SyncMode | PullMode | PushMode | SatisfyMode | AssistMode deriving (Eq, Show) data SyncOptions = SyncOptions @@ -143,15 +143,17 @@ optParser mode desc = SyncOptions ( metavar desc <> completeRemotes )) - <*> switch - ( long "only-annex" - <> short 'a' - <> help "do not operate on git branches" - ) - <*> switch - ( long "not-only-annex" - <> help "operate on git branches as well as annex" - ) + <*> whenmode [SatisfyMode] True + (switch + ( long "only-annex" + <> short 'a' + <> help "do not operate on git branches" + )) + <*> whenmode [SatisfyMode] False + ( switch + ( long "not-only-annex" + <> help "operate on git branches as well as annex" + )) <*> case mode of SyncMode -> switch ( long "commit" @@ -159,21 +161,25 @@ optParser mode desc = SyncOptions ) PushMode -> pure False PullMode -> pure False + SatisfyMode -> pure False AssistMode -> pure True - <*> unlessmode [SyncMode] True (switch - ( long "no-commit" - <> help "avoid git commit" - )) - <*> unlessmode [SyncMode, AssistMode] Nothing (optional (strOption - ( long "message" <> short 'm' <> metavar "MSG" - <> help "commit message" - ))) + <*> unlessmode [SyncMode] True + (switch + ( long "no-commit" + <> help "avoid git commit" + )) + <*> unlessmode [SyncMode, AssistMode] Nothing + (optional (strOption + ( long "message" <> short 'm' <> metavar "MSG" + <> help "commit message" + ))) <*> case mode of SyncMode -> invertableSwitch "pull" True ( help "avoid git pulls from remotes" ) PullMode -> pure True PushMode -> pure False + SatisfyMode -> pure False AssistMode -> pure True <*> case mode of SyncMode -> invertableSwitch "push" True @@ -181,31 +187,36 @@ optParser mode desc = SyncOptions ) PullMode -> pure False PushMode -> pure True + SatisfyMode -> pure False AssistMode -> pure True - <*> optional (flag' True - ( long "content" - <> help "transfer annexed file contents" - )) - <*> optional (flag' True - ( long "no-content" - <> short 'g' - <> help "do not transfer annexed file contents" - )) + <*> whenmode [SatisfyMode] (Just True) + (optional (flag' True + ( long "content" + <> help "transfer annexed file contents" + ))) + <*> whenmode [SatisfyMode] Nothing + (optional (flag' True + ( long "no-content" + <> short 'g' + <> help "do not transfer annexed file contents" + ))) <*> many (strOption ( long "content-of" <> short 'C' <> help "transfer contents of annexed files in a given location" <> metavar paramPath )) - <*> whenmode [PullMode] False (switch - ( long "cleanup" - <> help "remove synced/ branches" - )) + <*> whenmode [PullMode, SatisfyMode] False + (switch + ( long "cleanup" + <> help "remove synced/ branches" + )) <*> optional parseAllOption - <*> whenmode [PushMode] False (invertableSwitch "resolvemerge" True - ( help "do not automatically resolve merge conflicts" - )) - <*> whenmode [PushMode] False + <*> whenmode [PushMode, SatisfyMode] False + (invertableSwitch "resolvemerge" True + ( help "do not automatically resolve merge conflicts" + )) + <*> whenmode [PushMode, SatisfyMode] False parseUnrelatedHistoriesOption <*> pure mode where @@ -838,6 +849,7 @@ seekSyncContent o rs currbranch = do SyncMode -> "sync" PullMode -> "pull" PushMode -> "push" + SatisfyMode -> "satisfy" AssistMode -> "assist" gofile bloom mvar _ f k = @@ -910,7 +922,7 @@ syncFile o ebloom rs af k = do return (got || not (null putrs)) where wantget have inhere = allM id - [ pure (pullOption o) + [ pure (pullOption o || operationMode o == SatisfyMode) , pure (not $ null have) , pure (not inhere) , wantGet True (Just k) af @@ -924,7 +936,7 @@ syncFile o ebloom rs af k = do next $ return True wantput r - | pushOption o == False = return False + | pushOption o == False && operationMode o /= SatisfyMode = return False | Remote.readonly r || remoteAnnexReadOnly (Remote.gitconfig r) = return False | isExport r = return False | isThirdPartyPopulated r = return False @@ -949,6 +961,7 @@ syncFile o ebloom rs af k = do - of the branch. (If the branch is not currently checked out, anything - imported from the remote will not yet have been merged into it yet and - so exporting would delete files from the remote unexpectedly.) + - (This is not done in SatifyMode.) - - Otherwise, transfer any files that were part of a previous export - but are not in the remote yet. @@ -959,12 +972,14 @@ seekExportContent :: Maybe SyncOptions -> [Remote] -> CurrBranch -> Annex Bool seekExportContent o rs (currbranch, _) = or <$> forM rs go where go r + | maybe False (\o' -> operationMode o' == SatisfyMode) o = + case remoteAnnexTrackingBranch (Remote.gitconfig r) of + Nothing -> return False + Just b -> withdb r $ \db -> + cannotupdateexport r db (Just b) False | not (maybe True pushOption o) = return False | not (remoteAnnexPush (Remote.gitconfig r)) = return False - | otherwise = bracket - (Export.openDb (Remote.uuid r)) - Export.closeDb - (\db -> Export.writeLockDbWhile db (go' r db)) + | otherwise = withdb r (go' r) go' r db = case remoteAnnexTrackingBranch (Remote.gitconfig r) of Nothing -> cannotupdateexport r db Nothing True Just b -> do @@ -986,6 +1001,11 @@ seekExportContent o rs (currbranch, _) = or <$> forM rs go | otherwise -> cannotupdateexport r db (Just b) False _ -> cannotupdateexport r db (Just b) True + withdb r a = bracket + (Export.openDb (Remote.uuid r)) + Export.closeDb + (\db -> Export.writeLockDbWhile db (a db)) + cannotupdateexport r db mtb showwarning = do exported <- getExport (Remote.uuid r) when showwarning $ @@ -1056,6 +1076,7 @@ cleanupRemote remote (Just b, _) = shouldSyncContent :: SyncOptions -> Annex Bool shouldSyncContent o | fromMaybe False (noContentOption o) = pure False + | operationMode o == SatisfyMode = pure True -- For git-annex pull and git-annex push and git-annex assist, -- annex.syncontent defaults to True unless set | operationMode o /= SyncMode = annexsynccontent True diff --git a/doc/bugs/sync_-C_can_export_other_files.mdwn b/doc/bugs/sync_-C_can_export_other_files.mdwn new file mode 100644 index 0000000000..9ba19e225f --- /dev/null +++ b/doc/bugs/sync_-C_can_export_other_files.mdwn @@ -0,0 +1,14 @@ +When an exporttree=yes remote has had a tree exported to it, but eg with +`git-annex export master --to foo --fast`, so not all the files have been +sent to it yet, `git-annex sync -Cfoo` will also export files in other +directories to it. + +This seems like a bug, it ought to only export any files in the specified +directory. + +Another way this can happen is `git-annex sync -Cfoo`, when file bar +has changed. Currently this exports bar to the export remote, but does +not send it to any other remotes, since it's not in directory foo. + +Also this affects `git-annex satisfy`, and other related commands too. +--[[Joey]] diff --git a/doc/git-annex-pull.mdwn b/doc/git-annex-pull.mdwn index 5c8dec7027..c96355e7a5 100644 --- a/doc/git-annex-pull.mdwn +++ b/doc/git-annex-pull.mdwn @@ -33,7 +33,7 @@ to the parent branch or metadata. Normally this tries to download the content of each annexed file, from any remote that it's pulling from that has a copy. To control which files it downloads, configure the preferred -content of the local repository.It will also drop files from a +content of the local repository. It will also drop files from a remote that are not preferred content of the remote. See [[git-annex-preferred-content]](1). diff --git a/doc/git-annex-satisfy.mdwn b/doc/git-annex-satisfy.mdwn index c2211ce7f6..386b6d02f5 100644 --- a/doc/git-annex-satisfy.mdwn +++ b/doc/git-annex-satisfy.mdwn @@ -4,71 +4,50 @@ git-annex satisfy - transfer and drop content as configured # SYNOPSIS -git annex satisffy `[path ...]` +git annex satisfy `[remote ...]` # DESCRIPTION -This transfers and drops content to satisfy the numcopies and preferred -content settings of the local repository and remotes. +This transfers and drops content of annexed files to work toward satisfying +the preferred content settings of the local repository and remotes. It does the same thing as `git-annex sync --content` without the pulling -and pushing of git repositories. - -By default it operates on all annexed files in the current directory and -its subdirectories. Paths of files or directories can also be specified. +and pushing of git repositories, and without changing the trees that are +imported to or exported from special remotes. # OPTIONS -* `--jobs=N` `-JN` - - Enables parallel processing with up to the specified number of jobs - running at once. For example: `-J10` - - Setting this to "cpus" will run one job per CPU core. - -* `--all` `-A` - - Operate on all objects stored in the git annex, not only objects used by - currently existing files. - - However, this bypasses checking the .gitattributes annex.numcopies - setting when dropping files. - - This is the default behavior when running git-annex in a bare repository. +* `[remote]` -* `--branch=ref` + By default this command operates on all remotes, except for remotes + that have `remote..annex-sync` set to false. - Operate on files in the specified branch or treeish. + By specifying the names of remotes (or remote groups), you can control + which ones to operate on. - Like --all, this bypasses checking the .gitattributes annex.numcopies - setting when dropping files. +* `--content-of=path` `-C path` -* `--unused` + Operate on only files in the specified path. The default is to operate on + all files in the working tree. - Operate on files found by last run of git-annex unused. + This option can be repeated multiple times with different paths. -* `--failed` - - Operate on files that have recently failed to be transferred. - -* matching options - - The [[git-annex-matching-options]](1) - can be used to control what files to operate on. - -* `--json` +* `--jobs=N` `-JN` - Enable JSON output. This is intended to be parsed by programs that use - git-annex. Each line of output is a JSON object. + Enables parallel processing with up to the specified number of jobs + running at once. For example: `-J10` -* `--json-progress` + Setting this to "cpus" will run one job per CPU core. - Include progress objects in JSON output. +* `--all` `-A` -* `--json-error-messages` + Usually this command operates on annexed files in the current branch. + This option makes it operate on all available versions of all annexed files + (when preferred content settings allow). - Messages that would normally be output to standard error are included in - the JSON instead. + Note that preferred content settings that use `include=` or `exclude=` + will only match the version of files currently in the work tree, but not + past versions of files. * Also the [[git-annex-common-options]](1) can be used. @@ -78,8 +57,6 @@ its subdirectories. Paths of files or directories can also be specified. [[git-annex-sync]](1) -[[git-annex-preferred-numcopies]](1) - [[git-annex-preferred-content]](1) # AUTHOR diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e294d9a2ad..d8b73cf0b6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -138,10 +138,9 @@ content from the key-value store. See [[git-annex-assist]](1) for details. -* `satisfy [path ...]` +* `satisfy [remote ...]` - Satisfy numcopies and preferred content settings by transferring - and dropping content. + Satisfy preferred content settings by transferring and dropping content. See [[git-annex-satisfy]](1) for details. diff --git a/doc/todo/new_command_for_syncing_content_only.mdwn b/doc/todo/new_command_for_syncing_content_only.mdwn index 4b78c5c18d..834a9808c7 100644 --- a/doc/todo/new_command_for_syncing_content_only.mdwn +++ b/doc/todo/new_command_for_syncing_content_only.mdwn @@ -19,3 +19,5 @@ What would be a good name for such a command? --[[Joey]] > How about `git-annex port` as it carries content over like a luggage carrier aka. porter? > Another one that comes to mind is `git-annex shuttle` but that could be semantically too > similar to `git-annex assist`, perhaps. --[[jkniiv]] + +>> `git-annex satisfy` [[done]] --[[Joey]] diff --git a/git-annex.cabal b/git-annex.cabal index 46a554c0d1..72a4c0f4f5 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -790,6 +790,7 @@ Executable git-annex Command.ResolveMerge Command.Restage Command.RmUrl + Command.Satisfy Command.Schedule Command.Semitrust Command.SendKey -- 2.30.2