Add --url option and url= preferred content expression

To match content that is recorded as present in an url.

Note that, this cannot ask remotes to provide an url using whereisKey, like
whereis does. Because preferred content expressions need to match the same
from multiple perspectives, and the remote would not always be available.

That's why the docs say "recorded as present", but still this may be
surprising to some who see an url in whereis output and are surprised they
cannot match on it.

The use of getDownloader is to strip the downloader prefix from urls like
"yt:". Note that, when OtherDownloader is used, this strips the ":" prefix,
and allows matching on those urls too.

diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs
index 6157efa3f0a20a7d8a13673cbcd4fd9c1b4a7bf4..385e23a16e7a993131675625b4da7a3d32272699 100644 (file)
--- a/Annex/FileMatcher.hs
+++ b/Annex/FileMatcher.hs
@@ -1,6 +1,6 @@
 {- git-annex file matching
  -
- - Copyright 2012-2024 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2025 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -194,6 +194,7 @@ preferredContentTokens pcd =
        , ValueToken "approxlackingcopies" (usev $ limitLackingCopies "approxlackingcopies" True)
        , ValueToken "inbackend" (usev limitInBackend)
        , ValueToken "metadata" (usev limitMetaData)
+       , ValueToken "url" (usev limitUrl)
        , ValueToken "inallgroup" (usev $ limitInAllGroup $ getGroupMap pcd)
        , ValueToken "onlyingroup" (usev $ limitOnlyInGroup $ getGroupMap pcd)
        , ValueToken "balanced" (usev $ limitBalanced (repoUUID pcd) (getGroupMap pcd))

diff --git a/CHANGELOG b/CHANGELOG
index 7216b21fbb29306e2b1bee49bd74723610266846..e0407b39191b9d6fafda63597802b96700647b1b 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -10,6 +10,8 @@ git-annex (10.20250631) UNRELEASED; urgency=medium
     that have experienced the above bug.
   * Fix symlinks generated to annexed content when in adjusted unlocked
     branch in a linked worktree on a filesystem not supporting symlinks.
+  * Add --url option and url= preferred content expression, to match
+    content that is recorded as present in an url.   
 
  -- Joey Hess <id@joeyh.name>  Mon, 07 Jul 2025 15:59:42 -0400
 

diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs
index 890f9654de37cc2dc1e5a8694d97b1e47c60757b..4b44edda56e4a52b185a797ac795dc0d8c9e49f4 100644 (file)
--- a/CmdLine/GitAnnex/Options.hs
+++ b/CmdLine/GitAnnex/Options.hs
@@ -348,6 +348,11 @@ keyMatchingOptions' =
                <> help "match files with attached metadata"
                <> hidden
                )
+       , annexOption (setAnnexState . Limit.addUrl) $ strOption
+               ( long "url" <> metavar paramGlob
+               <> help "match files by url"
+               <> hidden
+               )
        , annexFlag (setAnnexState Limit.Wanted.addWantGet)
                ( long "want-get"
                <> help "match files the local repository wants to get"

diff --git a/Limit.hs b/Limit.hs
index d090e09d88ad02a5bb2ea1a3ee1f77b868b6e35b..1916a606d5c34a3b740347d4df5d469089d8775f 100644 (file)
--- a/Limit.hs
+++ b/Limit.hs
@@ -31,6 +31,7 @@ import Types.FileMatcher
 import Types.MetaData
 import Annex.MetaData
 import Logs.MetaData
+import Logs.Web
 import Logs.Group
 import Logs.Unused
 import Logs.Location
@@ -867,6 +868,26 @@ limitMetaData s = case parseMetaDataMatcher s of
                . S.filter matching
                . metaDataValues f <$> getCurrentMetaData k
 
+addUrl :: String -> Annex ()
+addUrl = addLimit . limitUrl
+
+limitUrl :: MkLimit Annex
+limitUrl glob = Right $ MatchFiles
+       { matchAction = const $ const $ checkKey check
+       , matchNeedsFileName = False
+       , matchNeedsFileContent = False
+       , matchNeedsKey = True
+       , matchNeedsLocationLog = False
+       , matchNeedsLiveRepoSize = False
+       , matchNegationUnstable = False
+       , matchDesc = "url" =? glob
+       }
+  where
+       check k = any (matchGlob cglob)
+               . map (fst . getDownloader)
+               <$> getUrls k
+       cglob = compileGlob glob CaseSensitive (GlobFilePath False) -- memoized
+
 addAccessedWithin :: Duration -> Annex ()
 addAccessedWithin duration = do
        now <- liftIO getPOSIXTime

diff --git a/doc/git-annex-matching-options.mdwn b/doc/git-annex-matching-options.mdwn
index ea29f98848ce732ee5d0c8e140be8479b3fdcffc..cf964cc71d3aaae4d5e162d1e7022b6b37fc059f 100644 (file)
--- a/doc/git-annex-matching-options.mdwn
+++ b/doc/git-annex-matching-options.mdwn
@@ -178,6 +178,11 @@ in either of two repositories.
   (Note that you will need to quote the second parameter to avoid
   the shell doing redirection.)
 
+* `--url=glob`
+
+  Matches when the content is recorded as being present in an url that
+  matches the glob.
+
 * `--want-get`
 
   Matches only when the preferred content settings for the local repository

diff --git a/doc/git-annex-preferred-content.mdwn b/doc/git-annex-preferred-content.mdwn
index 52c6ff225e0dbe9e57cb7f9d7450f0143278d441..6b9fc521ac82f12e75a0e073e915877d79f8476c 100644 (file)
--- a/doc/git-annex-preferred-content.mdwn
+++ b/doc/git-annex-preferred-content.mdwn
@@ -166,6 +166,11 @@ content not being configured.
   To match PDFs with between 100 and 200 pages (assuming something has set
   that metadata), use `metadata=pagecount>=100 and metadata=pagecount<=200`
 
+* `url=glob`
+
+  Matches when the content is recorded as being present in an url that
+  matches the glob. 
+
 * `present`
 
   Makes content be wanted if it's present, but not otherwise.

diff --git a/doc/todo/match_on_url.mdwn b/doc/todo/match_on_url.mdwn
index 6623debbed1d790cb7039c79255f402325047bfd..5ef885e02dd001aeb4a2152064130c145456a644 100644 (file)
--- a/doc/todo/match_on_url.mdwn
+++ b/doc/todo/match_on_url.mdwn
@@ -10,3 +10,5 @@ expression if adding that.
 An alternative way could be to populate a metadata field with the url,
 if that were done without increasing the size of the git repository.
 --[[Joey]]
+
+> [[done]] --[[Joey]]