make it easier to use curl for unusual url schemes

Use curl when annex.security.allowed-url-schemes includes an url scheme not
supported by git-annex internally, as long as
annex.security.allowed-ip-addresses is configured to allow using curl.

Sponsored-by: Luke Shumaker on Patreon

diff --git a/Annex/Url.hs b/Annex/Url.hs
index 0d74fcb20bf12b1d641796b913420b362407ad2a..d4942389160b8562c51f0f86ad9d171d8c23ac2d 100644 (file)
--- a/Annex/Url.hs
+++ b/Annex/Url.hs
@@ -1,7 +1,7 @@
 {- Url downloading, with git-annex user agent and configured http
  - headers, security restrictions, etc.
  -
- - Copyright 2013-2020 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2022 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -43,6 +43,7 @@ import Network.Socket
 import Network.HTTP.Client
 import Network.HTTP.Client.TLS
 import Text.Read
+import qualified Data.Set as S
 
 defaultUserAgent :: U.UserAgent
 defaultUserAgent = "git-annex/" ++ BuildInfo.packageversion
@@ -78,7 +79,8 @@ getUrlOptions = Annex.getState Annex.urloptions >>= \case
        checkallowedaddr = words . annexAllowedIPAddresses <$> Annex.getGitConfig >>= \case
                ["all"] -> do
                        curlopts <- map Param . annexWebOptions <$> Annex.getGitConfig
-                       let urldownloader = if null curlopts
+                       allowedurlschemes <- annexAllowedUrlSchemes <$> Annex.getGitConfig
+                       let urldownloader = if null curlopts && not (any (`S.member` U.conduitUrlSchemes) allowedurlschemes)
                                then U.DownloadWithConduit $
                                        U.DownloadWithCurlRestricted mempty
                                else U.DownloadWithCurl curlopts

diff --git a/CHANGELOG b/CHANGELOG
index 411bde1209fd9b81090ae1e11b3a5d84c6e0a3e5..e826f3db72bfe9986159c369d8c04ad859099b95 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -20,6 +20,9 @@ git-annex (10.20220725) UNRELEASED; urgency=medium
   * Added annex.dbdir config which can be used to move sqlite databases
     to a different filesystem than the git-annex repo, when the repo is on
     a filesystem that sqlite does not work well in.
+  * Use curl when annex.security.allowed-url-schemes includes an url
+    scheme not supported by git-annex internally, as long as
+    annex.security.allowed-ip-addresses is configured to allow using curl.
 
  -- Joey Hess <id@joeyh.name>  Mon, 25 Jul 2022 15:35:45 -0400
 

diff --git a/Utility/Url.hs b/Utility/Url.hs
index 88a8e6a71366a3c06f73ba0ed30e8dbc5f567df2..0dbf7c58d9a322cb323025fed827b7aa64e38958 100644 (file)
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -1,6 +1,6 @@
 {- Url downloading.
  -
- - Copyright 2011-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2022 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -40,6 +40,7 @@ module Utility.Url (
        noBasicAuth,
        applyBasicAuth',
        extractFromResourceT,
+       conduitUrlSchemes,
 ) where
 
 import Common
@@ -111,10 +112,13 @@ defUrlOptions = UrlOptions
        <*> pure (DownloadWithConduit (DownloadWithCurlRestricted mempty))
        <*> pure id
        <*> newManager tlsManagerSettings
-       <*> pure (S.fromList $ map mkScheme ["http", "https", "ftp"])
+       <*> pure conduitUrlSchemes
        <*> pure Nothing
        <*> pure noBasicAuth
 
+conduitUrlSchemes :: S.Set Scheme
+conduitUrlSchemes = S.fromList $ map mkScheme ["http", "https", "ftp"]
+
 mkUrlOptions :: Maybe UserAgent -> Headers -> UrlDownloader -> Manager -> S.Set Scheme -> Maybe (URI -> String) -> GetBasicAuth -> UrlOptions
 mkUrlOptions defuseragent reqheaders urldownloader =
        UrlOptions useragent reqheaders urldownloader applyrequest

diff --git a/doc/forum/Use_addurl_with_a_file_on_an_HPC_cluster/comment_1_bc5ffcf6d3b72c1ed7b7763549880560._comment b/doc/forum/Use_addurl_with_a_file_on_an_HPC_cluster/comment_1_bc5ffcf6d3b72c1ed7b7763549880560._comment
new file mode 100644 (file)
index 0000000..816bee6
--- /dev/null
+++ b/doc/forum/Use_addurl_with_a_file_on_an_HPC_cluster/comment_1_bc5ffcf6d3b72c1ed7b7763549880560._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-08-15T15:46:40Z"
+ content="""
+git-annex can be used with any url scheme that curl supports, but you have to
+configure it to allow using it. See the documentation 
+of annex.security.allowed-url-schemes in the git-annex man page.
+
+You will also have to set annex.security.allowed-ip-addresses
+to "all".
+
+It seems that even with both settings, git-annex still avoids using curl
+for unsupported url schemes, unless you also set annex.web-options
+to some option used by curl. That forces it to use curl. I set it to
+"--netrc". You will probably need to use that option anyway since I think
+curl needs configuration in a netrc file to authenticate for sftp.
+
+(I feel that it's a bug that annex.web-options needs to be set to make it
+use curl, and I've fixed that in master.)
+"""]]

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 373444d7965b7ae3eb0963b3e149e166663d91ea..66968cce116d0c027d0b9713ba9347c25931190a 100644 (file)
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -1745,6 +1745,9 @@ Remotes are configured using these settings in `.git/config`.
   repository, possibly causing it to be copied into your repository
   and transferred on to other remotes, exposing its content.
 
+  Any url schemes supported by curl can be listed here, but you will
+  also need to configure annex.allowed-ip-addresses to allow using curl.
+
   Some special remotes support their own domain-specific URL
   schemes; those are not affected by this configuration setting.