S3: Support signature=anonymous to access a S3 bucket anonymously

This can be used, for example, with importtree=yes to import from a public
bucket.

This needs a patch that has not yet landed in the aws library, and will
need to be adjusted to support compiling with old versions of the library,
so is not yet suitable for merging.
See https://github.com/aristidb/aws/pull/281

The stack.yaml changes are provided to show how to build against the aws
fork and will need to be reverted as well.

Sponsored-by: Dartmouth College's DANDI project

diff --git a/CHANGELOG b/CHANGELOG
index 8e8a0afb2b83711868222dd17d8aa153944ec52d..bd46e47235e68051df365cef6e9d185f8684b24a 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,9 @@ git-annex (10.20221004) UNRELEASED; urgency=medium
     do not operate on a repository that has an empty name.
   * move: Fix openFile crash with -J
     (Fixes a reversion in 8.20201103)
+  * S3: Support signature=anonymous to access a S3 bucket anonymously.
+    This can be used, for example, with importtree=yes to import from
+    a public bucket.
 
  -- Joey Hess <id@joeyh.name>  Mon, 03 Oct 2022 13:36:42 -0400
 

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 1f0ebd3d5a2f71172688d961ab73fed247ddd518..c0d99b541852875fe1b78f549fd9cb9a4a400512 100644 (file)
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -1,6 +1,6 @@
 {- S3 remotes
  -
- - Copyright 2011-2020 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2022 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -156,7 +156,9 @@ requeststyleField = Accepted "requeststyle"
 signatureField :: RemoteConfigField
 signatureField = Accepted "signature"
 
-newtype SignatureVersion = SignatureVersion Int
+data SignatureVersion 
+       = SignatureVersion Int
+       | Anonymous
 
 signatureVersionParser :: RemoteConfigField -> FieldDesc -> RemoteConfigFieldParser
 signatureVersionParser f fd =
@@ -165,10 +167,17 @@ signatureVersionParser f fd =
   where
        go "v2" = Just (SignatureVersion 2)
        go "v4" = Just (SignatureVersion 4)
+       go "anonymous" = Just Anonymous
        go _ = Nothing
 
        defver = SignatureVersion 2
 
+isAnonymous :: ParsedRemoteConfig -> Bool
+isAnonymous c = 
+       case getRemoteConfigValue signatureField c of
+               Just Anonymous -> True
+               _ -> False
+
 portField :: RemoteConfigField
 portField = Accepted "port"
 
@@ -272,7 +281,9 @@ s3Setup'  ss u mcreds c gc
                (c', encsetup) <- encryptionSetup (c `M.union` defaults) gc
                pc <- either giveup return . parseRemoteConfig c'
                        =<< configParser remote c'
-               c'' <- setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds
+               c'' <- if isAnonymous pc
+                       then pure c'
+                       else setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds
                pc' <- either giveup return . parseRemoteConfig c''
                        =<< configParser remote c''
                info <- extractS3Info pc'
@@ -286,7 +297,9 @@ s3Setup'  ss u mcreds c gc
                showNote "Internet Archive mode"
                pc <- either giveup return . parseRemoteConfig c
                        =<< configParser remote c
-               c' <- setRemoteCredPair ss noEncryptionUsed pc gc (AWS.creds u) mcreds
+               c' <- if isAnonymous pc
+                       then pure c
+                       else setRemoteCredPair ss noEncryptionUsed pc gc (AWS.creds u) mcreds
                -- Ensure user enters a valid bucket name, since
                -- this determines the name of the archive.org item.
                let validbucket = replace " " "-" $ map toLower $
@@ -841,17 +854,20 @@ type S3HandleVar = TVar (Either (Annex (Maybe S3Handle)) (Maybe S3Handle))
 {- Prepares a S3Handle for later use. Does not connect to S3 or do anything
  - else expensive. -}
 mkS3HandleVar :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar
-mkS3HandleVar c gc u = liftIO $ newTVarIO $ Left $ do
-       mcreds <- getRemoteCredPair c gc (AWS.creds u)
-       case mcreds of
-               Just creds -> do
-                       awscreds <- liftIO $ genCredentials creds
-                       let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper Nothing
-                       ou <- getUrlOptions
-                       return $ Just $ S3Handle (httpManager ou) awscfg s3cfg
-               Nothing -> return Nothing
+mkS3HandleVar c gc u = liftIO $ newTVarIO $ Left $
+       if isAnonymous c
+               then go =<< liftIO AWS.anonymousCredentials
+               else do
+                       mcreds <- getRemoteCredPair c gc (AWS.creds u)
+                       case mcreds of
+                               Just creds -> go =<< liftIO (genCredentials creds)
+                               Nothing -> return Nothing
   where
        s3cfg = s3Configuration c
+       go awscreds = do
+               let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper Nothing
+               ou <- getUrlOptions
+               return $ Just $ S3Handle (httpManager ou) awscfg s3cfg
 
 withS3Handle :: S3HandleVar -> (Maybe S3Handle -> Annex a) -> Annex a
 withS3Handle hv a = liftIO (readTVarIO hv) >>= \case

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index e34cbb7f9d413e11bdfd63b78f38c703a7bb8577..5bda70f98af5057209872b7de69128bcf40258ae 100644 (file)
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -82,6 +82,7 @@ the S3 remote.
 * `signature` - This controls the S3 signature version to use.
   "v2" is currently the default, "v4" is needed to use some S3 services.
   If you get some kind of authentication error, try "v4".
+  To access a S3 bucket anonymously, use "anonymous".
 
 * `bucket` - S3 requires that buckets have a globally unique name, 
   so by default, a bucket name is chosen based on the remote name

diff --git a/stack.yaml b/stack.yaml
index 7dbfb657ac1de1ca6196860366652ba7284f0a2d..988296933b4965f6a832bd78d5918eff7d123de7 100644 (file)
--- a/stack.yaml
+++ b/stack.yaml
@@ -11,10 +11,10 @@ flags:
     gitlfs: true
 packages:
 - '.'
-resolver: lts-18.13
+resolver: lts-19.16
 extra-deps:
 - IfElse-0.85
-- aws-0.22
+- /home/joey/tmp/aws
 - bloomfilter-2.0.1.0
 - git-lfs-1.2.0
 - http-client-restricted-0.0.4
@@ -24,4 +24,3 @@ extra-deps:
 - base16-bytestring-0.1.1.7
 - base64-bytestring-1.0.0.3
 - bencode-0.6.1.1
-- http-client-0.7.9