S3 region=

S3: Support a region= configuration useful for some non-Amazon S3
implementations. This feature needs git-annex to be built with aws-0.24.

datacenter= sets both the AWS hostname and region in one setting, which is
easy when using AWS, but not useful for other hosts. So kept datacenter
as-is, but added this additional config.

Sponsored-By: Brett Eisenberg on Patreon

diff --git a/CHANGELOG b/CHANGELOG
index ee9aa3961d14d6cac405a13b8c486ea52eaae146..6cbcaf5535029a3bce43b421e4e44fcdcd54a53b 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,10 @@
+git-annex (10.20230127) UNRELEASED; urgency=medium
+
+  * S3: Support a region= configuration useful for some non-Amazon S3
+    implementations. This feature needs git-annex to be built with aws-0.24.
+
+ -- Joey Hess <id@joeyh.name>  Mon, 06 Feb 2023 13:39:18 -0400
+
 git-annex (10.20230126) upstream; urgency=medium
 
   * Change --metadata comparisons < > <= and >= to fall back to

diff --git a/Remote/S3.hs b/Remote/S3.hs
index f5014202eb08fe3672d00072ac80236155592a6d..3ea9ed813d67ddb0076297ed08676673331f8ba0 100644 (file)
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -1,6 +1,6 @@
 {- S3 remotes
  -
- - Copyright 2011-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -83,6 +83,8 @@ remote = specialRemoteType $ RemoteType
                                (FieldDesc "S3 server hostname (default is Amazon S3)")
                        , optionalStringParser datacenterField
                                (FieldDesc "S3 datacenter to use (US, EU, us-west-1, ..)")
+                       , optionalStringParser regionField
+                               (FieldDesc "S3 region to use")
                        , optionalStringParser partsizeField
                                (FieldDesc "part size for multipart upload (eg 1GiB)")
                        , optionalStringParser storageclassField
@@ -129,6 +131,9 @@ hostField = Accepted "host"
 datacenterField :: RemoteConfigField
 datacenterField = Accepted "datacenter"
 
+regionField :: RemoteConfigField
+regionField = Accepted "region"
+
 partsizeField :: RemoteConfigField
 partsizeField = Accepted "partsize"
 
@@ -914,6 +919,7 @@ s3Configuration c = cfg
        }
   where
        h = fromJust $ getRemoteConfigValue hostField c
+       r = encodeBS <$> getRemoteConfigValue regionField c
        datacenter = fromJust $ getRemoteConfigValue datacenterField c
        -- When the default S3 host is configured, connect directly to
        -- the S3 endpoint for the configured datacenter.
@@ -948,8 +954,14 @@ s3Configuration c = cfg
                        | otherwise -> AWS.HTTP
        cfg = case getRemoteConfigValue signatureField c of
                Just (SignatureVersion 4) -> 
-                       S3.s3v4 proto endpoint False S3.SignWithEffort
-               _ -> S3.s3 proto endpoint False
+                       (S3.s3v4 proto endpoint False S3.SignWithEffort)
+#if MIN_VERSION_aws(0,24,0)
+                               { S3.s3Region = r }
+#endif
+               _ -> (S3.s3 proto endpoint False)
+#if MIN_VERSION_aws(0,24,0)
+                               { S3.s3Region = r }
+#endif
 
 data S3Info = S3Info
        { bucket :: S3.Bucket
@@ -964,6 +976,7 @@ data S3Info = S3Info
        , public :: Bool
        , publicurl :: Maybe URLString
        , host :: Maybe String
+       , region :: Maybe String
        }
 
 extractS3Info :: ParsedRemoteConfig -> Annex S3Info
@@ -987,6 +1000,7 @@ extractS3Info c = do
                        getRemoteConfigValue publicField c
                , publicurl = getRemoteConfigValue publicurlField c
                , host = getRemoteConfigValue hostField c
+               , region = getRemoteConfigValue regionField c
                }
 
 putObject :: S3Info -> T.Text -> RequestBody -> S3.PutObject
@@ -1126,7 +1140,12 @@ debugMapper level t = forward "Remote.S3" (T.unpack t)
 s3Info :: ParsedRemoteConfig -> S3Info -> [(String, String)]
 s3Info c info = catMaybes
        [ Just ("bucket", fromMaybe "unknown" (getBucketName c))
-       , Just ("endpoint", w82s (BS.unpack (S3.s3Endpoint s3c)))
+       , Just ("endpoint", decodeBS (S3.s3Endpoint s3c))
+#if MIN_VERSION_aws(0,24,0)
+       , case S3.s3Region s3c of
+               Nothing -> Nothing
+               Just r -> Just ("region", decodeBS r)
+#endif
        , Just ("port", show (S3.s3Port s3c))
        , Just ("protocol", map toLower (show (S3.s3Protocol s3c)))
        , Just ("storage class", showstorageclass (getStorageClass c))

diff --git a/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn b/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn
index 1bb1f330eb2566c39b3c543050ced601aa3ad734..a9a7c5a8f0bf645b7f64785c37d9d235de9e172d 100644 (file)
--- a/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn
+++ b/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn
@@ -28,3 +28,6 @@ Fedora Silverblue 37 / git-annex-10.20221212-1.fc37.x86_64
 Yes, many years ago - now trying to get it up and running with my self-hosted S3 endpoint.
 
 [1]: https://garagehq.deuxfleurs.fr/
+
+> [[fixed|done]] although it needs git-annex to be built against 
+> a not yet released version of aws. --[[Joey]]

diff --git a/doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment b/doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment
new file mode 100644 (file)
index 0000000..046002c
--- /dev/null
+++ b/doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2023-02-06T17:33:25Z"
+ content="""
+Implemented this in aws: https://github.com/aristidb/aws/pull/284
+Which should be released as version 0.24.
+
+git-annex will support region= when built with that version of aws.
+"""]]

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 5bda70f98af5057209872b7de69128bcf40258ae..6329bb093b070cca966444c0b02f07ef4003db0b 100644 (file)
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -34,9 +34,12 @@ the S3 remote.
   Think carefully about who can access your repository before using
   embedcreds without gpg encryption.
 
-* `datacenter` - Defaults to "US". Other values include "EU" (which is EU/Ireland),
-  "us-west-1", "us-west-2", "ap-southeast-1", "ap-southeast-2", and
-  "sa-east-1".
+* `datacenter` - Specifies which Amazon datacenter
+  to use for the bucket. Defaults to "US". Other values include "EU"
+  (which is EU/Ireland), "us-west-1", "us-west-2", "ap-southeast-1",
+  "ap-southeast-2", and "sa-east-1". See Amazon's documentation for a
+  complete list. Configuring this is equivilant to configuring both
+  `host` and `region`.
 
 * `storageclass` - Default is "STANDARD".  
   Consult S3 provider documentation for pricing details and available
@@ -63,6 +66,10 @@ the S3 remote.
 * `host` - Specify in order to use a different, S3 compatable
   service.
 
+* `region` - Specify the region to use. Only makes sense to use when
+  you also set `host`.
+  (Requires a git-annex built with aws-0.24.)
+
 * `protocol` - Either "http" (the default) or "https". Setting
   protocol=https implies port=443.