From 3b1702e6583d1093664f4fef74a2a755dbf9dce6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Aug 2025 15:21:22 -0400 Subject: [PATCH] probe AWS datacenter S3: When initremote is given the name of a bucket that already exists, automatically set datacenter to the right value, rather than needing it to be explicitly set. This needs aws-0.23. But, initremote stores the datacenter value, so a remote set up this way can be used with git-annex built with an older aws. This is not done when signature=anonymous, because in that case, using AWS.defaultRegion works fine for accessing buckets on other datacenters. It feels a bit round-about to need to do this probing. But without it, the problem seems to be that, with a v4 signature, the location constraint is included in the Authorization header. When that is the wrong location, AWS S3 rejects it. I do wonder though if there is an easier way that I am currently missing. Sponsored-by: Dartmouth College's DANDI project --- CHANGELOG | 3 ++ Remote/S3.hs | 42 +++++++++++++++---- ..._4f41ad125c1af26cb94bd40c61cfcd9f._comment | 12 ++++++ doc/special_remotes/S3.mdwn | 9 ++-- 4 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment diff --git a/CHANGELOG b/CHANGELOG index 434d2c160c..6fb6f99d6b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -10,6 +10,9 @@ git-annex (10.20250722) UNRELEASED; urgency=medium a different S3 host, the default remains signature=v2. * webapp: Support setting up S3 buckets in regions that need v4 signatures. + * S3: When initremote is given the name of a bucket that already exists, + automatically set datacenter to the right value, rather than needing it + to be explicitly set. (This needs aws-0.23) -- Joey Hess Wed, 30 Jul 2025 13:45:42 -0400 diff --git a/Remote/S3.hs b/Remote/S3.hs index 1fa8d44cd3..4da5ff3f36 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -193,7 +193,7 @@ gen r u rc gc rs = do c <- parsedRemoteConfig remote rc cst <- remoteCost gc c expensiveRemoteCost info <- extractS3Info c - hdl <- mkS3HandleVar c gc u + hdl <- mkS3HandleVar False c gc u magic <- liftIO initMagicMime return $ new c cst info hdl magic where @@ -287,7 +287,17 @@ s3Setup' ss u mcreds c gc =<< configParser remote c' c'' <- if isAnonymous pc then pure c' - else setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds + else do + v <- setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds + if M.member datacenterField c || M.member regionField c + then return v + -- Check if a bucket with this name + -- already exists, and if so, use + -- that location, rather than the + -- default datacenterField. + else getBucketLocation pc gc u >>= return . \case + Nothing -> v + Just loc -> M.insert datacenterField (Proposed $ T.unpack loc) v pc' <- either giveup return . parseRemoteConfig c'' =<< configParser remote c'' info <- extractS3Info pc' @@ -322,7 +332,7 @@ s3Setup' ss u mcreds c gc =<< configParser remote archiveconfig info <- extractS3Info pc' checkexportimportsafe pc' info - hdl <- mkS3HandleVar pc' gc u + hdl <- mkS3HandleVar False pc' gc u withS3HandleOrFail u hdl $ writeUUIDFile pc' u info use archiveconfig pc' info @@ -775,6 +785,22 @@ checkPresentExportWithContentIdentifierS3 hv r info _k loc knowncids = where o = T.pack $ bucketExportLocation info loc +getBucketLocation :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex (Maybe S3.LocationConstraint) +getBucketLocation c gc u = do +#if MIN_VERSION_aws(0,23,0) + info <- extractS3Info c + let info' = info { region = Nothing, host = Nothing } + -- Force anonymous access, because this API call does not work + -- when used in an authenticated context. + hdl <- mkS3HandleVar True c gc u + withS3HandleOrFail u hdl $ \h -> do + r <- liftIO $ tryNonAsync $ runResourceT $ + sendS3Handle h (S3.getBucketLocation $ bucket info') + return $ either (const Nothing) (Just . S3.gblrLocationConstraint) r +#else + return Nothing +#endif + {- Generate the bucket if it does not already exist, including creating the - UUID file within the bucket. - @@ -786,7 +812,7 @@ genBucket :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex () genBucket c gc u = do showAction "checking bucket" info <- extractS3Info c - hdl <- mkS3HandleVar c gc u + hdl <- mkS3HandleVar False c gc u withS3HandleOrFail u hdl $ \h -> go info h =<< checkUUIDFile c u info h where @@ -896,9 +922,9 @@ giveupS3HandleProblem S3HandleAnonymousOldAws _ = {- Prepares a S3Handle for later use. Does not connect to S3 or do anything - else expensive. -} -mkS3HandleVar :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar -mkS3HandleVar c gc u = liftIO $ newTVarIO $ Left $ - if isAnonymous c +mkS3HandleVar :: Bool -> ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar +mkS3HandleVar forceanonymous c gc u = liftIO $ newTVarIO $ Left $ + if forceanonymous || isAnonymous c then #if MIN_VERSION_aws(0,23,0) go =<< liftIO AWS.anonymousCredentials @@ -1380,7 +1406,7 @@ enableBucketVersioning ss info c gc u = do where enableversioning b = do showAction "checking bucket versioning" - hdl <- mkS3HandleVar c gc u + hdl <- mkS3HandleVar False c gc u let setversioning = S3.putBucketVersioning b S3.VersioningEnabled withS3HandleOrFail u hdl $ \h -> #if MIN_VERSION_aws(0,24,3) diff --git a/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment b/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment new file mode 100644 index 0000000000..fc1f37bcb8 --- /dev/null +++ b/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2025-08-13T18:22:53Z" + content=""" +I got GetBucketLocation to work, although only when git-annex is built with +aws-0.23 or newer. + + joey@darkstar:~/tmp/a9>git-annex initremote s3-originnew type=S3 importtree=yes encryption=none autoenable=true bucket=dandiarchive fileprefix=zarr-checksums/2ac71edb-738c-40ac-bd8c-8ca985adaa12/ + initremote s3-originnew (checking bucket...) ok + (recording state in git...) +"""]] diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn index 96dfdb56d7..935839a815 100644 --- a/doc/special_remotes/S3.mdwn +++ b/doc/special_remotes/S3.mdwn @@ -35,11 +35,10 @@ the S3 remote. embedcreds without gpg encryption. * `datacenter` - Specifies which Amazon datacenter - to use for the bucket. Defaults to "US". Other values include "EU" - (which is EU/Ireland), "us-west-1", "us-west-2", "ap-southeast-1", - "ap-southeast-2", and "sa-east-1". See Amazon's documentation for a - complete list. Configuring this is equivilant to configuring both - `host` and `region`. + to use when creating a bucket. Defaults to "US". Other values include "EU" + (which is EU/Ireland), "us-west-1", "us-west-2", etc. See Amazon's + documentation for a complete list. Configuring this is equivilant to + configuring both `host` and `region`. * `storageclass` - Default is "STANDARD". Consult S3 provider documentation for pricing details and available -- 2.30.2