From afff2bb47df7938bfc0a1e65f765394e14156d87 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Aug 2025 15:14:01 -0400 Subject: [PATCH] onlyencryptcreds=yes initremote: When onlyencryptcreds=yes is used along with embedcreds=yes, and encryption is enabled, only encrypt the embedded creds, without encrypting the content of the special remote. Useful for exporttree=yes/importtree=yes remotes. Sponsored-by: Joshua Antonishen --- Annex/SpecialRemote/Config.hs | 3 ++ CHANGELOG | 4 ++ Creds.hs | 23 ++++++---- Remote/Helper/Encryptable.hs | 42 ++++++++++++++----- ...ent_enableremote_changing_encryption.mdwn} | 6 ++- doc/encryption.mdwn | 4 ++ doc/special_remotes/S3.mdwn | 6 ++- doc/special_remotes/glacier.mdwn | 4 ++ doc/special_remotes/tahoe.mdwn | 4 ++ doc/special_remotes/webdav.mdwn | 4 ++ doc/todo/encrypt_only_the_credentials.mdwn | 2 + ..._18a06dcd03257332f2694a0c8b0e6a26._comment | 13 ++++++ 12 files changed, 94 insertions(+), 21 deletions(-) rename doc/bugs/{possible_to_enable_encryption_for_exporttree_remote.mdwn => prevent_enableremote_changing_encryption.mdwn} (63%) create mode 100644 doc/todo/encrypt_only_the_credentials/comment_6_18a06dcd03257332f2694a0c8b0e6a26._comment diff --git a/Annex/SpecialRemote/Config.hs b/Annex/SpecialRemote/Config.hs index 059a62f901..5f9d6db831 100644 --- a/Annex/SpecialRemote/Config.hs +++ b/Annex/SpecialRemote/Config.hs @@ -85,6 +85,9 @@ chunksizeField = Accepted "chunksize" embedCredsField :: RemoteConfigField embedCredsField = Accepted "embedcreds" +onlyEncryptCredsField :: RemoteConfigField +onlyEncryptCredsField = Accepted "onlyencryptcreds" + preferreddirField :: RemoteConfigField preferreddirField = Accepted "preferreddir" diff --git a/CHANGELOG b/CHANGELOG index a43bae3fd4..b7ce9fc91f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -20,6 +20,10 @@ git-annex (10.20250722) UNRELEASED; urgency=medium branch, re-adjusting errors out, rather than losing that merge commit. * sync: When another branch has been manually merged into an adjusted branch, error out rather than only displaying a warning. + * initremote: When onlyencryptcreds=yes is used along with + embedcreds=yes, and encryption is enabled, only encrypt the embedded + creds, without encrypting the content of the special remote. Useful + for exporttree=yes/importtree=yes remotes. * Bump aws build dependency to 0.24.1. * stack.yaml: Update to lts-24.2. diff --git a/Creds.hs b/Creds.hs index 4e197d7001..2003f691e2 100644 --- a/Creds.hs +++ b/Creds.hs @@ -33,7 +33,7 @@ import Annex.Perms import Utility.FileMode import Crypto import Types.ProposedAccepted -import Remote.Helper.Encryptable (remoteCipher, remoteCipher', embedCreds, EncryptionIsSetup, extractCipher) +import Remote.Helper.Encryptable (remoteCipher, remoteCipher', CipherPurpose(..), embedCreds, EncryptionIsSetup, extractCipher) import Utility.Env (getEnv) import Utility.Base64 import qualified Utility.FileIO as F @@ -95,14 +95,19 @@ setRemoteCredPair' pc encsetup gc storage mcreds = case mcreds of where localcache creds = writeCacheCredPair creds storage - storeconfig creds key (Just cipher) = do + storeconfig creds key (Just (CipherAllPurpose cipher)) = + storeconfigcipher creds key cipher + storeconfig creds key (Just (CipherOnlyCreds cipher)) = + storeconfigcipher creds key cipher + storeconfig creds key Nothing = + storeconfig' key (Accepted (decodeBS $ toB64 $ encodeBS $ encodeCredPair creds)) + + storeconfigcipher creds key cipher = do cmd <- gpgCmd <$> Annex.getGitConfig s <- liftIO $ encrypt cmd (pc, gc) cipher (feedBytes $ L8.pack $ encodeCredPair creds) (readBytesStrictly return) storeconfig' key (Accepted (decodeBS (toB64 s))) - storeconfig creds key Nothing = - storeconfig' key (Accepted (decodeBS $ toB64 $ encodeBS $ encodeCredPair creds)) storeconfig' key val = return $ pc { parsedRemoteConfigMap = M.insert key (RemoteConfigValue val) (parsedRemoteConfigMap pc) @@ -127,14 +132,16 @@ getRemoteCredPair c gc storage = maybe fromcache (return . Just) =<< fromenv <|> getRemoteConfigValue key c case (getval, mcipher) of (Nothing, _) -> return Nothing - (Just enccreds, Just (cipher, storablecipher)) -> - fromenccreds (encodeBS enccreds) cipher storablecipher + (Just enccreds, Just ((CipherAllPurpose cipher, storablecipher))) -> + fromenccreds enccreds cipher storablecipher + (Just enccreds, Just ((CipherOnlyCreds cipher, storablecipher))) -> + fromenccreds enccreds cipher storablecipher (Just bcreds, Nothing) -> fromcreds $ decodeBS $ fromB64 $ encodeBS bcreds fromenccreds enccreds cipher storablecipher = do cmd <- gpgCmd <$> Annex.getGitConfig mcreds <- liftIO $ catchMaybeIO $ decrypt cmd (c, gc) cipher - (feedBytes $ L8.fromStrict $ fromB64 enccreds) + (feedBytes $ L8.fromStrict $ fromB64 $ encodeBS enccreds) (readBytesStrictly $ return . S8.unpack) case mcreds of Just creds -> fromcreds creds @@ -145,7 +152,7 @@ getRemoteCredPair c gc storage = maybe fromcache (return . Just) =<< fromenv case storablecipher of SharedCipher {} -> showLongNote "gpg error above was caused by an old git-annex bug in credentials storage. Working around it.." _ -> giveup "*** Insecure credentials storage detected for this remote! See https://git-annex.branchable.com/upgrades/insecure_embedded_creds/" - fromcreds $ decodeBS $ fromB64 enccreds + fromcreds $ decodeBS $ fromB64 $ encodeBS enccreds fromcreds creds = case decodeCredPair creds of Just credpair -> do writeCacheCredPair credpair storage diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs index 33eb5b3837..46ab018f7a 100644 --- a/Remote/Helper/Encryptable.hs +++ b/Remote/Helper/Encryptable.hs @@ -15,6 +15,7 @@ module Remote.Helper.Encryptable ( encryptionConfigParsers, parseEncryptionConfig, parseEncryptionMethod, + CipherPurpose(..), remoteCipher, remoteCipher', embedCreds, @@ -63,6 +64,8 @@ encryptionConfigParsers = , optionalStringParser pubkeysField HiddenField , yesNoParser embedCredsField Nothing (FieldDesc "embed credentials into git repository") + , yesNoParser onlyEncryptCredsField Nothing + (FieldDesc "only encrypt embedded credentials, not annexed files") , macFieldParser , optionalStringParser (Accepted "keyid") (FieldDesc "gpg key id") @@ -217,12 +220,14 @@ encryptionSetup c gc = do -- remotes (while being backward-compatible). (map Accepted ["keyid", "keyid+", "keyid-", "highRandomQuality"]) -remoteCipher :: ParsedRemoteConfig -> RemoteGitConfig -> Annex (Maybe Cipher) -remoteCipher c gc = fmap fst <$> remoteCipher' c gc +data CipherPurpose t = CipherAllPurpose t | CipherOnlyCreds t {- Gets encryption Cipher. The decrypted Ciphers are cached in the Annex - state. -} -remoteCipher' :: ParsedRemoteConfig -> RemoteGitConfig -> Annex (Maybe (Cipher, StorableCipher)) +remoteCipher :: ParsedRemoteConfig -> RemoteGitConfig -> Annex (Maybe (CipherPurpose Cipher)) +remoteCipher c gc = fmap fst <$> remoteCipher' c gc + +remoteCipher' :: ParsedRemoteConfig -> RemoteGitConfig -> Annex (Maybe (CipherPurpose Cipher, StorableCipher)) remoteCipher' c gc = case extractCipher c of Nothing -> return Nothing Just encipher -> do @@ -230,7 +235,7 @@ remoteCipher' c gc = case extractCipher c of cachedciper <- liftIO $ atomically $ M.lookup encipher <$> readTMVar cachev case cachedciper of - Just cipher -> return $ Just (cipher, encipher) + Just cipher -> return $ Just (purpose cipher, encipher) -- Not cached; decrypt it, making sure -- to only decrypt one at a time. Avoids -- prompting for decrypting the same thing twice @@ -245,7 +250,10 @@ remoteCipher' c gc = case extractCipher c of cipher <- liftIO $ decryptCipher gpgcmd (c, gc) encipher liftIO $ atomically $ putTMVar cachev $ M.insert encipher cipher cache - return $ Just (cipher, encipher) + return $ Just (purpose cipher, encipher) + purpose + | onlyEncryptCreds c = CipherOnlyCreds + | otherwise = CipherAllPurpose {- Checks if the remote's config allows storing creds in the remote's config. - @@ -262,11 +270,19 @@ embedCreds c = case getRemoteConfigValue embedCredsField c of (Just (_ :: String), Just (_ :: String)) -> True _ -> False -{- Gets encryption Cipher, and key encryptor. -} +onlyEncryptCreds :: ParsedRemoteConfig -> Bool +onlyEncryptCreds c = case getRemoteConfigValue onlyEncryptCredsField c of + Just v -> v + Nothing -> False + +{- Gets key data encryption Cipher, and key encryptor. -} cipherKey :: ParsedRemoteConfig -> RemoteGitConfig -> Annex (Maybe (Cipher, EncKey)) -cipherKey c gc = fmap make <$> remoteCipher c gc +cipherKey c gc = go <$> remoteCipher c gc where - make ciphertext = (ciphertext, encryptKey mac ciphertext) + go (Just (CipherAllPurpose ciphertext)) = + Just (ciphertext, encryptKey mac ciphertext) + go (Just (CipherOnlyCreds _)) = Nothing + go Nothing = Nothing mac = fromMaybe defaultMac $ getRemoteConfigValue macField c {- Stores an StorableCipher in a remote's configuration. -} @@ -297,7 +313,7 @@ extractCipher c = case (getRemoteConfigValue cipherField c, readkeys = KeyIds . splitc ',' isEncrypted :: ParsedRemoteConfig -> Bool -isEncrypted = isJust . extractCipher +isEncrypted c = isJust (extractCipher c) && not (onlyEncryptCreds c) -- Check if encryption is enabled. This can be done before encryption -- is fully set up yet, so the cipher might not be present yet. @@ -305,12 +321,16 @@ encryptionIsEnabled :: ParsedRemoteConfig -> Bool encryptionIsEnabled c = case getRemoteConfigValue encryptionField c of Nothing -> False Just NoneEncryption -> False - Just _ -> True + Just _ -> not (onlyEncryptCreds c) describeEncryption :: ParsedRemoteConfig -> String describeEncryption c = case extractCipher c of Nothing -> "none" - Just cip -> nameCipher cip ++ " (" ++ describeCipher cip ++ ")" + Just cip + | onlyEncryptCreds c -> "creds only; " ++ desc cip + | otherwise -> desc cip + where + desc cip = nameCipher cip ++ " (" ++ describeCipher cip ++ ")" nameCipher :: StorableCipher -> String nameCipher (SharedCipher _) = "shared" diff --git a/doc/bugs/possible_to_enable_encryption_for_exporttree_remote.mdwn b/doc/bugs/prevent_enableremote_changing_encryption.mdwn similarity index 63% rename from doc/bugs/possible_to_enable_encryption_for_exporttree_remote.mdwn rename to doc/bugs/prevent_enableremote_changing_encryption.mdwn index 7976058550..f12bfcf862 100644 --- a/doc/bugs/possible_to_enable_encryption_for_exporttree_remote.mdwn +++ b/doc/bugs/prevent_enableremote_changing_encryption.mdwn @@ -7,4 +7,8 @@ encryption for such a remote: enableremote d (encryption setup) (encryption key stored in git repository) ok (recording state in git...) -This config change should not be allowed. --[[Joey]] +This config change should not be allowed. + +Also, the new onlyencryptcreds=yes setting can passed to enableremote, +which changes a previously encrypted remote to not use encryption for the +data stored on it. That should also not be allowed. --[[Joey]] diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn index 4ae705e680..5cf5656ebc 100644 --- a/doc/encryption.mdwn +++ b/doc/encryption.mdwn @@ -148,6 +148,7 @@ non-empty remote. Special remotes that need some form of credentials, such as a password, may support embedding the credentials in the git repository, using embedcreds=yes. See individual special remotes' documentation for details. + When credentials are embedded in the repository, they're also encrypted using whatever encryption setting has been selected for the repository. @@ -155,3 +156,6 @@ Such credentials are also cached locally in a file only you can read, in `.git/annex/creds/`. If you prefer to not expose the credentials on disk in unencrypted form, you can disable this cache, by setting the `annex.cachecreds` config to `false`. + +To only encrypt the credentials, without encrypting any data stored in the +special remote, use onlyencryptcreds=yes. diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn index 36c3b101a3..d36dfa1b36 100644 --- a/doc/special_remotes/S3.mdwn +++ b/doc/special_remotes/S3.mdwn @@ -25,7 +25,7 @@ the S3 remote. * `chunk` - Enables [[chunking]] when storing large files. `chunk=1MiB` is a good starting point for chunking. -* `embedcreds` - Optional. Set to "yes" embed the login credentials inside +* `embedcreds` - Optional. Set to "yes" to embed the login credentials inside the git repository, which allows other clones to also access them. This is the default when gpg encryption is enabled; the credentials are stored encrypted and only those with the repository's keys can access them. @@ -34,6 +34,10 @@ the S3 remote. Think carefully about who can access your repository before using embedcreds without gpg encryption. +* `onlyencryptcreds` - Optional. Set to "yes" to make the `encryption` + only be used for the embedded login credentials, but not used to encrypt + the content stored on the special remote. + * `datacenter` - Specifies which Amazon datacenter to use when creating a bucket. Defaults to "US". Other values include "EU" (which is EU/Ireland), "us-west-1", "us-west-2", etc. See Amazon's diff --git a/doc/special_remotes/glacier.mdwn b/doc/special_remotes/glacier.mdwn index 5dde37208c..8385749bc4 100644 --- a/doc/special_remotes/glacier.mdwn +++ b/doc/special_remotes/glacier.mdwn @@ -35,6 +35,10 @@ the Glacier remote. Think carefully about who can access your repository before using embedcreds without gpg encryption. +* `onlyencryptcreds` - Optional. Set to "yes" to make the `encryption` + only be used for the embedded login credentials, but not used to encrypt + the content stored on the special remote. + * `datacenter` - Defaults to "us-east-1". * `vault` - By default, a vault name is chosen based on the remote name diff --git a/doc/special_remotes/tahoe.mdwn b/doc/special_remotes/tahoe.mdwn index df1ca620fb..60e3bad619 100644 --- a/doc/special_remotes/tahoe.mdwn +++ b/doc/special_remotes/tahoe.mdwn @@ -35,6 +35,10 @@ the tahoe remote. whether you want to give them access to your tahoe system before using embedcreds! +* `onlyencryptcreds` - Optional. Set to "yes" to make the `encryption` + only be used for the embedded tahoe credentials, but not used to encrypt + the content stored on the special remote. + Setup example: # TAHOE_FURL=... git annex initremote tahoe type=tahoe embedcreds=yes diff --git a/doc/special_remotes/webdav.mdwn b/doc/special_remotes/webdav.mdwn index 752eab25b0..ae766f6be3 100644 --- a/doc/special_remotes/webdav.mdwn +++ b/doc/special_remotes/webdav.mdwn @@ -24,6 +24,10 @@ the webdav remote. Think carefully about who can access your repository before using embedcreds without gpg encryption. +* `onlyencryptcreds` - Optional. Set to "yes" to make the `encryption` + only be used for the embedded login credentials, but not used to encrypt + the content stored on the special remote. + * `url` - Required. The URL to the WebDAV directory where files will be stored. This can be a subdirectory of a larger WebDAV repository, and will be created as needed. Use of a https URL is strongly diff --git a/doc/todo/encrypt_only_the_credentials.mdwn b/doc/todo/encrypt_only_the_credentials.mdwn index d510f45002..cdfeba9772 100644 --- a/doc/todo/encrypt_only_the_credentials.mdwn +++ b/doc/todo/encrypt_only_the_credentials.mdwn @@ -1,3 +1,5 @@ Is it possible to add an option, for initremote/enableremote, to encrypt the credentials but not the contents? Then it would be possible to have an exporttree remote while using embedcreds. It would also be good if locally stored credentials could be stored in encrypted form, and decrypted for use as needed. I'm uneasy about keeping credentials accessible without a passphrase. [[!tag confirmed]] + +> [[done]] --[[Joey]] diff --git a/doc/todo/encrypt_only_the_credentials/comment_6_18a06dcd03257332f2694a0c8b0e6a26._comment b/doc/todo/encrypt_only_the_credentials/comment_6_18a06dcd03257332f2694a0c8b0e6a26._comment new file mode 100644 index 0000000000..b6ec442ced --- /dev/null +++ b/doc/todo/encrypt_only_the_credentials/comment_6_18a06dcd03257332f2694a0c8b0e6a26._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2025-08-20T19:07:50Z" + content=""" +I've implemented onlyencryptcreds=yes. + +Note that this bug needs to be addressed to avoid some foot shooting +with this new option: [[bugs/prevent_enableremote_changing_encryption]] +Since the foot shooting is no worse that previously possible foot shooting +shown in that bug, I went ahead and added this new feature before +addressing that bug. +"""]] -- 2.30.2