From 86e9e88530c98c4fffcb616fbb7502024f6b6106 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 30 Jul 2025 15:25:17 -0400 Subject: [PATCH] add git-remote-p2p-annex Added git-remote-p2p-annex, which allows git pull and push to P2P networks provided by external commands. This is a refactor of git-remote-tor-annex, and should just work. Except possibly for quirks with the address parsing. I've checked that the address parsing basically works. One thing I don't understand is why git-remote-tor-annex removes "/*" from the end of the address. The git history does not provide any hints. So I didn't make git-remote-p2p-annex do the same. Maybe that is needed in some situation? But, a P2P address could contain "/", so removing it would be a problem. I can't see anything in gitremote-helpers(7) about why the url might get such a thing added to the end of it. My guess is that is not needed for tor either (but does no harm there since onion addresses never contain "/"). At this point, the implementation of generic P2P transports needs only remotedaemon support. --- .gitignore | 1 + Build/Standalone.hs | 1 + CHANGELOG | 2 ++ CmdLine/GitRemoteP2PAnnex.hs | 62 ++++++++++++++++++++++++++++++++++ CmdLine/GitRemoteTorAnnex.hs | 53 +++++------------------------ CmdLine/Multicall.hs | 4 ++- Makefile | 6 +++- git-annex.cabal | 1 + git-annex.hs | 2 ++ standalone/linux/skel/runshell | 2 +- 10 files changed, 86 insertions(+), 48 deletions(-) create mode 100644 CmdLine/GitRemoteP2PAnnex.hs diff --git a/.gitignore b/.gitignore index a6daf2efd6..3c4a7a277c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ Build/MakeMans git-annex git-annex-shell git-remote-annex +git-remote-p2p-annex git-remote-tor-annex man doc/.ikiwiki diff --git a/Build/Standalone.hs b/Build/Standalone.hs index 2c0c8d1b55..b9a683fb0b 100644 --- a/Build/Standalone.hs +++ b/Build/Standalone.hs @@ -222,6 +222,7 @@ installGitAnnex topdir = go (topdir literalOsPath "bin") unlessM (boolSystem "strip" [File (fromOsPath (bindir literalOsPath "git-annex"))]) $ error "strip failed" createSymbolicLink "git-annex" (fromOsPath (bindir literalOsPath "git-annex-shell")) + createSymbolicLink "git-annex" (fromOsPath (bindir literalOsPath "git-remote-p2p-annex")) createSymbolicLink "git-annex" (fromOsPath (bindir literalOsPath "git-remote-tor-annex")) createSymbolicLink "git-annex" (fromOsPath (bindir literalOsPath "git-remote-annex")) diff --git a/CHANGELOG b/CHANGELOG index cb1223ee0d..29e18503ab 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,8 @@ git-annex (10.20250722) UNRELEASED; urgency=medium * p2p: Added --enable option, which can be used to enable P2P networks provided by external commands git-annex-p2p- + * Added git-remote-p2p-annex, which allows git pull and push to + P2P networks provided by external commands. -- Joey Hess Wed, 30 Jul 2025 13:45:42 -0400 diff --git a/CmdLine/GitRemoteP2PAnnex.hs b/CmdLine/GitRemoteP2PAnnex.hs new file mode 100644 index 0000000000..fcc77e1911 --- /dev/null +++ b/CmdLine/GitRemoteP2PAnnex.hs @@ -0,0 +1,62 @@ +{- git-remote-p2p-annex program + - + - Copyright 2016-2025 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module CmdLine.GitRemoteP2PAnnex where + +import Common +import qualified Annex +import qualified Git.CurrentRepo +import P2P.Protocol +import P2P.IO +import Utility.AuthToken +import Annex.UUID +import P2P.Address +import P2P.Auth +import Annex.Action + +run :: [String] -> IO () +run = runner mkaddress + where + mkaddress address = + fromMaybe (error $ "unable to parse address: " ++ address) $ + unformatP2PAddress (p2pAnnexScheme ++ ":" ++ address) + +runner :: (String -> P2PAddress) -> [String] -> IO () +runner mkaddress (_remotename:saddress:[]) = forever $ + getLine >>= \case + "capabilities" -> putStrLn "connect" >> ready + "connect git-upload-pack" -> go UploadPack + "connect git-receive-pack" -> go ReceivePack + l -> giveup $ "gitremote-helpers protocol error at " ++ show l + where + address = mkaddress saddress + go service = do + ready + connectService address service >>= \case + Right exitcode -> exitWith exitcode + Left e -> giveup $ describeProtoFailure e + ready = do + putStrLn "" + hFlush stdout +runner _ (_remotename:[]) = giveup "remote address not configured" +runner _ _ = giveup "expected remote name and address parameters" + +connectService :: P2PAddress -> Service -> IO (Either ProtoFailure ExitCode) +connectService address service = do + state <- Annex.new =<< Git.CurrentRepo.get + Annex.eval state $ do + authtoken <- fromMaybe nullAuthToken + <$> loadP2PRemoteAuthToken address + myuuid <- getUUID + g <- Annex.gitRepo + conn <- liftIO $ connectPeer (Just g) address + runst <- liftIO $ mkRunState Client + r <- liftIO $ runNetProto runst conn $ auth myuuid authtoken noop >>= \case + Just _theiruuid -> connect service stdin stdout + Nothing -> giveup $ "authentication failed, perhaps you need to set " ++ p2pAuthTokenEnv + quiesce False + return r diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs index 476eab0231..8ec9326d14 100644 --- a/CmdLine/GitRemoteTorAnnex.hs +++ b/CmdLine/GitRemoteTorAnnex.hs @@ -8,40 +8,19 @@ module CmdLine.GitRemoteTorAnnex where import Common -import qualified Annex -import qualified Git.CurrentRepo -import P2P.Protocol -import P2P.IO import Utility.Tor -import Utility.AuthToken -import Annex.UUID import P2P.Address -import P2P.Auth -import Annex.Action +import CmdLine.GitRemoteP2PAnnex (runner) run :: [String] -> IO () -run (_remotename:address:[]) = forever $ - getLine >>= \case - "capabilities" -> putStrLn "connect" >> ready - "connect git-upload-pack" -> go UploadPack - "connect git-receive-pack" -> go ReceivePack - l -> giveup $ "gitremote-helpers protocol error at " ++ show l +run = runner mkaddress where - (onionaddress, onionport) - | '/' `elem` address = parseAddressPort $ - reverse $ takeWhile (/= '/') $ reverse address - | otherwise = parseAddressPort address - go service = do - ready - connectService onionaddress onionport service >>= \case - Right exitcode -> exitWith exitcode - Left e -> giveup $ describeProtoFailure e - ready = do - putStrLn "" - hFlush stdout - -run (_remotename:[]) = giveup "remote address not configured" -run _ = giveup "expected remote name and address parameters" + mkaddress address = + let (onionaddress, onionport) + | '/' `elem` address = parseAddressPort $ + reverse $ takeWhile (/= '/') $ reverse address + | otherwise = parseAddressPort address + in TorAnnex onionaddress onionport parseAddressPort :: String -> (OnionAddress, OnionPort) parseAddressPort s = @@ -49,19 +28,3 @@ parseAddressPort s = in case readish sp of Nothing -> giveup "onion address must include port number" Just p -> (OnionAddress a, p) - -connectService :: OnionAddress -> OnionPort -> Service -> IO (Either ProtoFailure ExitCode) -connectService address port service = do - state <- Annex.new =<< Git.CurrentRepo.get - Annex.eval state $ do - authtoken <- fromMaybe nullAuthToken - <$> loadP2PRemoteAuthToken (TorAnnex address port) - myuuid <- getUUID - g <- Annex.gitRepo - conn <- liftIO $ connectPeer (Just g) (TorAnnex address port) - runst <- liftIO $ mkRunState Client - r <- liftIO $ runNetProto runst conn $ auth myuuid authtoken noop >>= \case - Just _theiruuid -> connect service stdin stdout - Nothing -> giveup $ "authentication failed, perhaps you need to set " ++ p2pAuthTokenEnv - quiesce False - return r diff --git a/CmdLine/Multicall.hs b/CmdLine/Multicall.hs index fa25836d37..566cd6d988 100644 --- a/CmdLine/Multicall.hs +++ b/CmdLine/Multicall.hs @@ -1,6 +1,6 @@ {- git-annex multicall binary - - - Copyright 2024 Joey Hess + - Copyright 2024-2025 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -17,12 +17,14 @@ import qualified Data.Map as M data OtherMultiCallCommand = GitAnnexShell | GitRemoteAnnex + | GitRemoteP2PAnnex | GitRemoteTorAnnex otherMulticallCommands :: M.Map String OtherMultiCallCommand otherMulticallCommands = M.fromList [ ("git-annex-shell", GitAnnexShell) , ("git-remote-annex", GitRemoteAnnex) + , ("git-remote-p2p-annex", GitRemoteP2PAnnex) , ("git-remote-tor-annex", GitRemoteTorAnnex) ] diff --git a/Makefile b/Makefile index e161d79bdd..e1ddb9d510 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all=git-annex git-annex-shell git-remote-annex git-remote-tor-annex mans docs +all=git-annex git-annex-shell git-remote-annex git-remote-p2p-annex git-remote-tor-annex mans docs # set to "./Setup" if you lack a cabal program. Or can be set to "stack" BUILDER?=cabal @@ -74,6 +74,9 @@ git-annex-shell: git-annex git-remote-annex: git-annex @ln -sf git-annex git-remote-annex +git-remote-p2p-annex: git-annex + @ln -sf git-annex git-remote-p2p-annex + git-remote-tor-annex: git-annex @ln -sf git-annex git-remote-tor-annex @@ -93,6 +96,7 @@ install-bins: build ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-remote-annex ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-remote-tor-annex + ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-remote-p2p-annex install-desktop: build Build/InstallDesktopFile ./Build/InstallDesktopFile $(PREFIX)/bin/git-annex || true diff --git a/git-annex.cabal b/git-annex.cabal index ac9941b902..199e97e692 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -649,6 +649,7 @@ Executable git-annex CmdLine.AnnexSetter CmdLine.Multicall CmdLine.GitRemoteAnnex + CmdLine.GitRemoteP2PAnnex CmdLine.GitRemoteTorAnnex CmdLine.Option CmdLine.Seek diff --git a/git-annex.hs b/git-annex.hs index 8f98d77265..5d1b1cf674 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -16,6 +16,7 @@ import CmdLine.Multicall import qualified CmdLine.GitAnnex import qualified CmdLine.GitAnnexShell import qualified CmdLine.GitRemoteAnnex +import qualified CmdLine.GitRemoteP2PAnnex import qualified CmdLine.GitRemoteTorAnnex import qualified Test import qualified Benchmark @@ -39,6 +40,7 @@ main = sanitizeTopLevelExceptionMessages $ withSocketsDo $ do run ps n = case M.lookup (takeFileName n) otherMulticallCommands of Just GitAnnexShell -> CmdLine.GitAnnexShell.run ps Just GitRemoteAnnex -> CmdLine.GitRemoteAnnex.run ps + Just GitRemoteP2PAnnex -> CmdLine.GitRemoteP2PAnnex.run ps Just GitRemoteTorAnnex -> CmdLine.GitRemoteTorAnnex.run ps Nothing -> CmdLine.GitAnnex.run Test.optParser diff --git a/standalone/linux/skel/runshell b/standalone/linux/skel/runshell index 1360d31fe8..63cc1ab9ea 100755 --- a/standalone/linux/skel/runshell +++ b/standalone/linux/skel/runshell @@ -206,7 +206,7 @@ case "$os" in # The bundled git does not work well on sdcard, so delete # it and use termux's git which works better. cd "$base" - find . | grep git | grep -v git-annex | grep -v git-remote-annex | grep -v git-remote-tor-annex | grep -v git-remote-gcrypt | xargs rm -rf + find . | grep git | grep -v git-annex | grep -v git-remote-annex | grep -v git-remote-p2p-annex | grep -v git-remote-tor-annex | grep -v git-remote-gcrypt | xargs rm -rf # Use termux's uname, which knows it's on android, # not the bundled one. rm -f bin/uname -- 2.30.2