From a93163d6f7c5192226604cb4fe1dc0715be77989 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 30 Aug 2022 15:20:04 -0400 Subject: [PATCH] optimise linker in linux standalone tarballs Trick the linker into not doing unncessary work searching for optimised libraries that are not present, by symlinking the directories where optimised libs would be to the main lib dir. This reduces the ENOENT of git-annex init by about 1/2. The linker always finds the files where it looks first time now. I have not looked at what the wall clock speedup might be, it's probably rather small. If a x86-64-v5 comes to be, the list will need to be extended. And there may be other directories used on some machines that I have missed. Not done for arm64 yet, or any uncommon architectures. Sponsored-by: Dartmouth College's Datalad project --- Build/LinuxMkLibs.hs | 49 +++++++++++++++++++ CHANGELOG | 1 + ..._growing_number_of_libpcre.so_lookups.mdwn | 2 + ..._91ab4b860a074c90e5cbeb03e3515ad4._comment | 12 +++++ 4 files changed, 64 insertions(+) create mode 100644 doc/todo/standlone__58___growing_number_of_libpcre.so_lookups/comment_4_91ab4b860a074c90e5cbeb03e3515ad4._comment diff --git a/Build/LinuxMkLibs.hs b/Build/LinuxMkLibs.hs index d6d7df62fa..da66a0153e 100644 --- a/Build/LinuxMkLibs.hs +++ b/Build/LinuxMkLibs.hs @@ -14,6 +14,7 @@ import Data.List import System.Posix.Files import Control.Monad.IfElse import Control.Applicative +import qualified System.Info import Prelude import Utility.LinuxMkLibs @@ -82,9 +83,57 @@ consolidateUsrLib top libdirs = go [] libdirs unless (dirCruft f) $ unlessM (doesDirectoryExist src) $ renameFile src dst + symlinkHwCapDirs top d go c rest _ -> go (x:c) rest +{- The linker looks for optimised versions of libraries depending on the + - hardware capabilities. That causes a lot of extra work searching for + - libraries, so to avoid it, make symlinks from the hwcap directories + - to the libdir. This way, the linker will find a library the first place + - it happens to look for it. + -} +symlinkHwCapDirs :: FilePath -> FilePath -> IO () +symlinkHwCapDirs top libdir = forM_ hwcapdirs $ \d -> + unlessM (doesDirectoryExist (top ++ libdir d)) $ do + createDirectoryIfMissing True (top ++ libdir takeDirectory d) + link <- relPathDirToFile + (toRawFilePath (top ++ takeDirectory (libdir d))) + (toRawFilePath (top ++ libdir)) + let link' = case fromRawFilePath link of + "" -> "." + l -> l + createSymbolicLink link' (top ++ libdir d) + where + hwcapdirs = case System.Info.arch of + "x86_64" -> + -- See glibc's sysdeps/x86_64/dl-hwcaps-subdirs.c + -- for list of subarchitecture directories. + [ "glibc-hwcaps/x86-64-v2" + , "glibc-hwcaps/x86-64-v3" + , "glibc-hwcaps/x86-64-v4" + -- The linker later checks these, and will check + -- them when none of the above subarchitectures + -- are supported by the processor, so make them + -- just in case. + , "tls/x86_64" + , "x86_64" + ] + "i386" -> + [ "tls/i686" + [ "tls/i586" + , "i686" + , + "i586" + ] + "arm" -> + -- Probably not complete, only what I have + -- observed. + [ "tls/v7l" + , "v7l" + ] + _ -> [] + {- Installs a linker shim script around a binary. - - Note that each binary is put into its own separate directory, diff --git a/CHANGELOG b/CHANGELOG index b387ebe52b..6961025c06 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ git-annex (10.20220823) UNRELEASED; urgency=medium * Include the assistant and webapp when building with cabal 3.4.1.0. * Merged the webapp build flag into the assistant build flag. + * Optimise linker in linux standalone tarballs. -- Joey Hess Mon, 29 Aug 2022 15:03:04 -0400 diff --git a/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups.mdwn b/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups.mdwn index 6ba1d0fb9b..2127dc66de 100644 --- a/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups.mdwn +++ b/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups.mdwn @@ -51,3 +51,5 @@ pr-55/build-ubuntu.yaml-799-32886238-failed/test-annex-more/6_Seek of dynlibs.tx anything to be done or should we just raise the bar to 140 in that test? [[!tag projects/datalad]] + +> [[done]] --[[Joey]] diff --git a/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups/comment_4_91ab4b860a074c90e5cbeb03e3515ad4._comment b/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups/comment_4_91ab4b860a074c90e5cbeb03e3515ad4._comment new file mode 100644 index 0000000000..efd7ddd917 --- /dev/null +++ b/doc/todo/standlone__58___growing_number_of_libpcre.so_lookups/comment_4_91ab4b860a074c90e5cbeb03e3515ad4._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2022-08-30T19:14:50Z" + content=""" +Implemented that idea now, so you may want to drop your test +back down to 40 or 50. + +So far it's only implemented for amd64, i386, and arm. Other architectures +might also have such directories that could be symlinked, I have not +investigated. +"""]] -- 2.30.2