From 4f6c6114fb487b5f84aa7e6922786b0e19806525 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 7 Nov 2022 14:44:51 -0400 Subject: [PATCH] avoid splitting repo tests into too small parts around -J16 The initTests have to be run once per part, and a point of diminishing returns can be reached where more work is being done to set up for 1 or 2 tests than to run them. This is better than a hard cap of -J8 or so, because it lets other things than these particular tests still be parallelized at -J16. Sponsored-by: Dartmouth College's Datalad project --- Test.hs | 10 ++++++- ..._e7da4a54b27f459d528a7651fd615913._comment | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment diff --git a/Test.hs b/Test.hs index cb0c515ee9..6d3c58d3c5 100644 --- a/Test.hs +++ b/Test.hs @@ -355,7 +355,15 @@ repoTests note numparts = map mk $ sep mk l = testGroup groupname (initTests : map adddep l) adddep = Test.Tasty.after AllSucceed (groupname ++ "." ++ initTestsName) groupname = "Repo Tests " ++ note - sep = sep' (replicate numparts []) + sep l = + -- Avoid separating into parts that contain less than + -- 5 tests each. Since the tests depend on the + -- initTests, this avoids spending too much work running + -- the initTests once per part. + let numparts' = if length l `div` numparts > 5 + then numparts + else length l `div` 5 + in sep' (replicate numparts' []) l sep' (p:ps) (l:ls) = sep' (ps++[l:p]) ls sep' ps [] = ps sep' [] _ = [] diff --git a/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment new file mode 100644 index 0000000000..2118ae0519 --- /dev/null +++ b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2022-11-07T17:07:52Z" + content=""" +I wondered if running the add and init tests once per worker was causing +a diminishing returns at higher -J levels. Since at -J16, each worker runs +the add and init tests followed by only 2 or 3 other tests, the time spent +on the add and init tests becomes more and more significant. + +Timings from my laptop (with 4 cores): + + -J16 250 seconds + -J8 212 seconds + -J4 214 seconds + -J2 307 seconds + +So a small diminishing returns at -J16. + +After some improvements: + + -J16 223 seconds + -J8 218 seconds + -J4 214 seconds + +That won't solve smaug being so overloaded though. +If anything, it will just make it get to some other test before it times out.. +"""]] -- 2.30.2