[WIP] towards better plots

Author: tbagrel1

.github/workflows/ci.yaml

@@ -87,7 +87,7 @@ jobs:
     - name: Build benchmarks
       run: nix-shell --arg installHls 'false' --pure --run "cabal build -w ${{ env.ghc-exe }} linear-base:bench:bench"
     - name: Run benchmarks in isolation
-      run: nix-shell --arg installHls 'false' --pure --run "echo $'=== Benchmarks (isolation) ===\n\n' > benchmark_${{ env.ghc-name }}.txt && cabal run -w ${{ env.ghc-exe }} -v0 linear-base:bench:bench -- -l | while read -r name; do cabal run -w ${{ env.ghc-exe }} -v0 linear-base:bench:bench -- --timeout 60 -p '"'$'"0 == \"'\""'$'"name\"'\"' 2>&1 | tee -a benchmark_${{ env.ghc-name }}.txt; done"
+      run: nix-shell --arg installHls 'false' --pure --run "echo $'=== Benchmarks (isolation) ===\n\n' > benchmark_${{ env.ghc-name }}.txt && cabal run -w ${{ env.ghc-exe }} -v0 linear-base:bench:bench -- -l | while read -r name; do cabal run -w ${{ env.ghc-exe }} -v0 linear-base:bench:bench -- --timeout 1200 --stdev 10 -p '"'$'"0 == \"'\""'$'"name\"'\"' 2>&1 | tee -a benchmark_${{ env.ghc-name }}.txt; done"
     - name: Upload benchmark results
       uses: actions/upload-artifact@v4
       with:

avg_manual_bench_and_patch.py

@@ -0,0 +1,97 @@
+import sys
+import pandas as pd
+from decimal import Decimal, getcontext
+
+def format_significant_decimal(x, digits=5):
+    if pd.isna(x):
+        return ''
+    if x == 0:
+        return "0"
+    
+    getcontext().prec = digits + 5
+    d = Decimal(str(x))
+    
+    exponent = d.adjusted()
+    shift = digits - exponent - 1
+    rounded = d.scaleb(shift).quantize(Decimal('1')).scaleb(-shift)
+    s = format(rounded.normalize(), 'f')
+    return s
+
+def format_dataframe_numbers(df, digits=5):
+    for col in df.select_dtypes(include=["number"]).columns:
+        df[col] = df[col].apply(lambda x: format_significant_decimal(x, digits))
+    return df
+
+def usage():
+    print("Usage:")
+    print("  python script.py manual_bench.csv averaged_manual_bench.csv")
+    print("  OR")
+    print("  python script.py manual_bench.csv averaged_manual_bench.csv auto_bench.csv updated_auto_bench.csv columns_to_patch")
+    print("    where columns_to_patch is a comma-separated list like: AutoCol1:ManualCol1,AutoCol2:ManualCol2 or just AutoCol1,AutoCol2 (for identity mapping)")
+    sys.exit(1)
+
+if not (4 <= len(sys.argv) <= 6):
+    usage()
+
+manual_path = sys.argv[1]
+averaged_manual_path = sys.argv[2]
+
+patching_enabled = False
+if len(sys.argv) == 6:
+    patching_enabled = True
+    auto_path = sys.argv[3]
+    updated_auto_path = sys.argv[4]
+    raw_column_mappings = sys.argv[5].split(",")
+
+    # Parse mappings like "A:B" or "A"
+    column_mappings = []
+    for entry in raw_column_mappings:
+        if ':' in entry:
+            auto_col, manual_col = entry.split(":", 1)
+        else:
+            auto_col = manual_col = entry
+        column_mappings.append((auto_col.strip(), manual_col.strip()))
+elif len(sys.argv) == 3:
+    column_mappings = []
+else:
+    usage()
+
+# Load and process manual benchmark
+manual_df = pd.read_csv(manual_path)
+if "Iteration" in manual_df.columns:
+    manual_df = manual_df.drop(columns=["Iteration"])
+
+aggregated = (
+    manual_df
+    .groupby(["Test Description", "Size", "Method"], as_index=False)
+    .mean(numeric_only=True)
+)
+
+aggregated = format_dataframe_numbers(aggregated, digits=5)
+aggregated.to_csv(averaged_manual_path, index=False)
+print(f"Averaged manual benchmark written to: {averaged_manual_path}")
+
+if patching_enabled:
+    auto_df = pd.read_csv(auto_path)
+
+    # Build lookup from aggregated benchmark
+    lookup = {}
+    for _, row in aggregated.iterrows():
+        key = (row["Test Description"], row["Size"], row["Method"])
+        lookup[key] = row
+
+    # Patch auto benchmark
+    for idx, row in auto_df.iterrows():
+        key = (row["Test Description"], row["Size"], row["Method"])
+        if key in lookup:
+            for auto_col, manual_col in column_mappings:
+                if manual_col in lookup[key]:
+                    auto_df.at[idx, auto_col] = lookup[key][manual_col]
+                else:
+                    print(f"Warning: Column '{manual_col}' not found for row {key}")
+        else:
+            print(f"Warning: No manual benchmark found for: {key}")
+
+    auto_df = format_dataframe_numbers(auto_df, digits=5)
+    auto_df.to_csv(updated_auto_path, index=False)
+    print(f"Patched auto benchmark written to: {updated_auto_path}")

bench-version-changes/ghc-dps-compact/after/Bench/Compact.hs

@@ -1,10 +1,11 @@
 module Bench.Compact where
 
-import Bench.Compact.BFTraversal (bftraversalBenchgroup)
-import Bench.Compact.DList (dlistBenchgroup)
-import Bench.Compact.Map (mapBenchgroup)
-import Bench.Compact.Queue (queueBenchgroup)
-import Bench.Compact.SExpr (sexprBenchgroup)
+import Bench.Compact.BFTraversal (bftraversalBenchgroup, bftraversalLaunch)
+import Bench.Compact.DList (dlistBenchgroup, dlistLaunch)
+import Bench.Compact.Map (mapBenchgroup, mapLaunch)
+import Bench.Compact.Queue (queueBenchgroup, queueLaunch)
+import Bench.Compact.SExpr (sexprBenchgroup, sexprLaunch)
+import System.Exit (exitFailure)
 import Test.Tasty.Bench
 
 benchmarks :: Benchmark
@@ -17,3 +18,26 @@ benchmarks =
       queueBenchgroup,
       sexprBenchgroup
     ]
+
+launchImpl' :: [(String -> Maybe (IO ()))] -> String -> IO ()
+launchImpl' launchers request = do
+  let tryLaunch [] = Nothing
+      tryLaunch (l : ls) = case l request of
+        Just action -> Just action
+        Nothing -> tryLaunch ls
+  case tryLaunch launchers of
+    Just action -> do
+      action
+    Nothing -> do
+      putStrLn $ "Error"
+      exitFailure
+
+launchImpl :: String -> IO ()
+launchImpl =
+  launchImpl'
+    [ bftraversalLaunch,
+      mapLaunch,
+      dlistLaunch,
+      queueLaunch,
+      sexprLaunch
+    ]

bench-version-changes/ghc-dps-compact/after/Bench/Compact/BFTraversal.hs

@@ -23,4 +23,7 @@ dataSets =
         else Node () (go (currentDepth + 1) maxDepth) (go (currentDepth + 1) maxDepth)
 
 bftraversalBenchgroup :: Benchmark
-bftraversalBenchgroup = Utils.benchImpls "Breadth-first tree traversal" BFTraversal.impls dataSets
+bftraversalBenchgroup = benchImpls "Breadth-first tree traversal" BFTraversal.impls dataSets
+
+bftraversalLaunch :: String -> Maybe (IO ())
+bftraversalLaunch = launchImpl "Breadth-first tree traversal" BFTraversal.impls dataSets

bench-version-changes/ghc-dps-compact/after/Bench/Compact/DList.hs

@@ -20,3 +20,6 @@ dataSets =
 
 dlistBenchgroup :: Benchmark
 dlistBenchgroup = benchImpls "List and DList concatenation" DList.impls dataSets
+
+dlistLaunch :: String -> Maybe (IO ())
+dlistLaunch = launchImpl "List and DList concatenation" DList.impls dataSets

bench-version-changes/ghc-dps-compact/after/Bench/Compact/Map.hs

@@ -22,3 +22,6 @@ dataSets =
 
 mapBenchgroup :: Benchmark
 mapBenchgroup = benchImpls "map on List" Map.impls dataSets
+
+mapLaunch :: String -> Maybe (IO ())
+mapLaunch = launchImpl "map on List" Map.impls dataSets

bench-version-changes/ghc-dps-compact/after/Bench/Compact/Queue.hs

@@ -19,3 +19,6 @@ dataSets =
 
 queueBenchgroup :: Benchmark
 queueBenchgroup = benchImpls "Queue enqueue operations" Queue.impls dataSets
+
+queueLaunch :: String -> Maybe (IO ())
+queueLaunch = launchImpl "Queue enqueue operations" Queue.impls dataSets

bench-version-changes/ghc-dps-compact/after/Bench/Compact/SExpr.hs

@@ -21,4 +21,7 @@ dataSets =
   ]
 
 sexprBenchgroup :: Benchmark
-sexprBenchgroup = Utils.benchImpls "S-expression parser" SExpr.impls dataSets
+sexprBenchgroup = benchImpls "S-expression parser" SExpr.impls dataSets
+
+sexprLaunch :: String -> Maybe (IO ())
+sexprLaunch = launchImpl "S-expression parser" SExpr.impls dataSets

bench-version-changes/ghc-dps-compact/after/Bench/Compact/Utils.hs

@@ -12,6 +12,8 @@ module Bench.Compact.Utils where
 import Control.DeepSeq
 import Control.Exception (evaluate)
 import Data.Functor ((<&>))
+import Data.List (find)
+import qualified Data.List.Split as Split
 import GHC.Compact (compact, getCompact)
 import Test.Tasty (testGroup)
 import Test.Tasty.Bench
@@ -82,3 +84,42 @@ benchImpls name impls datasets = do
 --   go' [] = error ("requested size '" ++ requestedSize ++ "' not found")
 --   go' ((loadSampleData, sizeName):_) | sizeName == requestedSize = loadSampleData
 --   go' (_:xs) = go' xs
+
+launchImpl ::
+  forall m a r.
+  (NFData a, NFData r) =>
+  String -> -- expected testName
+  [(a %m -> r, String, Bool)] ->
+  [(IO a, String)] ->
+  String -> -- full request string
+  Maybe (IO ())
+launchImpl name impls datasets request =
+  let segments = Split.splitOn "." request
+   in case segments of
+        ("All" : "DPS interface for compact regions" : testName : sizeName : rest)
+          | testName == name ->
+              case rest of
+                [implFullName] -> matchImpl sizeName implFullName Nothing
+                [implBase, suffix] -> matchImpl sizeName implBase (Just suffix)
+                _ -> Nothing
+          | otherwise -> Nothing
+        _ -> Nothing
+  where
+    matchImpl sizeName implBaseName suffix = do
+      (loadData, _) <- find ((== sizeName) . snd) datasets
+      (implFn, _, isLazy) <- find (\(_, n, _) -> n == implBaseName) impls
+      case (isLazy, suffix) of
+        (False, Nothing) ->
+          Just $
+            loadData >>= \sampleData ->
+              evaluate $ rwhnf $ implFn sampleData
+        (True, Just "force") ->
+          Just $
+            loadData >>= \sampleData ->
+              evaluate $ rwhnf $ force $ implFn sampleData
+        (True, Just "copyCR") ->
+          Just $
+            loadData >>= \sampleData ->
+              compact (implFn sampleData) >>= \res ->
+                evaluate $ rwhnf $ getCompact res
+        _ -> Nothing

bench-version-changes/ghc-dps-compact/after/Main.hs

@@ -0,0 +1,38 @@
+module Main where
+
+import qualified Bench.Compact as Compact
+import qualified Bench.Data.Array.Mutable as Array
+import qualified Bench.Data.Array.Mutable.Quicksort as Quicksort
+import qualified Bench.Data.HashMap.Mutable as HashMap
+import System.Environment (getArgs)
+import Test.Tasty.Bench (defaultMain)
+
+main :: IO ()
+main = do
+  args <- getArgs
+  case args of
+    ["manualLaunch", request] ->
+      Compact.launchImpl request
+    _ ->
+      defaultMain
+        [ Array.benchmarks,
+          Quicksort.benchmarks,
+          HashMap.benchmarks,
+          Compact.benchmarks
+        ]
+
+-- nix-shell --arg installHls 'false' --pure --run "
+-- echo \$'=== Benchmarks (manual launch, filtered) ===\n\n' > benchmark_ghc-dps-compact-95615577d7_prof.txt && \
+-- cabal run -w /home/thomas/tweag/linear-base/ghc-dps-compact-95615577d7/bin/ghc -v0 linear-base:bench:bench -- -l | \
+-- while read -r name; do \
+--   case \"\$name\" in \
+--     All.DPS\ interface\ for\ compact\ regions*) \
+--       for i in {1..5}; do \
+--         echo \"=== Running \$name (iteration \$i) ===\" | tee -a benchmark_ghc-dps-compact-95615577d7_prof.txt; \
+--         cabal run -w /home/thomas/tweag/linear-base/ghc-dps-compact-95615577d7/bin/ghc -v0 linear-base:bench:bench -- manualLaunch \"\$name\" +RTS -T -S -RTS 2>&1 | tee -a benchmark_ghc-dps-compact-95615577d7_prof.txt; \
+--       done \
+--       ;; \
+--     *) ;; \
+--   esac; \
+-- done
+-- "