Benchmark improvements (#1178)

* [test-ci] fix wibble

* [ghcide-bench] fix scrambled output

* [ghcide-bench] add a new experiment: getDefinition after edit

* [ghcide-bench] refine the position used for identifiers

* [ghcide-bench] Support examples with multiple FOIs

* [ghcide-bench] Allow identifierP to be optional

* [ghcide-bench] experiments: do all edits first, then query

* [ghcide-bench] Add examples with multiple FOIs

* [ghcide-bench] add a completions (without edit) experiment

* [ghcide-bench] Fix indentation

* [ghcide-bench] Fix incomplete pattern match

* [ghcide-bench] Include the documentContents setup in the "startup" metric

* [ghcide-bench] fix wibble

* [ghcide-bench] add more verbose output

* [shake-bench] Consolidate -s code

* [bench-hist] Fix: depend on # samples

* [ghcide-bench] cache searchSymbol

* [cabal-bench] --no-clean

Author: pepeiborra

.github/workflows/test.yml

@@ -49,7 +49,7 @@ jobs:
             ~/.cabal/store
           key: ${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
           restore-keys: |
-            ${{ runner.os }}-${{ matrix.ghc }}-bench-$${ hashFiles('cabal.project') }}
+            ${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
             ${{ runner.os }}-${{ matrix.ghc }}-build-
             ${{ runner.os }}-${{ matrix.ghc }}
 

ghcide/bench/config.yaml

@@ -14,20 +14,23 @@ examples:
   # Medium-sized project without TH
   - name: Cabal
     version: 3.0.0.0
-    module: Distribution/Simple.hs
+    modules:
+        - Distribution/Simple.hs
+        - Distribution/Types/Module.hs
   # Small-sized project with TH
-  - name: haskell-lsp-types
-    version: 0.22.0.0
-    module: src/Language/Haskell/LSP/Types/Lens.hs
-# - path: path-to-example
-#   module: path-to-module
+  - name: lsp-types
+    version: 1.0.0.1
+    modules:
+        - src/Language/LSP/VFS.hs
+        - src/Language/LSP/Types/Lens.hs
 
 # The set of experiments to execute
 experiments:
-    - hover
-    - edit
-    - getDefinition
+    - "edit"
+    - "hover"
     - "hover after edit"
+    - "getDefinition"
+    - "getDefinition after edit"
     - "completions after edit"
     - "code actions"
     - "code actions after edit"

ghcide/bench/exe/Main.hs

@@ -37,14 +37,23 @@
 import Control.Exception.Safe
 import Experiments
 import Options.Applicative
+import System.IO
+import           Control.Monad
+
+optsP :: Parser (Config, Bool)
+optsP = (,) <$> configP <*> switch (long "no-clean")
 
 main :: IO ()
 main = do
-  config <- execParser $ info (configP <**> helper) fullDesc
+  hSetBuffering stdout LineBuffering
+  hSetBuffering stderr LineBuffering
+  (config, noClean) <- execParser $ info (optsP <**> helper) fullDesc
   let ?config = config
 
+  hPrint stderr config
+
   output "starting test"
 
   SetupResult{..} <- setup
 
-  runBenchmarks experiments `finally` cleanUp
+  runBenchmarks experiments `finally` unless noClean cleanUp

ghcide/bench/hist/Main.hs

@@ -51,6 +51,7 @@ import Experiments.Types (Example, exampleToOptions)
 import qualified Experiments.Types as E
 import GHC.Generics (Generic)
 import Numeric.Natural (Natural)
+import Development.Shake.Classes
 
 
 config :: FilePath
@@ -70,7 +71,7 @@ main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest} $ do
       configStatic <- liftIO $ readConfigIO config
       let build = outputFolder configStatic
       buildRules build ghcideBuildRules
-      benchRules build resource (MkBenchRules (benchGhcide $ samples configStatic) "ghcide")
+      benchRules build resource (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide")
       csvRules build
       svgRules build
       action $ allTargets build
@@ -98,14 +99,18 @@ createBuildSystem userRules = do
 
   _ <- addOracle $ \GetExperiments {} -> experiments <$> readConfig config
   _ <- addOracle $ \GetVersions {} -> versions <$> readConfig config
-  _ <- addOracle $ \GetExamples{} -> examples <$> readConfig config
-  _ <- addOracle $ \(GetExample name) -> find (\e -> getExampleName e == name) . examples <$> readConfig config
+  _ <- versioned 1 $ addOracle $ \GetExamples{} -> examples <$> readConfig config
+  _ <- versioned 1 $ addOracle $ \(GetExample name) -> find (\e -> getExampleName e == name) . examples <$> readConfig config
   _ <- addOracle $ \GetBuildSystem {} -> buildTool <$> readConfig config
+  _ <- addOracle $ \GetSamples{} -> samples <$> readConfig config
 
   benchResource <- newResource "ghcide-bench" 1
 
   userRules benchResource
 
+newtype GetSamples = GetSamples () deriving newtype (Binary, Eq, Hashable, NFData, Show)
+type instance RuleResult GetSamples = Natural
+
 --------------------------------------------------------------------------------
 
 buildGhcide :: BuildSystem -> [CmdOption] -> FilePath -> Action ()
@@ -130,9 +135,10 @@ buildGhcide Stack args out =
 
 benchGhcide
   :: Natural -> BuildSystem -> [CmdOption] -> BenchProject Example -> Action ()
-benchGhcide samples buildSystem args BenchProject{..} =
+benchGhcide samples buildSystem args BenchProject{..} = do
   command_ args "ghcide-bench" $
     [ "--timeout=3000",
+      "--no-clean",
         "-v",
         "--samples=" <> show samples,
         "--csv="     <> outcsv,