From bf4f573aa5f7d6a07b4d7d2a804c68dc6ff0ac7e Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Sat, 22 Apr 2023 21:59:22 -0400 Subject: [PATCH] disasm-test: Overhaul treatment of LLVM version ranges This is a minor rewrite of the logic used to check for LLVM version ranges in `disasm-llvm` test cases. The new approach is similar to Case 3/3a from this `tasty-sugar` document: https://github.com/kquick/tasty-sugar/blob/1fc06bee124e02f49f6478bc1e1df13704cc4916/Ranges.org#case-3---explicit-and-a-weaker-match In particular: * We have adopted the convention that the test output for the most recent version of LLVM is always contained in a bare `.ll` file. * There are no longer any `at-least-llvm*` files, just `pre-llvm*`. This greatly simplifies the number of cases to consider and the number of checks to implement. * We now skip test configurations by having `SKIP_TEST` as the first line of the `.ll` file. Again, this greatly simplifies the logic needed to skip test cases on certain configurations. This is heavily inspired by a similar change made in GaloisInc/crucible#1083. --- disasm-test/Main.hs | 116 ++++++++++++++---- disasm-test/README.md | 45 +++++++ ...t-llvm14.ll => btf-tag-dicompositetype.ll} | 0 .../btf-tag-dicompositetype.pre-llvm14.ll | 1 + ...ast-llvm14.ll => btf-tag-diderivedtype.ll} | 0 .../tests/btf-tag-diderivedtype.pre-llvm14.ll | 1 + ...-llvm14.ll => btf-tag-diglobalvariable.ll} | 0 .../btf-tag-diglobalvariable.pre-llvm14.ll | 1 + ...t-llvm14.ll => btf-tag-dilocalvariable.ll} | 0 .../btf-tag-dilocalvariable.pre-llvm14.ll | 1 + ...east-llvm14.ll => btf-tag-disubprogram.ll} | 0 .../tests/btf-tag-disubprogram.pre-llvm14.ll | 1 + ...list.at-least-llvm13.ll => di-arg-list.ll} | 0 disasm-test/tests/di-arg-list.pre-llvm13.ll | 1 + ....at-least-llvm14.ll => dilocalvariable.ll} | 0 .../tests/dilocalvariable.pre-llvm14.ll | 1 + .../{poison.at-least-llvm12.ll => poison.ll} | 0 disasm-test/tests/poison.pre-llvm12.ll | 1 + llvm-pretty-bc-parser.cabal | 1 + 19 files changed, 146 insertions(+), 24 deletions(-) create mode 100644 disasm-test/README.md rename disasm-test/tests/{btf-tag-dicompositetype.at-least-llvm14.ll => btf-tag-dicompositetype.ll} (100%) create mode 100644 disasm-test/tests/btf-tag-dicompositetype.pre-llvm14.ll rename disasm-test/tests/{btf-tag-diderivedtype.at-least-llvm14.ll => btf-tag-diderivedtype.ll} (100%) create mode 100644 disasm-test/tests/btf-tag-diderivedtype.pre-llvm14.ll rename disasm-test/tests/{btf-tag-diglobalvariable.at-least-llvm14.ll => btf-tag-diglobalvariable.ll} (100%) create mode 100644 disasm-test/tests/btf-tag-diglobalvariable.pre-llvm14.ll rename disasm-test/tests/{btf-tag-dilocalvariable.at-least-llvm14.ll => btf-tag-dilocalvariable.ll} (100%) create mode 100644 disasm-test/tests/btf-tag-dilocalvariable.pre-llvm14.ll rename disasm-test/tests/{btf-tag-disubprogram.at-least-llvm14.ll => btf-tag-disubprogram.ll} (100%) create mode 100644 disasm-test/tests/btf-tag-disubprogram.pre-llvm14.ll rename disasm-test/tests/{di-arg-list.at-least-llvm13.ll => di-arg-list.ll} (100%) create mode 100644 disasm-test/tests/di-arg-list.pre-llvm13.ll rename disasm-test/tests/{dilocalvariable.at-least-llvm14.ll => dilocalvariable.ll} (100%) create mode 100644 disasm-test/tests/dilocalvariable.pre-llvm14.ll rename disasm-test/tests/{poison.at-least-llvm12.ll => poison.ll} (100%) create mode 100644 disasm-test/tests/poison.pre-llvm12.ll diff --git a/disasm-test/Main.hs b/disasm-test/Main.hs index 887fa964..9623c118 100644 --- a/disasm-test/Main.hs +++ b/disasm-test/Main.hs @@ -1,4 +1,5 @@ {-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -11,13 +12,16 @@ import Text.LLVM.PP (ppLLVM,ppModule) import qualified Control.Exception as X import Control.Lens ((^.), (^?), _Right, to) -import Control.Monad (unless, when) +import Control.Monad (guard, unless, when) import Data.Bifunctor (first) import qualified Data.ByteString.Lazy as L import Data.Char (ord,isLetter,isSpace,chr) import Data.Generics (everywhere, mkT) -- SYB -import Data.List (isInfixOf, sort, stripPrefix) +import Data.List (isInfixOf, isPrefixOf, sort, stripPrefix) +import Data.Maybe (mapMaybe) import Data.Proxy ( Proxy(..) ) +import qualified Data.Set as Set +import Data.Set (Set) import qualified Data.Text as T import Data.Typeable (Typeable) import Data.Versions (Versioning, versioning, prettyV, major) @@ -25,7 +29,7 @@ import qualified GHC.IO.Exception as GE import qualified Options.Applicative as OA import System.Directory (getTemporaryDirectory, removeFile) import System.Exit (ExitCode(..), exitFailure, exitSuccess) -import System.FilePath ((<.>)) +import System.FilePath ((<.>), takeFileName) import qualified System.IO as IO (stderr) import System.IO (openBinaryTempFile,hClose,openTempFile,hPutStrLn) @@ -35,6 +39,7 @@ import Test.Tasty.HUnit ( assertFailure, testCase ) import qualified Test.Tasty.Options as TO import qualified Test.Tasty.Runners as TR import qualified Test.Tasty.Sugar as TS +import Text.Read (readMaybe) import Text.Show.Pretty (ppShow) @@ -113,6 +118,16 @@ vcVersioning (VC _ v) = v mkVC :: String -> String -> VersionCheck mkVC nm raw = let r = T.pack raw in VC nm $ first (const r) $ versioning r +-- Check if a VersionCheck version is less than the numeric value of another +-- version (represented as a Word). +vcLT :: VersionCheck -> Word -> Bool +vcLT vc verNum = (vcVersioning vc ^? (_Right . major)) < Just verNum + +-- Check if a VersionCheck version is greater than or equal to the numeric +-- value of another version (represented as a Word). +vcGE :: VersionCheck -> Word -> Bool +vcGE vc verNum = (vcVersioning vc ^? (_Right . major)) >= Just verNum + getLLVMAsVersion :: LLVMAs -> IO VersionCheck getLLVMAsVersion (LLVMAs llvmAsPath) = getLLVMToolVersion "llvm-as" llvmAsPath @@ -205,10 +220,10 @@ cube = TS.mkCUBE { TS.inputDirs = ["disasm-test/tests"] , TS.rootName = "*.ll" , TS.separators = "." - , TS.validParams = [ ("llvm-range", Just [ "pre-llvm11" - , "at-least-llvm12" - , "at-least-llvm13" - , "at-least-llvm14" + , TS.validParams = [ ("llvm-range", Just [ "recent-llvm" + , "pre-llvm12" + , "pre-llvm13" + , "pre-llvm14" ]) ] -- Somewhat unusually for tasty-sugar, we make the expectedSuffix the same @@ -232,7 +247,14 @@ runTest llvmVer sweet expct | not llvmMatch = pure [] | otherwise - = do pure $ (:[]) $ + = do -- If an .ll file begins with SKIP_TEST, skip that test entirely. For + -- test cases that require a minimum LLVM version, this technique is + -- used to prevent running the test on older LLVM versions. + skipTest <- ("SKIP_TEST" `L.isPrefixOf`) <$> L.readFile (TS.expectedFile expct) + + if skipTest + then pure [] + else pure $ (:[]) $ askOption $ \llvmAs -> askOption $ \llvmDis -> askOption $ \roundtrip -> @@ -284,20 +306,53 @@ runTest llvmVer sweet expct else mapM_ putStrLn ["success: empty diff: ", file1, file2] -- Match any LLVM version range specification in the .ll - -- expected file against the current version of the LLVM tools. If the - -- current LLVM version doesn't match, no test should be - -- generated (i.e. only run tests for the version of LLVM tools - -- available). + -- expected file against the current version of the LLVM tools. + -- This implements a combination of Case 3 and Case 3a from this + -- tasty-sugar document: + -- https://github.com/kquick/tasty-sugar/blob/1fc06bee124e02f49f6478bc1e1df13704cc4916/Ranges.org#case-3---explicit-and-a-weaker-match + -- In particular, we use `recent-llvm` as an explicit super-supremum + -- (as in Case 3a), but we also consult the set of Expectations in the + -- full Sweets value to avoid generating duplicate tests for + -- `recent-llvm` (as described in Case 3). llvmMatch = - let specMatchesInstalled v = - or [ v == vcTag llvmVer - , and [ v == "pre-llvm11" - , vcMajor llvmVer `elem` (show <$> [3..10 :: Int]) - ] - , case stripPrefix "at-least-llvm" v of + let allMatchingExpectations = + filter + (\e -> (pfx ++ ".") `isPrefixOf` takeFileName (TS.expectedFile e)) + (TS.expected sweet) + + supportedPreLLVMs :: Set Word + supportedPreLLVMs = + Set.fromList $ + mapMaybe + (\e -> do + TS.Explicit v <- lookup "llvm-range" (TS.expParamsMatch e) + verStr <- stripPrefix "pre-llvm" v + ver <- readMaybe verStr + guard $ vcLT llvmVer ver + pure ver) + allMatchingExpectations + + -- Implement the "check" step described in Case 3/3a of the + -- tasty-sugar document linked above. + specMatchesInstalled v = + or [ case stripPrefix "pre-llvm" v of Nothing -> False - Just verStr -> - (vcVersioning llvmVer ^? (_Right . major)) >= Just (read verStr) + Just verStr + | Just ver <- readMaybe verStr + -- Check that the current LLVM version is less than + -- the in the `pre-llvm` file... + , vcLT llvmVer ver + -- ...moreover, also check that is the closest + -- `pre-llvm` version (without going over). For + -- instance, if the current LLVM version is 10 and + -- there are both `pre-llvm11` and `pre-llvm12` + -- `.ll` files, we only want to run with the + -- `pre-llvm11` configuration to avoid duplicate + -- tests. + , Just closestPreLLVM <- Set.lookupMin supportedPreLLVMs + -> ver == closestPreLLVM + | otherwise + -> False -- as a fallback, if the testing code here is -- unable to determine the version, run all -- tests. This is likely to cause a failure, but @@ -306,11 +361,24 @@ runTest llvmVer sweet expct -- done anything. , vcMajor llvmVer == "[missing]" ] - in case lookup "llvm-range" (TS.expParamsMatch expct) of - Just (TS.Explicit v) -> specMatchesInstalled v - Just (TS.Assumed v) - | v == "pre-llvm11" || v == "at-least-llvm12" + in -- Implement the "filter" step described in Case 3/3a of the + -- tasty-sugar document linked above. + case lookup "llvm-range" (TS.expParamsMatch expct) of + Just (TS.Explicit v) + -- Explicit matches are always allowed. -> specMatchesInstalled v + Just (TS.Assumed v) + -- The only allowable Assumed match is for `recent-llvm`, the + -- super-supremum value... + | v == "recent-llvm" + -> case Set.lookupMax supportedPreLLVMs of + -- ...if there are no `pre-llvm` .ll files, then allow + -- it... + Nothing -> True + -- ...otherwise, check that the current LLVM version is + -- larger than anything specified by a `pre-llvm` .ll + -- file. + Just largestPreLLVM -> vcGE llvmVer largestPreLLVM | otherwise -> False _ -> error "llvm-range unknown" diff --git a/disasm-test/README.md b/disasm-test/README.md new file mode 100644 index 00000000..be5d5093 --- /dev/null +++ b/disasm-test/README.md @@ -0,0 +1,45 @@ +# `disasm-test` + +This test suite ensures that for each `.ll` file under the `tests` directory: + +1. After using `llvm-as` to produce a `.bc` file, the `.bc` file can be parsed + using `llvm-pretty-bc-parser`. +2. The resulting `llvm-pretty` AST can be pretty-printed back out to an `.ll` + file using `llvm-pretty`'s pretty-printer. +3. The new `.ll` file is mostly equivalent to the original `.ll` file. + +Here, "mostly equivalent" means that the two files are syntactically +equivalent, ignoring minor differences in whitespace and the order of metadata +in the metadata list. + +## Conditional tests + +Some of the test cases have slightly different bitcode depending on which LLVM +version is used. These test cases will have accompanying +`.pre-llvm.ll` files, where `pre-llvm` indicates +that this test output is used for all LLVM versions up to (but not including) +``. Note that if a test case has multiple `pre-llvm.ll` +files, then the `` that is closest to the current LLVM version +(without going over) is picked. + +To illustrate this with a concrete example, consider suppose we have a test +case `foo` with the following `.ll` files + +* `foo.pre-llvm11.ll` +* `foo.pre-llvm13.ll` +* `foo.ll` + +The following `.ll` files would be used for the following LLVM versions: + +* LLVM 10: `foo.pre-llvm11.ll` +* LLVM 11: `foo.pre-llvm13.ll` +* LLVM 12: `foo.pre-llvm13.ll` +* LLVM 13 or later: `foo.ll` + +There are some test cases that require a sufficiently recent LLVM version to +run. To indicate that a test should not be run on LLVMs older than ``, +create a `pre-llvm.ll` file with `SKIP_TEST` as the first line. The +use of `SKIP_TEST` signals that this test should be skipped when using LLVMs +older than ``. Note that the test suite will not read anything past +`SKIP_TEST`, so the rest of the file can be used to document why the test is +skipped on that particular configuration. diff --git a/disasm-test/tests/btf-tag-dicompositetype.at-least-llvm14.ll b/disasm-test/tests/btf-tag-dicompositetype.ll similarity index 100% rename from disasm-test/tests/btf-tag-dicompositetype.at-least-llvm14.ll rename to disasm-test/tests/btf-tag-dicompositetype.ll diff --git a/disasm-test/tests/btf-tag-dicompositetype.pre-llvm14.ll b/disasm-test/tests/btf-tag-dicompositetype.pre-llvm14.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/btf-tag-dicompositetype.pre-llvm14.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/btf-tag-diderivedtype.at-least-llvm14.ll b/disasm-test/tests/btf-tag-diderivedtype.ll similarity index 100% rename from disasm-test/tests/btf-tag-diderivedtype.at-least-llvm14.ll rename to disasm-test/tests/btf-tag-diderivedtype.ll diff --git a/disasm-test/tests/btf-tag-diderivedtype.pre-llvm14.ll b/disasm-test/tests/btf-tag-diderivedtype.pre-llvm14.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/btf-tag-diderivedtype.pre-llvm14.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/btf-tag-diglobalvariable.at-least-llvm14.ll b/disasm-test/tests/btf-tag-diglobalvariable.ll similarity index 100% rename from disasm-test/tests/btf-tag-diglobalvariable.at-least-llvm14.ll rename to disasm-test/tests/btf-tag-diglobalvariable.ll diff --git a/disasm-test/tests/btf-tag-diglobalvariable.pre-llvm14.ll b/disasm-test/tests/btf-tag-diglobalvariable.pre-llvm14.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/btf-tag-diglobalvariable.pre-llvm14.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/btf-tag-dilocalvariable.at-least-llvm14.ll b/disasm-test/tests/btf-tag-dilocalvariable.ll similarity index 100% rename from disasm-test/tests/btf-tag-dilocalvariable.at-least-llvm14.ll rename to disasm-test/tests/btf-tag-dilocalvariable.ll diff --git a/disasm-test/tests/btf-tag-dilocalvariable.pre-llvm14.ll b/disasm-test/tests/btf-tag-dilocalvariable.pre-llvm14.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/btf-tag-dilocalvariable.pre-llvm14.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/btf-tag-disubprogram.at-least-llvm14.ll b/disasm-test/tests/btf-tag-disubprogram.ll similarity index 100% rename from disasm-test/tests/btf-tag-disubprogram.at-least-llvm14.ll rename to disasm-test/tests/btf-tag-disubprogram.ll diff --git a/disasm-test/tests/btf-tag-disubprogram.pre-llvm14.ll b/disasm-test/tests/btf-tag-disubprogram.pre-llvm14.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/btf-tag-disubprogram.pre-llvm14.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/di-arg-list.at-least-llvm13.ll b/disasm-test/tests/di-arg-list.ll similarity index 100% rename from disasm-test/tests/di-arg-list.at-least-llvm13.ll rename to disasm-test/tests/di-arg-list.ll diff --git a/disasm-test/tests/di-arg-list.pre-llvm13.ll b/disasm-test/tests/di-arg-list.pre-llvm13.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/di-arg-list.pre-llvm13.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/dilocalvariable.at-least-llvm14.ll b/disasm-test/tests/dilocalvariable.ll similarity index 100% rename from disasm-test/tests/dilocalvariable.at-least-llvm14.ll rename to disasm-test/tests/dilocalvariable.ll diff --git a/disasm-test/tests/dilocalvariable.pre-llvm14.ll b/disasm-test/tests/dilocalvariable.pre-llvm14.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/dilocalvariable.pre-llvm14.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/disasm-test/tests/poison.at-least-llvm12.ll b/disasm-test/tests/poison.ll similarity index 100% rename from disasm-test/tests/poison.at-least-llvm12.ll rename to disasm-test/tests/poison.ll diff --git a/disasm-test/tests/poison.pre-llvm12.ll b/disasm-test/tests/poison.pre-llvm12.ll new file mode 100644 index 00000000..99162696 --- /dev/null +++ b/disasm-test/tests/poison.pre-llvm12.ll @@ -0,0 +1 @@ +SKIP_TEST diff --git a/llvm-pretty-bc-parser.cabal b/llvm-pretty-bc-parser.cabal index cd3552f1..5ae803fa 100644 --- a/llvm-pretty-bc-parser.cabal +++ b/llvm-pretty-bc-parser.cabal @@ -122,6 +122,7 @@ Test-suite disasm-test hs-source-dirs: disasm-test Ghc-options: -Wall build-depends: base, + containers, process, directory, bytestring,