[llvm] Improved error handling if build_cmd fails.

compiler_gym/envs/llvm/service/Cost.cc

@@ -110,6 +110,16 @@ Status getTextSizeInBytes(const fs::path& file, int64_t* value) {
   return Status::OK;
 }
 
+#define TEXT_SIZE_RETURN_IF_ERROR(expr)                                                            \
+  while (1) {                                                                                      \
+    const auto status = expr;                                                                      \
+    if (!status.ok()) {                                                                            \
+      return Status(StatusCode::INVALID_ARGUMENT,                                                  \
+                    fmt::format("Failed to compute .text size cost: {}", status.error_message())); \
+    }                                                                                              \
+    break;                                                                                         \
+  }
+
 Status getTextSizeInBytes(llvm::Module& module, int64_t* value, const fs::path& workingDirectory,
                           const util::LocalShellCommand& buildCommand) {
   if (buildCommand.outfiles().size() != 1) {
@@ -122,20 +132,45 @@ Status getTextSizeInBytes(llvm::Module& module, int64_t* value, const fs::path&
   // Write the bitcode to the expected place.
   RETURN_IF_ERROR(writeBitcodeFile(module, "out.bc"));
 
-  RETURN_IF_ERROR(buildCommand.checkInfiles());
-  RETURN_IF_ERROR(buildCommand.checkCall());
-  RETURN_IF_ERROR(buildCommand.checkOutfiles());
+  TEXT_SIZE_RETURN_IF_ERROR(buildCommand.checkInfiles());
+  TEXT_SIZE_RETURN_IF_ERROR(buildCommand.checkCall());
+  TEXT_SIZE_RETURN_IF_ERROR(buildCommand.checkOutfiles());
 
   return getTextSizeInBytes(outfile, value);
 }
 
+#undef TEXT_SIZE_RETURN_IF_ERROR
+
 util::LocalShellCommand getBuildCommand(const BenchmarkDynamicConfig& dynamicConfig,
                                         bool compile_only) {
   // Append the '-c' flag to compile-only jobs.
   if (compile_only) {
     Command newCommand;
     newCommand.CopyFrom(dynamicConfig.buildCommand().proto());
+
+    // Determine if the compilation specifies an output file using the `-o
+    // <file>` flag. If not, then add one so that when we append the `-c` flag
+    // we still know where the generated object file will go.
+    bool outfileSpecified = false;
+    for (int i = 0; i < newCommand.argument_size() - 1; ++i) {
+      if (newCommand.argument(i) == "-o") {
+        outfileSpecified = true;
+        break;
+      }
+    }
+    if (!outfileSpecified) {
+      newCommand.add_argument("-o");
+      if (newCommand.outfile_size() < 1) {
+        newCommand.add_argument("a.out");
+        newCommand.add_outfile("a.out");
+      } else {
+        const auto& outfile = newCommand.outfile(0);
+        newCommand.add_argument(outfile);
+      }
+    }
+
     newCommand.add_argument("-c");
+
     return util::LocalShellCommand(newCommand);
   }
   return dynamicConfig.buildCommand();

tests/llvm/custom_benchmarks_test.py

@@ -11,7 +11,7 @@
 import gym
 import pytest
 
-from compiler_gym.datasets import Benchmark
+from compiler_gym.datasets import Benchmark, BenchmarkInitError
 from compiler_gym.envs import LlvmEnv, llvm
 from compiler_gym.service.proto import Benchmark as BenchmarkProto
 from compiler_gym.service.proto import File
@@ -311,5 +311,43 @@ def test_get_compiler_includes_output_parse_failure():
             os.environ["CXX"] = old_cxx
 
 
+def test_benchmark_with_invalid_build_cmd(env: LlvmEnv, tmpdir):
+    """Test that BenchmarkInitError raised if the build command fails."""
+    with open(tmpdir / "program.c", "w") as f:
+        f.write(
+            """
+    #include <stdio.h>
+
+    int main(int argc, char** argv) {
+        printf("Hello\\n");
+        for (int i = 0; i < 10; ++i) {
+            argc += 2;
+        }
+        return argc - argc;
+    }
+        """
+        )
+
+    benchmark = env.make_benchmark(Path(tmpdir) / "program.c")
+
+    benchmark.proto.dynamic_config.build_cmd.argument.extend(
+        ["$CC", "$IN", "-invalid-cc-argument"]
+    )
+    benchmark.proto.dynamic_config.build_cmd.outfile.extend(["a.out"])
+    benchmark.proto.dynamic_config.build_cmd.timeout_seconds = 10
+
+    benchmark.proto.dynamic_config.run_cmd.argument.extend(["./a.out"])
+    benchmark.proto.dynamic_config.run_cmd.timeout_seconds = 10
+
+    with pytest.raises(
+        BenchmarkInitError,
+        match=(
+            r"Command '\$CC \$IN -invalid-cc-argument .+' failed with exit code "
+            r"1: clang: error: unknown argument: '-invalid-cc-argument'"
+        ),
+    ):
+        env.reset(benchmark=benchmark)
+
+
 if __name__ == "__main__":
     main()

tests/llvm/runtime_test.py

@@ -11,7 +11,6 @@
 from flaky import flaky
 
 from compiler_gym.envs.llvm import LlvmEnv
-from compiler_gym.service.connection import ServiceError
 from tests.test_main import main
 
 pytest_plugins = ["tests.pytest_plugins.llvm"]
@@ -97,45 +96,6 @@ def test_custom_benchmark_runtimes_differ(env: LlvmEnv, tmpdir):
     assert not np.all(runtimes_a == runtimes_b)
 
 
-def test_failing_build_cmd(env: LlvmEnv, tmpdir):
-    """Test that Runtime observation raises an error if build command fails."""
-    with open(tmpdir / "program.c", "w") as f:
-        f.write(
-            """
-    #include <stdio.h>
-
-    int main(int argc, char** argv) {
-        printf("Hello\\n");
-        for (int i = 0; i < 10; ++i) {
-            argc += 2;
-        }
-        return argc - argc;
-    }
-        """
-        )
-
-    benchmark = env.make_benchmark(Path(tmpdir) / "program.c")
-
-    benchmark.proto.dynamic_config.build_cmd.argument.extend(
-        ["$CC", "$IN", "-invalid-cc-argument"]
-    )
-    benchmark.proto.dynamic_config.build_cmd.outfile.extend(["a.out"])
-    benchmark.proto.dynamic_config.build_cmd.timeout_seconds = 10
-
-    benchmark.proto.dynamic_config.run_cmd.argument.extend(["./a.out"])
-    benchmark.proto.dynamic_config.run_cmd.timeout_seconds = 10
-
-    env.reset(benchmark=benchmark)
-    with pytest.raises(
-        ServiceError,
-        match=(
-            r"Command '\$CC \$IN -invalid-cc-argument' failed with exit code 1: "
-            r"clang: error: unknown argument: '-invalid-cc-argument'"
-        ),
-    ):
-        env.observation.Runtime()
-
-
 def test_invalid_runtime_count(env: LlvmEnv):
     env.reset()
     with pytest.raises(