Implement llvm-foreach tool

llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll

@@ -0,0 +1,26 @@
+; UNSUPPORTED: system-windows
+;
+; RUN: echo 'Content of first file' > %t1.tgt
+; RUN: echo 'Content of second file' > %t2.tgt
+; RUN: echo "%t1.tgt" > %t.list
+; RUN: echo "%t2.tgt" >> %t.list
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: FileCheck < %t.res %s
+; CHECK: [[FIRST:.+1.tgt]]
+; CHECK: [[SECOND:.+2.tgt]]
+;
+; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
+; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
+; CHECK-LIST: [[FIRST:.+\.out]]
+; CHECK-LIST: [[SECOND:.+\.out]]
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
+; CHECK-CONTENT: Content of
+
+; RUN: echo 'something' > %t3.tgt
+; RUN: echo 'something again' > %t4.tgt
+; RUN: echo "%t3.tgt" > %t1.list
+; RUN: echo "%t4.tgt" >> %t1.list
+; RUN: llvm-foreach --in-replace="{}" --in-replace="in" --in-file-list=%t.list --in-file-list=%t1.list -- echo -first-part-of-arg={}.out -first-part-of-arg=in.out > %t1.res
+; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
+; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]]
+; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]]

llvm/test/tools/llvm-foreach/llvm-foreach-win.ll

@@ -0,0 +1,26 @@
+; REQUIRES: system-windows
+;
+; RUN: echo 'Content of first file' > %t1.tgt
+; RUN: echo 'Content of second file' > %t2.tgt
+; RUN: echo "%t1.tgt" > %t.list
+; RUN: echo "%t2.tgt" >> %t.list
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: FileCheck < %t.res %s
+; CHECK: [[FIRST:.+1.tgt]]
+; CHECK: [[SECOND:.+2.tgt]]
+;
+; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- copy "{}" %t
+; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
+; CHECK-LIST: [[FIRST:.+\.out]]
+; CHECK-LIST: [[SECOND:.+\.out]]
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
+; CHECK-CONTENT: Content of
+
+; RUN: echo 'something' > %t3.tgt
+; RUN: echo 'something again' > %t4.tgt
+; RUN: echo "%t3.tgt" > %t1.list
+; RUN: echo "%t4.tgt" >> %t1.list
+; RUN: llvm-foreach --in-replace="{}" --in-replace="in" --in-file-list=%t.list --in-file-list=%t1.list -- echo -first-part-of-arg={}.out -first-part-of-arg=in.out > %t1.res
+; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
+; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]]
+; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]]

llvm/tools/LLVMBuild.txt

@@ -35,6 +35,7 @@ subdirectories =
  llvm-ifs
  llvm-exegesis
  llvm-extract
+ llvm-foreach
  llvm-jitlistener
  llvm-jitlink
  llvm-link

llvm/tools/llvm-foreach/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_llvm_tool(llvm-foreach
+  llvm-foreach.cpp
+)

llvm/tools/llvm-foreach/LLVMBuild.txt

@@ -0,0 +1,21 @@
+;===- ./tools/llvm-foreach/LLVMBuild.txt ---------------------*- Conf -*--===;
+;
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Tool
+name = llvm-foreach
+parent = Tools
+required_libraries = Support

llvm/tools/llvm-foreach/llvm-foreach.cpp

@@ -0,0 +1,214 @@
+//===- llvm-foreach.cpp - Command lines execution ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This source is utility to execute command lines. The specified command will
+// be invoked as many times as necessary to use up the list of input items.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/SystemUtils.h"
+
+#include <vector>
+
+using namespace llvm;
+
+static cl::list<std::string> InputFileLists{
+    "in-file-list", cl::OneOrMore,
+    cl::desc("Input list of file names, file names must be delimited by a "
+             "newline character."),
+    cl::value_desc("filename")};
+
+static cl::list<std::string> InputCommandArgs{
+    cl::Positional, cl::OneOrMore, cl::desc("<command>"),
+    cl::value_desc("command")};
+
+static cl::list<std::string> Replaces{
+    "in-replace", cl::OneOrMore,
+    cl::desc("Specify input path in input command, this will be replaced with "
+             "names read from corresponding input list of files."),
+    cl::value_desc("R")};
+
+static cl::opt<std::string> OutReplace{
+    "out-replace",
+    cl::desc("Specify output path in input command, this will be replaced with "
+             "name of temporary file created for writing command's outputs."),
+    cl::init(""), cl::value_desc("R")};
+
+static cl::opt<std::string> OutDirectory{
+    "out-dir",
+    cl::desc("Specify directory for output files; If unspecified, assume "
+             "system temporary directory."),
+    cl::init(""), cl::value_desc("R")};
+
+static cl::opt<std::string> OutFilesExt{
+    "out-ext",
+    cl::desc("Specify extenstion for output files; If unspecified, assume "
+             ".out"),
+    cl::init("out"), cl::value_desc("R")};
+
+// Emit list of produced files for better integration with other tools.
+static cl::opt<std::string> OutputFileList{
+    "out-file-list", cl::desc("Specify filename for list of outputs."),
+    cl::value_desc("filename"), cl::init("")};
+
+static void error(const Twine &Msg) {
+  errs() << "llvm-foreach: " << Msg << '\n';
+  exit(1);
+}
+
+static void error(std::error_code EC, const Twine &Prefix) {
+  if (EC)
+    error(Prefix + ": " + EC.message());
+}
+
+int main(int argc, char **argv) {
+  cl::ParseCommandLineOptions(
+      argc, argv,
+      "llvm-foreach: Execute specified command as many times as\n"
+      "necessary to use up the list of input items.\n"
+      "Usage:\n"
+      "llvm-foreach --in-file-list=a.list --in-replace='{}' -- echo '{}'\n"
+      "NOTE: commands containig redirects are not supported by llvm-foreach\n"
+      "yet.\n");
+
+  ExitOnError ExitOnErr("llvm-foreach: ");
+
+  if (InputFileLists.size() != Replaces.size())
+    error("Number of input file lists and input path replaces don't match.");
+
+  std::vector<std::unique_ptr<MemoryBuffer>> MBs;
+  std::vector<line_iterator> LineIterators;
+  for (auto &InputFileList : InputFileLists) {
+    std::unique_ptr<MemoryBuffer> MB = ExitOnErr(
+        errorOrToExpected(MemoryBuffer::getFileOrSTDIN(InputFileList)));
+    LineIterators.push_back(line_iterator(*MB));
+    MBs.push_back(std::move(MB));
+  }
+
+  SmallVector<StringRef, 8> Args(InputCommandArgs.begin(),
+                                 InputCommandArgs.end());
+
+  if (Args.empty())
+    error("No command?");
+
+  struct ArgumentReplace {
+    size_t ArgNum = 0;
+    // Index in argument string where replace length starts.
+    size_t Start = 0;
+    size_t ReplaceLen = 0;
+  };
+
+  // Find args to replace with filenames from input list.
+  std::vector<ArgumentReplace> InReplaceArgs;
+  ArgumentReplace OutReplaceArg;
+  for (size_t i = 1; i < Args.size(); ++i) {
+    for (auto &Replace : Replaces) {
+      size_t ReplaceStart = Args[i].find(Replace);
+      if (ReplaceStart != StringRef::npos)
+        InReplaceArgs.push_back({i, ReplaceStart, Replace.size()});
+    }
+
+    if (!OutReplace.empty() && Args[i].contains(OutReplace)) {
+      size_t ReplaceStart = Args[i].find(OutReplace);
+      if (ReplaceStart != StringRef::npos)
+        OutReplaceArg = {i, ReplaceStart, OutReplace.size()};
+    }
+  }
+
+  // Emit an error if user requested replace output file in the command but
+  // replace string is not found.
+  if (!OutReplace.empty() && OutReplaceArg.ArgNum == 0)
+    error("Couldn't find replace string for output in the command.");
+
+  // Make sure that specified program exists, emit an error if not.
+  std::string Prog =
+      ExitOnErr(errorOrToExpected(sys::findProgramByName(Args[0])));
+
+  std::vector<std::vector<std::string>> FileLists(LineIterators.size());
+  size_t PrevNumOfLines = 0;
+  for (size_t i = 0; i < FileLists.size(); ++i) {
+    for (; !LineIterators[i].is_at_eof(); ++LineIterators[i]) {
+      FileLists[i].push_back(LineIterators[i]->str());
+    }
+    if (i != 0 && FileLists[i].size() != PrevNumOfLines)
+      error("All input file lists must have same number of lines!");
+    PrevNumOfLines = FileLists[i].size();
+  }
+
+  std::error_code EC;
+  raw_fd_ostream OS{OutputFileList, EC, sys::fs::OpenFlags::OF_None};
+  if (!OutputFileList.empty())
+    error(EC, "error opening the file '" + OutputFileList + "'");
+
+  std::string ResOutArg;
+  std::vector<std::string> ResInArgs(InReplaceArgs.size());
+  std::string ResFileList = "";
+  for (size_t j = 0; j != FileLists[0].size(); ++j) {
+    for (size_t i = 0; i < InReplaceArgs.size(); ++i) {
+      ArgumentReplace CurReplace = InReplaceArgs[i];
+      std::string OriginalString = InputCommandArgs[CurReplace.ArgNum];
+      ResInArgs[i] = (Twine(OriginalString.substr(0, CurReplace.Start)) +
+                      Twine(FileLists[i][j]) +
+                      Twine(OriginalString.substr(CurReplace.Start +
+                                                  CurReplace.ReplaceLen)))
+                         .str();
+      Args[CurReplace.ArgNum] = ResInArgs[i];
+    }
+
+    SmallString<128> Path;
+    if (!OutReplace.empty()) {
+      // Create a file for command result. Add file name to output
+      // file list if needed.
+      std::string TempFileNameBase = sys::path::stem(OutReplace);
+      if (OutDirectory.empty())
+        EC = sys::fs::createTemporaryFile(TempFileNameBase, OutFilesExt, Path);
+      else {
+        SmallString<128> PathPrefix(OutDirectory);
+        // "CreateUniqueFile" functions accepts "Model" - special string with
+        // substring containing sequence of "%" symbols. In the resulting
+        // filename "%" symbols sequence from "Model" string will be replaced
+        // with random chars to make it unique.
+        llvm::sys::path::append(PathPrefix,
+                                TempFileNameBase + "-%%%%%%." + OutFilesExt);
+        EC = sys::fs::createUniqueFile(PathPrefix, Path);
+      }
+      error(EC, "Could not create a file for command output.");
+
+      std::string OriginalString = InputCommandArgs[OutReplaceArg.ArgNum];
+      ResOutArg =
+          (Twine(OriginalString.substr(0, OutReplaceArg.Start)) + Twine(Path) +
+           Twine(OriginalString.substr(OutReplaceArg.Start +
+                                       OutReplaceArg.ReplaceLen)))
+              .str();
+      Args[OutReplaceArg.ArgNum] = ResOutArg;
+
+      if (!OutputFileList.empty())
+        OS << Path << "\n";
+    }
+
+    std::string ErrMsg;
+    // TODO: Add possibility to execute commands in parallel.
+    int Result =
+        sys::ExecuteAndWait(Prog, Args, /*Env=*/None, /*Redirects=*/None,
+                            /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg);
+    if (Result != 0)
+      error(ErrMsg);
+  }
+
+  if (!OutputFileList.empty()) {
+    OS.close();
+  }
+
+  return 0;
+}

sycl/CMakeLists.txt

@@ -186,6 +186,7 @@ add_custom_target( sycl-toolchain
           llvm-as
           llvm-ar
           llvm-dis
+          llvm-foreach
           llvm-no-spir-kernel
           llvm-spirv
           llvm-link
@@ -227,6 +228,7 @@ set( SYCL_TOOLCHAIN_DEPLOY_COMPONENTS
      llvm-as
      llvm-ar
      llvm-dis
+     llvm-foreach
      llvm-no-spir-kernel
      llvm-spirv
      llvm-link