diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll new file mode 100644 index 000000000000..3adcba2b4188 --- /dev/null +++ b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll @@ -0,0 +1,26 @@ +; UNSUPPORTED: system-windows +; +; RUN: echo 'Content of first file' > %t1.tgt +; RUN: echo 'Content of second file' > %t2.tgt +; RUN: echo "%t1.tgt" > %t.list +; RUN: echo "%t2.tgt" >> %t.list +; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res +; RUN: FileCheck < %t.res %s +; CHECK: [[FIRST:.+1.tgt]] +; CHECK: [[SECOND:.+2.tgt]] +; +; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t +; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST +; CHECK-LIST: [[FIRST:.+\.out]] +; CHECK-LIST: [[SECOND:.+\.out]] +; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT +; CHECK-CONTENT: Content of + +; RUN: echo 'something' > %t3.tgt +; RUN: echo 'something again' > %t4.tgt +; RUN: echo "%t3.tgt" > %t1.list +; RUN: echo "%t4.tgt" >> %t1.list +; RUN: llvm-foreach --in-replace="{}" --in-replace="in" --in-file-list=%t.list --in-file-list=%t1.list -- echo -first-part-of-arg={}.out -first-part-of-arg=in.out > %t1.res +; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS +; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] +; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll new file mode 100644 index 000000000000..a8b30abca7d9 --- /dev/null +++ b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll @@ -0,0 +1,26 @@ +; REQUIRES: system-windows +; +; RUN: echo 'Content of first file' > %t1.tgt +; RUN: echo 'Content of second file' > %t2.tgt +; RUN: echo "%t1.tgt" > %t.list +; RUN: echo "%t2.tgt" >> %t.list +; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res +; RUN: FileCheck < %t.res %s +; CHECK: [[FIRST:.+1.tgt]] +; CHECK: [[SECOND:.+2.tgt]] +; +; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- copy "{}" %t +; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST +; CHECK-LIST: [[FIRST:.+\.out]] +; CHECK-LIST: [[SECOND:.+\.out]] +; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT +; CHECK-CONTENT: Content of + +; RUN: echo 'something' > %t3.tgt +; RUN: echo 'something again' > %t4.tgt +; RUN: echo "%t3.tgt" > %t1.list +; RUN: echo "%t4.tgt" >> %t1.list +; RUN: llvm-foreach --in-replace="{}" --in-replace="in" --in-file-list=%t.list --in-file-list=%t1.list -- echo -first-part-of-arg={}.out -first-part-of-arg=in.out > %t1.res +; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS +; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] +; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] diff --git a/llvm/tools/LLVMBuild.txt b/llvm/tools/LLVMBuild.txt index 6674783ba96e..7121c0bc517f 100644 --- a/llvm/tools/LLVMBuild.txt +++ b/llvm/tools/LLVMBuild.txt @@ -35,6 +35,7 @@ subdirectories = llvm-ifs llvm-exegesis llvm-extract + llvm-foreach llvm-jitlistener llvm-jitlink llvm-link diff --git a/llvm/tools/llvm-foreach/CMakeLists.txt b/llvm/tools/llvm-foreach/CMakeLists.txt new file mode 100644 index 000000000000..12f037101a6d --- /dev/null +++ b/llvm/tools/llvm-foreach/CMakeLists.txt @@ -0,0 +1,7 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_llvm_tool(llvm-foreach + llvm-foreach.cpp +) diff --git a/llvm/tools/llvm-foreach/LLVMBuild.txt b/llvm/tools/llvm-foreach/LLVMBuild.txt new file mode 100644 index 000000000000..0767d6c7f51e --- /dev/null +++ b/llvm/tools/llvm-foreach/LLVMBuild.txt @@ -0,0 +1,21 @@ +;===- ./tools/llvm-foreach/LLVMBuild.txt ---------------------*- Conf -*--===; +; +; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Tool +name = llvm-foreach +parent = Tools +required_libraries = Support diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp new file mode 100644 index 000000000000..6e815205592f --- /dev/null +++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp @@ -0,0 +1,214 @@ +//===- llvm-foreach.cpp - Command lines execution ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This source is utility to execute command lines. The specified command will +// be invoked as many times as necessary to use up the list of input items. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/SystemUtils.h" + +#include + +using namespace llvm; + +static cl::list InputFileLists{ + "in-file-list", cl::OneOrMore, + cl::desc("Input list of file names, file names must be delimited by a " + "newline character."), + cl::value_desc("filename")}; + +static cl::list InputCommandArgs{ + cl::Positional, cl::OneOrMore, cl::desc(""), + cl::value_desc("command")}; + +static cl::list Replaces{ + "in-replace", cl::OneOrMore, + cl::desc("Specify input path in input command, this will be replaced with " + "names read from corresponding input list of files."), + cl::value_desc("R")}; + +static cl::opt OutReplace{ + "out-replace", + cl::desc("Specify output path in input command, this will be replaced with " + "name of temporary file created for writing command's outputs."), + cl::init(""), cl::value_desc("R")}; + +static cl::opt OutDirectory{ + "out-dir", + cl::desc("Specify directory for output files; If unspecified, assume " + "system temporary directory."), + cl::init(""), cl::value_desc("R")}; + +static cl::opt OutFilesExt{ + "out-ext", + cl::desc("Specify extenstion for output files; If unspecified, assume " + ".out"), + cl::init("out"), cl::value_desc("R")}; + +// Emit list of produced files for better integration with other tools. +static cl::opt OutputFileList{ + "out-file-list", cl::desc("Specify filename for list of outputs."), + cl::value_desc("filename"), cl::init("")}; + +static void error(const Twine &Msg) { + errs() << "llvm-foreach: " << Msg << '\n'; + exit(1); +} + +static void error(std::error_code EC, const Twine &Prefix) { + if (EC) + error(Prefix + ": " + EC.message()); +} + +int main(int argc, char **argv) { + cl::ParseCommandLineOptions( + argc, argv, + "llvm-foreach: Execute specified command as many times as\n" + "necessary to use up the list of input items.\n" + "Usage:\n" + "llvm-foreach --in-file-list=a.list --in-replace='{}' -- echo '{}'\n" + "NOTE: commands containig redirects are not supported by llvm-foreach\n" + "yet.\n"); + + ExitOnError ExitOnErr("llvm-foreach: "); + + if (InputFileLists.size() != Replaces.size()) + error("Number of input file lists and input path replaces don't match."); + + std::vector> MBs; + std::vector LineIterators; + for (auto &InputFileList : InputFileLists) { + std::unique_ptr MB = ExitOnErr( + errorOrToExpected(MemoryBuffer::getFileOrSTDIN(InputFileList))); + LineIterators.push_back(line_iterator(*MB)); + MBs.push_back(std::move(MB)); + } + + SmallVector Args(InputCommandArgs.begin(), + InputCommandArgs.end()); + + if (Args.empty()) + error("No command?"); + + struct ArgumentReplace { + size_t ArgNum = 0; + // Index in argument string where replace length starts. + size_t Start = 0; + size_t ReplaceLen = 0; + }; + + // Find args to replace with filenames from input list. + std::vector InReplaceArgs; + ArgumentReplace OutReplaceArg; + for (size_t i = 1; i < Args.size(); ++i) { + for (auto &Replace : Replaces) { + size_t ReplaceStart = Args[i].find(Replace); + if (ReplaceStart != StringRef::npos) + InReplaceArgs.push_back({i, ReplaceStart, Replace.size()}); + } + + if (!OutReplace.empty() && Args[i].contains(OutReplace)) { + size_t ReplaceStart = Args[i].find(OutReplace); + if (ReplaceStart != StringRef::npos) + OutReplaceArg = {i, ReplaceStart, OutReplace.size()}; + } + } + + // Emit an error if user requested replace output file in the command but + // replace string is not found. + if (!OutReplace.empty() && OutReplaceArg.ArgNum == 0) + error("Couldn't find replace string for output in the command."); + + // Make sure that specified program exists, emit an error if not. + std::string Prog = + ExitOnErr(errorOrToExpected(sys::findProgramByName(Args[0]))); + + std::vector> FileLists(LineIterators.size()); + size_t PrevNumOfLines = 0; + for (size_t i = 0; i < FileLists.size(); ++i) { + for (; !LineIterators[i].is_at_eof(); ++LineIterators[i]) { + FileLists[i].push_back(LineIterators[i]->str()); + } + if (i != 0 && FileLists[i].size() != PrevNumOfLines) + error("All input file lists must have same number of lines!"); + PrevNumOfLines = FileLists[i].size(); + } + + std::error_code EC; + raw_fd_ostream OS{OutputFileList, EC, sys::fs::OpenFlags::OF_None}; + if (!OutputFileList.empty()) + error(EC, "error opening the file '" + OutputFileList + "'"); + + std::string ResOutArg; + std::vector ResInArgs(InReplaceArgs.size()); + std::string ResFileList = ""; + for (size_t j = 0; j != FileLists[0].size(); ++j) { + for (size_t i = 0; i < InReplaceArgs.size(); ++i) { + ArgumentReplace CurReplace = InReplaceArgs[i]; + std::string OriginalString = InputCommandArgs[CurReplace.ArgNum]; + ResInArgs[i] = (Twine(OriginalString.substr(0, CurReplace.Start)) + + Twine(FileLists[i][j]) + + Twine(OriginalString.substr(CurReplace.Start + + CurReplace.ReplaceLen))) + .str(); + Args[CurReplace.ArgNum] = ResInArgs[i]; + } + + SmallString<128> Path; + if (!OutReplace.empty()) { + // Create a file for command result. Add file name to output + // file list if needed. + std::string TempFileNameBase = sys::path::stem(OutReplace); + if (OutDirectory.empty()) + EC = sys::fs::createTemporaryFile(TempFileNameBase, OutFilesExt, Path); + else { + SmallString<128> PathPrefix(OutDirectory); + // "CreateUniqueFile" functions accepts "Model" - special string with + // substring containing sequence of "%" symbols. In the resulting + // filename "%" symbols sequence from "Model" string will be replaced + // with random chars to make it unique. + llvm::sys::path::append(PathPrefix, + TempFileNameBase + "-%%%%%%." + OutFilesExt); + EC = sys::fs::createUniqueFile(PathPrefix, Path); + } + error(EC, "Could not create a file for command output."); + + std::string OriginalString = InputCommandArgs[OutReplaceArg.ArgNum]; + ResOutArg = + (Twine(OriginalString.substr(0, OutReplaceArg.Start)) + Twine(Path) + + Twine(OriginalString.substr(OutReplaceArg.Start + + OutReplaceArg.ReplaceLen))) + .str(); + Args[OutReplaceArg.ArgNum] = ResOutArg; + + if (!OutputFileList.empty()) + OS << Path << "\n"; + } + + std::string ErrMsg; + // TODO: Add possibility to execute commands in parallel. + int Result = + sys::ExecuteAndWait(Prog, Args, /*Env=*/None, /*Redirects=*/None, + /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg); + if (Result != 0) + error(ErrMsg); + } + + if (!OutputFileList.empty()) { + OS.close(); + } + + return 0; +} diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt index fc62b446ee9f..3d2cdac471b3 100644 --- a/sycl/CMakeLists.txt +++ b/sycl/CMakeLists.txt @@ -186,6 +186,7 @@ add_custom_target( sycl-toolchain llvm-as llvm-ar llvm-dis + llvm-foreach llvm-no-spir-kernel llvm-spirv llvm-link @@ -227,6 +228,7 @@ set( SYCL_TOOLCHAIN_DEPLOY_COMPONENTS llvm-as llvm-ar llvm-dis + llvm-foreach llvm-no-spir-kernel llvm-spirv llvm-link