[Python Migration] Separated common IPEX and PyTorch rules

clang/lib/DPCT/UserDefinedRules/UserDefinedRules.cpp

@@ -344,17 +344,79 @@ MetaRuleObject::PatternRewriter::PatternRewriter(
   Subrules = S;
 }
 
+// Read a YAML file recursively and substitute any "!include <filename>"
+// directive with the contents of the referenced file.
+std::unique_ptr<llvm::MemoryBuffer>
+readYAMLFile(const llvm::StringRef &ruleFilePath) {
+  // Load the rule file into a MemoryBuffer
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
+      llvm::MemoryBuffer::getFile(ruleFilePath);
+  if (!Buffer) {
+    llvm::errs() << "Error: failed to read " << ruleFilePath << ": "
+                 << Buffer.getError().message() << "\n";
+    clang::dpct::ShowStatus(MigrationErrorInvalidRuleFilePath);
+    dpctExit(MigrationErrorInvalidRuleFilePath);
+  }
+
+  // Get the directory path of the rule file.
+  llvm::SmallString<128> directoryPath(ruleFilePath);
+  llvm::sys::path::remove_filename(directoryPath);
+
+  // Split the file content into lines using newline as the delimiter.
+  llvm::StringRef buffContent = std::move(*Buffer)->getBuffer();
+  llvm::SmallVector<StringRef, 16> lines;
+  buffContent.split(lines, "\n");
+
+  std::stringstream output;
+  for (auto line : lines) {
+    auto lineStr = line.str();
+
+    // Extract the filename following !include.
+    std::istringstream iss(lineStr);
+    std::string incDirective, incRuleFilePath;
+    // iss >> incDirective >> incRuleFilePath;
+    iss >> incDirective;
+    std::getline(iss >> std::ws, incRuleFilePath);
+
+    // Trim trailing spaces from incRuleFilePath
+    incRuleFilePath.erase(incRuleFilePath.find_last_not_of(" \t\n\r\f\v") + 1);
+
+    if (incDirective == "!include") {
+      if (!incRuleFilePath.empty()) {
+        // Remove surrounding quotes if they exist.
+        if (incRuleFilePath.front() == '"' && incRuleFilePath.back() == '"') {
+          incRuleFilePath =
+              incRuleFilePath.substr(1, incRuleFilePath.size() - 2);
+        }
+
+        // Calculate the absolute path of the included file based on its parent
+        // rule file
+        llvm::SmallString<128> incRuleFileAbsPath = directoryPath;
+        llvm::sys::path::append(incRuleFileAbsPath, incRuleFilePath);
+
+        // Recursively process the included file.
+        if (llvm::sys::fs::exists(incRuleFileAbsPath)) {
+          output << readYAMLFile(incRuleFileAbsPath.str())->getBuffer().str()
+                 << "\n";
+        } else {
+          output << readYAMLFile(incRuleFilePath)->getBuffer().str() << "\n";
+        }
+      } else {
+        continue;
+      }
+    } else {
+      output << lineStr << "\n";
+    }
+  }
+
+  return llvm::MemoryBuffer::getMemBufferCopy(output.str(), ruleFilePath);
+}
+
 void importRules(std::vector<clang::tooling::UnifiedPath> &RuleFiles) {
   for (auto &RuleFile : RuleFiles) {
     // open the yaml file
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
-        llvm::MemoryBuffer::getFile(RuleFile.getCanonicalPath());
-    if (!Buffer) {
-      llvm::errs() << "Error: failed to read " << RuleFile << ": "
-                   << Buffer.getError().message() << "\n";
-      clang::dpct::ShowStatus(MigrationErrorInvalidRuleFilePath);
-      dpctExit(MigrationErrorInvalidRuleFilePath);
-    }
+        readYAMLFile(RuleFile.getCanonicalPath());
 
     // load rules
     std::vector<std::shared_ptr<MetaRuleObject>> CurrentRules;

clang/test/dpct/python_migration/case_001/include_rule_file/expected.py

@@ -0,0 +1,10 @@
+import a_repl
+import b_repl
+
+a_repl()
+b_repl()
+b1_repl()
+b2_repl()
+b3_repl()
+b4_repl()
+b5_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/include_rule_file.cpp

@@ -0,0 +1,13 @@
+// RUN: rm -rf %T && mkdir -p %T
+// RUN: cd %T
+// RUN: cp %S/input.py ./
+// RUN: cp %S/expected.py ./
+// RUN: cp -r %S/rules ./
+// RUN: echo "!include %T/rules/b/b5.yaml" >> %T/rules/python_build_script_migration_rule_inc.yaml
+
+// RUN: dpct -in-root ./ --migrate-build-script-only --migrate-build-script=Python --rule-file=%T/rules/python_build_script_migration_rule_inc.yaml input.py
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/dpct_output/input.py >> %T/diff.txt
+// RUN: echo "end" >> %T/diff.txt
+// CHECK: begin
+// CHECK-NEXT: end

clang/test/dpct/python_migration/case_001/include_rule_file/input.py

@@ -0,0 +1,10 @@
+import a
+import b
+
+a()
+b()
+b1()
+b2()
+b3()
+b4()
+b5()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/a.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_a
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_a
+  In: a()
+  Out: a_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b 4.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b4
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b4
+  In: b4()
+  Out: b4_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b
+  In: b()
+  Out: b_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b1.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b1
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b1
+  In: b1()
+  Out: b1_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b2.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b2
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b2
+  In: b2()
+  Out: b2_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b3.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b3
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b3
+  In: b3()
+  Out: b3_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b5.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b5
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b5
+  In: b5()
+  Out: b5_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/python_build_script_migration_rule_inc.yaml

@@ -0,0 +1,29 @@
+!include a.yaml
+
+- Rule: rule_import_a
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: import_a
+  In: import a
+  Out: import a_repl
+
+!include b/b.yaml
+
+- Rule: rule_import_b
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: import_b
+  In: import b
+  Out: import b_repl
+
+!include "b/b1.yaml"
+
+!include b/b2.yaml    
+
+    !include b/b3.yaml
+
+!include "b/b 4.yaml"
+
+!include

clang/tools/dpct/extensions/CMakeLists.txt

@@ -12,6 +12,7 @@ set(dpct_cmake_rule_files
 set(dpct_python_rule_files
   ${CMAKE_SOURCE_DIR}/../clang/tools/dpct/extensions/python_rules/python_build_script_migration_rule_ipex.yaml
   ${CMAKE_SOURCE_DIR}/../clang/tools/dpct/extensions/python_rules/python_build_script_migration_rule_pytorch.yaml
+  ${CMAKE_SOURCE_DIR}/../clang/tools/dpct/extensions/python_rules/python_build_script_migration_rule_pytorch_common.yaml
 )
 
 set(dpct_pytorch_api_rule_files