diff --git a/.ci-config.json b/.ci-config.json
index c3ea25f1fd8d..59995fd1ff4b 100644
--- a/.ci-config.json
+++ b/.ci-config.json
@@ -61,9 +61,9 @@
         "LICENSE.txt",
         "README.md",
         "ChangeLog.md",
-        "**/ChangeLog.md",
-        "**/readme.md",
-        "src/**/document/*",
+        "*/ChangeLog.md",
+        "*/readme.md",
+        "src/*/document/*",
         ".ci-config.json",
         "tools/PrepareAutorestModule.ps1",
         "tools/SyncFromMainBranch.ps1",
@@ -84,7 +84,7 @@
     },
     {
       "patterns": [
-        "src/{ModuleName}/**/*.md$"
+        "src/{ModuleName}/*/*.md$"
       ],
       "phases": [
         "build:module",
@@ -93,7 +93,7 @@
     },
     {
       "patterns": [
-        "src/{ModuleName}/**/*.csproj$"
+        "src/{ModuleName}/*/*.csproj$"
       ],
       "phases": [
         "build:related-module",
diff --git a/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs b/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs
index ac5734d3099d..576695e2906a 100644
--- a/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs
+++ b/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs
@@ -240,7 +240,7 @@ private bool ProcessTargetModule(Dictionary<string, string[]> csprojMap)
 
         private string ProcessSinglePattern(string pattern)
         {
-            return pattern.Replace("**", ".*").Replace("{ModuleName}", "(?<ModuleName>[^/]+)");
+            return pattern.Replace(".", "\\.").Replace("*", ".*").Replace("{ModuleName}", "(?<ModuleName>[^/]+)");
         }
 
         private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEachPhase(List<(Regex, List<string>)> ruleList, Dictionary<string, string[]> csprojMap)
@@ -462,17 +462,7 @@ public override bool Execute()
                 }
                 else
                 {
-                    Dictionary<string, HashSet<string>> influencedModuleInfo = new Dictionary<string, HashSet<string>>
-                    {
-                        [BUILD_PHASE] = new HashSet<string>() { ACCOUNT_MODULE_NAME },
-                        [ANALYSIS_BREAKING_CHANGE_PHASE] = new HashSet<string>() { ACCOUNT_MODULE_NAME },
-                        [ANALYSIS_DEPENDENCY_PHASE] = new HashSet<string>() { ACCOUNT_MODULE_NAME },
-                        [ANALYSIS_HELP_PHASE] = new HashSet<string>() { ACCOUNT_MODULE_NAME },
-                        [ANALYSIS_SIGNATURE_PHASE] = new HashSet<string>() { ACCOUNT_MODULE_NAME },
-                        [TEST_PHASE] = new HashSet<string>() { ACCOUNT_MODULE_NAME }
-                    };
-                    FilterTaskResult.PhaseInfo = CalculateCsprojForBuildAndTest(influencedModuleInfo, csprojMap);
-                    return true;
+                    return false;
                 }
             }
             return true;
diff --git a/tools/PrepareAutorestModule.ps1 b/tools/PrepareAutorestModule.ps1
index d9f266d5100b..660ca32b74b0 100644
--- a/tools/PrepareAutorestModule.ps1
+++ b/tools/PrepareAutorestModule.ps1
@@ -46,7 +46,10 @@ foreach ($file in $ChangedFiles)
 }
 if ($ModuleSet.Contains($ALL_MODULE))
 {
-    $ModuleList = (Get-ChildItem "$PSScriptRoot\..\src\" -Directory -Exclude helpers,lib).Name | Where-Object { $SKIP_MODULES -notcontains $_ -and (Get-Item env:SELECTEDMODULELIST).Value.Split(';') -contains $_ }
+    $Null = $ModuleSet.Remove($ALL_MODULE)
+    $SelectedModuleList = (Get-ChildItem "$PSScriptRoot\..\src\").Name | Where-Object { (Get-Item env:SELECTEDMODULELIST).Value.Split(';') -contains $_ }
+    $Null = $ModuleSet.Add($SelectedModuleList)
+    $ModuleList = $ModuleSet | Where-Object { $SKIP_MODULES -notcontains $_ }
 }
 else
 {