diff --git a/eng/tools/lint-diff/src/processChanges.ts b/eng/tools/lint-diff/src/processChanges.ts
index 6824683d5776..d1a29c184934 100644
--- a/eng/tools/lint-diff/src/processChanges.ts
+++ b/eng/tools/lint-diff/src/processChanges.ts
@@ -135,9 +135,18 @@ export async function buildState(
   // For readme files that have changed but there are no affected swaggers,
   // add them to the map with no tags
   for (const changedReadme of existingChangedFiles.filter(readme)) {
+    const readmePath = resolve(rootPath, changedReadme);
+
+    // Skip readme.md files that don't have "input-file:" as autorest cannot
+    // scan them.
+    const readmeContent = await readFile(readmePath, { encoding: "utf-8" });
+    if (!readmeContent.includes("input-file:")) {
+      continue;
+    }
+
     const service = specModels.get(getService(changedReadme))!;
     const readmes = await service.getReadmes();
-    const readmeObject = readmes.get(resolve(rootPath, changedReadme))!;
+    const readmeObject = readmes.get(readmePath)!;
 
     if (!changedFileAndTagsMap.has(changedReadme)) {
       changedFileAndTagsMap.set(changedReadme, {
diff --git a/eng/tools/lint-diff/test/fixtures/buildState/specification/no-input-file/readme.md b/eng/tools/lint-diff/test/fixtures/buildState/specification/no-input-file/readme.md
new file mode 100644
index 000000000000..38902d5bade1
--- /dev/null
+++ b/eng/tools/lint-diff/test/fixtures/buildState/specification/no-input-file/readme.md
@@ -0,0 +1,13 @@
+# Widget
+
+> see https://aka.ms/autorest
+> This is the AutoRest configuration file for Widget.
+
+## Configuration
+
+Required if any services under this folder are RPaaS.
+
+```yaml
+openapi-type: arm
+openapi-subtype: rpaas
+```
\ No newline at end of file
diff --git a/eng/tools/lint-diff/test/processChanges.test.ts b/eng/tools/lint-diff/test/processChanges.test.ts
index 5fad4a37997f..294f4c3c5de1 100644
--- a/eng/tools/lint-diff/test/processChanges.test.ts
+++ b/eng/tools/lint-diff/test/processChanges.test.ts
@@ -249,4 +249,13 @@ describe("buildState", () => {
       ),
     ).not.toThrow();
   });
+
+  test.skipIf(isWindows())("does not include readme files that has no input-file:", async () => {
+    const actual = await buildState(
+      ["specification/no-input-file/readme.md"],
+      "test/fixtures/buildState/",
+    );
+
+    expect(actual).toEqual([new Map<string, ReadmeAffectedTags>(), []]);
+  });
 });