From aa7ac5d8d53ba8db5ed99aef99ca5add1d14b29e Mon Sep 17 00:00:00 2001 From: Ajetunmobi Isaac Date: Sun, 24 Nov 2024 11:20:24 +0100 Subject: [PATCH 1/4] implemented include empty directory option --- repomix.config.json | 3 +- src/config/configSchema.ts | 2 + src/core/file/fileSearch.ts | 64 ++++++++++++++++++++++++++++-- src/core/file/fileTreeGenerate.ts | 43 ++++++++++++-------- src/core/output/outputGenerate.ts | 7 +++- src/core/packager.ts | 2 +- tests/core/file/fileSearch.test.ts | 58 +++++++++++++++++++++++++-- tests/core/packager.test.ts | 7 +++- 8 files changed, 157 insertions(+), 29 deletions(-) diff --git a/repomix.config.json b/repomix.config.json index 6f3128730..e868ec9a9 100644 --- a/repomix.config.json +++ b/repomix.config.json @@ -7,7 +7,8 @@ "removeComments": false, "removeEmptyLines": false, "topFilesLength": 5, - "showLineNumbers": false + "showLineNumbers": false, + "includeEmptyDirectories": true }, "include": [], "ignore": { diff --git a/src/config/configSchema.ts b/src/config/configSchema.ts index 428b10f91..f40d84e16 100644 --- a/src/config/configSchema.ts +++ b/src/config/configSchema.ts @@ -24,6 +24,7 @@ export const repomixConfigBaseSchema = z.object({ topFilesLength: z.number().optional(), showLineNumbers: z.boolean().optional(), copyToClipboard: z.boolean().optional(), + includeEmptyDirectories: z.boolean().optional() }) .optional(), include: z.array(z.string()).optional(), @@ -54,6 +55,7 @@ export const repomixConfigDefaultSchema = z.object({ topFilesLength: z.number().int().min(0).default(5), showLineNumbers: z.boolean().default(false), copyToClipboard: z.boolean().default(false), + includeEmptyDirectories: z.boolean().optional() }) .default({}), include: z.array(z.string()).default([]), diff --git a/src/core/file/fileSearch.ts b/src/core/file/fileSearch.ts index 6ae8a2517..9ccba5116 100644 --- a/src/core/file/fileSearch.ts +++ b/src/core/file/fileSearch.ts @@ -4,8 +4,47 @@ import { defaultIgnoreList } from '../../config/defaultIgnore.js'; import { logger } from '../../shared/logger.js'; import { sortPaths } from './filePathSort.js'; import { PermissionError, checkDirectoryPermissions } from './permissionCheck.js'; +import { minimatch } from 'minimatch'; +import path from 'path'; +import fs from 'node:fs/promises'; + +export interface FileSearchResult { + filePaths: string[]; + emptyDirPaths: string[]; +} + +const findEmptyDirectories = async ( + rootDir: string, + directories: string[], + ignorePatterns: string[], +): Promise => { + const emptyDirs: string[] = []; + + for (const dir of directories) { + const fullPath = path.join(rootDir, dir); + try { + const entries = await fs.readdir(fullPath); + const hasVisibleContents = entries.some(entry => !entry.startsWith('.')); + + if (!hasVisibleContents) { + // This checks if the directory itself matches any ignore patterns + const shouldIgnore = ignorePatterns.some(pattern => + minimatch(dir, pattern) || minimatch(`${dir}/`, pattern) + ); + + if (!shouldIgnore) { + emptyDirs.push(dir); + } + } + } catch (error) { + logger.debug(`Error checking directory ${dir}:`, error); + } + } + + return emptyDirs; +}; -export const searchFiles = async (rootDir: string, config: RepomixConfigMerged): Promise => { +export const searchFiles = async (rootDir: string, config: RepomixConfigMerged): Promise => { // First check directory permissions const permissionCheck = await checkDirectoryPermissions(rootDir); @@ -47,10 +86,29 @@ export const searchFiles = async (rootDir: string, config: RepomixConfigMerged): throw error; }); + + let emptyDirPaths: string[] = []; + if (config.output.includeEmptyDirectories) { + const directories = await globby(includePatterns, { + cwd: rootDir, + ignore: [...ignorePatterns], + ignoreFiles: [...ignoreFilePatterns], + onlyDirectories: true, + absolute: false, + dot: true, + followSymbolicLinks: false, + }); + + emptyDirPaths = await findEmptyDirectories(rootDir, directories, ignorePatterns); + } + logger.trace(`Filtered ${filePaths.length} files`); - const sortedPaths = sortPaths(filePaths); - return sortedPaths; + return { + filePaths: sortPaths(filePaths), + emptyDirPaths: sortPaths(emptyDirPaths), + }; + } catch (error: unknown) { // Re-throw PermissionError as is if (error instanceof PermissionError) { diff --git a/src/core/file/fileTreeGenerate.ts b/src/core/file/fileTreeGenerate.ts index b20c4628c..a0f07dc95 100644 --- a/src/core/file/fileTreeGenerate.ts +++ b/src/core/file/fileTreeGenerate.ts @@ -1,4 +1,4 @@ -import path from 'node:path'; +import nodepath from 'node:path'; interface TreeNode { name: string; @@ -8,28 +8,37 @@ interface TreeNode { const createTreeNode = (name: string, isDirectory: boolean): TreeNode => ({ name, children: [], isDirectory }); -export const generateFileTree = (files: string[]): TreeNode => { +export const generateFileTree = (files: string[], emptyDirPaths: string[] = []): TreeNode => { const root: TreeNode = createTreeNode('root', true); for (const file of files) { - const parts = file.split(path.sep); - let currentNode = root; + addPathToTree(root, file, false); + } + + // Add empty directories + for (const dir of emptyDirPaths) { + addPathToTree(root, dir, true); + } - for (let i = 0; i < parts.length; i++) { - const part = parts[i]; - const isLastPart = i === parts.length - 1; - let child = currentNode.children.find((c) => c.name === part); + return root; +}; - if (!child) { - child = createTreeNode(part, !isLastPart); - currentNode.children.push(child); - } +const addPathToTree = (root: TreeNode, path: string, isDirectory: boolean): void => { + const parts = path.split(nodepath.sep); + let currentNode = root; - currentNode = child; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const isLastPart = i === parts.length - 1; + let child = currentNode.children.find((c) => c.name === part); + + if (!child) { + child = createTreeNode(part, !isLastPart || isDirectory); + currentNode.children.push(child); } - } - return root; + currentNode = child; + } }; const sortTreeNodes = (node: TreeNode) => { @@ -59,7 +68,7 @@ export const treeToString = (node: TreeNode, prefix = ''): string => { return result; }; -export const generateTreeString = (files: string[]): string => { - const tree = generateFileTree(files); +export const generateTreeString = (files: string[], emptyDirPaths: string[] = []): string => { + const tree = generateFileTree(files, emptyDirPaths); return treeToString(tree).trim(); }; diff --git a/src/core/output/outputGenerate.ts b/src/core/output/outputGenerate.ts index 906402495..7a848329d 100644 --- a/src/core/output/outputGenerate.ts +++ b/src/core/output/outputGenerate.ts @@ -17,6 +17,7 @@ import { import { getMarkdownTemplate } from './outputStyles/markdownStyle.js'; import { getPlainTemplate } from './outputStyles/plainStyle.js'; import { getXmlTemplate } from './outputStyles/xmlStyle.js'; +import { searchFiles } from '../file/fileSearch.js'; const createRenderContext = (outputGeneratorContext: OutputGeneratorContext) => { return { @@ -78,9 +79,13 @@ export const buildOutputGeneratorContext = async ( } } + const emptyDirPaths = config.output.includeEmptyDirectories ? + (await searchFiles(rootDir, config)).emptyDirPaths : + []; + return { generationDate: new Date().toISOString(), - treeString: generateTreeString(allFilePaths), + treeString: generateTreeString(allFilePaths, emptyDirPaths), processedFiles, config, instruction: repositoryInstruction, diff --git a/src/core/packager.ts b/src/core/packager.ts index c3492cda7..fcad58bcd 100644 --- a/src/core/packager.ts +++ b/src/core/packager.ts @@ -38,7 +38,7 @@ export const pack = async ( ): Promise => { // Get all file paths considering the config progressCallback('Searching for files...'); - const filePaths = await deps.searchFiles(rootDir, config); + const { filePaths } = await deps.searchFiles(rootDir, config); // Collect raw files progressCallback('Collecting files...'); diff --git a/tests/core/file/fileSearch.test.ts b/tests/core/file/fileSearch.test.ts index 892bfe73f..b6c4f465b 100644 --- a/tests/core/file/fileSearch.test.ts +++ b/tests/core/file/fileSearch.test.ts @@ -46,6 +46,54 @@ describe('fileSearch', () => { const filePatterns = await getIgnoreFilePatterns(mockConfig); expect(filePatterns).toEqual(['**/.repomixignore']); }); + + test('should handle empty directories when enabled', async () => { + const mockConfig = createMockConfig({ + output: { + includeEmptyDirectories: true, + }, + }); + + const mockFilePaths = ['src/file1.js', 'src/file2.js']; + const mockEmptyDirs = ['src/empty', 'empty-root']; + + vi.mocked(globby).mockImplementation(async (_, options) => { + if (options?.onlyDirectories) { + return mockEmptyDirs; + } + return mockFilePaths; + }); + + vi.mocked(fs.readdir).mockResolvedValue([]); + + const result = await searchFiles('/mock/root', mockConfig); + + expect(result.filePaths).toEqual(mockFilePaths); + expect(result.emptyDirPaths).toEqual(mockEmptyDirs); + }); + + test('should not collect empty directories when disabled', async () => { + const mockConfig = createMockConfig({ + output: { + includeEmptyDirectories: false, + }, + }); + + const mockFilePaths = ['src/file1.js', 'src/file2.js']; + + vi.mocked(globby).mockImplementation(async (_, options) => { + if (options?.onlyDirectories) { + throw new Error('Should not search for directories when disabled'); + } + return mockFilePaths; + }); + + const result = await searchFiles('/mock/root', mockConfig); + + expect(result.filePaths).toEqual(mockFilePaths); + expect(result.emptyDirPaths).toEqual([]); + expect(globby).toHaveBeenCalledTimes(1); + }); }); describe('getIgnorePatterns', () => { @@ -194,8 +242,9 @@ node_modules }); const result = await searchFiles('/mock/root', mockConfig); - expect(result).toEqual(['root/another/file3.js', 'root/subdir/file2.js', 'root/file1.js']); - expect(result).not.toContain('root/subdir/ignored.js'); + expect(result.filePaths).toEqual(['root/another/file3.js', 'root/subdir/file2.js', 'root/file1.js']); + expect(result.filePaths).not.toContain('root/subdir/ignored.js'); + expect(result.emptyDirPaths).toEqual([]); }); test('should not apply .gitignore when useGitignore is false', async () => { @@ -219,8 +268,9 @@ node_modules const result = await searchFiles('/mock/root', mockConfig); - expect(result).toEqual(mockFileStructure); - expect(result).toContain('root/subdir/ignored.js'); + expect(result.filePaths).toEqual(mockFileStructure); + expect(result.filePaths).toContain('root/subdir/ignored.js'); + expect(result.emptyDirPaths).toEqual([]); }); }); }); diff --git a/tests/core/packager.test.ts b/tests/core/packager.test.ts index 57b059b87..332aedb11 100644 --- a/tests/core/packager.test.ts +++ b/tests/core/packager.test.ts @@ -34,7 +34,10 @@ describe('packager', () => { vi.resetAllMocks(); const file2Path = path.join('dir1', 'file2.txt'); mockDeps = { - searchFiles: vi.fn().mockResolvedValue(['file1.txt', file2Path]), + searchFiles: vi.fn().mockResolvedValue({ + filePaths: ['file1.txt', file2Path], + emptyDirPaths: [], + }), collectFiles: vi.fn().mockResolvedValue([ { path: 'file1.txt', content: 'raw content 1' }, { path: file2Path, content: 'raw content 2' }, @@ -111,7 +114,7 @@ describe('packager', () => { const mockConfig = createMockConfig(); const suspiciousFile = 'suspicious.txt'; const file2Path = path.join('dir1', 'file2.txt'); - vi.mocked(mockDeps.searchFiles).mockResolvedValue(['file1.txt', file2Path, suspiciousFile]); + vi.mocked(mockDeps.searchFiles).mockResolvedValue({ emptyDirPaths: [], filePaths: ['file1.txt', file2Path, suspiciousFile]}); vi.mocked(mockDeps.collectFiles).mockResolvedValue([ { path: 'file1.txt', content: 'raw content 1' }, { path: file2Path, content: 'raw content 2' }, From 53bb5005ffa6432d419be00a0b163313bdebafdd Mon Sep 17 00:00:00 2001 From: Ajetunmobi Isaac Date: Sun, 24 Nov 2024 11:27:09 +0100 Subject: [PATCH 2/4] fixed node path import --- src/config/configSchema.ts | 4 ++-- src/core/file/fileSearch.ts | 14 +++++--------- src/core/output/outputGenerate.ts | 6 ++---- tests/core/packager.test.ts | 5 ++++- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/config/configSchema.ts b/src/config/configSchema.ts index f40d84e16..6243fba27 100644 --- a/src/config/configSchema.ts +++ b/src/config/configSchema.ts @@ -24,7 +24,7 @@ export const repomixConfigBaseSchema = z.object({ topFilesLength: z.number().optional(), showLineNumbers: z.boolean().optional(), copyToClipboard: z.boolean().optional(), - includeEmptyDirectories: z.boolean().optional() + includeEmptyDirectories: z.boolean().optional(), }) .optional(), include: z.array(z.string()).optional(), @@ -55,7 +55,7 @@ export const repomixConfigDefaultSchema = z.object({ topFilesLength: z.number().int().min(0).default(5), showLineNumbers: z.boolean().default(false), copyToClipboard: z.boolean().default(false), - includeEmptyDirectories: z.boolean().optional() + includeEmptyDirectories: z.boolean().optional(), }) .default({}), include: z.array(z.string()).default([]), diff --git a/src/core/file/fileSearch.ts b/src/core/file/fileSearch.ts index 9ccba5116..8efe5b962 100644 --- a/src/core/file/fileSearch.ts +++ b/src/core/file/fileSearch.ts @@ -1,12 +1,12 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; import { globby } from 'globby'; +import { minimatch } from 'minimatch'; import type { RepomixConfigMerged } from '../../config/configSchema.js'; import { defaultIgnoreList } from '../../config/defaultIgnore.js'; import { logger } from '../../shared/logger.js'; import { sortPaths } from './filePathSort.js'; import { PermissionError, checkDirectoryPermissions } from './permissionCheck.js'; -import { minimatch } from 'minimatch'; -import path from 'path'; -import fs from 'node:fs/promises'; export interface FileSearchResult { filePaths: string[]; @@ -24,13 +24,11 @@ const findEmptyDirectories = async ( const fullPath = path.join(rootDir, dir); try { const entries = await fs.readdir(fullPath); - const hasVisibleContents = entries.some(entry => !entry.startsWith('.')); + const hasVisibleContents = entries.some((entry) => !entry.startsWith('.')); if (!hasVisibleContents) { // This checks if the directory itself matches any ignore patterns - const shouldIgnore = ignorePatterns.some(pattern => - minimatch(dir, pattern) || minimatch(`${dir}/`, pattern) - ); + const shouldIgnore = ignorePatterns.some((pattern) => minimatch(dir, pattern) || minimatch(`${dir}/`, pattern)); if (!shouldIgnore) { emptyDirs.push(dir); @@ -86,7 +84,6 @@ export const searchFiles = async (rootDir: string, config: RepomixConfigMerged): throw error; }); - let emptyDirPaths: string[] = []; if (config.output.includeEmptyDirectories) { const directories = await globby(includePatterns, { @@ -108,7 +105,6 @@ export const searchFiles = async (rootDir: string, config: RepomixConfigMerged): filePaths: sortPaths(filePaths), emptyDirPaths: sortPaths(emptyDirPaths), }; - } catch (error: unknown) { // Re-throw PermissionError as is if (error instanceof PermissionError) { diff --git a/src/core/output/outputGenerate.ts b/src/core/output/outputGenerate.ts index 7a848329d..dc5d739a6 100644 --- a/src/core/output/outputGenerate.ts +++ b/src/core/output/outputGenerate.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import Handlebars from 'handlebars'; import type { RepomixConfigMerged } from '../../config/configSchema.js'; import { RepomixError } from '../../shared/errorHandle.js'; +import { searchFiles } from '../file/fileSearch.js'; import { generateTreeString } from '../file/fileTreeGenerate.js'; import type { ProcessedFile } from '../file/fileTypes.js'; import type { OutputGeneratorContext } from './outputGeneratorTypes.js'; @@ -17,7 +18,6 @@ import { import { getMarkdownTemplate } from './outputStyles/markdownStyle.js'; import { getPlainTemplate } from './outputStyles/plainStyle.js'; import { getXmlTemplate } from './outputStyles/xmlStyle.js'; -import { searchFiles } from '../file/fileSearch.js'; const createRenderContext = (outputGeneratorContext: OutputGeneratorContext) => { return { @@ -79,9 +79,7 @@ export const buildOutputGeneratorContext = async ( } } - const emptyDirPaths = config.output.includeEmptyDirectories ? - (await searchFiles(rootDir, config)).emptyDirPaths : - []; + const emptyDirPaths = config.output.includeEmptyDirectories ? (await searchFiles(rootDir, config)).emptyDirPaths : []; return { generationDate: new Date().toISOString(), diff --git a/tests/core/packager.test.ts b/tests/core/packager.test.ts index 332aedb11..b11589c37 100644 --- a/tests/core/packager.test.ts +++ b/tests/core/packager.test.ts @@ -114,7 +114,10 @@ describe('packager', () => { const mockConfig = createMockConfig(); const suspiciousFile = 'suspicious.txt'; const file2Path = path.join('dir1', 'file2.txt'); - vi.mocked(mockDeps.searchFiles).mockResolvedValue({ emptyDirPaths: [], filePaths: ['file1.txt', file2Path, suspiciousFile]}); + vi.mocked(mockDeps.searchFiles).mockResolvedValue({ + emptyDirPaths: [], + filePaths: ['file1.txt', file2Path, suspiciousFile], + }); vi.mocked(mockDeps.collectFiles).mockResolvedValue([ { path: 'file1.txt', content: 'raw content 1' }, { path: file2Path, content: 'raw content 2' }, From 540a0ce3ba8f973dad2bb388fdabfef559c5e320 Mon Sep 17 00:00:00 2001 From: Ajetunmobi Isaac Date: Sun, 24 Nov 2024 11:48:11 +0100 Subject: [PATCH 3/4] fixed failed test & added minor improvements --- src/core/output/outputGenerate.ts | 14 ++++++++++++-- tests/core/file/fileSearch.test.ts | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/core/output/outputGenerate.ts b/src/core/output/outputGenerate.ts index dc5d739a6..1986d325a 100644 --- a/src/core/output/outputGenerate.ts +++ b/src/core/output/outputGenerate.ts @@ -79,8 +79,18 @@ export const buildOutputGeneratorContext = async ( } } - const emptyDirPaths = config.output.includeEmptyDirectories ? (await searchFiles(rootDir, config)).emptyDirPaths : []; - + // const emptyDirPaths = config.output.includeEmptyDirectories ? (await searchFiles(rootDir, config)).emptyDirPaths : []; + let emptyDirPaths: string[] = []; + if (config.output.includeEmptyDirectories) { + try { + const searchResult = await searchFiles(rootDir, config); + emptyDirPaths = searchResult.emptyDirPaths; + } catch (error) { + if (error instanceof Error) { + throw new RepomixError(`Failed to search for empty directories: ${error.message}`); + } + } + } return { generationDate: new Date().toISOString(), treeString: generateTreeString(allFilePaths, emptyDirPaths), diff --git a/tests/core/file/fileSearch.test.ts b/tests/core/file/fileSearch.test.ts index b6c4f465b..65b9e227d 100644 --- a/tests/core/file/fileSearch.test.ts +++ b/tests/core/file/fileSearch.test.ts @@ -69,7 +69,7 @@ describe('fileSearch', () => { const result = await searchFiles('/mock/root', mockConfig); expect(result.filePaths).toEqual(mockFilePaths); - expect(result.emptyDirPaths).toEqual(mockEmptyDirs); + expect(result.emptyDirPaths.sort()).toEqual(mockEmptyDirs.sort()); }); test('should not collect empty directories when disabled', async () => { From be993d847ee5d9eb57eb5f439424b2194dc32d23 Mon Sep 17 00:00:00 2001 From: Kazuki Yamada Date: Mon, 25 Nov 2024 00:20:08 +0900 Subject: [PATCH 4/4] refactor(output): remove comment out --- src/core/output/outputGenerate.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/output/outputGenerate.ts b/src/core/output/outputGenerate.ts index 1986d325a..36ae6e1ba 100644 --- a/src/core/output/outputGenerate.ts +++ b/src/core/output/outputGenerate.ts @@ -79,7 +79,6 @@ export const buildOutputGeneratorContext = async ( } } - // const emptyDirPaths = config.output.includeEmptyDirectories ? (await searchFiles(rootDir, config)).emptyDirPaths : []; let emptyDirPaths: string[] = []; if (config.output.includeEmptyDirectories) { try {