From 7a02eb9e589f55f270e7ba12c52b020f72ff3a64 Mon Sep 17 00:00:00 2001 From: Yamada Dev Date: Sat, 1 Feb 2025 17:38:05 +0900 Subject: [PATCH 1/3] chore(website): Increase CPU for website from 1 to 2 --- website/server/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/server/cloudbuild.yaml b/website/server/cloudbuild.yaml index 5a3355826..a0b2c6534 100644 --- a/website/server/cloudbuild.yaml +++ b/website/server/cloudbuild.yaml @@ -38,7 +38,7 @@ steps: - '--memory' - '2048Mi' - '--cpu' - - '1' + - '2' - '--min-instances' - '0' - '--max-instances' From 5ffef1f96ec87beabd7b00edc71c307706644d91 Mon Sep 17 00:00:00 2001 From: Yamada Dev Date: Sat, 1 Feb 2025 18:23:22 +0900 Subject: [PATCH 2/3] feat: Add accurate codebase scope description to output header --- src/core/output/outputGenerate.ts | 21 +-- src/core/output/outputGeneratorTypes.ts | 16 ++ src/core/output/outputStyleDecorate.ts | 129 +++++++++++++-- tests/core/output/outputStyleDecorate.test.ts | 153 ++++++++++++++++++ .../outputs/simple-project-output.txt | 11 +- .../outputs/simple-project-output.xml | 11 +- 6 files changed, 297 insertions(+), 44 deletions(-) create mode 100644 tests/core/output/outputStyleDecorate.test.ts diff --git a/src/core/output/outputGenerate.ts b/src/core/output/outputGenerate.ts index 69d1b5ba9..79b28ed77 100644 --- a/src/core/output/outputGenerate.ts +++ b/src/core/output/outputGenerate.ts @@ -7,7 +7,7 @@ import { RepomixError } from '../../shared/errorHandle.js'; import { searchFiles } from '../file/fileSearch.js'; import { generateTreeString } from '../file/fileTreeGenerate.js'; import type { ProcessedFile } from '../file/fileTypes.js'; -import type { OutputGeneratorContext } from './outputGeneratorTypes.js'; +import type { OutputGeneratorContext, RenderContext } from './outputGeneratorTypes.js'; import { generateHeader, generateSummaryFileFormat, @@ -19,22 +19,6 @@ import { getMarkdownTemplate } from './outputStyles/markdownStyle.js'; import { getPlainTemplate } from './outputStyles/plainStyle.js'; import { getXmlTemplate } from './outputStyles/xmlStyle.js'; -interface RenderContext { - readonly generationHeader: string; - readonly summaryPurpose: string; - readonly summaryFileFormat: string; - readonly summaryUsageGuidelines: string; - readonly summaryNotes: string; - readonly headerText: string | undefined; - readonly instruction: string; - readonly treeString: string; - readonly processedFiles: ReadonlyArray; - readonly fileSummaryEnabled: boolean; - readonly directoryStructureEnabled: boolean; - readonly escapeFileContent: boolean; - readonly markdownCodeBlockDelimiter: string; -} - const calculateMarkdownDelimiter = (files: ReadonlyArray): string => { const maxBackticks = files .flatMap((file) => file.content.match(/`+/g) ?? []) @@ -44,7 +28,7 @@ const calculateMarkdownDelimiter = (files: ReadonlyArray): string const createRenderContext = (outputGeneratorContext: OutputGeneratorContext): RenderContext => { return { - generationHeader: generateHeader(outputGeneratorContext.generationDate), + generationHeader: generateHeader(outputGeneratorContext.config, outputGeneratorContext.generationDate), // configを追加 summaryPurpose: generateSummaryPurpose(), summaryFileFormat: generateSummaryFileFormat(), summaryUsageGuidelines: generateSummaryUsageGuidelines( @@ -175,6 +159,7 @@ export const buildOutputGeneratorContext = async ( } } } + return { generationDate: new Date().toISOString(), treeString: generateTreeString(allFilePaths, emptyDirPaths), diff --git a/src/core/output/outputGeneratorTypes.ts b/src/core/output/outputGeneratorTypes.ts index 5ff261c72..ce082b56d 100644 --- a/src/core/output/outputGeneratorTypes.ts +++ b/src/core/output/outputGeneratorTypes.ts @@ -8,3 +8,19 @@ export interface OutputGeneratorContext { config: RepomixConfigMerged; instruction: string; } + +export interface RenderContext { + readonly generationHeader: string; + readonly summaryPurpose: string; + readonly summaryFileFormat: string; + readonly summaryUsageGuidelines: string; + readonly summaryNotes: string; + readonly headerText: string | undefined; + readonly instruction: string; + readonly treeString: string; + readonly processedFiles: ReadonlyArray; + readonly fileSummaryEnabled: boolean; + readonly directoryStructureEnabled: boolean; + readonly escapeFileContent: boolean; + readonly markdownCodeBlockDelimiter: string; +} diff --git a/src/core/output/outputStyleDecorate.ts b/src/core/output/outputStyleDecorate.ts index 98fabf776..2304e140d 100644 --- a/src/core/output/outputStyleDecorate.ts +++ b/src/core/output/outputStyleDecorate.ts @@ -1,10 +1,82 @@ import type { RepomixConfigMerged } from '../../config/configSchema.js'; -export const generateHeader = (generationDate: string): string => { - return ` -This file is a merged representation of the entire codebase, combining all repository files into a single document. -Generated by Repomix on: ${generationDate} -`.trim(); +interface ContentInfo { + selection: { + isEntireCodebase: boolean; + include?: boolean; + ignore?: boolean; + gitignore?: boolean; + defaultIgnore?: boolean; + }; + processing: { + commentsRemoved: boolean; + emptyLinesRemoved: boolean; + securityCheckEnabled: boolean; + showLineNumbers: boolean; + parsableStyle: boolean; + }; +} + +export const analyzeContent = (config: RepomixConfigMerged): ContentInfo => { + return { + selection: { + isEntireCodebase: !config.include.length && !config.ignore.customPatterns.length, + include: config.include.length > 0, + ignore: config.ignore.customPatterns.length > 0, + gitignore: config.ignore.useGitignore, + defaultIgnore: config.ignore.useDefaultPatterns, + }, + processing: { + commentsRemoved: config.output.removeComments, + emptyLinesRemoved: config.output.removeEmptyLines, + securityCheckEnabled: config.security.enableSecurityCheck, + showLineNumbers: config.output.showLineNumbers, + parsableStyle: config.output.parsableStyle, + }, + }; +}; + +export const generateHeader = (config: RepomixConfigMerged, generationDate: string): string => { + const info = analyzeContent(config); + + // Generate selection description + let description: string; + if (info.selection.isEntireCodebase) { + description = 'This file is a merged representation of the entire codebase'; + } else { + const parts = []; + if (info.selection.include) { + parts.push('specifically included files'); + } + if (info.selection.ignore) { + parts.push('files not matching ignore patterns'); + } + description = `This file is a merged representation of a subset of the codebase, containing ${parts.join(' and ')}`; + } + + // Add processing information + const processingNotes = []; + if (info.processing.commentsRemoved) { + processingNotes.push('comments have been removed'); + } + if (info.processing.emptyLinesRemoved) { + processingNotes.push('empty lines have been removed'); + } + if (info.processing.showLineNumbers) { + processingNotes.push('line numbers have been added'); + } + if (info.processing.parsableStyle) { + processingNotes.push('content has been formatted for parsing'); + } + if (!info.processing.securityCheckEnabled) { + processingNotes.push('security check has been disabled'); + } + + const processingInfo = + processingNotes.length > 0 ? ` The content has been processed where ${processingNotes.join(', ')}.` : ''; + + return `${description}, combined into a single document.${processingInfo} +Generated by Repomix on: ${generationDate}`; }; export const generateSummaryPurpose = (): string => { @@ -38,13 +110,42 @@ ${repositoryInstruction ? '- Pay special attention to the Repository Instruction }; export const generateSummaryNotes = (config: RepomixConfigMerged): string => { - return ` -- Some files may have been excluded based on .gitignore rules and Repomix's - configuration. -- Binary files are not included in this packed representation. Please refer to - the Repository Structure section for a complete list of file paths, including - binary files. -${config.output.removeComments ? '- Code comments have been removed.\n' : ''} -${config.output.showLineNumbers ? '- Line numbers have been added to the beginning of each line.\n' : ''} -`.trim(); + const info = analyzeContent(config); + const notes = [ + "- Some files may have been excluded based on .gitignore rules and Repomix's configuration", + '- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files', + ]; + + // File selection notes + if (info.selection.include) { + notes.push(`- Only files matching these patterns are included: ${config.include.join(', ')}`); + } + if (info.selection.ignore) { + notes.push(`- Files matching these patterns are excluded: ${config.ignore.customPatterns.join(', ')}`); + } + if (info.selection.gitignore) { + notes.push('- Files matching patterns in .gitignore are excluded'); + } + if (info.selection.defaultIgnore) { + notes.push('- Files matching default ignore patterns are excluded'); + } + + // Processing notes + if (info.processing.commentsRemoved) { + notes.push('- Code comments have been removed from supported file types'); + } + if (info.processing.emptyLinesRemoved) { + notes.push('- Empty lines have been removed from all files'); + } + if (info.processing.showLineNumbers) { + notes.push('- Line numbers have been added to the beginning of each line'); + } + if (info.processing.parsableStyle) { + notes.push(`- Content has been formatted for parsing in ${config.output.style} style`); + } + if (!info.processing.securityCheckEnabled) { + notes.push('- Security check has been disabled - content may contain sensitive information'); + } + + return notes.join('\n'); }; diff --git a/tests/core/output/outputStyleDecorate.test.ts b/tests/core/output/outputStyleDecorate.test.ts new file mode 100644 index 000000000..6b5b27be9 --- /dev/null +++ b/tests/core/output/outputStyleDecorate.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vitest'; +import { + analyzeContent, + generateHeader, + generateSummaryNotes, + generateSummaryPurpose, + generateSummaryUsageGuidelines, +} from '../../../src/core/output/outputStyleDecorate.js'; +import { createMockConfig } from '../../testing/testUtils.js'; + +describe('analyzeContent', () => { + it('should detect entire codebase when using default settings', () => { + const config = createMockConfig(); + const result = analyzeContent(config); + expect(result.selection.isEntireCodebase).toBe(true); + }); + + it('should detect subset when using include patterns', () => { + const config = createMockConfig({ + include: ['src/**/*.ts'], + }); + const result = analyzeContent(config); + expect(result.selection.isEntireCodebase).toBe(false); + expect(result.selection.include).toBe(true); + }); + + it('should detect processing states', () => { + const config = createMockConfig({ + output: { + removeComments: true, + removeEmptyLines: true, + }, + }); + const result = analyzeContent(config); + expect(result.processing.commentsRemoved).toBe(true); + expect(result.processing.emptyLinesRemoved).toBe(true); + }); +}); + +describe('generateHeader', () => { + const mockDate = '2025-01-29T11:23:01.763Z'; + + it('should generate header for entire codebase', () => { + const config = createMockConfig(); + const header = generateHeader(config, mockDate); + expect(header).toContain('entire codebase'); + expect(header).not.toContain('subset'); + }); + + it('should generate header for subset with processing', () => { + const config = createMockConfig({ + include: ['src/**/*.ts'], + output: { + removeComments: true, + }, + }); + const header = generateHeader(config, mockDate); + expect(header).toContain('subset of the codebase'); + expect(header).toContain('comments have been removed'); + }); + + it('should include security check disabled warning', () => { + const config = createMockConfig({ + security: { + enableSecurityCheck: false, + }, + }); + const header = generateHeader(config, mockDate); + expect(header).toContain('security check has been disabled'); + }); + + it('should include multiple processing states', () => { + const config = createMockConfig({ + output: { + removeComments: true, + removeEmptyLines: true, + showLineNumbers: true, + }, + }); + const header = generateHeader(config, mockDate); + expect(header).toContain('comments have been removed'); + expect(header).toContain('empty lines have been removed'); + expect(header).toContain('line numbers have been added'); + }); +}); + +describe('generateSummaryPurpose', () => { + it('should generate consistent purpose text', () => { + const purpose = generateSummaryPurpose(); + expect(purpose).toContain('packed representation'); + expect(purpose).toContain('AI systems'); + expect(purpose).toContain('code review'); + }); +}); + +describe('generateSummaryUsageGuidelines', () => { + it('should include header text note when headerText is provided', () => { + const config = createMockConfig({ + output: { + headerText: 'Custom header', + }, + }); + const guidelines = generateSummaryUsageGuidelines(config, ''); + expect(guidelines).toContain('Repository Description'); + }); + + it('should include instruction note when instruction is provided', () => { + const config = createMockConfig(); + const guidelines = generateSummaryUsageGuidelines(config, 'Custom instruction'); + expect(guidelines).toContain('Repository Instruction'); + }); +}); + +describe('generateSummaryNotes', () => { + it('should include selection information', () => { + const config = createMockConfig({ + include: ['src/**/*.ts'], + ignore: { + customPatterns: ['*.test.ts'], + }, + }); + const notes = generateSummaryNotes(config); + expect(notes).toContain('Only files matching these patterns are included: src/**/*.ts'); + expect(notes).toContain('Files matching these patterns are excluded: *.test.ts'); + }); + + it('should include processing notes', () => { + const config = createMockConfig({ + output: { + removeComments: true, + showLineNumbers: true, + style: 'xml', + parsableStyle: true, + }, + security: { + enableSecurityCheck: false, + }, + }); + const notes = generateSummaryNotes(config); + expect(notes).toContain('Code comments have been removed'); + expect(notes).toContain('Line numbers have been added'); + expect(notes).toContain('Content has been formatted for parsing in xml style'); + expect(notes).toContain('Security check has been disabled'); + }); + + it('should handle case with minimal processing', () => { + const config = createMockConfig(); + const notes = generateSummaryNotes(config); + expect(notes).toContain('Files matching patterns in .gitignore are excluded'); + expect(notes).toContain('Files matching default ignore patterns are excluded'); + expect(notes).not.toContain('Code comments have been removed'); + }); +}); diff --git a/tests/integration-tests/fixtures/packager/outputs/simple-project-output.txt b/tests/integration-tests/fixtures/packager/outputs/simple-project-output.txt index 82b882435..5330b377a 100644 --- a/tests/integration-tests/fixtures/packager/outputs/simple-project-output.txt +++ b/tests/integration-tests/fixtures/packager/outputs/simple-project-output.txt @@ -1,4 +1,4 @@ -This file is a merged representation of the entire codebase, combining all repository files into a single document. +This file is a merged representation of the entire codebase, combined into a single document. ================================================================ File Summary @@ -35,11 +35,10 @@ Usage Guidelines: Notes: ------ -- Some files may have been excluded based on .gitignore rules and Repomix's - configuration. -- Binary files are not included in this packed representation. Please refer to - the Repository Structure section for a complete list of file paths, including - binary files. +- Some files may have been excluded based on .gitignore rules and Repomix's configuration +- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files +- Files matching patterns in .gitignore are excluded +- Files matching default ignore patterns are excluded Additional Info: ---------------- diff --git a/tests/integration-tests/fixtures/packager/outputs/simple-project-output.xml b/tests/integration-tests/fixtures/packager/outputs/simple-project-output.xml index 271d3da40..94d10f407 100644 --- a/tests/integration-tests/fixtures/packager/outputs/simple-project-output.xml +++ b/tests/integration-tests/fixtures/packager/outputs/simple-project-output.xml @@ -1,4 +1,4 @@ -This file is a merged representation of the entire codebase, combining all repository files into a single document. +This file is a merged representation of the entire codebase, combined into a single document. This section contains a summary of this file. @@ -30,11 +30,10 @@ The content is organized as follows: -- Some files may have been excluded based on .gitignore rules and Repomix's - configuration. -- Binary files are not included in this packed representation. Please refer to - the Repository Structure section for a complete list of file paths, including - binary files. +- Some files may have been excluded based on .gitignore rules and Repomix's configuration +- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files +- Files matching patterns in .gitignore are excluded +- Files matching default ignore patterns are excluded From f9ef4271be776bc138339ebd2fa53d5b77423df6 Mon Sep 17 00:00:00 2001 From: Yamada Dev Date: Sat, 1 Feb 2025 18:41:12 +0900 Subject: [PATCH 3/3] docs(readme): update model list in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fdec31f22..8ded8687f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ 📦 Repomix is a powerful tool that packs your entire repository into a single, AI-friendly file. It is perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools like Claude, -ChatGPT, and Gemini. +ChatGPT, DeepSeek, Perplexity, Gemini, Gemma, Llama, Grok, and more. ## 🎉 New: Repomix Website & Discord Community!