diff --git a/README.md b/README.md index c1b9021ad..98ab7a5e5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![npm](https://img.shields.io/npm/l/repopack.svg?maxAge=1000)](https://github.com/yamadashy/repopack/blob/master/LICENSE.md) [![node](https://img.shields.io/node/v/repopack.svg?maxAge=1000)](https://www.npmjs.com/package/repopack) -Repopack is a powerful tool that packs your entire repository into a single, AI-friendly file. Perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools. +Repopack is a powerful tool that packs your entire repository into a single, AI-friendly file. Perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools. It now includes a security check feature to detect potentially sensitive information in your files. @@ -15,6 +15,7 @@ Repopack is a powerful tool that packs your entire repository into a single, AI- - **Simple to Use**: Just one command to pack your entire repository. - **Customizable**: Easily configure what to include or exclude. - **Git-Aware**: Automatically respects your .gitignore files. +- **Security Check**: Detects potentially sensitive information in your files. @@ -91,6 +92,26 @@ npx repopack src +## 🔍 Security Check + +Repopack now includes a security check feature that uses SecretLint to detect potentially sensitive information in your files. This feature helps you identify possible security risks before sharing your packed repository. + +The security check results will be displayed in the CLI output after the packing process is complete. If any suspicious files are detected, you'll see a list of these files along with a warning message. + +Example output: + +``` +🔍 Security Check: +────────────────── +2 suspicious file(s) detected: +1. src/config.js +2. tests/testData.json + +Please review these files for potential sensitive information. +``` + + + ## ⚙️ Configuration Create a `repopack.config.json` file in your project root for custom configurations. Here's an explanation of the configuration options: @@ -99,7 +120,7 @@ Create a `repopack.config.json` file in your project root for custom configurati |--------|-------------|---------| |`output.filePath`| The name of the output file | `"repopack-output.txt"` | |`output.headerText`| Custom text to include in the file header |`null`| -|`output.removeComments`| Whether to remove comments from supported file types. Suppurts python | `false` | +|`output.removeComments`| Whether to remove comments from supported file types | `false` | |`output.topFilesLength`| Number of top files to display in the summary. If set to 0, no summary will be displayed |`5`| |`ignore.useDefaultPatterns`| Whether to use default ignore patterns |`true`| |`ignore.customPatterns`| Additional patterns to ignore |`[]`| diff --git a/package-lock.json b/package-lock.json index 8b899b1b3..6ba041856 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,8 @@ "version": "0.1.7", "license": "MIT", "dependencies": { + "@secretlint/core": "^8.2.4", + "@secretlint/secretlint-rule-preset-recommend": "^8.2.4", "cli-spinners": "^2.9.2", "commander": "^7.1.0", "iconv-lite": "^0.6.3", @@ -951,6 +953,41 @@ "win32" ] }, + "node_modules/@secretlint/core": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/@secretlint/core/-/core-8.2.4.tgz", + "integrity": "sha512-Ws/jX/It7O5kRlvYXM6tHgvLmbSTvQTG7G+vQ0FWb1KjS14+5CbuRdxcQQLkD8shD/87tHu53lOmIlvn/Rc/YA==", + "dependencies": { + "@secretlint/profiler": "^8.2.4", + "@secretlint/types": "^8.2.4", + "debug": "^4.3.4", + "structured-source": "^4.0.0" + }, + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, + "node_modules/@secretlint/profiler": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/@secretlint/profiler/-/profiler-8.2.4.tgz", + "integrity": "sha512-KfRGWf7R4tAxJwk7Ojoa8n53yLh3zmM1hmG1Nh/xWkSXsatTF5qi7bfyi3+QjAxhBZRaz6aR9JbX8PS3JGon1w==" + }, + "node_modules/@secretlint/secretlint-rule-preset-recommend": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-preset-recommend/-/secretlint-rule-preset-recommend-8.2.4.tgz", + "integrity": "sha512-Ifaz0ngkgP911TfJkkIXrP9dCpxQGCDAISZ/sG0mdOOg9KO8jF9pnGKzCOuVX4q97v6MDtELXjYAxnPa8xV4Ow==", + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, + "node_modules/@secretlint/types": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/@secretlint/types/-/types-8.2.4.tgz", + "integrity": "sha512-Pf+ArQmx4+K75TpMRhUgqw2FL8DGNf0OkT9g6L1t4HoYESiVlfZw3quNTvO+GTWzRxssUjnWoQ4+sEg3MNgEHA==", + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, "node_modules/@types/eslint": { "version": "8.56.10", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.10.tgz", @@ -1548,6 +1585,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/boundary": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/boundary/-/boundary-2.0.0.tgz", + "integrity": "sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA==" + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -1788,7 +1830,6 @@ "version": "4.3.5", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -3710,8 +3751,7 @@ "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/nanoid": { "version": "3.3.7", @@ -4696,6 +4736,14 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/structured-source": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-4.0.0.tgz", + "integrity": "sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==", + "dependencies": { + "boundary": "^2.0.0" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/package.json b/package.json index 159f8f8d6..92366621d 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,8 @@ }, "type": "module", "dependencies": { + "@secretlint/core": "^8.2.4", + "@secretlint/secretlint-rule-preset-recommend": "^8.2.4", "cli-spinners": "^2.9.2", "commander": "^7.1.0", "iconv-lite": "^0.6.3", @@ -62,8 +64,8 @@ "devDependencies": { "@eslint/js": "^9.7.0", "@types/eslint": "~8.56.10", - "@types/eslint__js": "~8.42.3", "@types/eslint-config-prettier": "~6.11.3", + "@types/eslint__js": "~8.42.3", "@types/node": "^20.14.10", "@types/strip-comments": "^2.0.4", "@typescript-eslint/eslint-plugin": "^7.16.0", diff --git a/src/cli/cliOutput.ts b/src/cli/cliOutput.ts index 9ff00255e..d52756c1e 100644 --- a/src/cli/cliOutput.ts +++ b/src/cli/cliOutput.ts @@ -8,6 +8,21 @@ export function printSummary(totalFiles: number, totalCharacters: number, output console.log(`${pc.white(' Output:')} ${pc.white(outputPath)}`); } +export function printSecurityCheck(suspiciousFiles: string[]) { + console.log(pc.white('🔎 Security Check:')); + console.log(pc.dim('──────────────────')); + + if (suspiciousFiles.length === 0) { + console.log(pc.green('✔') + ' ' + pc.white('No suspicious files detected.')); + } else { + console.log(pc.yellow(`${suspiciousFiles.length} suspicious file(s) detected:`)); + suspiciousFiles.forEach((file, index) => { + console.log(`${pc.white(`${index + 1}.`)} ${pc.white(file)}`); + }); + console.log(pc.yellow('\nPlease review these files for potential sensitive information.')); + } +} + export function printTopFiles(fileCharCounts: Record, topFilesLength: number) { console.log(pc.white(`📈 Top ${topFilesLength} Files by Character Count:`)); console.log(pc.dim('──────────────────────────────────')); diff --git a/src/cli/index.ts b/src/cli/index.ts index 588a3032f..4330b3703 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -8,7 +8,7 @@ import { getVersion } from '../utils/packageJsonUtils.js'; import Spinner from '../utils/spinner.js'; import pc from 'picocolors'; import { handleError } from '../utils/errorHandler.js'; -import { printSummary, printTopFiles, printCompletion } from './cliOutput.js'; +import { printSummary, printTopFiles, printCompletion, printSecurityCheck } from './cliOutput.js'; interface CliOptions extends OptionValues { version?: boolean; @@ -59,7 +59,7 @@ async function executeAction(directory: string, options: CliOptions) { spinner.start(); try { - const { totalFiles, totalCharacters, fileCharCounts } = await pack(targetPath, config); + const { totalFiles, totalCharacters, fileCharCounts, suspiciousFiles } = await pack(targetPath, config); spinner.succeed('Packing completed successfully!'); console.log(''); @@ -68,6 +68,9 @@ async function executeAction(directory: string, options: CliOptions) { console.log(''); } + printSecurityCheck(suspiciousFiles); + console.log(''); + printSummary(totalFiles, totalCharacters, config.output.filePath); console.log(''); diff --git a/src/core/packager.ts b/src/core/packager.ts index 42b032b54..771e7f3f0 100644 --- a/src/core/packager.ts +++ b/src/core/packager.ts @@ -9,6 +9,7 @@ import { } from '../utils/gitignoreUtils.js'; import { generateOutput as defaultGenerateOutput } from './outputGenerator.js'; import { defaultIgnoreList } from '../utils/defaultIgnore.js'; +import { checkFileWithSecretLint, createSecretLintConfig } from '../utils/secretLintUtils.js'; export interface Dependencies { getGitignorePatterns: typeof defaultGetGitignorePatterns; @@ -21,6 +22,7 @@ export interface PackResult { totalFiles: number; totalCharacters: number; fileCharCounts: Record; + suspiciousFiles: string[]; } export async function pack( @@ -33,18 +35,24 @@ export async function pack( generateOutput: defaultGenerateOutput, }, ): Promise { + // Get ignore patterns const gitignorePatterns = await deps.getGitignorePatterns(rootDir); - const ignorePatterns = getIgnorePatterns(gitignorePatterns, config); const ignoreFilter = deps.createIgnoreFilter(ignorePatterns); - const packedFiles = await packDirectory(rootDir, '', config, ignoreFilter, deps); + // Get all file paths in the directory + const filePaths = await getFilePaths(rootDir, '', ignoreFilter); - const totalFiles = packedFiles.length; - const totalCharacters = packedFiles.reduce((sum, file) => sum + file.content.length, 0); + // Perform security check + const suspiciousFiles = await performSecurityCheck(filePaths, rootDir); + // Pack files and generate output + const packedFiles = await packFiles(filePaths, rootDir, config, deps); await deps.generateOutput(rootDir, config, packedFiles); + // Metrics + const totalFiles = packedFiles.length; + const totalCharacters = packedFiles.reduce((sum, file) => sum + file.content.length, 0); const fileCharCounts: Record = {}; packedFiles.forEach((file) => { fileCharCounts[file.path] = file.content.length; @@ -54,6 +62,7 @@ export async function pack( totalFiles, totalCharacters, fileCharCounts, + suspiciousFiles, }; } @@ -68,30 +77,55 @@ function getIgnorePatterns(gitignorePatterns: string[], config: RepopackConfigMe return ignorePatterns; } -async function packDirectory( - dir: string, - relativePath: string, - config: RepopackConfigMerged, - ignoreFilter: IgnoreFilter, - deps: Dependencies, -): Promise<{ path: string; content: string }[]> { +async function getFilePaths(dir: string, relativePath: string, ignoreFilter: IgnoreFilter): Promise { const entries = await fs.readdir(dir, { withFileTypes: true }); - const packedFiles: { path: string; content: string }[] = []; + const filePaths: string[] = []; for (const entry of entries) { - const fullPath = path.join(dir, entry.name); const entryRelativePath = path.join(relativePath, entry.name); if (!ignoreFilter(entryRelativePath)) continue; if (entry.isDirectory()) { - const subDirFiles = await packDirectory(fullPath, entryRelativePath, config, ignoreFilter, deps); - packedFiles.push(...subDirFiles); + const subDirPaths = await getFilePaths(path.join(dir, entry.name), entryRelativePath, ignoreFilter); + filePaths.push(...subDirPaths); } else { - const content = await deps.processFile(fullPath, config); - if (content) { - packedFiles.push({ path: entryRelativePath, content }); - } + filePaths.push(entryRelativePath); + } + } + + return filePaths; +} + +async function performSecurityCheck(filePaths: string[], rootDir: string): Promise { + const secretLintConfig = createSecretLintConfig(); + const suspiciousFiles: string[] = []; + + for (const filePath of filePaths) { + const fullPath = path.join(rootDir, filePath); + const content = await fs.readFile(fullPath, 'utf-8'); + const isSuspicious = await checkFileWithSecretLint(fullPath, content, secretLintConfig); + if (isSuspicious) { + suspiciousFiles.push(filePath); + } + } + + return suspiciousFiles; +} + +async function packFiles( + filePaths: string[], + rootDir: string, + config: RepopackConfigMerged, + deps: Dependencies, +): Promise<{ path: string; content: string }[]> { + const packedFiles: { path: string; content: string }[] = []; + + for (const filePath of filePaths) { + const fullPath = path.join(rootDir, filePath); + const content = await deps.processFile(fullPath, config); + if (content) { + packedFiles.push({ path: filePath, content }); } } diff --git a/src/utils/secretLintUtils.ts b/src/utils/secretLintUtils.ts new file mode 100644 index 000000000..a5660db52 --- /dev/null +++ b/src/utils/secretLintUtils.ts @@ -0,0 +1,34 @@ +import type { SecretLintCoreConfig } from '@secretlint/types'; +import { lintSource } from '@secretlint/core'; +import { creator } from '@secretlint/secretlint-rule-preset-recommend'; + +export async function checkFileWithSecretLint( + filePath: string, + content: string, + config: SecretLintCoreConfig, +): Promise { + const result = await lintSource({ + source: { + filePath: filePath, + content: content, + ext: filePath.split('.').pop() || '', + contentType: 'text', + }, + options: { + config: config, + }, + }); + + return result.messages.length > 0; +} + +export function createSecretLintConfig(): SecretLintCoreConfig { + return { + rules: [ + { + id: '@secretlint/secretlint-rule-preset-recommend', + rule: creator, + }, + ], + }; +} diff --git a/tests/utils/secretLintUtils.test.ts b/tests/utils/secretLintUtils.test.ts new file mode 100644 index 000000000..62b2bf4bc --- /dev/null +++ b/tests/utils/secretLintUtils.test.ts @@ -0,0 +1,74 @@ +import { expect, test, describe } from 'vitest'; +import { checkFileWithSecretLint, createSecretLintConfig } from '../../src/utils/secretLintUtils.js'; +import type { SecretLintCoreConfig } from '@secretlint/types'; + +describe('secretLintUtils', () => { + const config: SecretLintCoreConfig = createSecretLintConfig(); + + test('should detect sensitive information', async () => { + const sensitiveContent = ` +# Secretlint Demo + +URL: https://user:pass@example.com + +GitHub Token: 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 + +SendGrid: "SG.APhb3zgjtx3hajdas1TjBB.H7Sgbba3afgKSDyB442aDK0kpGO3SD332313-L5528Kewhere" + +AWS_SECRET_ACCESS_KEY = wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETSKEY + +Slack: +xoxa-23984754863-2348975623103 +xoxb-23984754863-2348975623103 +xoxo-23984754863-2348975623103 + +Private Key: + +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQCYdGaf5uYMsilGHfnx/zxXtihdGFr3hCWwebHGhgEAVn0xlsTd +1QwoKi+rpI1O6hzyVOuoQtboODsONGRlHbNl6yJ936Yhmr8PiNwpA5qIxZAdmFv2 +tqEllWr0dGPPm3B/2NbjuMpSiJNAcBQa46X++doG5yNMY8NCgTsjBZIBKwIDAQAB +AoGAN+Pkg5aIm/rsurHeoeMqYhV7srVtE/S0RIA4tkkGMPOELhvRzGmAbXEZzNkk +nNujBQww4JywYK3MqKZ4b8F1tMG3infs1w8V7INAYY/c8HzfrT3f+MVxijoKV2Fl +JlUXCclztoZhxAxhCR+WC1Upe1wIrWNwad+JA0Vws/mwrEECQQDxiT/Q0lK+gYaa ++riFeZmOaqwhlFlYNSK2hCnLz0vbnvnZE5ITQoV+yiy2+BhpMktNFsYNCfb0pdKN +D87x+jr7AkEAoZWITvqErh1RbMCXd26QXZEfZyrvVZMpYf8BmWFaBXIbrVGme0/Q +d7amI6B8Vrowyt+qgcUk7rYYaA39jYB7kQJAdaX2sY5gw25v1Dlfe5Q5WYdYBJsv +0alAGUrS2PVF69nJtRS1SDBUuedcVFsP+N2IlCoNmfhKk+vZXOBgWrkZ1QJAGJlE +FAntUvhhofW72VG6ppPmPPV7VALARQvmOWxpoPSbJAqPFqyy5tamejv/UdCshuX/ +9huGINUV6BlhJT6PEQJAF/aqQTwZqJdwwJqYEQArSmyOW7UDAlQMmKMofjBbeBvd +H4PSJT5bvaEhxRj7QCwonoX4ZpV0beTnzloS55Z65g== +-----END RSA PRIVATE KEY----- + `; + + const result = await checkFileWithSecretLint('test.md', sensitiveContent, config); + expect(result).toBe(true); + }); + + test('should not detect sensitive information in normal content', async () => { + const normalContent = ` +# Normal Content + +This is a regular markdown file with no sensitive information. + +Here's some code: + +\`\`\`javascript +function greet(name) { + console.log(\`Hello, \${name}!\`); +} +\`\`\` + +And here's a list: + +1. Item 1 +2. Item 2 +3. Item 3 + +That's all! + `; + + const result = await checkFileWithSecretLint('normal.md', normalContent, config); + expect(result).toBe(false); + }); +});