feat(repo): manage deps automatically

package.json

@@ -43,7 +43,7 @@
         "g:rimraf": "cd $INIT_CWD && rimraf",
         "g:tsc": "cd $INIT_CWD && tsc",
         "g:tsx": "cd $INIT_CWD && tsx",
-        "g:depcheck": "cd $INIT_CWD && depcheck",
+        "g:depcheck": "cd $INIT_CWD && tsx $PROJECT_CWD/scripts/fixDependencies.ts",
         "_______ Nx testing _______": "Nx wrapped commands for testing, linting, type checking...",
         "nx:build:libs": "yarn nx affected --target=build:lib",
         "nx:type-check": "yarn nx affected --target=type-check",
@@ -58,7 +58,7 @@
         "check-workspace-resolutions": "yarn tsx ./scripts/check-workspace-resolutions.ts",
         "generate-package": "yarn tsx ./scripts/generatePackage.js",
         "deps": "rimraf **/node_modules && yarn",
-        "depcheck": "yarn nx affected --target=depcheck",
+        "depcheck": "yarn nx affected --target=depcheck --output-style=static",
         "list-outdated": "./scripts/list-outdated-dependencies/list-outdated-dependencies.sh",
         "message-system-sign-config": "yarn workspace @suite-common/message-system sign-config",
         "format": "yarn nx format:write",

scripts/fixDependencies.ts

@@ -0,0 +1,191 @@
+/*
+    This script fixes dependencies in package.json.
+    1. It removes unused dependencies.
+    2. It adds missing dependencies
+        - if a dependency is used in the monorepo, but not directly in any of our packages, it adds it to package.json with the workspace:* version.
+        - if a dependency is used in the monorepo, but not in any of our packages, it gets the latest version that is used anywhere in node_modules.
+        - if dependency is not used anywhere in the monorepo, it gets the latest version from npm.
+*/
+
+import { A, D } from '@mobily/ts-belt';
+import chalk from 'chalk';
+import depcheck from 'depcheck';
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import semver from 'semver';
+import { formatObjectToJson } from './utils/getPrettierConfig';
+import { getAffectedWorkspaces, getWorkspacesList, WorkspaceItem } from './utils/getWorkspacesList';
+import { getPackageVersionInMonorepo, getLatestVersionFromNpm } from './utils/packageVersionsUtils';
+import path from 'node:path';
+
+const { defaultOptions } = require('depcheck/dist/constants');
+
+if (!process.env.PROJECT_CWD) {
+    console.error(
+        'PROJECT_CWD environment variable is not set. This variable should be automatically set by Yarn.',
+    );
+    process.exit(1);
+}
+
+// If there arg --verify, only run depcheck in shell and pipe output to stdout
+// const isVerifyOnly = process.argv.includes('--verify');
+
+const options = {
+    ...defaultOptions,
+    skipMissing: false, // skip calculation of missing dependencies
+    ignorePatterns: [
+        ...defaultOptions.ignorePatterns,
+        'dist',
+        'build',
+        'coverage',
+        'public',
+        'lib',
+        'libDev',
+        '*.json',
+        'tsconfig.json',
+        'tsconfig.lib.json',
+        // webpack configs
+        '**/webpack.config.js',
+        '**/webpack.config.ts',
+        '**/*.webpack.config.js',
+        '**/*.webpack.config.ts',
+    ],
+    ignoreMatches: [
+        ...defaultOptions.ignoreMatches,
+        // alias that is used in @trezor/suite package
+        'src',
+        // invity-api is package only for typescript types and it's imported from @types/invity-api
+        'invity-api',
+    ],
+} satisfies depcheck.Options;
+
+const transformPathToRelative = (filePath: string) => {
+    return filePath.replace(process.cwd(), '.').replace(/^\//, '');
+};
+// Execute the command and split the output by newlines
+const ourWorkspaces = getWorkspacesList();
+const ourPackages = Object.keys(ourWorkspaces);
+
+const ourPackagesScopes = new Set(
+    ourPackages.map(pkg => pkg.split('/')[0]).filter(pkg => pkg.startsWith('@')),
+);
+
+async function fixDependencies(workspace: WorkspaceItem) {
+    const workspaceFullPath = path.join(process.env.PROJECT_CWD!, workspace.location);
+    const packageJsonPath = path.join(workspaceFullPath, 'package.json');
+    const originalPackageJsonContent = fs.readFileSync(packageJsonPath, 'utf8');
+    const originalPackageJson = JSON.parse(originalPackageJsonContent);
+
+    console.log('Running depcheck for', workspace.name);
+    const result = await depcheck(workspaceFullPath, {
+        ...options,
+        package: originalPackageJson,
+    });
+    console.log('Depcheck result:', result);
+
+    const newPackageJson = JSON.parse(originalPackageJsonContent);
+
+    result.dependencies.forEach(depName => {
+        // remove dep from package.json
+        console.log(
+            chalk.red(`${chalk.bold(depName)} - Removing unused dependency from package.json.`),
+        );
+        console.log('');
+        delete newPackageJson.dependencies[depName];
+    });
+
+    // remove self from missing deps in case package references itself
+    delete result.missing[newPackageJson.name];
+
+    const missingDeps = Object.keys(result.missing);
+
+    if (A.isEmpty(missingDeps)) {
+        // process.exit(0);
+        return;
+    }
+
+    missingDeps.forEach(depName => {
+        if (depName === newPackageJson.name) return;
+
+        const files = result.missing[depName];
+
+        console.error(chalk.red(`${chalk.bold(depName)} is missing in package.json.`));
+        console.log(`Used in ${files.length} files:`);
+        files.forEach(file => {
+            console.log(` - ${depName}/${transformPathToRelative(file)}`);
+        });
+
+        let version = null;
+        const depScope = depName.startsWith('@') ? depName.split('/')[0] : null;
+        if (depScope && ourPackagesScopes.has(depScope)) {
+            if (ourPackages.includes(depName)) {
+                // add dep to package.json
+                version = 'workspace:*';
+            } else {
+                console.error(
+                    chalk.red(
+                        `Using dependency with our scope ${depName} but this package doesn't exist in monorepo. This is probably a mistake, verify package name.`,
+                    ),
+                );
+                process.exit(1);
+            }
+        } else {
+            const versionFromMonorepo = getPackageVersionInMonorepo(depName);
+            if (versionFromMonorepo) {
+                console.log(
+                    chalk.green(
+                        `${chalk.bold(depName)}@${chalk.bold(versionFromMonorepo)} - adding to package.json ${chalk.bold(
+                            '(version from monorepo)',
+                        )}`,
+                    ),
+                );
+                version = versionFromMonorepo;
+            } else {
+                console.log(
+                    chalk.red(
+                        ` ${chalk.bold(depName)} - no version of found in monorepo. Getting latest version from NPM.`,
+                    ),
+                );
+                const versionFromNpm = getLatestVersionFromNpm(depName);
+                if (versionFromNpm) {
+                    console.log(
+                        chalk.green(
+                            `${chalk.bold(depName)}@${chalk.bold(versionFromNpm)} - adding to package.json ${chalk.bold(
+                                '(version from npm)',
+                            )}`,
+                        ),
+                    );
+                    version = versionFromNpm;
+                } else {
+                    console.error(
+                        chalk.red(
+                            `${chalk.bold(depName)} - no version of found in npm. Please add it to package.json manually.`,
+                        ),
+                    );
+                }
+            }
+        }
+
+        if (version) {
+            newPackageJson.dependencies = {
+                ...newPackageJson.dependencies,
+                [depName]: version,
+            };
+        }
+        // empty line
+        console.log('');
+    });
+
+    fs.writeFileSync(packageJsonPath, await formatObjectToJson(newPackageJson, 2));
+}
+
+const affectedWorkspaces = getAffectedWorkspaces();
+
+console.log(
+    'Running for affected workspaces:\n',
+    affectedWorkspaces.map(w => w.name),
+);
+
+affectedWorkspaces.forEach(workspace => {
+    fixDependencies(workspace);
+});

scripts/utils/getAffectedPackages.ts

@@ -0,0 +1,18 @@
+import { execSync } from 'node:child_process';
+
+export const getAffectedPackages = (): string[] => {
+    let affectedPackages: string[] = [];
+
+    try {
+        const affectedPackagesOutput = execSync(
+            `cd ${process.env.PROJECT_CWD} && yarn nx show projects --affected`,
+            { encoding: 'utf-8' },
+        );
+        affectedPackages = affectedPackagesOutput.split('\n').map(pkg => pkg.trim());
+    } catch (error) {
+        console.error('Error getting affected packages:', error.toString());
+        process.exit(1);
+    }
+
+    return affectedPackages;
+};

scripts/utils/getPrettierConfig.ts

@@ -10,3 +10,16 @@ export const getPrettierConfig = async () => {
 
     return prettierConfig;
 };
+
+export const formatObjectToJson = async (value: any, stringifySpaces?: number) => {
+    const prettierConfig = await getPrettierConfig();
+    try {
+        return prettier.format(
+            JSON.stringify(value, null, stringifySpaces).replace(/\\\\/g, '/'),
+            prettierConfig,
+        );
+    } catch (error) {
+        console.error(error);
+        process.exit(1);
+    }
+};

scripts/utils/getWorkspacesList.ts

@@ -1,15 +1,22 @@
 import { A, D, pipe } from '@mobily/ts-belt';
 import { execSync } from 'child_process';
+import { getAffectedPackages } from './getAffectedPackages';
 
 type WorkspacePackageName = string;
-type WorkspaceItem = {
+export type WorkspaceItem = {
     location: string;
     name: WorkspacePackageName;
     workspaceDependencies: WorkspacePackageName[];
     mismatchedWorkspaceDependencies: WorkspacePackageName[];
 };
 
+let workspacesList: Record<WorkspacePackageName, WorkspaceItem> | null = null;
 export const getWorkspacesList = (): Record<WorkspacePackageName, WorkspaceItem> => {
+    if (workspacesList) {
+        // Cache the results because this could be slow and it's always the same
+        return workspacesList;
+    }
+
     const rawList = execSync('yarn workspaces list --json --verbose')
         .toString()
         .replaceAll('}', '},');
@@ -26,5 +33,16 @@ export const getWorkspacesList = (): Record<WorkspacePackageName, WorkspaceItem>
         D.fromPairs,
     );
 
+    workspacesList = workspaces;
+
     return workspaces;
 };
+
+export const getAffectedWorkspaces = (): WorkspaceItem[] => {
+    const affectedPackages = getAffectedPackages();
+    const workspaces = getWorkspacesList();
+
+    return affectedPackages
+        .map(packageName => workspaces[packageName])
+        .filter(workspace => !!workspace);
+};

scripts/utils/packageVersionsUtils.ts

@@ -0,0 +1,110 @@
+import { A } from '@mobily/ts-belt';
+import chalk from 'chalk';
+import { execSync } from 'node:child_process';
+import semver from 'semver';
+import { getWorkspacesList } from './getWorkspacesList';
+
+// Execute the command and split the output by newlines
+const ourWorkspaces = getWorkspacesList();
+const ourPackages = Object.keys(ourWorkspaces);
+
+export const findPackageUsagesInMonorepo = (depName: string): string[] => {
+    const whyOutput = execSync(`yarn why ${depName} --json`, { encoding: 'utf-8' });
+    const whyOutputJson = whyOutput
+        .trim()
+        .split('\n')
+        .filter(line => line.trim() !== '')
+        .map(line => JSON.parse(line));
+
+    const directVersions = new Set<string>();
+    const allVersions = new Set<string>();
+
+    const isSpecificVersion = (version: string) => {
+        // Regular expression to match specific versions like "7.1.2" or "^7.1.2"
+        return /^(\^)?(\d+\.){2}\d+(-.*)?$/.test(version);
+    };
+
+    const getPackageName = (value: string) => {
+        const parts = value.split('@');
+        if (parts[0] === '') {
+            // Scoped package
+            return `@${parts[1].split(':')[0]}`;
+        }
+
+        return parts[0];
+    };
+
+    const extractVersion = (descriptor: string) => {
+        if (descriptor.includes('#npm:')) {
+            return descriptor.split('#npm:')[1];
+        }
+
+        return descriptor.split('@npm:')[1];
+    };
+
+    whyOutputJson.forEach(item => {
+        if (item.children) {
+            Object.entries(item.children).forEach(([childName, child]: [string, any]) => {
+                if (
+                    child.descriptor &&
+                    (childName.startsWith(`${depName}@npm:`) ||
+                        childName.startsWith(`${depName}@virtual:`))
+                ) {
+                    const version = extractVersion(child.descriptor);
+                    if (isSpecificVersion(version)) {
+                        allVersions.add(version);
+
+                        // Check if dependency is used in one of our packages
+                        const parentPackage = getPackageName(item.value);
+                        if (ourPackages.includes(parentPackage)) {
+                            directVersions.add(version);
+                        }
+                    }
+                }
+            });
+        }
+    });
+
+    // If we found direct versions in ourPackages, return those
+    if (directVersions.size > 0) {
+        return Array.from(directVersions);
+    }
+
+    if (allVersions.size > 0) {
+        console.log(
+            chalk.yellow(
+                `Package ${chalk.bold(depName)} is used in monorepo but not directly in our packages.`,
+            ),
+        );
+    }
+
+    // Otherwise, return all specific versions found
+    return Array.from(allVersions);
+};
+
+export const getPackageVersionInMonorepo = (packageName: string) => {
+    const packageVersionInMonorepo = findPackageUsagesInMonorepo(packageName);
+    if (packageVersionInMonorepo.length === 0) {
+        return null;
+    }
+
+    if (packageVersionInMonorepo.length === 1) {
+        return packageVersionInMonorepo[0];
+    }
+
+    // remove ^ prefix only for semver comparison (it doesn't work)
+    const cleanVersion = (version: string) => version.replace(/^[\^]/, '');
+
+    const sortedVersions = A.sort(packageVersionInMonorepo, (a, b) => {
+        return semver.compare(cleanVersion(a), cleanVersion(b));
+    });
+
+    return A.last(sortedVersions);
+};
+
+export const getLatestVersionFromNpm = (packageName: string) => {
+    const npmInfo = execSync(`yarn npm info ${packageName} --json`, { encoding: 'utf-8' });
+    const npmInfoJson = JSON.parse(npmInfo);
+
+    return npmInfoJson.version;
+};