feat: Pre-fetch all package.json files (#642)

Repository initialisation is refactored so that all package files and their content is retrieved up-front before an renovating begins. This allows us to know both how many package files as well as determine if there is renovate json in a package.json before triggering the onboarding PR.

Closes #634

lib/logger/config-serializer.js

@@ -17,5 +17,8 @@ function configSerializer(config) {
     if (val && templateFields.indexOf(this.key) !== -1) {
       this.update('[Template]');
     }
+    if (this.key === 'content') {
+      this.update('[content]');
+    }
   });
 }

lib/workers/package-file/index.js

@@ -1,8 +1,5 @@
-const path = require('path');
 const configParser = require('../../config');
 const depTypeWorker = require('../dep-type');
-const configMigration = require('../../config/migration');
-const configValidation = require('../../config/validation');
 
 let logger = require('../../logger');
 
@@ -11,25 +8,19 @@ module.exports = {
 };
 
 async function renovatePackageFile(packageFileConfig) {
-  let config = { ...packageFileConfig };
+  const config = { ...packageFileConfig };
   let upgrades = [];
   logger = config.logger;
   logger.info(`Processing package file`);
-  // If onboarding, use the package.json in onboarding branch unless a custom base branch was defined
-  const packageContent = await config.api.getFileJson(
-    config.packageFile,
-    config.contentBaseBranch
-  );
 
-  if (!packageContent) {
-    config.depName = config.packageFile;
-    config.type = 'error';
-    config.message = 'No json content found';
-    logger.warn(config.message);
-    return [config];
+  // Check if config is disabled
+  if (config.enabled === false) {
+    logger.info('packageFile is disabled');
+    return upgrades;
   }
 
-  if (packageContent.workspaces) {
+  // Warn if workspaces found
+  if (config.content.workspaces) {
     logger.warn('Found workspaces');
     const warn = { ...config };
     warn.depName = 'workspaces';
@@ -39,61 +30,6 @@ async function renovatePackageFile(packageFileConfig) {
     upgrades.push(warn);
   }
 
-  // Check for renovate config inside the package.json
-  if (packageContent.renovate) {
-    logger.trace(
-      { config: packageContent.renovate },
-      'package.json>renovate config'
-    );
-    const { isMigrated, migratedConfig } = configMigration.migrateConfig(
-      packageContent.renovate,
-      config
-    );
-    if (isMigrated) {
-      config.logger.info(
-        { oldConfig: packageContent.renovate, newConfig: migratedConfig },
-        'Config migration necessary'
-      );
-    } else {
-      config.logger.debug('No config migration necessary');
-    }
-    const { warnings, errors } = configValidation.validateConfig(
-      migratedConfig
-    );
-    // istanbul ignore if
-    if (warnings.length) {
-      logger.debug(
-        { warnings },
-        'Found package.json>renovate configuration warnings'
-      );
-    }
-    if (errors.length) {
-      logger.warn(
-        { errors },
-        'Found package.json>renovate configuration errors'
-      );
-      /* TODO #556
-      errors.forEach(error => {
-        upgrades.push(
-          { ...error, {
-            depName: `${config.packageFile}(renovate)`,
-            type: 'error',
-          })
-        );
-      });
-      */
-    }
-    // package.json>renovate config takes precedence over existing config
-    config = configParser.mergeChildConfig(config, migratedConfig);
-  } else {
-    logger.debug('Package file has no renovate configuration');
-  }
-  // Now check if config is disabled
-  if (config.enabled === false) {
-    logger.info('packageFile is disabled');
-    return upgrades;
-  }
-
   const depTypes = [
     'dependencies',
     'devDependencies',
@@ -116,25 +52,9 @@ async function renovatePackageFile(packageFileConfig) {
   logger.trace({ config: depTypeConfigs }, `depTypeConfigs`);
   for (const depTypeConfig of depTypeConfigs) {
     upgrades = upgrades.concat(
-      await depTypeWorker.renovateDepType(packageContent, depTypeConfig)
+      await depTypeWorker.renovateDepType(config.content, depTypeConfig)
     );
   }
-
-  // Detect if a yarn.lock file is in use
-  const yarnLockFileName = path.join(
-    path.dirname(config.packageFile),
-    'yarn.lock'
-  );
-  if (await config.api.getFileContent(yarnLockFileName)) {
-    config.hasYarnLock = true;
-  }
-  const packageLockFileName = path.join(
-    path.dirname(config.packageFile),
-    'package-lock.json'
-  );
-  if (await config.api.getFileContent(packageLockFileName)) {
-    config.hasPackageLock = true;
-  }
   if (
     config.lockFileMaintenance.enabled &&
     (config.hasYarnLock || config.hasPackageLock)

lib/workers/repository/apis.js

@@ -1,5 +1,6 @@
 const conventionalCommitsDetector = require('conventional-commits-detector');
 const ini = require('ini');
+const path = require('path');
 const jsonValidator = require('json-dup-key-validator');
 const configParser = require('../../config');
 const configMigration = require('../../config/migration');
@@ -16,6 +17,7 @@ module.exports = {
   mergeRenovateJson,
   checkForLerna,
   detectPackageFiles,
+  resolvePackageFiles,
 };
 
 async function detectSemanticCommits(config) {
@@ -93,6 +95,34 @@ async function initApis(inputConfig, token) {
   return config;
 }
 
+function migrateAndValidate(config, input) {
+  const { isMigrated, migratedConfig } = configMigration.migrateConfig(
+    input,
+    config
+  );
+  if (isMigrated) {
+    config.logger.info(
+      { oldConfig: input, newConfig: migratedConfig },
+      'Config migration necessary'
+    );
+  }
+  const { warnings, errors } = configValidation.validateConfig(migratedConfig);
+  // istanbul ignore if
+  if (warnings.length) {
+    config.logger.debug({ warnings }, 'Found renovate.json warnings');
+  }
+  if (errors.length) {
+    config.logger.warn({ errors }, 'Found renovate.json errors');
+    /* TODO #556
+    renovateJsonErrors.forEach(error => {
+      config.errors.push(
+        { ...error, ...{ depName: 'renovate.json' } }
+      );
+    }); */
+  }
+  return migratedConfig;
+}
+
 // Check for config in `renovate.json`
 async function mergeRenovateJson(config, branchName) {
   let returnConfig = { ...config };
@@ -138,32 +168,7 @@ async function mergeRenovateJson(config, branchName) {
     }
     renovateJson = JSON.parse(renovateJsonContent);
     config.logger.debug({ config: renovateJson }, 'renovate.json config');
-    const { isMigrated, migratedConfig } = configMigration.migrateConfig(
-      renovateJson,
-      config
-    );
-    if (isMigrated) {
-      config.logger.info(
-        { oldConfig: renovateJson, newConfig: migratedConfig },
-        'Config migration necessary'
-      );
-    }
-    const { warnings, errors } = configValidation.validateConfig(
-      migratedConfig
-    );
-    // istanbul ignore if
-    if (warnings.length) {
-      config.logger.debug({ warnings }, 'Found renovate.json warnings');
-    }
-    if (errors.length) {
-      config.logger.warn({ errors }, 'Found renovate.json errors');
-      /* TODO #556
-      renovateJsonErrors.forEach(error => {
-        config.errors.push(
-          { ...error, ...{ depName: 'renovate.json' } }
-        );
-      }); */
-    }
+    const migratedConfig = migrateAndValidate(config, renovateJson);
     returnConfig = configParser.mergeChildConfig(returnConfig, migratedConfig);
     returnConfig.renovateJsonPresent = true;
   } catch (err) {
@@ -206,5 +211,85 @@ async function detectPackageFiles(input) {
       );
     }
   }
+  if (config.packageFiles.length === 0) {
+    config.logger.debug('Checking manually if repository has a package.json');
+    if (await config.api.getFileJson('package.json')) {
+      config.packageFiles = ['package.json'];
+    }
+  }
+  return config;
+}
+
+async function resolvePackageFiles(inputConfig) {
+  const config = { ...inputConfig };
+  config.logger.trace({ config }, 'resolvePackageFiles()');
+  const packageFiles = [];
+  for (let packageFile of config.packageFiles) {
+    packageFile =
+      typeof packageFile === 'string' ? { packageFile } : packageFile;
+    config.logger.debug(`Resolving packageFile ${JSON.stringify(packageFile)}`);
+    packageFile.content = await config.api.getFileJson(
+      packageFile.packageFile,
+      config.baseBranch
+    );
+    if (packageFile.content) {
+      // hoist renovate config if exists
+      if (packageFile.content.renovate) {
+        config.logger.debug(
+          { packageFile: packageFile.packageFile },
+          `Found renovate config`
+        );
+        config.hasPackageJsonRenovateConfig = true;
+        Object.assign(
+          packageFile,
+          migrateAndValidate(config, packageFile.content.renovate)
+        );
+        delete packageFile.content.renovate;
+      } else {
+        config.logger.debug(
+          { packageFile: packageFile.packageFile },
+          `No renovate config`
+        );
+      }
+      // Detect if lock files are used
+      const yarnLockFileName = path.join(
+        path.dirname(packageFile.packageFile),
+        'yarn.lock'
+      );
+      if (
+        await config.api.getFileContent(yarnLockFileName, config.baseBranch)
+      ) {
+        config.logger.debug(
+          { packageFile: packageFile.packageFile },
+          'Found yarn.lock'
+        );
+        packageFile.hasYarnLock = true;
+      } else {
+        packageFile.hasYarnLock = false;
+      }
+      const packageLockFileName = path.join(
+        path.dirname(packageFile.packageFile),
+        'package-lock.json'
+      );
+      if (
+        await config.api.getFileContent(packageLockFileName, config.baseBranch)
+      ) {
+        config.logger.debug(
+          { packageFile: packageFile.packageFile },
+          'Found yarn.lock'
+        );
+        packageFile.hasPackageLock = true;
+      } else {
+        packageFile.hasPackageLock = false;
+      }
+      packageFiles.push(packageFile);
+    } else {
+      config.logger.warn(
+        { packageFile: packageFile.packageFile },
+        'package file not found'
+      );
+    }
+  }
+  config.packageFiles = packageFiles;
   return config;
 }

lib/workers/repository/index.js

@@ -25,13 +25,14 @@ async function renovateRepository(repoConfig, token) {
     }
     if (config.isFork && !config.renovateJsonPresent) {
       config.logger.debug('repository is a fork and not manually configured');
-      await cleanup.pruneStaleBranches(config, []);
       return;
     }
     if (config.baseBranch) {
+      // Renovate should read content and target PRs here
       if (await config.api.branchExists(config.baseBranch)) {
-        await config.api.setBaseBranch(config.baseBranch);
+        config.api.setBaseBranch(config.baseBranch);
       } else {
+        // Warn and ignore setting (use default branch)
         const message = `The configured baseBranch "${config.baseBranch}" is not present. Ignoring`;
         config.errors.push({
           depName: 'baseBranch',
@@ -40,29 +41,32 @@ async function renovateRepository(repoConfig, token) {
         config.logger.warn(message);
       }
     }
+    // Detect package files in default branch if not manually provisioned
     if (config.packageFiles.length === 0) {
       config.logger.debug('Detecting package files');
       config = await apis.detectPackageFiles(config);
+      // If we can't detect any package.json then return
       if (config.packageFiles.length === 0) {
-        if (!config.hasRenovateJson) {
-          config.logger.debug('Checking if repository has a package.json');
-          const pJson = await config.api.getFileJson('package.json');
-          if (!pJson) {
-            config.logger.info('Repository has no package.json');
-            return;
-          }
-        }
-        config.packageFiles.push('package.json');
+        config.logger.info('Cannot detect package.json');
+        return;
       }
+      config.logger.debug(
+        `Detected ${config.packageFiles
+          .length} package files: ${config.packageFiles}`
+      );
     }
+    config = await apis.resolvePackageFiles(config);
+    config.logger.trace({ config }, 'post-packageFiles config');
     config.repoIsOnboarded = await onboarding.getOnboardingStatus(config);
     if (!config.repoIsOnboarded) {
       config.contentBaseBranch = `${config.branchPrefix}configure`;
-      const packageFiles = config.packageFiles;
-      config = await apis.mergeRenovateJson(
-        config,
-        `${config.branchPrefix}configure`
+      // Remove packageFile list in case they are provisioned in renovate.json
+      const packageFiles = config.packageFiles.map(
+        packageFile => packageFile.packageFile
       );
+      config.packageFiles = [];
+      config = await apis.mergeRenovateJson(config, config.contentBaseBranch);
+      // Restore previous packageFile list if not provisioned manually
       if (config.packageFiles.length === 0) {
         config.packageFiles = packageFiles;
       }
@@ -78,6 +82,8 @@ async function renovateRepository(repoConfig, token) {
           config.logger.warn(message);
         }
       }
+      config = await apis.resolvePackageFiles(config);
+      config.logger.trace({ config }, 'onboarding config');
     }
     const allUpgrades = await upgrades.determineRepoUpgrades(config);
     const res = await upgrades.branchifyUpgrades(allUpgrades, config.logger);
@@ -106,10 +112,10 @@ async function renovateRepository(repoConfig, token) {
   } catch (err) {
     // Swallow this error so that other repositories can be processed
     if (err.message === 'uninitiated') {
-      config.logger.info('Repository is unitiated - skipping');
+      repoConfig.logger.info('Repository is unitiated - skipping');
     } else {
-      config.logger.error(`Failed to process repository: ${err.message}`);
-      config.logger.debug({ err });
+      repoConfig.logger.error(`Failed to process repository: ${err.message}`);
+      repoConfig.logger.debug({ err });
     }
   }
 }