Skip to content

Commit

Permalink
server(plugins): upate without downtime
Browse files Browse the repository at this point in the history 
Create a new folder each time a new plugin/theme is installed or
updated. The folder name is created based on the package.json content
hash.

closes Chocobozzz#4828
  • Loading branch information
@kontrollanten
kontrollanten committed Jun 19, 2024
1 parent ca26687 commit 3919659
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 38 deletions.
16 changes: 15 additions & 1 deletion server/core/helpers/core-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

import { promisify1, promisify2, promisify3 } from '@peertube/peertube-core-utils'
import { exec, ExecOptions } from 'child_process'
import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
import { createHash, ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
import { createReadStream } from 'fs'
import truncate from 'lodash-es/truncate.js'
import { pipeline } from 'stream'
import { URL } from 'url'
Expand Down Expand Up @@ -261,6 +262,17 @@ function generateED25519KeyPairPromise () {
})
}

function getContentHash (filePath: string): Promise<string> {
return new Promise((resolve, reject) => {
const hash = createHash('md5')
hash.update(new Date().toString())
const stream = createReadStream(filePath)
stream.on('error', err => reject(err))
stream.on('data', chunk => hash.update(chunk))
stream.on('end', () => resolve(hash.digest('hex')))
})
}

// ---------------------------------------------------------------------------

const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
Expand All @@ -285,6 +297,8 @@ export {

scryptPromise,

getContentHash,

randomBytesPromise,

generateRSAKeyPairPromise,
Expand Down
15 changes: 1 addition & 14 deletions server/core/helpers/decache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,6 @@
import { Module } from 'module'
import { extname } from 'path'

function decachePlugin (require: NodeRequire, libraryPath: string) {
const moduleName = find(require, libraryPath)

if (!moduleName) return

searchCache(require, moduleName, function (mod) {
delete require.cache[mod.id]

removeCachedPath(mod.path)
})
}

function decacheModule (require: NodeRequire, name: string) {
const moduleName = find(require, name)

Expand All @@ -31,8 +19,7 @@ function decacheModule (require: NodeRequire, name: string) {
// ---------------------------------------------------------------------------

export {
decacheModule,
decachePlugin
decacheModule
}

// ---------------------------------------------------------------------------
Expand Down
48 changes: 33 additions & 15 deletions server/core/lib/plugins/plugin-manager.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import express from 'express'
import { createReadStream, createWriteStream } from 'fs'
import { ensureDir, outputFile, readJSON } from 'fs-extra/esm'
import { createReadStream, createWriteStream, statSync } from 'fs'
import { mkdir, readlink } from 'fs/promises'
import { ensureDir, outputFile, readJSON, ensureSymlink } from 'fs-extra/esm'
import { Server } from 'http'
import { createRequire } from 'module'
import { basename, join } from 'path'
Expand All @@ -16,7 +17,6 @@ import {
ServerHook,
ServerHookName
} from '@peertube/peertube-models'
import { decachePlugin } from '@server/helpers/decache.js'
import { ApplicationModel } from '@server/models/application/application.js'
import { MOAuthTokenUser, MUser } from '@server/types/models/index.js'
import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins.js'
Expand Down Expand Up @@ -77,13 +77,25 @@ export class PluginManager implements ServerHook {
private hooks: { [name: string]: HookInformationValue[] } = {}
private translations: PluginLocalesTranslations = {}

private readonly latestDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, 'latest')

private server: Server

private constructor () {
}

init (server: Server) {
async init (server: Server) {
this.server = server

try {
statSync(this.latestDirectory)
} catch (err) {
const workingDir = join(CONFIG.STORAGE.PLUGINS_DIR, Date.now().toString())
await mkdir(workingDir)
await ensureSymlink(workingDir, this.latestDirectory)
// await writeJSON(join(this.latestDirectory, 'package.json'), {})
// await writeFile(join(this.latestDirectory, 'yarn.lock'), '')
}
}

registerWebSocketRouter () {
Expand Down Expand Up @@ -374,6 +386,11 @@ export class PluginManager implements ServerHook {
logger.info('Successful installation of plugin %s.', toInstall)

if (register) {
// Unregister old hooks if it's an update
try {
await this.unregister(npmName)
} catch (err) {}

await this.registerPluginOrTheme(plugin)
}
} catch (rootErr) {
Expand All @@ -394,6 +411,11 @@ export class PluginManager implements ServerHook {
}

throw rootErr
} finally {
// Update plugin paths
for (const npmName in this.registeredPlugins) {
this.registeredPlugins[npmName].path = await this.getPluginPath(this.registeredPlugins[npmName].name, this.registeredPlugins[npmName].type)
}
}

return plugin
Expand All @@ -411,9 +433,6 @@ export class PluginManager implements ServerHook {
version = plugin.latestVersion
}

// Unregister old hooks
await this.unregister(npmName)

return this.install({ toInstall: toUpdate, version, fromDisk })
}

Expand Down Expand Up @@ -463,7 +482,7 @@ export class PluginManager implements ServerHook {
logger.info('Registering plugin or theme %s.', npmName)

const packageJSON = await this.getPackageJSON(plugin.name, plugin.type)
const pluginPath = this.getPluginPath(plugin.name, plugin.type)
const pluginPath = await this.getPluginPath(plugin.name, plugin.type)

this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type)

Expand Down Expand Up @@ -503,9 +522,7 @@ export class PluginManager implements ServerHook {
private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) {
const npmName = PluginModel.buildNpmName(plugin.name, plugin.type)

// Delete cache if needed
const modulePath = join(pluginPath, packageJSON.library)
decachePlugin(require, modulePath)
const library: PluginLibrary = require(modulePath)

if (!isLibraryCodeValid(library)) {
Expand All @@ -530,7 +547,7 @@ export class PluginManager implements ServerHook {
private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) {
for (const locale of Object.keys(translationPaths)) {
const path = translationPaths[locale]
const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path))
const json = await readJSON(join(await this.getPluginPath(plugin.name, plugin.type), path))

const completeLocale = getCompleteLocale(locale)

Expand Down Expand Up @@ -591,16 +608,17 @@ export class PluginManager implements ServerHook {
}
}

private getPackageJSON (pluginName: string, pluginType: PluginType_Type) {
const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json')
private async getPackageJSON (pluginName: string, pluginType: PluginType_Type) {
const pluginPath = join(await this.getPluginPath(pluginName, pluginType), 'package.json')

return readJSON(pluginPath) as Promise<PluginPackageJSON>
}

private getPluginPath (pluginName: string, pluginType: PluginType_Type) {
private async getPluginPath (pluginName: string, pluginType: PluginType_Type) {
const npmName = PluginModel.buildNpmName(pluginName, pluginType)
const currentDirectory = await readlink(join(CONFIG.STORAGE.PLUGINS_DIR, 'latest'))

return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName)
return join(currentDirectory, 'node_modules', npmName)
}

private getAuth (npmName: string, authName: string) {
Expand Down
49 changes: 42 additions & 7 deletions server/core/lib/plugins/yarn.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { outputJSON, pathExists } from 'fs-extra/esm'
import { copy, ensureSymlink, remove, outputJSON, pathExists } from 'fs-extra/esm'
import { mkdir, readlink } from 'fs/promises'
import { join } from 'path'
import { execShell } from '../../helpers/core-utils.js'
import { execShell, getContentHash } from '../../helpers/core-utils.js'
import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js'
import { logger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
Expand All @@ -16,7 +17,7 @@ async function installNpmPlugin (npmName: string, versionArg?: string) {
let toInstall = npmName
if (version) toInstall += `@${version}`

const { stdout } = await execYarn('add ' + toInstall)
const stdout = await execYarn('add ' + toInstall)

logger.debug('Added a yarn package.', { yarnStdout: stdout })
}
Expand Down Expand Up @@ -47,21 +48,55 @@ export {
// ############################################################################

async function execYarn (command: string) {
const latestDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, 'latest')
const currentDirectory = await readlink(latestDirectory)
let workingDirectory: string
let stdout: string

try {
const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR
const pluginPackageJSON = join(pluginDirectory, 'package.json')
const pluginPackageJSON = join(currentDirectory, 'package.json')

// Create empty package.json file if needed
if (!await pathExists(pluginPackageJSON)) {
await outputJSON(pluginPackageJSON, {})
}

return execShell(`yarn ${command}`, { cwd: pluginDirectory })
const hash = await getContentHash(pluginPackageJSON)

workingDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, hash)
await mkdir(workingDirectory)
await copy(join(currentDirectory, 'package.json'), join(workingDirectory, 'package.json'))

try {
await copy(join(currentDirectory, 'yarn.lock'), join(workingDirectory, 'yarn.lock'))
} catch (err) {
logger.debug('No yarn.lock file to copy, will continue without.')
}

const result = await execShell(`yarn ${command}`, { cwd: workingDirectory })
stdout = result.stdout
} catch (result) {
logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr })
logger.error('Cannot exec yarn.', { command, err: result, stderr: result.stderr })

await remove(workingDirectory)

throw result.err
}

try {
await remove(latestDirectory)
await ensureSymlink(workingDirectory, latestDirectory)
} catch (err) {
logger.error('Cannot create symlink for new plugin set. Trying to restore the old one.', { err })
await ensureSymlink(currentDirectory, latestDirectory)
logger.info('Succeeded to restore old plugin set.')

throw err
}

await remove(currentDirectory)

return stdout
}

function checkNpmPluginNameOrThrow (name: string) {
Expand Down
2 changes: 1 addition & 1 deletion server/server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ async function startApplication () {

OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })

PluginManager.Instance.init(server)
await PluginManager.Instance.init(server)
// Before PeerTubeSocket init
PluginManager.Instance.registerWebSocketRouter()

Expand Down

0 comments on commit 3919659

Please sign in to comment.