From 11d1b2d5c88e2b62ebe0643b036c2e046fa0c020 Mon Sep 17 00:00:00 2001 From: Daniel Roe Date: Mon, 6 Feb 2023 09:35:50 +0000 Subject: [PATCH] fix: work around hotfix cloudflare kv storage bug --- server/cloudflare-driver.ts | 194 ++++++++++++++++++++++++++++++++++++ server/shared.ts | 4 +- 2 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 server/cloudflare-driver.ts diff --git a/server/cloudflare-driver.ts b/server/cloudflare-driver.ts new file mode 100644 index 0000000000..5fe7441686 --- /dev/null +++ b/server/cloudflare-driver.ts @@ -0,0 +1,194 @@ +// Temporary hotfix of https://github.com/unjs/unstorage/blob/4d637a117667ae638a6cac657aac139d88a78027/src/drivers/cloudflare-kv-http.ts#L6 + +import { $fetch } from 'ofetch' +import { defineDriver } from 'unstorage' + +const LOG_TAG = '[unstorage] [cloudflare-http] ' + +interface KVAuthAPIToken { + /** + * API Token generated from the [User Profile 'API Tokens' page](https://dash.cloudflare.com/profile/api-tokens) + * of the Cloudflare console. + * @see https://api.cloudflare.com/#getting-started-requests + */ + apiToken: string +} + +interface KVAuthServiceKey { + /** + * A special Cloudflare API key good for a restricted set of endpoints. + * Always begins with "v1.0-", may vary in length. + * May be used to authenticate in place of `apiToken` or `apiKey` and `email`. + * @see https://api.cloudflare.com/#getting-started-requests + */ + userServiceKey: string +} + +interface KVAuthEmailKey { + /** + * Email address associated with your account. + * Should be used along with `apiKey` to authenticate in place of `apiToken`. + */ + email: string + /** + * API key generated on the "My Account" page of the Cloudflare console. + * Should be used along with `email` to authenticate in place of `apiToken`. + * @see https://api.cloudflare.com/#getting-started-requests + */ + apiKey: string +} + +export type KVHTTPOptions = { + /** + * Cloudflare account ID (required) + */ + accountId: string + /** + * The ID of the KV namespace to target (required) + */ + namespaceId: string + /** + * The URL of the Cloudflare API. + * @default https://api.cloudflare.com + */ + apiURL?: string +} & (KVAuthServiceKey | KVAuthAPIToken | KVAuthEmailKey) + +type CloudflareAuthorizationHeaders = { + 'X-Auth-Email': string + 'X-Auth-Key': string + 'X-Auth-User-Service-Key'?: string + Authorization?: `Bearer ${string}` +} | { + 'X-Auth-Email'?: string + 'X-Auth-Key'?: string + 'X-Auth-User-Service-Key': string + Authorization?: `Bearer ${string}` +} | { + 'X-Auth-Email'?: string + 'X-Auth-Key'?: string + 'X-Auth-User-Service-Key'?: string + Authorization: `Bearer ${string}` +} + +export default defineDriver((opts) => { + if (!opts) + throw new Error('Options must be provided.') + + if (!opts.accountId) + throw new Error(`${LOG_TAG}\`accountId\` is required.`) + + if (!opts.namespaceId) + throw new Error(`${LOG_TAG}\`namespaceId\` is required.`) + + let headers: CloudflareAuthorizationHeaders + + if ('apiToken' in opts) { + headers = { Authorization: `Bearer ${opts.apiToken}` } + } + else if ('userServiceKey' in opts) { + headers = { 'X-Auth-User-Service-Key': opts.userServiceKey } + } + else if (opts.email && opts.apiKey) { + headers = { 'X-Auth-Email': opts.email, 'X-Auth-Key': opts.apiKey } + } + else { + throw new Error( + `${LOG_TAG}One of the \`apiToken\`, \`userServiceKey\`, or a combination of \`email\` and \`apiKey\` is required.`, + ) + } + + const apiURL = opts.apiURL || 'https://api.cloudflare.com' + const baseURL = `${apiURL}/client/v4/accounts/${opts.accountId}/storage/kv/namespaces/${opts.namespaceId}` + const kvFetch = $fetch.create({ baseURL, headers }) + + const hasItem = async (key: string) => { + try { + const res = await kvFetch(`/metadata/${key}`) + return res?.success === true + } + catch (err: any) { + if (!err.response) + throw err + if (err.response.status === 404) + return false + throw err + } + } + + const getItem = async (key: string) => { + try { + // Cloudflare API returns with `content-type: application/octet-stream` + return await kvFetch(`/values/${key}`).then(r => r.text()) + } + catch (err: any) { + if (!err.response) + throw err + if (err.response.status === 404) + return null + throw err + } + } + + const setItem = async (key: string, value: any) => { + return await kvFetch(`/values/${key}`, { method: 'PUT', body: value }) + } + + const removeItem = async (key: string) => { + return await kvFetch(`/values/${key}`, { method: 'DELETE' }) + } + + const getKeys = async (base?: string) => { + const keys: string[] = [] + + const params = new URLSearchParams() + if (base) + params.set('prefix', base) + + const firstPage = await kvFetch('/keys', { params }) + firstPage.result.forEach(({ name }: { name: string }) => keys.push(name)) + + const cursor = firstPage.result_info.cursor + if (cursor) + params.set('cursor', cursor) + + while (params.has('cursor')) { + const pageResult = await kvFetch('/keys', { params: Object.fromEntries(params.entries()) }) + pageResult.result.forEach(({ name }: { name: string }) => keys.push(name)) + const pageCursor = pageResult.result_info.cursor + if (pageCursor) + params.set('cursor', pageCursor) + + else + params.delete('cursor') + } + return keys + } + + const clear = async () => { + const keys: string[] = await getKeys() + // Split into chunks of 10000, as the API only allows for 10,000 keys at a time + const chunks = keys.reduce((acc, key, i) => { + if (i % 10000 === 0) + acc.push([]) + acc[acc.length - 1].push(key) + return acc + }, [[]] as string[][]) + // Call bulk delete endpoint with each chunk + await Promise.all(chunks.map((chunk) => { + return kvFetch('/bulk', { + method: 'DELETE', + body: { keys: chunk }, + }) + })) + } + + return { + hasItem, + getItem, + setItem, + removeItem, + getKeys, + clear, + } +}) diff --git a/server/shared.ts b/server/shared.ts index 5a0d97e58c..304007a3e4 100644 --- a/server/shared.ts +++ b/server/shared.ts @@ -1,8 +1,6 @@ // @ts-expect-error unstorage needs to provide backwards-compatible subpath types import _fs from 'unstorage/drivers/fs' // @ts-expect-error unstorage needs to provide backwards-compatible subpath types -import _kv from 'unstorage/drivers/cloudflare-kv-http' -// @ts-expect-error unstorage needs to provide backwards-compatible subpath types import _memory from 'unstorage/drivers/memory' import { stringifyQuery } from 'ufo' @@ -11,6 +9,7 @@ import { $fetch } from 'ofetch' import type { Storage } from 'unstorage' import cached from './cache-driver' +import kv from './cloudflare-driver' // @ts-expect-error virtual import import { env } from '#build-info' @@ -21,7 +20,6 @@ import type { AppInfo } from '~/types' import { APP_NAME } from '~/constants' const fs = _fs as typeof import('unstorage/dist/drivers/fs')['default'] -const kv = _kv as typeof import('unstorage/dist/drivers/cloudflare-kv-http')['default'] const memory = _memory as typeof import('unstorage/dist/drivers/memory')['default'] const storage = useStorage() as Storage