diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index ed8aadad2c4..36ea348eab4 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -294,7 +294,8 @@ jobs: --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n - --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer + --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log + --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder --storage=ddb --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder diff --git a/.github/workflows/pushDev.yml b/.github/workflows/pushDev.yml index 4d462a8a163..04ee0cdad69 100644 --- a/.github/workflows/pushDev.yml +++ b/.github/workflows/pushDev.yml @@ -208,7 +208,8 @@ jobs: --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n - --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer + --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log + --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder --storage=ddb --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder diff --git a/.github/workflows/pushNext.yml b/.github/workflows/pushNext.yml index 24d2227030a..044aba6a3bd 100644 --- a/.github/workflows/pushNext.yml +++ b/.github/workflows/pushNext.yml @@ -208,7 +208,8 @@ jobs: --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n - --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer + --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log + --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder --storage=ddb --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder diff --git a/.github/workflows/wac/utils/listPackagesWithJestTests.ts b/.github/workflows/wac/utils/listPackagesWithJestTests.ts index 7b4fb256a56..bc7f2e6f6a9 100644 --- a/.github/workflows/wac/utils/listPackagesWithJestTests.ts +++ b/.github/workflows/wac/utils/listPackagesWithJestTests.ts @@ -25,7 +25,7 @@ interface PackageWithTestsWithId extends PackageWithTests { // Takes a PackageWithTests object and returns an array of commands, where each // command is just running a subset of tests. This is achieved by using the // Jest's `--shard` option. -const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount: number = 6) => { +const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount = 6) => { const commands: PackageWithTests[] = []; for (let currentShard = 1; currentShard <= shardsCount; currentShard++) { commands.push({ ...pkg, cmd: pkg.cmd + ` --shard=${currentShard}/${shardsCount}` }); @@ -61,6 +61,10 @@ const CUSTOM_HANDLERS: Record Array> = { return [{ cmd: "packages/api-tenant-manager --storage=ddb", storage: "ddb" }]; }, + "api-log": () => { + return [{ cmd: "packages/api-log --storage=ddb", storage: "ddb" }]; + }, + "api-file-manager": () => { return [ { cmd: "packages/api-file-manager --storage=ddb", storage: "ddb" }, diff --git a/apps/api/graphql/package.json b/apps/api/graphql/package.json index b0a9ffc10ee..866d5f6807c 100644 --- a/apps/api/graphql/package.json +++ b/apps/api/graphql/package.json @@ -25,6 +25,7 @@ "@webiny/api-i18n": "0.0.0", "@webiny/api-i18n-content": "0.0.0", "@webiny/api-i18n-ddb": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/api-page-builder": "0.0.0", "@webiny/api-page-builder-aco": "0.0.0", "@webiny/api-page-builder-import-export": "0.0.0", diff --git a/apps/api/graphql/src/index.ts b/apps/api/graphql/src/index.ts index a957791fc2b..2669b3a8efb 100644 --- a/apps/api/graphql/src/index.ts +++ b/apps/api/graphql/src/index.ts @@ -43,6 +43,7 @@ import { createCountDynamoDbTask } from "~/plugins/countDynamoDbTask"; import { createContinuingTask } from "~/plugins/continuingTask"; import { createWebsockets } from "@webiny/api-websockets"; import { createRecordLocking } from "@webiny/api-record-locking"; +import { createLogger } from "@webiny/api-log"; import scaffoldsPlugins from "./plugins/scaffolds"; import { extensions } from "./extensions"; @@ -63,6 +64,9 @@ export const handler = createHandler({ driver: new DynamoDbDriver({ documentClient }) }), securityPlugins({ documentClient }), + createLogger({ + documentClient + }), tenantManager(), i18nPlugins(), i18nDynamoDbStorageOperations(), diff --git a/apps/api/graphql/tsconfig.json b/apps/api/graphql/tsconfig.json index 21d0a32fc01..0c9b4fe533d 100644 --- a/apps/api/graphql/tsconfig.json +++ b/apps/api/graphql/tsconfig.json @@ -127,6 +127,9 @@ }, { "path": "../../../packages/tasks/tsconfig.build.json" + }, + { + "path": "../../../packages/api-log/tsconfig.build.json" } ], "compilerOptions": { @@ -215,7 +218,9 @@ "@webiny/tasks/*": ["../../../packages/tasks/src/*"], "@webiny/tasks": ["../../../packages/tasks/src"], "@webiny/api-websockets/*": ["../../../packages/api-websockets/src/*"], - "@webiny/api-websockets": ["../../../packages/api-websockets/src"] + "@webiny/api-websockets": ["../../../packages/api-websockets/src"], + "@webiny/api-log/*": ["../../../packages/api-log/src/*"], + "@webiny/api-log": ["../../../packages/api-log/src"] }, "baseUrl": "." } diff --git a/jest.config.base.js b/jest.config.base.js index 6057ad86fe2..c46a0ab9fef 100644 --- a/jest.config.base.js +++ b/jest.config.base.js @@ -91,9 +91,45 @@ module.exports = function ({ path }, presets = []) { process.env.DB_TABLE = "DynamoDB"; process.env.DB_TABLE_ELASTICSEARCH = "ElasticsearchStream"; +process.env.DB_TABLE_LOG = "DynamoDBLog"; process.env.WEBINY_VERSION = version; process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true"; +const createGlobalSecondaryIndexesAttributeDefinitions = amount => { + const attributes = []; + + for (let current = 1; current <= amount; current++) { + attributes.push({ AttributeName: `GSI${current}_PK`, AttributeType: "S" }); + attributes.push({ AttributeName: `GSI${current}_SK`, AttributeType: "S" }); + } + return attributes; +}; + +const createGlobalSecondaryIndexes = options => { + if (!options.amount) { + return []; + } + const indexes = []; + for (let current = 1; current <= options.amount; current++) { + indexes.push({ + IndexName: `GSI${current}`, + KeySchema: [ + { AttributeName: `GSI${current}_PK`, KeyType: "HASH" }, + { AttributeName: `GSI${current}_SK`, KeyType: "RANGE" } + ], + Projection: { + ProjectionType: "ALL" + }, + ProvisionedThroughput: { + ReadCapacityUnits: 1, + WriteCapacityUnits: 1 + } + }); + } + + return indexes; +}; + const createDynaliteTables = (options = {}) => { return { tables: [ @@ -106,42 +142,12 @@ const createDynaliteTables = (options = {}) => { AttributeDefinitions: [ { AttributeName: "PK", AttributeType: "S" }, { AttributeName: "SK", AttributeType: "S" }, - { AttributeName: "GSI1_PK", AttributeType: "S" }, - { AttributeName: "GSI1_SK", AttributeType: "S" }, - { AttributeName: "GSI2_PK", AttributeType: "S" }, - { AttributeName: "GSI2_SK", AttributeType: "S" } + ...createGlobalSecondaryIndexesAttributeDefinitions(2) ], ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 }, - GlobalSecondaryIndexes: [ - { - IndexName: "GSI1", - KeySchema: [ - { AttributeName: "GSI1_PK", KeyType: "HASH" }, - { AttributeName: "GSI1_SK", KeyType: "RANGE" } - ], - Projection: { - ProjectionType: "ALL" - }, - ProvisionedThroughput: { - ReadCapacityUnits: 1, - WriteCapacityUnits: 1 - } - }, - { - IndexName: "GSI2", - KeySchema: [ - { AttributeName: "GSI2_PK", KeyType: "HASH" }, - { AttributeName: "GSI2_SK", KeyType: "RANGE" } - ], - Projection: { - ProjectionType: "ALL" - }, - ProvisionedThroughput: { - ReadCapacityUnits: 1, - WriteCapacityUnits: 1 - } - } - ], + GlobalSecondaryIndexes: createGlobalSecondaryIndexes({ + amount: 2 + }), data: options.data || [] }, { @@ -155,6 +161,23 @@ const createDynaliteTables = (options = {}) => { { AttributeName: "SK", AttributeType: "S" } ], ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 } + }, + { + TableName: process.env.DB_TABLE_LOG, + KeySchema: [ + { AttributeName: "PK", KeyType: "HASH" }, + { AttributeName: "SK", KeyType: "RANGE" } + ], + AttributeDefinitions: [ + { AttributeName: "PK", AttributeType: "S" }, + { AttributeName: "SK", AttributeType: "S" }, + ...createGlobalSecondaryIndexesAttributeDefinitions(5) + ], + ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 }, + GlobalSecondaryIndexes: createGlobalSecondaryIndexes({ + amount: 5 + }), + data: options.data || [] } ], basePort: 8000 diff --git a/packages/api-dynamodb-to-elasticsearch/__tests__/transfer.test.ts b/packages/api-dynamodb-to-elasticsearch/__tests__/transfer.test.ts index daff9ce40eb..244ed7c6d51 100644 --- a/packages/api-dynamodb-to-elasticsearch/__tests__/transfer.test.ts +++ b/packages/api-dynamodb-to-elasticsearch/__tests__/transfer.test.ts @@ -1,5 +1,7 @@ import { createEventHandler, OperationType } from "~/index"; import { createElasticsearchClient } from "@webiny/project-utils/testing/elasticsearch/createClient"; +// @ts-expect-error +import { createMockApiLog } from "@webiny/project-utils/testing/mockApiLog"; import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; import { Context, LambdaContext, Reply, Request } from "@webiny/handler-aws/types"; import { marshall } from "@webiny/aws-sdk/client-dynamodb"; @@ -13,6 +15,7 @@ describe("transfer data", () => { const context = { elasticsearch, + logger: createMockApiLog(), plugins: new PluginsContainer() } as unknown as ElasticsearchContext & Context; /** diff --git a/packages/api-dynamodb-to-elasticsearch/package.json b/packages/api-dynamodb-to-elasticsearch/package.json index 02bb3b9ebd3..05831ee2dd5 100644 --- a/packages/api-dynamodb-to-elasticsearch/package.json +++ b/packages/api-dynamodb-to-elasticsearch/package.json @@ -12,6 +12,7 @@ "author": "Webiny Ltd.", "dependencies": { "@webiny/api-elasticsearch": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/aws-sdk": "0.0.0", "@webiny/error": "0.0.0", "@webiny/handler-aws": "0.0.0", diff --git a/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts b/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts index ef4b1e0f58f..dcd6e4fb87f 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts @@ -1,9 +1,9 @@ import { getNumberEnvVariable } from "~/helpers/getNumberEnvVariable"; import { createDynamoDBEventHandler, timerFactory } from "@webiny/handler-aws"; -import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; import { Decompressor } from "~/Decompressor"; import { OperationsBuilder } from "~/OperationsBuilder"; import { executeWithRetry } from "~/executeWithRetry"; +import { Context } from "~/types"; const MAX_PROCESSOR_PERCENT = getNumberEnvVariable( "MAX_ES_PROCESSOR", @@ -20,7 +20,7 @@ const MAX_RUNNING_TIME = 900; export const createEventHandler = () => { return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => { const timer = timerFactory(lambdaContext); - const context = ctx as unknown as ElasticsearchContext; + const context = ctx as unknown as Context; if (!context.elasticsearch) { console.error("Missing elasticsearch definition on context."); return null; diff --git a/packages/api-dynamodb-to-elasticsearch/src/execute.ts b/packages/api-dynamodb-to-elasticsearch/src/execute.ts index 2bbcd6b5569..44db8ce76b6 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/execute.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/execute.ts @@ -5,9 +5,9 @@ import { WaitingHealthyClusterAbortedError } from "@webiny/api-elasticsearch"; import { ITimer } from "@webiny/handler-aws"; -import { ApiResponse, ElasticsearchContext } from "@webiny/api-elasticsearch/types"; +import { ApiResponse } from "@webiny/api-elasticsearch/types"; import { WebinyError } from "@webiny/error"; -import { IOperations } from "./types"; +import { Context, IOperations } from "./types"; export interface BulkOperationsResponseBodyItemIndexError { reason?: string; @@ -30,7 +30,7 @@ export interface IExecuteParams { timer: ITimer; maxRunningTime: number; maxProcessorPercent: number; - context: Pick; + context: Context; operations: IOperations; } @@ -84,6 +84,8 @@ export const execute = (params: IExecuteParams) => { maxWaitingTime }); + const log = context.logger.withSource("dynamodbToElasticsearch"); + try { await healthCheck.wait({ async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) { @@ -120,6 +122,11 @@ export const execute = (params: IExecuteParams) => { }); checkErrors(res); } catch (error) { + log.error(error, { + tenant: "root", + locale: "unknown" + }); + if (process.env.DEBUG !== "true") { throw error; } diff --git a/packages/api-dynamodb-to-elasticsearch/src/types.ts b/packages/api-dynamodb-to-elasticsearch/src/types.ts index 2707505d00e..142ba3f53b7 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/types.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/types.ts @@ -1,5 +1,7 @@ import { GenericRecord } from "@webiny/cli/types"; import { DynamoDBRecord } from "@webiny/handler-aws/types"; +import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; +import { Context as LoggerContext } from "@webiny/api-log/types"; export interface IOperationsBuilderBuildParams { records: DynamoDBRecord[]; @@ -33,3 +35,5 @@ export interface IOperations { export interface IDecompressor { decompress(data: GenericRecord): Promise; } + +export interface Context extends ElasticsearchContext, Pick {} diff --git a/packages/api-dynamodb-to-elasticsearch/tsconfig.build.json b/packages/api-dynamodb-to-elasticsearch/tsconfig.build.json index 712444ad798..3900c679862 100644 --- a/packages/api-dynamodb-to-elasticsearch/tsconfig.build.json +++ b/packages/api-dynamodb-to-elasticsearch/tsconfig.build.json @@ -3,6 +3,7 @@ "include": ["src"], "references": [ { "path": "../api-elasticsearch/tsconfig.build.json" }, + { "path": "../api-log/tsconfig.build.json" }, { "path": "../aws-sdk/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, { "path": "../handler-aws/tsconfig.build.json" }, diff --git a/packages/api-dynamodb-to-elasticsearch/tsconfig.json b/packages/api-dynamodb-to-elasticsearch/tsconfig.json index 7ea79de6f53..fd5e9d8ef0e 100644 --- a/packages/api-dynamodb-to-elasticsearch/tsconfig.json +++ b/packages/api-dynamodb-to-elasticsearch/tsconfig.json @@ -3,6 +3,7 @@ "include": ["src", "__tests__"], "references": [ { "path": "../api-elasticsearch" }, + { "path": "../api-log" }, { "path": "../aws-sdk" }, { "path": "../error" }, { "path": "../handler-aws" }, @@ -17,6 +18,8 @@ "~tests/*": ["./__tests__/*"], "@webiny/api-elasticsearch/*": ["../api-elasticsearch/src/*"], "@webiny/api-elasticsearch": ["../api-elasticsearch/src"], + "@webiny/api-log/*": ["../api-log/src/*"], + "@webiny/api-log": ["../api-log/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], "@webiny/aws-sdk": ["../aws-sdk/src"], "@webiny/error/*": ["../error/src/*"], diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts index b7f5658f75c..d2270120bfb 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts @@ -2,6 +2,7 @@ import { ContextPlugin } from "@webiny/api"; import { HcmsBulkActionsContext } from "~/types"; import { CmsGraphQLSchemaPlugin, isHeadlessCmsReady } from "@webiny/api-headless-cms"; import { Response } from "@webiny/handler-graphql"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/api-headless-cms/constants"; export interface CreateBulkActionGraphQL { name: string; @@ -16,11 +17,19 @@ export const createBulkActionGraphQL = (config: CreateBulkActionGraphQL) => { const models = await context.security.withoutAuthorization(async () => { const allModels = await context.cms.listModels(); - return allModels.filter( - model => - !model.isPrivate && - (!config.modelIds?.length || config.modelIds.includes(model.modelId)) - ); + return allModels.filter(model => { + if (model.isPrivate) { + return false; + } + const tags = Array.isArray(model.tags) ? model.tags : []; + if (tags.includes(CMS_MODEL_SINGLETON_TAG)) { + return false; + } + if (config.modelIds?.length) { + return config.modelIds.includes(model.modelId); + } + return true; + }); }); const plugins: CmsGraphQLSchemaPlugin[] = []; diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts index 1dba3223372..40aa2d64e50 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts @@ -1,6 +1,7 @@ import { ContextPlugin } from "@webiny/api"; import { HcmsBulkActionsContext } from "~/types"; import { CmsGraphQLSchemaPlugin, isHeadlessCmsReady } from "@webiny/api-headless-cms"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/api-headless-cms/constants"; export const createDefaultGraphQL = () => { return new ContextPlugin(async context => { @@ -24,7 +25,16 @@ export const createDefaultGraphQL = () => { const models = await context.security.withoutAuthorization(async () => { const allModels = await context.cms.listModels(); - return allModels.filter(model => !model.isPrivate); + return allModels.filter(model => { + if (model.isPrivate) { + return false; + } + const tags = Array.isArray(model.tags) ? model.tags : []; + if (tags.includes(CMS_MODEL_SINGLETON_TAG)) { + return false; + } + return true; + }); }); const modelPlugins: CmsGraphQLSchemaPlugin[] = []; diff --git a/packages/api-headless-cms-import-export/src/graphql/typeDefs.ts b/packages/api-headless-cms-import-export/src/graphql/typeDefs.ts index 0211879aba7..7c4a34d30de 100644 --- a/packages/api-headless-cms-import-export/src/graphql/typeDefs.ts +++ b/packages/api-headless-cms-import-export/src/graphql/typeDefs.ts @@ -1,17 +1,25 @@ import type { NonEmptyArray } from "@webiny/api/types"; import { CmsModel } from "@webiny/api-headless-cms/types"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/api-headless-cms/constants"; const createExportContentEntriesByModel = (models: NonEmptyArray): string => { return models .map(model => { + let whereCondition = ""; + const tags = model.tags || []; + if (tags.includes(CMS_MODEL_SINGLETON_TAG) === false) { + whereCondition = /* GraphQL */ ` + # filter the entries by providing a where input + where: ${model.singularApiName}ListWhereInput + `; + } return /* GraphQL */ ` export${model.pluralApiName}ContentEntries( # limit on how much entries will be fetched in a single batch - mostly used for testing limit: Int # do we export assets as well? default is false exportAssets: Boolean - # filter the entries by providing a where input - where: ${model.singularApiName}ListWhereInput + ${whereCondition} # if after is provided, export will start after the provided cursor after: String ): ExportContentEntriesResponse! diff --git a/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts b/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts index cac7f5e45c6..51111c74f13 100644 --- a/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts +++ b/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts @@ -1,4 +1,4 @@ -import { CmsContext, CmsModel } from "~/types"; +import { CmsContext, CmsModel, ApiEndpoint } from "~/types"; import { createManageSDL } from "./createManageSDL"; import { createReadSDL } from "./createReadSDL"; import { createManageResolvers } from "./createManageResolvers"; @@ -50,19 +50,23 @@ export const generateSchemaPlugins = async ( models.forEach(model => { if (model.tags?.includes(CMS_MODEL_SINGLETON_TAG)) { + /** + * We always need to send either manage or read. + */ + const singularType: ApiEndpoint = type === "manage" ? "manage" : "read"; const plugin = createCmsGraphQLSchemaPlugin({ typeDefs: createSingularSDL({ models, model, fieldTypePlugins, - type + type: singularType }), resolvers: createSingularResolvers({ context, models, model, fieldTypePlugins, - type + type: singularType }) }); plugin.name = `headless-cms.graphql.schema.singular.${model.modelId}`; diff --git a/packages/api-headless-cms/src/utils/renderFields.ts b/packages/api-headless-cms/src/utils/renderFields.ts index 362c9167484..86bd2d250c2 100644 --- a/packages/api-headless-cms/src/utils/renderFields.ts +++ b/packages/api-headless-cms/src/utils/renderFields.ts @@ -47,6 +47,8 @@ export const renderField = ({ if (!plugin) { // Let's not render the field if it does not exist in the field plugins. return null; + } else if (!plugin[type]) { + throw new Error(`Missing "${type}" plugin for field type "${field.type}".`); } const { createTypeField } = plugin[type] as CmsModelFieldToGraphQLPlugin["manage"]; const defs = createTypeField({ diff --git a/packages/api-log/.babelrc.js b/packages/api-log/.babelrc.js new file mode 100644 index 00000000000..9da7674cb52 --- /dev/null +++ b/packages/api-log/.babelrc.js @@ -0,0 +1 @@ +module.exports = require("@webiny/project-utils").createBabelConfigForNode({ path: __dirname }); diff --git a/packages/api-log/LICENSE b/packages/api-log/LICENSE new file mode 100644 index 00000000000..f772d04d4db --- /dev/null +++ b/packages/api-log/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Webiny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/api-log/README.md b/packages/api-log/README.md new file mode 100644 index 00000000000..2c45d0c818b --- /dev/null +++ b/packages/api-log/README.md @@ -0,0 +1,12 @@ +# @webiny/api-logs + +[![](https://img.shields.io/npm/dw/@webiny/api-logs.svg)](https://www.npmjs.com/package/@webiny/api-logs) +[![](https://img.shields.io/npm/v/@webiny/api-logs.svg)](https://www.npmjs.com/package/@webiny/api-logs) +[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) + +## Install + +``` +yarn add @webiny/api-logs +``` diff --git a/packages/api-log/__tests__/createLogger.test.ts b/packages/api-log/__tests__/createLogger.test.ts new file mode 100644 index 00000000000..bfb5a311fdb --- /dev/null +++ b/packages/api-log/__tests__/createLogger.test.ts @@ -0,0 +1,79 @@ +import { Context } from "~/types"; +import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { getTenant } from "~tests/mocks/getTenant"; +import { getLocale } from "~tests/mocks/getLocale"; +import { createLogger } from "~/index"; +import { PluginsContainer } from "@webiny/plugins"; + +describe("createLogger", () => { + const documentClient = getDocumentClient(); + + it("should create logger context with db driver", async () => { + const context: Context = { + plugins: new PluginsContainer(), + db: { + driver: { + // @ts-expect-error + documentClient + } + }, + tenancy: { + // @ts-expect-error + getCurrentTenant: () => { + return { id: getTenant() }; + } + }, + i18n: { + // @ts-expect-error + getContentLocale: () => { + return { + code: getLocale() + }; + } + } + }; + expect(context.logger).toBeUndefined(); + + const plugins = createLogger(); + + for (const plugin of plugins) { + // @ts-expect-error + if (!plugin.apply) { + continue; + } + // @ts-expect-error + await plugin.apply(context); + } + + expect(context.logger).toBeDefined(); + }); + + it("should create logger context with direct params", async () => { + // @ts-expect-error + const context: Context = { + plugins: new PluginsContainer() + }; + expect(context.logger).toBeUndefined(); + + const plugins = createLogger({ + documentClient, + getTenant: () => { + return "root"; + }, + getLocale: () => { + return "en-US"; + } + }); + + for (const plugin of plugins) { + // @ts-expect-error + if (!plugin.apply) { + continue; + } + // @ts-expect-error + await plugin.apply(context); + } + + expect(context.logger).toBeDefined(); + }); +}); diff --git a/packages/api-log/__tests__/crud/crud.test.ts b/packages/api-log/__tests__/crud/crud.test.ts new file mode 100644 index 00000000000..c7da5c4d62a --- /dev/null +++ b/packages/api-log/__tests__/crud/crud.test.ts @@ -0,0 +1,160 @@ +import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { getTenant } from "~tests/mocks/getTenant"; +import { getLocale } from "~tests/mocks/getLocale"; +import { DynamoDbLoggerKeys, DynamoDbStorageOperations } from "~/logger"; +import { create } from "~/db"; +import { Context, ILoggerStorageOperations, LogType } from "~/types"; +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { createCrud as baseCreateCrud } from "~/crud"; +import { NotFoundError } from "@webiny/handler-graphql"; +import { loggerFactory } from "~/logger/factory"; + +describe("crud", () => { + const documentClient = getDocumentClient(); + + let entity: Entity; + let storageOperations: ILoggerStorageOperations; + + const checkPermission = async (): Promise => { + return; + }; + + const createCrud = () => { + return baseCreateCrud({ + storageOperations, + checkPermission + }); + }; + + beforeEach(() => { + const result = create({ + documentClient + }); + entity = result.entity; + storageOperations = new DynamoDbStorageOperations({ + keys: new DynamoDbLoggerKeys(), + entity + }); + }); + + it("should create crud methods", async () => { + const crud = createCrud(); + + expect(crud).toHaveProperty("getLog"); + expect(crud).toHaveProperty("deleteLog"); + expect(crud).toHaveProperty("deleteLogs"); + expect(crud).toHaveProperty("listLogs"); + expect(crud).toHaveProperty("withSource"); + }); + + it("should run getLog method", async () => { + const crud = createCrud(); + + try { + const result = await crud.getLog({ + where: { + id: "1" + } + }); + expect(result).toEqual("Should not happen."); + } catch (ex) { + expect(ex).toBeInstanceOf(NotFoundError); + } + }); + + it("should run deleteLog method", async () => { + const crud = createCrud(); + + try { + const result = await crud.deleteLog({ + where: { + id: "1" + } + }); + expect(result).toEqual("Should not happen."); + } catch (ex) { + expect(ex).toBeInstanceOf(NotFoundError); + } + }); + + it("should run deleteLogs method", async () => { + const crud = createCrud(); + + const result = await crud.deleteLogs({ + where: { + items: ["1"] + } + }); + expect(result).toEqual([]); + }); + + it("should run listLogs method", async () => { + const crud = createCrud(); + + const result = await crud.listLogs({}); + expect(result).toEqual({ + items: [], + meta: { + totalCount: -1, + cursor: null, + hasMoreItems: false + } + }); + }); + + it("should run withSource method", async () => { + const crud = createCrud(); + + const result = crud.withSource("source"); + expect(result).toHaveProperty("info"); + expect(result).toHaveProperty("notice"); + expect(result).toHaveProperty("debug"); + expect(result).toHaveProperty("warn"); + expect(result).toHaveProperty("error"); + expect(result).toHaveProperty("flush"); + }); + + it("should log via withSource method", async () => { + const { logger: masterLogger } = loggerFactory({ + documentClient, + getLocale, + getTenant + }); + + const context: Context["logger"] = { + log: masterLogger, + ...createCrud() + }; + + const logger = context.withSource("source"); + logger.info({ message: "test" }); + logger.notice({ message: "test" }); + logger.debug({ message: "test" }); + logger.warn({ message: "test" }); + logger.error({ message: "test" }); + + const result = await logger.flush(); + expect(result).toMatchObject([ + { + id: expect.any(String), + type: LogType.INFO + }, + { + id: expect.any(String), + type: LogType.NOTICE + }, + { + id: expect.any(String), + type: LogType.DEBUG + }, + { + id: expect.any(String), + type: LogType.WARN + }, + { + id: expect.any(String), + type: LogType.ERROR + } + ]); + }); +}); diff --git a/packages/api-log/__tests__/logger/dynamodb/DynamoDbLogger.test.ts b/packages/api-log/__tests__/logger/dynamodb/DynamoDbLogger.test.ts new file mode 100644 index 00000000000..1a86ff356e3 --- /dev/null +++ b/packages/api-log/__tests__/logger/dynamodb/DynamoDbLogger.test.ts @@ -0,0 +1,214 @@ +import { createDynamoDbLogger } from "~/logger"; +import { getTenant } from "~tests/mocks/getTenant"; +import { getLocale } from "~tests/mocks/getLocale"; +import { ILoggerLog, LogType } from "~/types"; + +jest.setTimeout(5000); + +describe("DynamoDbLogger", () => { + it("should one log per type log - autoflush", async () => { + const logs: ILoggerLog[] = []; + + const onFlush = jest.fn(async (items: ILoggerLog[]): Promise => { + logs.push(...items); + return items; + }); + + const tenant = getTenant(); + const locale = getLocale(); + + const logger = createDynamoDbLogger({ + onFlush, + getLocale, + getTenant + }); + + logger.debug("source1", "log1"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.info("source2", "log2"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.warn("source3", "log3"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.notice("source4", "log4"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.error("source5", "log5"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + /** + * Should be empty as logs are flushed after N ms. + */ + expect(logs).toEqual([]); + /** + * TODO: Increase when default timeout is changed. + */ + await new Promise(resolve => setTimeout(resolve, 2000)); + + expect(logs).toEqual([ + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source1", + data: "log1", + tenant, + locale, + type: LogType.DEBUG + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source2", + data: "log2", + tenant, + locale, + type: LogType.INFO + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source3", + data: "log3", + tenant, + locale, + type: LogType.WARN + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source4", + data: "log4", + tenant, + locale, + type: LogType.NOTICE + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source5", + data: "log5", + tenant, + locale, + type: LogType.ERROR + } + ]); + /** + * Make sure all types of logs are represented. + */ + const types = Object.values(LogType); + expect(types.length).toBeGreaterThan(1); + for (const type of types) { + expect(logs.filter(log => log.type === type)).toHaveLength(1); + } + + const result = await logger.flush(); + /** + * Should be empty as logs have already been flushed. + */ + expect(result).toEqual([]); + }); + + it("should one log per type log - manual flush", async () => { + const logs: ILoggerLog[] = []; + + const onFlush = jest.fn(async (items: ILoggerLog[]): Promise => { + logs.push(...items); + return items; + }); + + const tenant = getTenant(); + const locale = getLocale(); + + const logger = createDynamoDbLogger({ + onFlush, + getLocale, + getTenant + }); + + logger.debug("source1", "log1"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.info("source2", "log2"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.warn("source3", "log3"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.notice("source4", "log4"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + logger.error("source5", "log5"); + expect(logs).toEqual([]); + expect(onFlush).toHaveBeenCalledTimes(0); + + await logger.flush(); + + expect(onFlush).toHaveBeenCalledTimes(1); + + expect(logs).toEqual([ + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source1", + data: "log1", + tenant, + locale, + type: LogType.DEBUG + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source2", + data: "log2", + tenant, + locale, + type: LogType.INFO + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source3", + data: "log3", + tenant, + locale, + type: LogType.WARN + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source4", + data: "log4", + tenant, + locale, + type: LogType.NOTICE + }, + { + id: expect.any(String), + createdOn: expect.toBeDateString(), + source: "source5", + data: "log5", + tenant, + locale, + type: LogType.ERROR + } + ]); + /** + * Make sure all types of logs are represented. + */ + const types = Object.values(LogType); + expect(types.length).toBeGreaterThan(1); + for (const type of types) { + expect(logs.filter(log => log.type === type)).toHaveLength(1); + } + + const result = await logger.flush(); + + expect(onFlush).toHaveBeenCalledTimes(1); + /** + * Should be empty as logs have already been flushed. + */ + expect(result).toEqual([]); + }); +}); diff --git a/packages/api-log/__tests__/logger/dynamodb/DynamoDbLoggerKeys.test.ts b/packages/api-log/__tests__/logger/dynamodb/DynamoDbLoggerKeys.test.ts new file mode 100644 index 00000000000..01c759189f6 --- /dev/null +++ b/packages/api-log/__tests__/logger/dynamodb/DynamoDbLoggerKeys.test.ts @@ -0,0 +1,36 @@ +import { mdbid } from "@webiny/utils"; +import { DynamoDbLoggerKeys } from "~/logger"; +import { LogType } from "~/types"; +import { getTenant } from "~tests/mocks/getTenant"; + +describe("DynamoDbLoggerKeys", () => { + it("should create keys", async () => { + const keys = new DynamoDbLoggerKeys(); + + const id = mdbid(); + const tenant = getTenant(); + const source = "some-source"; + const type = LogType.ERROR; + const result = keys.create({ + id, + tenant, + source, + type + }); + + expect(result).toEqual({ + PK: "LOG", + SK: id, + GSI1_PK: `SOURCE#${source}#LOG`, + GSI1_SK: id, + GSI2_PK: `TYPE#${type}#LOG`, + GSI2_SK: id, + GSI3_PK: `T#${tenant}#LOG`, + GSI3_SK: id, + GSI4_PK: `T#${tenant}#SOURCE#${source}#LOG`, + GSI4_SK: id, + GSI5_PK: `T#${tenant}#TYPE#${type}#LOG`, + GSI5_SK: id + }); + }); +}); diff --git a/packages/api-log/__tests__/logger/dynamodb/DynamoDbStorageOperations.test.ts b/packages/api-log/__tests__/logger/dynamodb/DynamoDbStorageOperations.test.ts new file mode 100644 index 00000000000..0bbef5b8af2 --- /dev/null +++ b/packages/api-log/__tests__/logger/dynamodb/DynamoDbStorageOperations.test.ts @@ -0,0 +1,158 @@ +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { create } from "~/db"; +import { DynamoDbStorageOperations } from "~/logger/dynamodb/DynamoDbStorageOperations"; +import { DynamoDbLoggerKeys } from "~/logger"; +import { mdbid } from "@webiny/utils"; +import { ILoggerLog, LogType } from "~/types"; +import { getTenant } from "~tests/mocks/getTenant"; +import { getLocale } from "~tests/mocks/getLocale"; + +describe("DynamoDbStorageOperations", () => { + const documentClient = getDocumentClient(); + + const tenant = getTenant(); + const locale = getLocale(); + + let entity: Entity; + let keys: DynamoDbLoggerKeys; + beforeEach(() => { + const result = create({ + documentClient + }); + entity = result.entity; + keys = new DynamoDbLoggerKeys(); + }); + + it("should insert, get, list and delete log", async () => { + const storageOperations = new DynamoDbStorageOperations({ + entity, + keys + }); + + const log: ILoggerLog = { + id: mdbid(), + createdOn: new Date().toISOString(), + tenant, + locale, + source: "some-source", + type: LogType.ERROR, + data: { + someData: true, + withSomeMoreNestedData: true + } + }; + + const inserted = await storageOperations.insert({ + items: [log] + }); + + expect(inserted).toEqual([log]); + expect(inserted).toHaveLength(1); + + const getLogResult = await storageOperations.getLog({ + where: { + id: log.id + } + }); + expect(getLogResult).toEqual(inserted[0]); + + const getWrongTenantLogResult = await storageOperations.getLog({ + where: { + tenant: "unknown-tenant", + id: log.id + } + }); + expect(getWrongTenantLogResult).toBeNull(); + + const listLogsResult = await storageOperations.listLogs({}); + + expect(listLogsResult).toEqual({ + items: [log], + meta: { + hasMoreItems: false, + totalCount: -1, + cursor: null + } + }); + + const deleteWrongTenantLogResult = await storageOperations.deleteLog({ + where: { + tenant: "unknown-tenant", + id: log.id + } + }); + expect(deleteWrongTenantLogResult).toBeNull(); + + const getLogAfterWrongTenantDeleteResult = await storageOperations.getLog({ + where: { + id: log.id + } + }); + expect(getLogAfterWrongTenantDeleteResult).toEqual(inserted[0]); + + const deleteLogResult = await storageOperations.deleteLog({ + where: { + id: log.id + } + }); + expect(deleteLogResult).toEqual(inserted[0]); + + const getLogAfterDeleteResult = await storageOperations.getLog({ + where: { + id: log.id + } + }); + expect(getLogAfterDeleteResult).toBeNull(); + const listLogsAfterDeleteResult = await storageOperations.listLogs({}); + expect(listLogsAfterDeleteResult).toEqual({ + items: [], + meta: { + hasMoreItems: false, + totalCount: -1, + cursor: null + } + }); + + await storageOperations.insert({ + items: [ + { + ...log, + id: mdbid() + }, + { + ...log, + id: mdbid() + }, + { + ...log, + id: mdbid() + } + ] + }); + + const listLogsAfterInsertResult = await storageOperations.listLogs({}); + expect(listLogsAfterInsertResult.items).toHaveLength(3); + + const deleteLogsResult = await storageOperations.deleteLogs({ + where: { + items: listLogsAfterInsertResult.items.map(item => { + return item.id; + }) + } + }); + expect(deleteLogsResult.sort((a, b) => a.id.localeCompare(b.id))).toEqual( + listLogsAfterInsertResult.items.sort((a, b) => a.id.localeCompare(b.id)) + ); + + const listLogsAfterDeleteLogsResult = await storageOperations.listLogs({}); + expect(listLogsAfterDeleteLogsResult).toEqual({ + items: [], + meta: { + hasMoreItems: false, + totalCount: -1, + cursor: null + } + }); + }); +}); diff --git a/packages/api-log/__tests__/mocks/getLocale.ts b/packages/api-log/__tests__/mocks/getLocale.ts new file mode 100644 index 00000000000..c1f87805528 --- /dev/null +++ b/packages/api-log/__tests__/mocks/getLocale.ts @@ -0,0 +1,3 @@ +export const getLocale = () => { + return "en-US"; +}; diff --git a/packages/api-log/__tests__/mocks/getTenant.ts b/packages/api-log/__tests__/mocks/getTenant.ts new file mode 100644 index 00000000000..177f9cbdb35 --- /dev/null +++ b/packages/api-log/__tests__/mocks/getTenant.ts @@ -0,0 +1,3 @@ +export const getTenant = () => { + return "root"; +}; diff --git a/packages/api-log/__tests__/mocks/logger.ts b/packages/api-log/__tests__/mocks/logger.ts new file mode 100644 index 00000000000..099f5f62173 --- /dev/null +++ b/packages/api-log/__tests__/mocks/logger.ts @@ -0,0 +1,27 @@ +import { DynamoDbLogger } from "~/logger"; +import { ILogger, ILoggerStorageOperations } from "~/types"; + +export interface ICreateMockLoggerParams { + locale?: string; + tenant?: string; + storageOperations: ILoggerStorageOperations; +} + +export const createMockLogger = (params: ICreateMockLoggerParams): ILogger => { + return new DynamoDbLogger({ + getLocale: () => { + return params.locale || "en-US"; + }, + getTenant: () => { + return params.tenant || "root"; + }, + options: { + waitForFlushMs: 500 + }, + onFlush: async items => { + return await params.storageOperations.insert({ + items + }); + } + }); +}; diff --git a/packages/api-log/__tests__/tasks/pruneLogs/PruneLogs.test.ts b/packages/api-log/__tests__/tasks/pruneLogs/PruneLogs.test.ts new file mode 100644 index 00000000000..15b8898f9c2 --- /dev/null +++ b/packages/api-log/__tests__/tasks/pruneLogs/PruneLogs.test.ts @@ -0,0 +1,282 @@ +import { PruneLogs } from "~/tasks/pruneLogs/PruneLogs"; +import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { DynamoDbLoggerKeys, DynamoDbStorageOperations } from "~/logger"; +import { Response, TaskResponse } from "@webiny/tasks"; +import { + ILogger, + ILoggerCrudListLogsCallable, + ILoggerStorageOperations, + ILoggerStorageOperationsListLogsCallable, + LogType +} from "~/types"; +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { create } from "~/db"; +import { createMockLogger } from "~tests/mocks/logger"; + +describe("PruneLogs", () => { + let prune: PruneLogs; + const documentClient = getDocumentClient(); + let response: TaskResponse; + + let entity: Entity; + let storageOperations: ILoggerStorageOperations; + + let logger: ILogger; + + beforeEach(async () => { + prune = new PruneLogs({ + documentClient, + keys: new DynamoDbLoggerKeys() + }); + response = new TaskResponse( + new Response({ + endpoint: "manage", + locale: "en-US", + tenant: "root", + executionName: "test", + stateMachineId: "test", + webinyTaskDefinitionId: "test", + webinyTaskId: "test" + }) + ); + const result = create({ + documentClient + }); + entity = result.entity; + storageOperations = new DynamoDbStorageOperations({ + keys: new DynamoDbLoggerKeys(), + entity + }); + logger = createMockLogger({ + storageOperations + }); + }); + + it("should run prune logs with no logs to actually prune", async () => { + const list: ILoggerCrudListLogsCallable = async () => { + return { + items: [], + meta: { + cursor: null, + totalCount: 0, + hasMoreItems: false + } + }; + }; + const result = await prune.execute({ + list, + response, + input: {}, + isAborted: () => false, + isCloseToTimeout: () => false + }); + + expect(result).toEqual({ + locale: "en-US", + message: undefined, + output: { + items: 0 + }, + status: "done", + tenant: "root", + webinyTaskDefinitionId: "test", + webinyTaskId: "test" + }); + }); + + it("should prune logs in `anotherTenant`", async () => { + const source = "myCustomSource"; + const rootTenant = "root"; + const anotherTenant = "anotherTenant"; + const locale = "en-US"; + // create some logs in root and anotherTenant + logger.debug(source, "debug-message", { + locale, + tenant: rootTenant + }); + logger.notice(source, "notice-message", { + locale, + tenant: rootTenant + }); + logger.info(source, "info-message", { + locale, + tenant: rootTenant + }); + logger.warn(source, "warn-message", { + locale, + tenant: rootTenant + }); + + logger.error(source, "error-message", { + locale, + tenant: rootTenant + }); + + logger.debug(source, "debug-message", { + locale, + tenant: anotherTenant + }); + + await logger.flush(); + + const result = await storageOperations.listLogs({}); + const expected = [ + { + type: LogType.DEBUG, + createdOn: expect.toBeDateString(), + source, + tenant: rootTenant, + locale, + data: "debug-message", + id: expect.any(String) + }, + { + type: LogType.NOTICE, + createdOn: expect.toBeDateString(), + source, + tenant: rootTenant, + locale, + data: "notice-message", + id: expect.any(String) + }, + { + type: LogType.INFO, + createdOn: expect.toBeDateString(), + source, + tenant: rootTenant, + locale, + data: "info-message", + id: expect.any(String) + }, + { + type: LogType.WARN, + createdOn: expect.toBeDateString(), + source, + tenant: rootTenant, + locale, + data: "warn-message", + id: expect.any(String) + }, + { + type: LogType.ERROR, + createdOn: expect.toBeDateString(), + source, + tenant: rootTenant, + locale, + data: "error-message", + id: expect.any(String) + }, + { + type: LogType.DEBUG, + createdOn: expect.toBeDateString(), + source, + tenant: anotherTenant, + locale, + data: "debug-message", + id: expect.any(String) + } + ]; + expect(result.items.length).toBe(expected.length); + expect(result.items).toEqual(expected); + + const list: ILoggerStorageOperationsListLogsCallable = async params => { + return storageOperations.listLogs(params); + }; + /** + * Should not prune anything because the default date is too far into the past. + */ + const pruneNothingResult = await prune.execute({ + list, + response, + input: {}, + isAborted: () => false, + isCloseToTimeout: () => false + }); + + expect(pruneNothingResult).toEqual({ + locale: "en-US", + message: undefined, + output: { + items: 0 + }, + status: "done", + tenant: "root", + webinyTaskDefinitionId: "test", + webinyTaskId: "test" + }); + + /** + * Only prune from anotherTenant. + */ + const pruneResult = await prune.execute({ + list, + response, + input: { + createdAfter: new Date().toISOString(), + tenant: anotherTenant + }, + isAborted: () => false, + isCloseToTimeout: () => false + }); + expect(pruneResult).toEqual({ + locale: "en-US", + message: undefined, + output: { + items: 1 + }, + status: "done", + tenant: "root", + webinyTaskDefinitionId: "test", + webinyTaskId: "test" + }); + + const resultAfterPrune = await storageOperations.listLogs({}); + expect(resultAfterPrune.items.length).toBe(expected.length - 1); + expect(resultAfterPrune.items).toEqual( + expected.filter(item => item.tenant !== anotherTenant) + ); + + /** + * After prune of anotherTenant, let's add some more logs to another tenant. + */ + + logger.info(source, "info-message", { + tenant: anotherTenant, + locale + }); + logger.error(source, "error-message", { + tenant: anotherTenant, + locale + }); + await logger.flush(); + + const resultAfterFlush = await storageOperations.listLogs({}); + expect(resultAfterFlush.items.length).toBe(7); + /** + * And then prune everything. + */ + const pruneAllResult = await prune.execute({ + list, + response, + input: { + createdAfter: new Date().toISOString() + }, + isAborted: () => false, + isCloseToTimeout: () => false + }); + expect(pruneAllResult).toEqual({ + locale: "en-US", + message: undefined, + output: { + items: 7 + }, + status: "done", + tenant: "root", + webinyTaskDefinitionId: "test", + webinyTaskId: "test" + }); + + const resultAfterPruneAll = await storageOperations.listLogs({}); + expect(resultAfterPruneAll.items.length).toBe(0); + }); +}); diff --git a/packages/api-log/jest-dynalite-config.js b/packages/api-log/jest-dynalite-config.js new file mode 100644 index 00000000000..6d5840539c8 --- /dev/null +++ b/packages/api-log/jest-dynalite-config.js @@ -0,0 +1,3 @@ +const { createDynaliteTables } = require("../../jest.config.base"); + +module.exports = createDynaliteTables(); diff --git a/packages/api-log/jest.setup.js b/packages/api-log/jest.setup.js new file mode 100644 index 00000000000..7f2a525a5b5 --- /dev/null +++ b/packages/api-log/jest.setup.js @@ -0,0 +1,6 @@ +const base = require("../../jest.config.base"); +const presets = require("@webiny/project-utils/testing/presets")([]); + +module.exports = { + ...base({ path: __dirname }, presets) +}; diff --git a/packages/api-log/package.json b/packages/api-log/package.json new file mode 100644 index 00000000000..2f171869d7d --- /dev/null +++ b/packages/api-log/package.json @@ -0,0 +1,47 @@ +{ + "name": "@webiny/api-log", + "version": "0.0.0", + "main": "index.js", + "repository": { + "type": "git", + "url": "https://github.com/webiny/webiny-js.git", + "directory": "packages/api-log" + }, + "keywords": [ + "log:base" + ], + "author": "Webiny Ltd", + "description": "API to store logs.", + "license": "MIT", + "dependencies": { + "@webiny/api": "0.0.0", + "@webiny/api-i18n": "0.0.0", + "@webiny/api-security": "0.0.0", + "@webiny/api-tenancy": "0.0.0", + "@webiny/aws-sdk": "0.0.0", + "@webiny/db-dynamodb": "0.0.0", + "@webiny/handler": "0.0.0", + "@webiny/handler-graphql": "0.0.0", + "@webiny/plugins": "0.0.0", + "@webiny/tasks": "0.0.0", + "@webiny/utils": "0.0.0", + "zod": "^3.22.4" + }, + "devDependencies": { + "@webiny/cli": "0.0.0", + "@webiny/project-utils": "0.0.0", + "jest": "^29.7.0", + "jest-dynalite": "^3.6.1", + "rimraf": "^5.0.5", + "ttypescript": "^1.5.12", + "typescript": "4.9.5" + }, + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "scripts": { + "build": "yarn webiny run build", + "watch": "yarn webiny run watch" + } +} diff --git a/packages/api-log/src/context.ts b/packages/api-log/src/context.ts new file mode 100644 index 00000000000..b27254dac42 --- /dev/null +++ b/packages/api-log/src/context.ts @@ -0,0 +1,67 @@ +import { ContextPlugin } from "@webiny/api/plugins/ContextPlugin"; +import { Context } from "~/types"; +import { loggerFactory } from "~/logger/factory"; +import { createCrud } from "~/crud"; +import { checkPermissionFactory } from "~/security/checkPermission"; +import { createGraphQl } from "~/graphql"; +import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; + +export interface ICreateLoggerContextParams { + documentClient?: DynamoDBDocument; + getTenant?: () => string; + getLocale?: () => string; +} + +const getDocumentClient = (context: Context) => { + // @ts-expect-error + const { documentClient } = context.db?.driver || {}; + if (!documentClient) { + throw new Error("Missing document client on the context."); + } + return documentClient; +}; + +export const createContextPlugin = (params?: ICreateLoggerContextParams) => { + const plugin = new ContextPlugin(async context => { + const getTenant = () => { + if (params?.getTenant) { + return params.getTenant(); + } + const tenant = context.tenancy?.getCurrentTenant?.(); + if (!tenant) { + throw new Error("Missing tenant."); + } + return tenant.id; + }; + const getLocale = () => { + if (params?.getLocale) { + return params.getLocale(); + } + const locale = context.i18n?.getContentLocale?.(); + if (!locale) { + throw new Error("Missing locale."); + } + return locale.code; + }; + + const getContext = () => context; + + const { logger, storageOperations } = loggerFactory({ + documentClient: params?.documentClient || getDocumentClient(context), + getLocale, + getTenant + }); + + context.logger = { + log: logger, + ...createCrud({ + storageOperations, + checkPermission: checkPermissionFactory({ getContext }) + }) + }; + context.plugins.register(createGraphQl()); + }); + + plugin.name = "logger.createContext"; + return plugin; +}; diff --git a/packages/api-log/src/crud/index.ts b/packages/api-log/src/crud/index.ts new file mode 100644 index 00000000000..0ebff2bd64e --- /dev/null +++ b/packages/api-log/src/crud/index.ts @@ -0,0 +1,83 @@ +import { + Context, + ILoggerCrud, + ILoggerCrudDeleteLogParams, + ILoggerCrudDeleteLogResponse, + ILoggerCrudDeleteLogsParams, + ILoggerCrudGetLogResponse, + ILoggerCrudGetLogsParams, + ILoggerCrudListLogsParams, + ILoggerCrudListLogsResponse, + ILoggerLog, + ILoggerStorageOperations, + ILoggerWithSource +} from "~/types"; +import { NotFoundError } from "@webiny/handler-graphql"; + +export interface ICreateCrudParams { + storageOperations: ILoggerStorageOperations; + checkPermission(): Promise; +} + +export const createCrud = (params: ICreateCrudParams): ILoggerCrud => { + const { storageOperations, checkPermission } = params; + + return { + async getLog(params: ILoggerCrudGetLogsParams): Promise { + await checkPermission(); + const item = await storageOperations.getLog(params); + if (!item) { + throw new NotFoundError(); + } + return { + item + }; + }, + async deleteLog(params: ILoggerCrudDeleteLogParams): Promise { + await checkPermission(); + const item = await storageOperations.deleteLog({ + ...params + }); + if (!item) { + throw new NotFoundError(); + } + return { + item + }; + }, + async deleteLogs(params: ILoggerCrudDeleteLogsParams): Promise { + await checkPermission(); + return storageOperations.deleteLogs(params); + }, + async listLogs(params: ILoggerCrudListLogsParams): Promise { + await checkPermission(); + const { items, meta } = await storageOperations.listLogs(params); + return { + items, + meta + }; + }, + withSource(this: Context["logger"], source: string): ILoggerWithSource { + return { + info: (data, options) => { + return this.log.info(source, data, options); + }, + notice: (data, options) => { + return this.log.notice(source, data, options); + }, + debug: (data, options) => { + return this.log.debug(source, data, options); + }, + warn: (data, options) => { + return this.log.warn(source, data, options); + }, + error: (data, options) => { + return this.log.error(source, data, options); + }, + flush: () => { + return this.log.flush(); + } + }; + } + }; +}; diff --git a/packages/api-log/src/db/entity.ts b/packages/api-log/src/db/entity.ts new file mode 100644 index 00000000000..cadd2522392 --- /dev/null +++ b/packages/api-log/src/db/entity.ts @@ -0,0 +1,109 @@ +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { ITable } from "~/db/types"; + +interface IParams { + table: ITable; +} + +export interface IEntityAttributes { + PK: string; + SK: string; + GSI1_PK: string; + GSI1_SK: string; + GSI2_PK: string; + GSI2_SK: string; + GSI3_PK: string; + GSI3_SK: string; + GSI4_PK: string; + GSI4_SK: string; + id: string; + tenant: string; + locale: string; + source: string; + type: string; + data: string; +} + +export const createEntity = (params: IParams): Entity => { + const { table } = params; + return new Entity({ + name: "Log", + table, + attributes: { + PK: { + partitionKey: true + }, + SK: { + sortKey: true + }, + GSI1_PK: { + type: "string", + required: true + }, + GSI1_SK: { + type: "string", + required: true + }, + GSI2_PK: { + type: "string", + required: true + }, + GSI2_SK: { + type: "string", + required: true + }, + GSI3_PK: { + type: "string", + required: true + }, + GSI3_SK: { + type: "string", + required: true + }, + GSI4_PK: { + type: "string", + required: true + }, + GSI4_SK: { + type: "string", + required: true + }, + GSI5_PK: { + type: "string", + required: true + }, + GSI5_SK: { + type: "string", + required: true + }, + id: { + type: "string", + required: true + }, + createdOn: { + type: "string", + required: true + }, + tenant: { + type: "string", + required: true + }, + locale: { + type: "string", + required: true + }, + source: { + type: "string", + required: true + }, + type: { + type: "string", + required: true + }, + data: { + type: "string", + required: true + } + } + }); +}; diff --git a/packages/api-log/src/db/index.ts b/packages/api-log/src/db/index.ts new file mode 100644 index 00000000000..5f0f54e7178 --- /dev/null +++ b/packages/api-log/src/db/index.ts @@ -0,0 +1,27 @@ +import { createTable } from "~/db/table"; +import { createEntity } from "~/db/entity"; +import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; + +interface IParams { + documentClient: DynamoDBDocument; + table?: string; +} + +export const create = (params: IParams) => { + const name = params.table || process.env.DB_TABLE_LOG; + if (!name) { + throw new Error("Missing table name when creating a logger table."); + } + const table = createTable({ + documentClient: params.documentClient, + name + }); + + const entity = createEntity({ + table + }); + return { + table, + entity + }; +}; diff --git a/packages/api-log/src/db/table.ts b/packages/api-log/src/db/table.ts new file mode 100644 index 00000000000..16570173f3f --- /dev/null +++ b/packages/api-log/src/db/table.ts @@ -0,0 +1,49 @@ +import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; +import { Table } from "@webiny/db-dynamodb/toolbox"; +import { ITable } from "~/db/types"; + +interface Params { + name: string; + documentClient: DynamoDBDocument; +} + +export const createTable = ({ name, documentClient }: Params): ITable => { + return new Table({ + name, + partitionKey: "PK", + sortKey: "SK", + DocumentClient: documentClient, + /** + * @see DynamoDbLoggerKeys.create + */ + indexes: { + // source + GSI1: { + partitionKey: "GSI1_PK", + sortKey: "GSI1_SK" + }, + // type + GSI2: { + partitionKey: "GSI2_PK", + sortKey: "GSI2_SK" + }, + // tenant + GSI3: { + partitionKey: "GSI3_PK", + sortKey: "GSI3_SK" + }, + // tenant and source + GSI4: { + partitionKey: "GSI4_PK", + sortKey: "GSI4_SK" + }, + // tenant and type + GSI5: { + partitionKey: "GSI5_PK", + sortKey: "GSI5_SK" + } + }, + autoExecute: true, + autoParse: true + }); +}; diff --git a/packages/api-log/src/db/types.ts b/packages/api-log/src/db/types.ts new file mode 100644 index 00000000000..df3198f806b --- /dev/null +++ b/packages/api-log/src/db/types.ts @@ -0,0 +1,3 @@ +import { Table as DynamoDbTable } from "@webiny/db-dynamodb/toolbox"; + +export type ITable = DynamoDbTable; diff --git a/packages/api-log/src/graphql/index.ts b/packages/api-log/src/graphql/index.ts new file mode 100644 index 00000000000..6d630d4ba8a --- /dev/null +++ b/packages/api-log/src/graphql/index.ts @@ -0,0 +1,10 @@ +import { Plugin } from "@webiny/plugins/types"; +import { createGraphQlPlugin } from "~/graphql/plugin"; + +export const createGraphQl = (): Plugin[] => { + if (process.env.DEBUG !== "true") { + return []; + } + + return [createGraphQlPlugin()]; +}; diff --git a/packages/api-log/src/graphql/plugin.ts b/packages/api-log/src/graphql/plugin.ts new file mode 100644 index 00000000000..29b1d8df9da --- /dev/null +++ b/packages/api-log/src/graphql/plugin.ts @@ -0,0 +1,177 @@ +import { GraphQLSchemaPlugin, resolve, resolveList } from "@webiny/handler-graphql"; +import { Context, LogType } from "~/types"; +import zod from "zod"; +import { createZodError } from "@webiny/utils"; + +const getLogArgsSchema = zod.object({ + where: zod.object({ + id: zod.string() + }) +}); + +const listLogsArgsSchema = zod.object({ + where: zod.object({ + tenant: zod.string().optional(), + source: zod.string().optional(), + type: zod + .enum([LogType.DEBUG, LogType.NOTICE, LogType.INFO, LogType.WARN, LogType.ERROR]) + .optional() + }), + sort: zod.array(zod.enum(["ASC", "DESC"])).optional(), + limit: zod.number().optional(), + after: zod.string().optional() +}); + +const deleteLogArgsSchema = zod.object({ + where: zod.object({ + tenant: zod.string().optional(), + id: zod.string() + }) +}); + +const deleteLogsArgsSchema = zod.object({ + where: zod.object({ + tenant: zod.string().optional(), + items: zod.array(zod.string()) + }) +}); + +export const createGraphQlPlugin = () => { + return new GraphQLSchemaPlugin({ + typeDefs: /* GraphQL */ ` + extend type Query { + log: LogQuery + } + + extend type Mutation { + log: LogMutation + } + + enum LogType { + ${LogType.DEBUG} + ${LogType.NOTICE} + ${LogType.INFO} + ${LogType.WARN} + ${LogType.ERROR} + } + + type LogQueryResponseItem { + id: ID! + type: LogType! + source: String! + data: JSON! + createdOn: DateTime! + } + + type LogQueryGetResponse { + data: LogQueryResponseItem + error: LogQueryResponseError + } + + type LogQueryListResponseMeta { + cursor: String + hasMoreItems: Boolean! + totalCount: Int! + } + + type LogQueryResponseError { + message: String! + code: String + data: JSON + stack: String + } + + type LogQueryListResponse { + data: [LogQueryResponseItem!] + meta: LogQueryListResponseMeta + error: LogQueryResponseError + } + + input ListLogsWhereInput { + tenant: String + source: String + type: LogType + } + + enum ListLogsSortEnum { + ASC + DESC + } + + type LogQuery { + getLog(id: ID!): LogQueryGetResponse! + listLogs( + where: ListLogsWhereInput + sort: ListLogsSortEnum + limit: Int + after: String + ): LogQueryListResponse! + } + + type LogMutationDeleteLogResponse { + data: LogQueryResponseItem + error: LogQueryResponseError + } + + type LogMutationDeleteLogsResponse { + data: Int + error: LogQueryResponseError + } + + input DeleteLogWhereInput { + id: ID! + } + + input DeleteLogsWhereInput { + items: [ID!]! + } + + type LogMutation { + deleteLog(where: DeleteLogWhereInput!): LogMutationDeleteLogResponse! + deleteLogs(where: DeleteLogsWhereInput!): LogMutationDeleteLogsResponse! + } + `, + resolvers: { + LogQuery: { + getLog: async (_: unknown, args: unknown, context) => { + return resolve(async () => { + const result = getLogArgsSchema.safeParse(args); + if (result.error) { + throw createZodError(result.error); + } + return await context.logger.getLog(result.data); + }); + }, + listLogs: async (_, args, context) => { + return resolveList(async () => { + const result = listLogsArgsSchema.safeParse(args); + if (result.error) { + throw createZodError(result.error); + } + return await context.logger.listLogs(result.data); + }); + } + }, + LogMutation: { + deleteLog: async (_, args, context) => { + return resolve(async () => { + const result = deleteLogArgsSchema.safeParse(args); + if (result.error) { + throw createZodError(result.error); + } + return await context.logger.deleteLog(result.data); + }); + }, + deleteLogs: async (_, args, context) => { + return resolve(async () => { + const result = deleteLogsArgsSchema.safeParse(args); + if (result.error) { + throw createZodError(result.error); + } + return await context.logger.deleteLogs(result.data); + }); + } + } + } + }); +}; diff --git a/packages/api-log/src/index.ts b/packages/api-log/src/index.ts new file mode 100644 index 00000000000..16a0984ae37 --- /dev/null +++ b/packages/api-log/src/index.ts @@ -0,0 +1,7 @@ +import { createContextPlugin, ICreateLoggerContextParams } from "./context"; +import { createLifecycle } from "./lifecycle"; +import { createPruneLogsTask } from "~/tasks/createPruneLogsTask"; + +export const createLogger = (params?: ICreateLoggerContextParams) => { + return [createLifecycle(), createContextPlugin(params), createPruneLogsTask()]; +}; diff --git a/packages/api-log/src/lifecycle.ts b/packages/api-log/src/lifecycle.ts new file mode 100644 index 00000000000..10fc7bc96b3 --- /dev/null +++ b/packages/api-log/src/lifecycle.ts @@ -0,0 +1,30 @@ +import { createModifyFastifyPlugin } from "@webiny/handler"; +import { Context } from "./types"; + +export const createLifecycle = () => { + return createModifyFastifyPlugin(app => { + const execute = async () => { + // @ts-expect-error + if (app.webiny.___flushedLogs) { + return; + } + // @ts-expect-error + app.webiny.___flushedLogs = true; + const context = app.webiny as Context; + try { + await context.logger.log.flush(); + } catch (ex) { + console.error("Error flushing logs."); + console.log(ex); + } + }; + + app.addHook("onTimeout", async () => { + execute(); + }); + + app.addHook("onResponse", async () => { + execute(); + }); + }); +}; diff --git a/packages/api-log/src/logger/dynamodb/DynamoDbLogger.ts b/packages/api-log/src/logger/dynamodb/DynamoDbLogger.ts new file mode 100644 index 00000000000..1a474139561 --- /dev/null +++ b/packages/api-log/src/logger/dynamodb/DynamoDbLogger.ts @@ -0,0 +1,166 @@ +import { ILogger, ILoggerLog, ILoggerLogCallableOptions, LogType } from "~/types"; +import { GenericRecord } from "@webiny/api/types"; +import { mdbid } from "@webiny/utils"; + +interface IDynamoDbLoggerAddParams { + source: string; + data: T; + type: LogType; + options?: ILoggerLogCallableOptions; +} + +export interface IDynamoDbLoggerParamsOnFlushCallable { + (items: ILoggerLog[]): Promise; +} + +export interface IDynamoDbLoggerOptions { + waitForFlushMs?: number; +} + +export interface IDynamoDbLoggerParams { + readonly getTenant: () => string; + readonly getLocale: () => string; + readonly onFlush: IDynamoDbLoggerParamsOnFlushCallable; + readonly options?: IDynamoDbLoggerOptions; +} + +type IAwaiter = Promise; +/** + * Milliseconds to wait before flushing logs. + */ +const defaultWaitForFlushMs = 1000; + +export class DynamoDbLogger implements ILogger { + private readonly items = new Set(); + private readonly getTenant: () => string; + private readonly getLocale: () => string; + private readonly onFlush: IDynamoDbLoggerParamsOnFlushCallable; + private readonly options?: IDynamoDbLoggerOptions; + + private awaiter: IAwaiter | null = null; + private timeout: NodeJS.Timeout | null = null; + + public constructor(params: IDynamoDbLoggerParams) { + this.getTenant = params.getTenant; + this.getLocale = params.getLocale; + this.onFlush = params.onFlush; + this.options = params.options; + } + public debug( + source: string, + data: T, + options?: ILoggerLogCallableOptions + ): void { + return this.add({ + data, + source, + type: LogType.DEBUG, + options + }); + } + + public info( + source: string, + data: T, + options?: ILoggerLogCallableOptions + ): void { + return this.add({ + data, + source, + type: LogType.INFO, + options + }); + } + + public warn( + source: string, + data: T, + options?: ILoggerLogCallableOptions + ): void { + return this.add({ + data, + source, + type: LogType.WARN, + options + }); + } + + public notice( + source: string, + data: T, + options?: ILoggerLogCallableOptions + ): void { + return this.add({ + data, + source, + type: LogType.NOTICE, + options + }); + } + + public error( + source: string, + data: T, + options?: ILoggerLogCallableOptions + ): void { + return this.add({ + data, + source, + type: LogType.ERROR, + options + }); + } + + public async flush(): Promise { + if (this.timeout) { + clearTimeout(this.timeout); + this.timeout = null; + } + if (this.awaiter) { + return this.awaiter; + } else if (this.items.size === 0) { + return []; + } + const items = Array.from(this.items); + this.items.clear(); + + this.awaiter = new Promise(resolve => { + this.onFlush(items).then(items => { + resolve(items); + this.awaiter = null; + }); + }); + + return this.awaiter; + } + + private startFlush(): void { + if (this.items.size === 0) { + return; + } else if (this.awaiter || this.timeout) { + return; + } + const ms = this.options?.waitForFlushMs || defaultWaitForFlushMs; + + this.timeout = setTimeout(() => { + this.flush(); + }, ms); + } + + private add(params: IDynamoDbLoggerAddParams): void { + this.items.add({ + id: mdbid(), + createdOn: new Date().toISOString(), + tenant: params.options?.tenant || this.getTenant(), + locale: params.options?.locale || this.getLocale(), + data: params.data, + source: params.source, + type: params.type + }); + this.startFlush(); + } +} + +export const createDynamoDbLogger = (params: IDynamoDbLoggerParams): ILogger => { + return new DynamoDbLogger(params); +}; diff --git a/packages/api-log/src/logger/dynamodb/DynamoDbLoggerKeys.ts b/packages/api-log/src/logger/dynamodb/DynamoDbLoggerKeys.ts new file mode 100644 index 00000000000..a80ccc22d95 --- /dev/null +++ b/packages/api-log/src/logger/dynamodb/DynamoDbLoggerKeys.ts @@ -0,0 +1,102 @@ +import { ILoggerLog } from "~/types"; + +export interface IDynamoDbLoggerKeysCreateResponse { + PK: string; + SK: string; + GSI1_PK: string; + GSI1_SK: string; + GSI2_PK: string; + GSI2_SK: string; + GSI3_PK: string; + GSI3_SK: string; + GSI4_PK: string; + GSI4_SK: string; + GSI5_PK: string; + GSI5_SK: string; +} + +export class DynamoDbLoggerKeys { + public create( + item: Pick + ): IDynamoDbLoggerKeysCreateResponse { + return { + PK: this.createPartitionKey(), + SK: this.createSortKey(item), + GSI1_PK: this.createSourcePartitionKey(item), + GSI1_SK: this.createSourceSortKey(item), + GSI2_PK: this.createTypePartitionKey(item), + GSI2_SK: this.createTypeSortKey(item), + /** + * Tenant specific. + */ + GSI3_PK: this.createTenantPartitionKey(item), + GSI3_SK: this.createTenantSortKey(item), + GSI4_PK: this.createTenantAndSourcePartitionKey(item), + GSI4_SK: this.createTenantAndSourceSortKey(item), + GSI5_PK: this.createTenantAndTypePartitionKey(item), + GSI5_SK: this.createTenantAndTypeSortKey(item) + }; + } + /** + * Query for all logs or one by ID. + */ + public createPartitionKey(): string { + return `LOG`; + } + + public createSortKey(item: Pick): string { + return item.id; + } + /** + * Query for all logs from a source. + */ + public createSourcePartitionKey(item: Pick): string { + return `SOURCE#${item.source}#LOG`; + } + + public createSourceSortKey(item: Pick): string { + return item.id; + } + /** + * Query for all logs of a certain type. + */ + public createTypePartitionKey(item: Pick): string { + return `TYPE#${item.type}#LOG`; + } + + public createTypeSortKey(item: Pick): string { + return item.id; + } + /** + * Query for all logs by tenant. + */ + public createTenantPartitionKey(item: Pick): string { + return `T#${item.tenant}#LOG`; + } + + public createTenantSortKey(item: Pick): string { + return item.id; + } + + /** + * Query for all logs by tenant + source. + */ + public createTenantAndSourcePartitionKey(item: Pick): string { + return `T#${item.tenant}#SOURCE#${item.source}#LOG`; + } + + public createTenantAndSourceSortKey(item: Pick): string { + return item.id; + } + + /** + * Query for all logs by tenant + type. + */ + public createTenantAndTypePartitionKey(item: Pick): string { + return `T#${item.tenant}#TYPE#${item.type}#LOG`; + } + + public createTenantAndTypeSortKey(item: Pick): string { + return item.id; + } +} diff --git a/packages/api-log/src/logger/dynamodb/DynamoDbStorageOperations.ts b/packages/api-log/src/logger/dynamodb/DynamoDbStorageOperations.ts new file mode 100644 index 00000000000..09daab30423 --- /dev/null +++ b/packages/api-log/src/logger/dynamodb/DynamoDbStorageOperations.ts @@ -0,0 +1,299 @@ +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { + ILoggerCrudDeleteLogParams, + ILoggerCrudDeleteLogsParams, + ILoggerCrudGetLogsParams, + ILoggerCrudListLogsParams, + ILoggerCrudListLogsResponse, + ILoggerCrudListSort, + ILoggerLog, + ILoggerStorageOperations, + ILoggerStorageOperationsInsertParams, + LogType +} from "~/types"; +import { DynamoDbLoggerKeys } from "./DynamoDbLoggerKeys"; +import { + batchReadAll, + batchWriteAll, + cleanupItem, + getClean, + queryPerPageClean +} from "@webiny/db-dynamodb"; +import { GenericRecord } from "@webiny/api/types"; +import { compress, decompress } from "@webiny/utils/compression/gzip"; +import { convertAfterToStartKey, convertLastEvaluatedKeyToAfterKey } from "./convertKeys"; + +const TO_STORAGE_ENCODING = "base64"; +const FROM_STORAGE_ENCODING = "utf8"; + +export interface IDynamoDbStorageOperationsParams { + entity: Entity; + keys: DynamoDbLoggerKeys; +} + +interface ICreateQueryParamsInput { + tenant: string | undefined; + source: string | undefined; + type: LogType | undefined; + after: string | undefined; + sort: ILoggerCrudListSort[] | undefined; +} + +interface ICreateQueryParamsResult { + index: string | undefined; + partitionKey: string; + reverse: boolean; + startKey: GenericRecord | undefined; +} + +interface ILoggerLogDb extends ILoggerLog { + data: string; +} + +export class DynamoDbStorageOperations implements ILoggerStorageOperations { + private readonly entity: Entity; + private readonly keys: DynamoDbLoggerKeys; + + public constructor(params: IDynamoDbStorageOperationsParams) { + this.entity = params.entity; + this.keys = params.keys; + } + + public async insert(params: ILoggerStorageOperationsInsertParams): Promise { + const compressed = await Promise.all( + params.items.map(async item => { + return { + ...item, + data: await this.compress(item.data) + }; + }) + ); + const items = compressed.map(item => { + return this.entity.putBatch({ + ...item, + ...this.keys.create(item) + }); + }); + try { + await batchWriteAll({ + items, + table: this.entity.table + }); + return params.items; + } catch (ex) { + console.error("Failed to insert logs."); + console.log(ex); + throw ex; + } + } + + public async listLogs(params: ILoggerCrudListLogsParams): Promise { + const { where = {}, sort, limit = 50, after } = params; + + const { source, type, tenant } = where; + + const queryParams = this.createQueryParams({ + tenant, + source, + type, + sort, + after + }); + + const result = await queryPerPageClean({ + entity: this.entity, + partitionKey: queryParams.partitionKey, + options: { + index: queryParams.index, + reverse: queryParams.reverse, + startKey: queryParams.startKey, + limit + } + }); + + const cursor = convertLastEvaluatedKeyToAfterKey(result.lastEvaluatedKey); + + return { + items: await Promise.all( + result.items.map(async item => { + return { + ...item, + data: await this.decompress(item.data) + }; + }) + ), + meta: { + cursor, + totalCount: -1, + hasMoreItems: !!cursor + } + }; + } + + public async getLog(params: ILoggerCrudGetLogsParams): Promise { + const { where } = params; + try { + const item = await getClean({ + entity: this.entity, + keys: { + PK: this.keys.createPartitionKey(), + SK: this.keys.createSortKey(where) + } + }); + if (!item || (where.tenant && item.tenant !== where.tenant)) { + return null; + } + return { + ...item, + data: await this.decompress(item.data) + }; + } catch (ex) { + console.error("Failed to get log."); + throw ex; + } + } + + public async deleteLog(params: ILoggerCrudDeleteLogParams): Promise { + const item = await this.getLog(params); + if (!item) { + return null; + } + try { + await this.entity.delete({ + PK: this.keys.createPartitionKey(), + SK: this.keys.createSortKey(params.where) + }); + return item; + } catch (ex) { + console.error("Failed to delete log."); + throw ex; + } + } + + public async deleteLogs(params: ILoggerCrudDeleteLogsParams): Promise { + const items = params.where.items.map(id => { + return this.entity.getBatch({ + PK: this.keys.createPartitionKey(), + SK: this.keys.createSortKey({ id }) + }); + }); + const compressedResults = await batchReadAll({ + items, + table: this.entity.table + }); + const cleanedResults = compressedResults + .map(item => cleanupItem(this.entity, item)) + .filter((item): item is ILoggerLogDb => !!item); + + const results = await Promise.all( + cleanedResults.map(async item => { + return { + ...item, + data: await this.decompress(item.data) + }; + }) + ); + try { + await batchWriteAll({ + items: results.map(item => { + return this.entity.deleteBatch({ + PK: this.keys.createPartitionKey(), + SK: this.keys.createSortKey({ id: item.id }) + }); + }), + table: this.entity.table + }); + return results; + } catch (ex) { + console.error("Failed to delete logs."); + throw ex; + } + } + + private async compress(input: unknown): Promise { + if (!input) { + return undefined; + } + const str = JSON.stringify(input); + const data = await compress(str); + + return data.toString(TO_STORAGE_ENCODING); + } + + private async decompress(input?: string): Promise { + if (!input) { + return undefined; + } + try { + const data = await decompress(Buffer.from(input, TO_STORAGE_ENCODING)); + return JSON.parse(data.toString(FROM_STORAGE_ENCODING)) as T; + } catch (ex) { + console.error("Failed to decompress data."); + console.log(ex); + return undefined; + } + } + + private createQueryParams(input: ICreateQueryParamsInput): ICreateQueryParamsResult { + const { tenant, source, type, after, sort } = input; + + const reverse = sort ? sort.includes("DESC") : false; + const startKey = convertAfterToStartKey(after); + /** + * Tenant related queries. + */ + if (tenant) { + if (source) { + return { + index: "GSI4", + partitionKey: this.keys.createTenantAndSourcePartitionKey({ tenant, source }), + reverse, + startKey + }; + } else if (type) { + return { + index: "GSI5", + partitionKey: this.keys.createTenantAndTypePartitionKey({ tenant, type }), + reverse, + startKey + }; + } + return { + index: "GSI3", + partitionKey: this.keys.createTenantPartitionKey({ tenant }), + reverse, + startKey + }; + } + /** + * All tenants queries. + */ + if (source) { + return { + index: "GSI1", + partitionKey: this.keys.createSourcePartitionKey({ source }), + reverse, + startKey + }; + } else if (type) { + return { + index: "GSI2", + partitionKey: this.keys.createTypePartitionKey({ type }), + reverse, + startKey + }; + } + return { + index: undefined, + partitionKey: this.keys.createPartitionKey(), + reverse, + startKey + }; + } +} + +export const createStorageOperations = ( + params: IDynamoDbStorageOperationsParams +): ILoggerStorageOperations => { + return new DynamoDbStorageOperations(params); +}; diff --git a/packages/api-log/src/logger/dynamodb/convertKeys.ts b/packages/api-log/src/logger/dynamodb/convertKeys.ts new file mode 100644 index 00000000000..86d037ebb93 --- /dev/null +++ b/packages/api-log/src/logger/dynamodb/convertKeys.ts @@ -0,0 +1,35 @@ +import { GenericRecord } from "@webiny/api/types"; + +export const convertLastEvaluatedKeyToAfterKey = ( + lastEvaluatedKey?: GenericRecord +): string | null => { + if (!lastEvaluatedKey) { + return null; + } + try { + const json = JSON.stringify(lastEvaluatedKey); + return Buffer.from(json).toString("base64"); + } catch (ex) { + console.error("Failed to convert last evaluated key to after."); + console.log(ex); + return null; + } +}; + +export const convertAfterToStartKey = (after?: string | null): GenericRecord | undefined => { + if (!after) { + return undefined; + } + try { + const json = Buffer.from(after, "base64").toString("utf-8"); + const result = JSON.parse(json); + if (!result || typeof result !== "object") { + return undefined; + } + return result || undefined; + } catch (ex) { + console.error("Failed to convert after to start key."); + console.log(ex); + return undefined; + } +}; diff --git a/packages/api-log/src/logger/dynamodb/index.ts b/packages/api-log/src/logger/dynamodb/index.ts new file mode 100644 index 00000000000..0bac78548f9 --- /dev/null +++ b/packages/api-log/src/logger/dynamodb/index.ts @@ -0,0 +1,3 @@ +export * from "./DynamoDbLogger"; +export * from "./DynamoDbLoggerKeys"; +export * from "./DynamoDbStorageOperations"; diff --git a/packages/api-log/src/logger/factory.ts b/packages/api-log/src/logger/factory.ts new file mode 100644 index 00000000000..80f0a6a3465 --- /dev/null +++ b/packages/api-log/src/logger/factory.ts @@ -0,0 +1,34 @@ +import { createDynamoDbLogger, createStorageOperations, DynamoDbLoggerKeys } from "./dynamodb"; +import { create } from "~/db"; +import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; + +export interface ILoggerFactoryParams { + documentClient: DynamoDBDocument; + getTenant: () => string; + getLocale: () => string; +} + +export const loggerFactory = ({ getTenant, getLocale, documentClient }: ILoggerFactoryParams) => { + const keys = new DynamoDbLoggerKeys(); + const { entity } = create({ + documentClient + }); + + const storageOperations = createStorageOperations({ + entity, + keys + }); + + return { + logger: createDynamoDbLogger({ + onFlush: async items => { + return await storageOperations.insert({ + items + }); + }, + getLocale, + getTenant + }), + storageOperations + }; +}; diff --git a/packages/api-log/src/logger/index.ts b/packages/api-log/src/logger/index.ts new file mode 100644 index 00000000000..6f7d116a0bb --- /dev/null +++ b/packages/api-log/src/logger/index.ts @@ -0,0 +1 @@ +export * from "./dynamodb"; diff --git a/packages/api-log/src/security/checkPermission.ts b/packages/api-log/src/security/checkPermission.ts new file mode 100644 index 00000000000..81536096a7d --- /dev/null +++ b/packages/api-log/src/security/checkPermission.ts @@ -0,0 +1,20 @@ +import { Context } from "~/types"; +import { NotAuthorizedError } from "@webiny/api-security"; + +export interface ICheckPermissionFactoryParams { + getContext(): Pick; +} + +export const checkPermissionFactory = (params: ICheckPermissionFactoryParams) => { + return async () => { + const context = params.getContext(); + if (!context.security) { + throw new Error("Missing security context."); + } + const permission = await context.security.getPermission("logs"); + if (permission) { + return; + } + throw new NotAuthorizedError(); + }; +}; diff --git a/packages/api-log/src/tasks/createPruneLogsTask.ts b/packages/api-log/src/tasks/createPruneLogsTask.ts new file mode 100644 index 00000000000..52f5bf214fe --- /dev/null +++ b/packages/api-log/src/tasks/createPruneLogsTask.ts @@ -0,0 +1,70 @@ +import { createTaskDefinition } from "@webiny/tasks"; +import { Context, IPruneLogsInput, IPruneLogsOutput } from "~/tasks/pruneLogs/types"; +import { LogType } from "~/types"; +import { NonEmptyArray } from "@webiny/api/types"; + +export const PRUNE_LOGS_TASK = "pruneLogs"; + +export const createPruneLogsTask = () => { + return createTaskDefinition({ + id: PRUNE_LOGS_TASK, + maxIterations: 5, + disableDatabaseLogs: true, + isPrivate: false, + description: "Prune logs from the database, but keep the ones created in last 1 minute.", + title: "Prune Logs", + run: async params => { + const { PruneLogs } = await import( + /* webpackChunkName: "PruneLogs" */ "./pruneLogs/PruneLogs" + ); + + const { DynamoDbLoggerKeys } = await import( + /* webpackChunkName: "DynamoDbLoggerKeys" */ "~/logger/dynamodb/DynamoDbLoggerKeys" + ); + + try { + const prune = new PruneLogs({ + // @ts-expect-error + documentClient: params.context.db.driver.documentClient, + keys: new DynamoDbLoggerKeys() + }); + return await prune.execute({ + input: params.input, + list: params.context.logger.listLogs, + response: params.response, + isAborted: params.isAborted, + isCloseToTimeout: params.isCloseToTimeout + }); + } catch (ex) { + console.log("Error executing the task.", ex); + return params.response.error({ + message: ex.message, + code: ex.code || "PRUNE_LOGS_TASK_ERROR", + data: ex.data + }); + } + }, + createInputValidation: ({ validator }) => { + return { + tenant: validator.string().optional(), + source: validator.string().optional(), + keys: validator.object({}).passthrough(), + type: validator.enum(Object.keys(LogType) as NonEmptyArray).optional(), + createdAfter: validator + .string() + .optional() + .transform(value => { + if (!value) { + return undefined; + } + try { + return new Date(value).toISOString(); + } catch (ex) { + return undefined; + } + }), + items: validator.number().optional() + }; + } + }); +}; diff --git a/packages/api-log/src/tasks/pruneLogs/PruneLogs.ts b/packages/api-log/src/tasks/pruneLogs/PruneLogs.ts new file mode 100644 index 00000000000..7e06aa23828 --- /dev/null +++ b/packages/api-log/src/tasks/pruneLogs/PruneLogs.ts @@ -0,0 +1,121 @@ +import { ITaskResponse, ITaskResponseResult } from "@webiny/tasks"; +import { IPruneLogsInput, IPruneLogsOutput } from "~/tasks/pruneLogs/types"; +import { create } from "~/db"; +import { ILoggerCrudListLogsCallable, ILoggerCrudListLogsResponse, ILoggerLog } from "~/types"; +import { batchWriteAll } from "@webiny/db-dynamodb"; +import { DynamoDbLoggerKeys } from "~/logger"; +import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; + +const getDate = (input: string | undefined, reduceSeconds = 60): Date => { + if (input) { + return new Date(input); + } + const current = new Date().getTime(); + const next = current - reduceSeconds * 1000; + return new Date(next); +}; + +export interface IPruneLogsParams { + documentClient: DynamoDBDocument; + keys: DynamoDbLoggerKeys; +} + +export interface IPruneLogsExecuteParams< + I extends IPruneLogsInput = IPruneLogsInput, + O extends IPruneLogsOutput = IPruneLogsOutput +> { + list: ILoggerCrudListLogsCallable; + input: I; + response: ITaskResponse; + isAborted: () => boolean; + isCloseToTimeout: () => boolean; +} + +export class PruneLogs< + I extends IPruneLogsInput = IPruneLogsInput, + O extends IPruneLogsOutput = IPruneLogsOutput +> { + private readonly documentClient: DynamoDBDocument; + private readonly keys: DynamoDbLoggerKeys; + + public constructor(params: IPruneLogsParams) { + this.documentClient = params.documentClient; + this.keys = params.keys; + } + + public async execute(params: IPruneLogsExecuteParams): Promise { + const { list, response, input, isAborted, isCloseToTimeout } = params; + + const { entity, table } = create({ + documentClient: this.documentClient + }); + + let startKey = input.keys || undefined; + + const createdAfter = getDate(input.createdAfter); + + let totalItems = input.items || 0; + + const filter = (item: Pick): boolean => { + /** + * We always check the date first. We do not need to go any further if the date is not older than the provided date. + */ + const date = new Date(item.createdOn); + const isDeletable = date.getTime() <= createdAfter.getTime(); + if (!isDeletable || (!input.source && !input.type)) { + return isDeletable; + } else if (input.source && input.type) { + return (input.source === item.source && input.type === item.type) || isDeletable; + } else if (input.source) { + return input.source === item.source || isDeletable; + } + return input.type === item.type || isDeletable; + }; + + let result: ILoggerCrudListLogsResponse; + do { + if (isAborted()) { + return response.aborted(); + } else if (isCloseToTimeout()) { + const inputOutput: IPruneLogsInput = { + ...input, + createdAfter: createdAfter.toISOString(), + items: totalItems, + keys: startKey + }; + return response.continue(inputOutput as I); + } + result = await list({ + where: { + tenant: input.tenant, + source: input.source, + type: input.type + }, + limit: 100 + }); + + const items = result.items.filter(filter); + + if (items.length > 0) { + await batchWriteAll({ + items: result.items.map(item => { + return entity.deleteBatch({ + PK: this.keys.createPartitionKey(), + SK: this.keys.createSortKey(item) + }); + }), + table + }); + totalItems += items.length; + } + + if (result?.meta?.hasMoreItems) { + startKey = result.meta.cursor || undefined; + } + } while (startKey); + const output: IPruneLogsOutput = { + items: totalItems + }; + return response.done(output as O); + } +} diff --git a/packages/api-log/src/tasks/pruneLogs/types.ts b/packages/api-log/src/tasks/pruneLogs/types.ts new file mode 100644 index 00000000000..95bb5d593c5 --- /dev/null +++ b/packages/api-log/src/tasks/pruneLogs/types.ts @@ -0,0 +1,17 @@ +import { Context as TaskContext, ITaskResponseDoneResultOutput } from "@webiny/tasks"; +import { Context as LoggerContext, LogType } from "~/types"; + +export interface IPruneLogsInput { + tenant?: string; + source?: string; + type?: LogType; + createdAfter?: string; + keys?: string; + items?: number; +} + +export interface IPruneLogsOutput extends ITaskResponseDoneResultOutput { + items: number; +} + +export interface Context extends LoggerContext, TaskContext {} diff --git a/packages/api-log/src/types.ts b/packages/api-log/src/types.ts new file mode 100644 index 00000000000..9efaacc77e4 --- /dev/null +++ b/packages/api-log/src/types.ts @@ -0,0 +1,147 @@ +import { TenancyContext } from "@webiny/api-tenancy/types"; +import { I18NContext } from "@webiny/api-i18n/types"; +import { Context as HandlerContext } from "@webiny/handler/types"; + +export interface ILoggerLogCallableOptions { + tenant?: string; + locale?: string; +} +export interface ILoggerLogCallable { + (source: string, data: unknown, options?: ILoggerLogCallableOptions): void; +} + +export enum LogType { + DEBUG = "debug", + NOTICE = "notice", + INFO = "info", + WARN = "warn", + ERROR = "error" +} + +export interface ILoggerLog { + id: string; + createdOn: string; + tenant: string; + locale: string; + source: string; + type: string; + data: unknown; +} + +export interface ILoggerCrudListLogsParamsWhere { + tenant?: string; + source?: string; + type?: LogType; +} + +export type ILoggerCrudListSort = "ASC" | "DESC"; + +export interface ILoggerCrudListLogsParams { + where?: ILoggerCrudListLogsParamsWhere; + sort?: ILoggerCrudListSort[]; + limit?: number; + after?: string; +} + +export interface ILoggerCrudListLogsResponse { + items: ILoggerLog[]; + meta: { + cursor: string | null; + totalCount: number; + hasMoreItems: boolean; + }; +} + +export interface ILoggerCrudGetLogsParamsWhere { + id: string; + tenant?: string; +} + +export interface ILoggerCrudGetLogsParams { + where: ILoggerCrudGetLogsParamsWhere; +} + +export interface ILoggerCrudGetLogResponse { + item: ILoggerLog; +} + +export interface ILoggerCrudDeleteLogResponse { + item: ILoggerLog; +} + +export interface ILoggerCrudDeleteLogParamsWhere { + tenant?: string; + id: string; +} + +export interface ILoggerCrudDeleteLogParams { + where: ILoggerCrudDeleteLogParamsWhere; +} + +export interface ILoggerCrudDeleteLogsParamsWhere { + tenant?: string; + items: string[]; +} + +export interface ILoggerCrudDeleteLogsParams { + where: ILoggerCrudDeleteLogsParamsWhere; +} + +export interface ILoggerWithSourceLogCallable { + (data: unknown, options?: ILoggerLogCallableOptions): void; +} + +export interface ILoggerWithSource { + info: ILoggerWithSourceLogCallable; + notice: ILoggerWithSourceLogCallable; + debug: ILoggerWithSourceLogCallable; + warn: ILoggerWithSourceLogCallable; + error: ILoggerWithSourceLogCallable; + flush(): Promise; +} + +export interface ILoggerCrudListLogsCallable { + (params: ILoggerCrudListLogsParams): Promise; +} + +export interface ILoggerCrud { + withSource(source: string): ILoggerWithSource; + listLogs: ILoggerCrudListLogsCallable; + getLog(params: ILoggerCrudGetLogsParams): Promise; + deleteLog(params: ILoggerCrudDeleteLogParams): Promise; + deleteLogs(params: ILoggerCrudDeleteLogsParams): Promise; +} + +export interface ILoggerStorageOperationsInsertParams { + items: ILoggerLog[]; +} + +export interface ILoggerStorageOperationsListLogsCallable { + (params: ILoggerCrudListLogsParams): Promise; +} + +export interface ILoggerStorageOperations { + insert(params: ILoggerStorageOperationsInsertParams): Promise; + listLogs: ILoggerStorageOperationsListLogsCallable; + getLog(params: ILoggerCrudGetLogsParams): Promise; + deleteLog(params: ILoggerCrudDeleteLogParams): Promise; + deleteLogs(params: ILoggerCrudDeleteLogsParams): Promise; +} + +export interface ILogger { + debug: ILoggerLogCallable; + info: ILoggerLogCallable; + warn: ILoggerLogCallable; + notice: ILoggerLogCallable; + error: ILoggerLogCallable; + flush(): Promise; +} + +export interface Context + extends Pick, + Pick, + HandlerContext { + logger: ILoggerCrud & { + log: ILogger; + }; +} diff --git a/packages/api-log/tsconfig.build.json b/packages/api-log/tsconfig.build.json new file mode 100644 index 00000000000..7948a6dddf4 --- /dev/null +++ b/packages/api-log/tsconfig.build.json @@ -0,0 +1,24 @@ +{ + "extends": "../../tsconfig.build.json", + "include": ["src"], + "references": [ + { "path": "../api/tsconfig.build.json" }, + { "path": "../api-i18n/tsconfig.build.json" }, + { "path": "../api-security/tsconfig.build.json" }, + { "path": "../api-tenancy/tsconfig.build.json" }, + { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../db-dynamodb/tsconfig.build.json" }, + { "path": "../handler/tsconfig.build.json" }, + { "path": "../handler-graphql/tsconfig.build.json" }, + { "path": "../plugins/tsconfig.build.json" }, + { "path": "../tasks/tsconfig.build.json" }, + { "path": "../utils/tsconfig.build.json" } + ], + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "declarationDir": "./dist", + "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, + "baseUrl": "." + } +} diff --git a/packages/api-log/tsconfig.json b/packages/api-log/tsconfig.json new file mode 100644 index 00000000000..87fd73d93dc --- /dev/null +++ b/packages/api-log/tsconfig.json @@ -0,0 +1,49 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src", "__tests__"], + "references": [ + { "path": "../api" }, + { "path": "../api-i18n" }, + { "path": "../api-security" }, + { "path": "../api-tenancy" }, + { "path": "../aws-sdk" }, + { "path": "../db-dynamodb" }, + { "path": "../handler" }, + { "path": "../handler-graphql" }, + { "path": "../plugins" }, + { "path": "../tasks" }, + { "path": "../utils" } + ], + "compilerOptions": { + "rootDirs": ["./src", "./__tests__"], + "outDir": "./dist", + "declarationDir": "./dist", + "paths": { + "~/*": ["./src/*"], + "~tests/*": ["./__tests__/*"], + "@webiny/api/*": ["../api/src/*"], + "@webiny/api": ["../api/src"], + "@webiny/api-i18n/*": ["../api-i18n/src/*"], + "@webiny/api-i18n": ["../api-i18n/src"], + "@webiny/api-security/*": ["../api-security/src/*"], + "@webiny/api-security": ["../api-security/src"], + "@webiny/api-tenancy/*": ["../api-tenancy/src/*"], + "@webiny/api-tenancy": ["../api-tenancy/src"], + "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], + "@webiny/aws-sdk": ["../aws-sdk/src"], + "@webiny/db-dynamodb/*": ["../db-dynamodb/src/*"], + "@webiny/db-dynamodb": ["../db-dynamodb/src"], + "@webiny/handler/*": ["../handler/src/*"], + "@webiny/handler": ["../handler/src"], + "@webiny/handler-graphql/*": ["../handler-graphql/src/*"], + "@webiny/handler-graphql": ["../handler-graphql/src"], + "@webiny/plugins/*": ["../plugins/src/*"], + "@webiny/plugins": ["../plugins/src"], + "@webiny/tasks/*": ["../tasks/src/*"], + "@webiny/tasks": ["../tasks/src"], + "@webiny/utils/*": ["../utils/src/*"], + "@webiny/utils": ["../utils/src"] + }, + "baseUrl": "." + } +} diff --git a/packages/api-log/webiny.config.js b/packages/api-log/webiny.config.js new file mode 100644 index 00000000000..6dff86766c9 --- /dev/null +++ b/packages/api-log/webiny.config.js @@ -0,0 +1,8 @@ +const { createWatchPackage, createBuildPackage } = require("@webiny/project-utils"); + +module.exports = { + commands: { + build: createBuildPackage({ cwd: __dirname }), + watch: createWatchPackage({ cwd: __dirname }) + } +}; diff --git a/packages/api-prerendering-service-aws/package.json b/packages/api-prerendering-service-aws/package.json index 04699dc4333..d91b236612e 100644 --- a/packages/api-prerendering-service-aws/package.json +++ b/packages/api-prerendering-service-aws/package.json @@ -12,6 +12,7 @@ "license": "MIT", "dependencies": { "@webiny/api": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/api-prerendering-service": "0.0.0", "@webiny/aws-sdk": "0.0.0", "@webiny/error": "0.0.0", diff --git a/packages/api-prerendering-service-aws/src/render/sqsRender.ts b/packages/api-prerendering-service-aws/src/render/sqsRender.ts index 8ac54450ca5..934ddf7ce47 100644 --- a/packages/api-prerendering-service-aws/src/render/sqsRender.ts +++ b/packages/api-prerendering-service-aws/src/render/sqsRender.ts @@ -1,6 +1,7 @@ import plugin, { RenderParams } from "@webiny/api-prerendering-service/render"; import { createSQSEventHandler } from "@webiny/handler-aws"; import { HandlerPayload } from "@webiny/api-prerendering-service/render/types"; +import { Context as LoggerContext } from "@webiny/api-log/types"; export default (params: RenderParams) => { const render = plugin(params); @@ -9,7 +10,7 @@ export default (params: RenderParams) => { const events: HandlerPayload = event.Records.map(r => JSON.parse(r.body)); return render.cb({ - context, + context: context as LoggerContext, payload: events, request, reply diff --git a/packages/api-prerendering-service-aws/tsconfig.build.json b/packages/api-prerendering-service-aws/tsconfig.build.json index 3ee6512ce99..4b4d42789a1 100644 --- a/packages/api-prerendering-service-aws/tsconfig.build.json +++ b/packages/api-prerendering-service-aws/tsconfig.build.json @@ -3,6 +3,7 @@ "include": ["src"], "references": [ { "path": "../api/tsconfig.build.json" }, + { "path": "../api-log/tsconfig.build.json" }, { "path": "../api-prerendering-service/tsconfig.build.json" }, { "path": "../aws-sdk/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, diff --git a/packages/api-prerendering-service-aws/tsconfig.json b/packages/api-prerendering-service-aws/tsconfig.json index 645ad0cb85b..b3a34eb8502 100644 --- a/packages/api-prerendering-service-aws/tsconfig.json +++ b/packages/api-prerendering-service-aws/tsconfig.json @@ -3,6 +3,7 @@ "include": ["src", "__tests__"], "references": [ { "path": "../api" }, + { "path": "../api-log" }, { "path": "../api-prerendering-service" }, { "path": "../aws-sdk" }, { "path": "../error" }, @@ -19,6 +20,8 @@ "~tests/*": ["./__tests__/*"], "@webiny/api/*": ["../api/src/*"], "@webiny/api": ["../api/src"], + "@webiny/api-log/*": ["../api-log/src/*"], + "@webiny/api-log": ["../api-log/src"], "@webiny/api-prerendering-service/*": ["../api-prerendering-service/src/*"], "@webiny/api-prerendering-service": ["../api-prerendering-service/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], diff --git a/packages/api-prerendering-service/package.json b/packages/api-prerendering-service/package.json index 7bfd8601f30..384f38c3df4 100644 --- a/packages/api-prerendering-service/package.json +++ b/packages/api-prerendering-service/package.json @@ -17,6 +17,7 @@ "dependencies": { "@sparticuz/chromium": "123.0.1", "@webiny/api": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/aws-sdk": "0.0.0", "@webiny/error": "0.0.0", "@webiny/handler": "0.0.0", diff --git a/packages/api-prerendering-service/src/render/index.ts b/packages/api-prerendering-service/src/render/index.ts index a7de5ffbfcc..b54bbe82e91 100644 --- a/packages/api-prerendering-service/src/render/index.ts +++ b/packages/api-prerendering-service/src/render/index.ts @@ -2,7 +2,7 @@ import renderUrl, { File } from "./renderUrl"; import { join } from "path"; import { S3, PutObjectCommandInput } from "@webiny/aws-sdk/client-s3"; import { getStorageFolder, getRenderUrl, getIsNotFoundPage, isMultiTenancyEnabled } from "~/utils"; -import { HandlerPayload, RenderHookPlugin } from "./types"; +import { Context, HandlerPayload, RenderHookPlugin } from "./types"; import { PrerenderingServiceStorageOperations, Render, TagPathLink } from "~/types"; import omit from "lodash/omit"; import { EventPlugin } from "@webiny/handler"; @@ -46,7 +46,7 @@ export default (params: RenderParams) => { const isMultiTenant = isMultiTenancyEnabled(); const log = console.log; - return new EventPlugin(async ({ payload, context }) => { + return new EventPlugin(async ({ payload, context }) => { const handlerArgs = Array.isArray(payload) ? payload : [payload]; const handlerHookPlugins = context.plugins.byType("ps-render-hook"); diff --git a/packages/api-prerendering-service/src/render/types.ts b/packages/api-prerendering-service/src/render/types.ts index b629aa8cee5..98731bc11ae 100644 --- a/packages/api-prerendering-service/src/render/types.ts +++ b/packages/api-prerendering-service/src/render/types.ts @@ -1,6 +1,8 @@ -import { Context } from "@webiny/handler/types"; +import { Context as LoggerContext } from "@webiny/api-log/types"; import { Plugin } from "@webiny/plugins/types"; -import { RenderEvent, PrerenderingSettings, Render } from "~/types"; +import { PrerenderingSettings, Render, RenderEvent } from "~/types"; + +export type Context = LoggerContext; export type HandlerPayload = RenderEvent | RenderEvent[]; diff --git a/packages/api-prerendering-service/tsconfig.build.json b/packages/api-prerendering-service/tsconfig.build.json index ee4b305f1a2..043a2ea30f2 100644 --- a/packages/api-prerendering-service/tsconfig.build.json +++ b/packages/api-prerendering-service/tsconfig.build.json @@ -3,6 +3,7 @@ "include": ["src"], "references": [ { "path": "../api/tsconfig.build.json" }, + { "path": "../api-log/tsconfig.build.json" }, { "path": "../aws-sdk/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, { "path": "../handler/tsconfig.build.json" }, diff --git a/packages/api-prerendering-service/tsconfig.json b/packages/api-prerendering-service/tsconfig.json index be3a0af37bd..46ea17f9f40 100644 --- a/packages/api-prerendering-service/tsconfig.json +++ b/packages/api-prerendering-service/tsconfig.json @@ -3,6 +3,7 @@ "include": ["src", "__tests__"], "references": [ { "path": "../api" }, + { "path": "../api-log" }, { "path": "../aws-sdk" }, { "path": "../error" }, { "path": "../handler" }, @@ -20,6 +21,8 @@ "~tests/*": ["./__tests__/*"], "@webiny/api/*": ["../api/src/*"], "@webiny/api": ["../api/src"], + "@webiny/api-log/*": ["../api-log/src/*"], + "@webiny/api-log": ["../api-log/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], "@webiny/aws-sdk": ["../aws-sdk/src"], "@webiny/error/*": ["../error/src/*"], diff --git a/packages/api-serverless-cms/__tests__/handlers/cms/index.ts b/packages/api-serverless-cms/__tests__/handlers/cms/index.ts new file mode 100644 index 00000000000..ed8e6b5cb60 --- /dev/null +++ b/packages/api-serverless-cms/__tests__/handlers/cms/index.ts @@ -0,0 +1,5 @@ +import { createCmsModelPlugins } from "./models"; + +export const createCmsPlugins = () => { + return [...createCmsModelPlugins()]; +}; diff --git a/packages/api-serverless-cms/__tests__/handlers/cms/models.ts b/packages/api-serverless-cms/__tests__/handlers/cms/models.ts new file mode 100644 index 00000000000..8e19e091761 --- /dev/null +++ b/packages/api-serverless-cms/__tests__/handlers/cms/models.ts @@ -0,0 +1,55 @@ +import type { CmsGroupPlugin, CmsModelPlugin } from "@webiny/api-headless-cms"; +import { + createCmsGroupPlugin, + createCmsModelPlugin, + createSingleEntryModel +} from "@webiny/api-headless-cms"; + +export const createCmsModelPlugins = (): (CmsGroupPlugin | CmsModelPlugin)[] => { + const group = createCmsGroupPlugin({ + id: "aTestGroupId", + name: "A Test Group Name", + icon: "icon", + description: "A test description.", + slug: "a-test-group-slug" + }); + return [ + group, + createCmsModelPlugin({ + modelId: "category", + singularApiName: "Category", + pluralApiName: "Categories", + name: "Category", + group: group.contentModelGroup, + fields: [ + { + id: "title", + fieldId: "title", + label: "Title", + type: "text" + } + ], + layout: [["title"]], + description: "A test description.", + titleFieldId: "title" + }), + createSingleEntryModel({ + modelId: "book", + singularApiName: "Book", + pluralApiName: "Books", + name: "Book", + group: group.contentModelGroup, + fields: [ + { + id: "title", + fieldId: "title", + label: "Title", + type: "text" + } + ], + layout: [["title"]], + description: "A test description.", + titleFieldId: "title" + }) + ]; +}; diff --git a/packages/api-serverless-cms/__tests__/handlers/helpers/core.ts b/packages/api-serverless-cms/__tests__/handlers/helpers/core.ts index 46d81960218..6cdcc75575f 100644 --- a/packages/api-serverless-cms/__tests__/handlers/helpers/core.ts +++ b/packages/api-serverless-cms/__tests__/handlers/helpers/core.ts @@ -43,6 +43,8 @@ import pageBuilderImportExportPlugins from "@webiny/api-page-builder-import-expo import { createStorageOperations as createPageBuilderImportExportStorageOperations } from "@webiny/api-page-builder-import-export-so-ddb"; import { Context } from "~/index"; import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { createLogger } from "@webiny/api-log"; +import { createCmsPlugins } from "../cms"; export interface ICreateCoreParams { plugins?: Plugin[]; @@ -149,6 +151,9 @@ export const createCore = (params: ICreateCoreParams): ICreateCoreResult => { ...tenancyStorage.plugins, ...adminUsersStorage.plugins, ...security.plugins, + createLogger({ + documentClient + }), createAdminUsersApp({ storageOperations: adminUsersStorage.storageOperations }), @@ -167,6 +172,7 @@ export const createCore = (params: ICreateCoreParams): ICreateCoreResult => { storageOperations: cmsStorage.storageOperations }), createHeadlessCmsGraphQL(), + ...createCmsPlugins(), createPageBuilderContext({ storageOperations: pageBuilderStorage.storageOperations }), diff --git a/packages/api-serverless-cms/package.json b/packages/api-serverless-cms/package.json index bff76b3298d..27d7f488e6f 100644 --- a/packages/api-serverless-cms/package.json +++ b/packages/api-serverless-cms/package.json @@ -31,6 +31,7 @@ "@webiny/api-audit-logs": "0.0.0", "@webiny/api-headless-cms-aco": "0.0.0", "@webiny/api-headless-cms-tasks": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/api-page-builder-import-export": "0.0.0", "@webiny/api-page-builder-import-export-so-ddb": "0.0.0", "@webiny/api-record-locking": "0.0.0", diff --git a/packages/api-serverless-cms/tsconfig.build.json b/packages/api-serverless-cms/tsconfig.build.json index 710533d52f4..8546fb63978 100644 --- a/packages/api-serverless-cms/tsconfig.build.json +++ b/packages/api-serverless-cms/tsconfig.build.json @@ -22,6 +22,7 @@ { "path": "../api-audit-logs/tsconfig.build.json" }, { "path": "../api-headless-cms-aco/tsconfig.build.json" }, { "path": "../api-headless-cms-tasks/tsconfig.build.json" }, + { "path": "../api-log/tsconfig.build.json" }, { "path": "../api-page-builder-import-export/tsconfig.build.json" }, { "path": "../api-page-builder-import-export-so-ddb/tsconfig.build.json" }, { "path": "../api-record-locking/tsconfig.build.json" }, diff --git a/packages/api-serverless-cms/tsconfig.json b/packages/api-serverless-cms/tsconfig.json index 6cc8b697587..7ef989ac80f 100644 --- a/packages/api-serverless-cms/tsconfig.json +++ b/packages/api-serverless-cms/tsconfig.json @@ -22,6 +22,7 @@ { "path": "../api-audit-logs" }, { "path": "../api-headless-cms-aco" }, { "path": "../api-headless-cms-tasks" }, + { "path": "../api-log" }, { "path": "../api-page-builder-import-export" }, { "path": "../api-page-builder-import-export-so-ddb" }, { "path": "../api-record-locking" }, @@ -79,6 +80,8 @@ "@webiny/api-headless-cms-aco": ["../api-headless-cms-aco/src"], "@webiny/api-headless-cms-tasks/*": ["../api-headless-cms-tasks/src/*"], "@webiny/api-headless-cms-tasks": ["../api-headless-cms-tasks/src"], + "@webiny/api-log/*": ["../api-log/src/*"], + "@webiny/api-log": ["../api-log/src"], "@webiny/api-page-builder-import-export/*": ["../api-page-builder-import-export/src/*"], "@webiny/api-page-builder-import-export": ["../api-page-builder-import-export/src"], "@webiny/api-page-builder-import-export-so-ddb/*": [ diff --git a/packages/cli-plugin-extensions/tsconfig.build.json b/packages/cli-plugin-extensions/tsconfig.build.json index 0c0500749d2..5fd85bfd03d 100644 --- a/packages/cli-plugin-extensions/tsconfig.build.json +++ b/packages/cli-plugin-extensions/tsconfig.build.json @@ -2,27 +2,16 @@ "extends": "../../tsconfig.build.json", "include": ["src"], "references": [ - { - "path": "../plugins/tsconfig.build.json" - }, - { - "path": "../aws-sdk/tsconfig.build.json" - }, - { - "path": "../cli-plugin-scaffold/tsconfig.build.json" - }, - { - "path": "../error/tsconfig.build.json" - } + { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../cli-plugin-scaffold/tsconfig.build.json" }, + { "path": "../error/tsconfig.build.json" }, + { "path": "../plugins/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", "outDir": "./dist", "declarationDir": "./dist", - "paths": { - "~/*": ["./src/*"], - "~tests/*": ["./__tests__/*"] - }, + "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, "baseUrl": "." } } diff --git a/packages/cli-plugin-extensions/tsconfig.json b/packages/cli-plugin-extensions/tsconfig.json index b21ff1a398f..564aad64d1e 100644 --- a/packages/cli-plugin-extensions/tsconfig.json +++ b/packages/cli-plugin-extensions/tsconfig.json @@ -2,18 +2,10 @@ "extends": "../../tsconfig.json", "include": ["src", "__tests__"], "references": [ - { - "path": "../aws-sdk" - }, - { - "path": "../cli-plugin-scaffold" - }, - { - "path": "../error" - }, - { - "path": "../plugins" - } + { "path": "../aws-sdk" }, + { "path": "../cli-plugin-scaffold" }, + { "path": "../error" }, + { "path": "../plugins" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], diff --git a/packages/cli-plugin-scaffold-extensions/tsconfig.build.json b/packages/cli-plugin-scaffold-extensions/tsconfig.build.json index fbaa95c4e0b..d0a312dc64a 100644 --- a/packages/cli-plugin-scaffold-extensions/tsconfig.build.json +++ b/packages/cli-plugin-scaffold-extensions/tsconfig.build.json @@ -2,21 +2,14 @@ "extends": "../../tsconfig.build.json", "include": ["src"], "references": [ - { - "path": "../cli-plugin-extensions/tsconfig.build.json" - }, - { - "path": "../cli-plugin-scaffold/tsconfig.build.json" - } + { "path": "../cli-plugin-extensions/tsconfig.build.json" }, + { "path": "../cli-plugin-scaffold/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", "outDir": "./dist", "declarationDir": "./dist", - "paths": { - "~/*": ["./src/*"], - "~tests/*": ["./__tests__/*"] - }, + "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, "baseUrl": "." } } diff --git a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json index 9293936e261..525154247ab 100644 --- a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json +++ b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json @@ -23,6 +23,7 @@ "@webiny/api-headless-cms-aco": "latest", "@webiny/api-headless-cms-ddb-es": "latest", "@webiny/api-headless-cms-tasks-ddb-es": "latest", + "@webiny/api-log": "latest", "@webiny/api-record-locking": "latest", "@webiny/api-page-builder": "latest", "@webiny/api-page-builder-aco": "latest", diff --git a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts index 93f67110ae8..6c8054f61b7 100644 --- a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts +++ b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts @@ -40,6 +40,7 @@ import { createApwGraphQL, createApwPageBuilderContext } from "@webiny/api-apw"; import { createStorageOperations as createApwSaStorageOperations } from "@webiny/api-apw-scheduler-so-ddb"; import { createWebsockets } from "@webiny/api-websockets"; import { createRecordLocking } from "@webiny/api-record-locking"; +import { createLogger } from "@webiny/api-log"; // Imports plugins created via scaffolding utilities. import scaffoldsPlugins from "./plugins/scaffolds"; @@ -66,6 +67,9 @@ export const handler = createHandler({ driver: new DynamoDbDriver({ documentClient }) }), securityPlugins({ documentClient }), + createLogger({ + documentClient + }), tenantManager(), i18nPlugins(), i18nDynamoDbStorageOperations(), diff --git a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json index 3a8dc68fb07..6cbf29f1ba4 100644 --- a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json +++ b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json @@ -23,6 +23,7 @@ "@webiny/api-headless-cms-aco": "latest", "@webiny/api-headless-cms-ddb-es": "latest", "@webiny/api-headless-cms-tasks-ddb-es": "latest", + "@webiny/api-log": "latest", "@webiny/api-record-locking": "latest", "@webiny/api-page-builder": "latest", "@webiny/api-page-builder-aco": "latest", diff --git a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts index d01c107bdac..b2973364a13 100644 --- a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts +++ b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts @@ -40,6 +40,7 @@ import { createApwGraphQL, createApwPageBuilderContext } from "@webiny/api-apw"; import { createStorageOperations as createApwSaStorageOperations } from "@webiny/api-apw-scheduler-so-ddb"; import { createWebsockets } from "@webiny/api-websockets"; import { createRecordLocking } from "@webiny/api-record-locking"; +import { createLogger } from "@webiny/api-log"; // Imports plugins created via scaffolding utilities. import scaffoldsPlugins from "./plugins/scaffolds"; @@ -66,6 +67,9 @@ export const handler = createHandler({ driver: new DynamoDbDriver({ documentClient }) }), securityPlugins({ documentClient }), + createLogger({ + documentClient + }), tenantManager(), i18nPlugins(), i18nDynamoDbStorageOperations(), diff --git a/packages/cwp-template-aws/template/ddb/apps/api/graphql/package.json b/packages/cwp-template-aws/template/ddb/apps/api/graphql/package.json index c906b6ef7e7..e6a55c33772 100644 --- a/packages/cwp-template-aws/template/ddb/apps/api/graphql/package.json +++ b/packages/cwp-template-aws/template/ddb/apps/api/graphql/package.json @@ -23,6 +23,7 @@ "@webiny/api-headless-cms-aco": "latest", "@webiny/api-headless-cms-ddb": "latest", "@webiny/api-headless-cms-tasks": "latest", + "@webiny/api-log": "latest", "@webiny/api-record-locking": "latest", "@webiny/api-page-builder": "latest", "@webiny/api-page-builder-aco": "latest", diff --git a/packages/cwp-template-aws/template/ddb/apps/api/graphql/src/index.ts b/packages/cwp-template-aws/template/ddb/apps/api/graphql/src/index.ts index ac96cd22ddd..a5ba267cb6b 100644 --- a/packages/cwp-template-aws/template/ddb/apps/api/graphql/src/index.ts +++ b/packages/cwp-template-aws/template/ddb/apps/api/graphql/src/index.ts @@ -36,6 +36,7 @@ import { createApwGraphQL, createApwPageBuilderContext } from "@webiny/api-apw"; import { createStorageOperations as createApwSaStorageOperations } from "@webiny/api-apw-scheduler-so-ddb"; import { createWebsockets } from "@webiny/api-websockets"; import { createRecordLocking } from "@webiny/api-record-locking"; +import { createLogger } from "@webiny/api-log"; // Imports plugins created via scaffolding utilities. import scaffoldsPlugins from "./plugins/scaffolds"; @@ -57,6 +58,9 @@ export const handler = createHandler({ driver: new DynamoDbDriver({ documentClient }) }), securityPlugins({ documentClient }), + createLogger({ + documentClient + }), tenantManager(), i18nPlugins(), i18nDynamoDbStorageOperations(), diff --git a/packages/db-dynamodb/src/utils/batchWrite.ts b/packages/db-dynamodb/src/utils/batchWrite.ts index b65e5afe94a..3bd50ef0e00 100644 --- a/packages/db-dynamodb/src/utils/batchWrite.ts +++ b/packages/db-dynamodb/src/utils/batchWrite.ts @@ -7,7 +7,7 @@ export interface BatchWriteItem { } export interface BatchWriteParams { - table?: TableDef; + table: TableDef | undefined; items: BatchWriteItem[]; } diff --git a/packages/db-dynamodb/src/utils/cleanup.ts b/packages/db-dynamodb/src/utils/cleanup.ts index f2e05fa5d30..2626d424ec5 100644 --- a/packages/db-dynamodb/src/utils/cleanup.ts +++ b/packages/db-dynamodb/src/utils/cleanup.ts @@ -17,12 +17,18 @@ const attributesToRemove = [ "GSI1_SK", "GSI2_PK", "GSI2_SK", + "GSI3_PK", + "GSI3_SK", + "GSI4_PK", + "GSI4_SK", + "GSI5_PK", + "GSI5_SK", "TYPE" ]; export function cleanupItem( entity: Entity, - item?: (T & Record) | null, + item?: T | null, removeAttributes: string[] = [] ): T | null { if (!item) { @@ -47,7 +53,7 @@ export function cleanupItem( export function cleanupItems( entity: Entity, - items: (T & Record)[], + items: T[], removeAttributes: string[] = [] ): T[] { return items.map(item => cleanupItem(entity, item, removeAttributes) as T); diff --git a/packages/db-dynamodb/src/utils/query.ts b/packages/db-dynamodb/src/utils/query.ts index eec01194058..6b6401c259c 100644 --- a/packages/db-dynamodb/src/utils/query.ts +++ b/packages/db-dynamodb/src/utils/query.ts @@ -1,7 +1,7 @@ import WebinyError from "@webiny/error"; -import { Entity } from "~/toolbox"; -import { EntityQueryOptions } from "~/toolbox"; +import { Entity, EntityQueryOptions } from "~/toolbox"; import { cleanupItem, cleanupItems } from "~/utils/cleanup"; +import { GenericRecord } from "@webiny/api/types"; export interface QueryAllParams { entity: Entity; @@ -130,6 +130,36 @@ export const queryAllClean = async (params: QueryAllParams): Promise => return cleanupItems(params.entity, results); }; +export interface IQueryPageResponse { + items: T[]; + lastEvaluatedKey: GenericRecord; +} + +export const queryPerPage = async (params: QueryAllParams): Promise> => { + const result = await query({ + ...params, + options: { + ...params.options, + limit: params.options?.limit || 50 + } + }); + + return { + items: result.items, + lastEvaluatedKey: result.result?.LastEvaluatedKey + }; +}; + +export const queryPerPageClean = async ( + params: QueryAllParams +): Promise> => { + const result = await queryPerPage(params); + return { + items: cleanupItems(params.entity, result.items), + lastEvaluatedKey: result.lastEvaluatedKey + }; +}; + /** * Will run the query to fetch the results no matter how many iterations it needs to go through. * Results of each iteration will be passed to the provided callback diff --git a/packages/project-utils/package.json b/packages/project-utils/package.json index 5ad7b99d973..6b8d380cfb1 100644 --- a/packages/project-utils/package.json +++ b/packages/project-utils/package.json @@ -87,6 +87,7 @@ }, "devDependencies": { "@elastic/elasticsearch": "7.12.0", + "@webiny/api-log": "0.0.0", "jest-dynalite": "^3.6.1", "listr2": "^5.0.8", "load-json-file": "6.2.0", diff --git a/packages/project-utils/testing/elasticsearch/getElasticsearchClient.ts b/packages/project-utils/testing/elasticsearch/getElasticsearchClient.ts index cbed5994f8d..bada294013a 100644 --- a/packages/project-utils/testing/elasticsearch/getElasticsearchClient.ts +++ b/packages/project-utils/testing/elasticsearch/getElasticsearchClient.ts @@ -13,6 +13,7 @@ import { getDocumentClient, simulateStream } from "../dynamodb"; import { PluginCollection } from "../environment"; import { ElasticsearchContext } from "../../../api-elasticsearch/src/types"; import { getElasticsearchIndexPrefix } from "../../../api-elasticsearch/src/indexPrefix"; +import { createLogger } from "@webiny/api-log"; interface GetElasticsearchClientParams { name: string; @@ -79,7 +80,19 @@ export class ElasticsearchClientConfig { }); const dynamoDbHandler = createHandler({ - plugins: [simulationContext, createDynamoDBToElasticsearchEventHandler()] + plugins: [ + simulationContext, + createLogger({ + documentClient, + getTenant: () => { + return "root"; + }, + getLocale: () => { + return "unknown"; + } + }), + createDynamoDBToElasticsearchEventHandler() + ] }); simulateStream(documentClient, dynamoDbHandler); diff --git a/packages/project-utils/testing/mockApiLog.js b/packages/project-utils/testing/mockApiLog.js new file mode 100644 index 00000000000..6ac4862c5de --- /dev/null +++ b/packages/project-utils/testing/mockApiLog.js @@ -0,0 +1,28 @@ +export const createMockApiLog = () => { + const logging = { + notice: () => { + return; + }, + debug: () => { + return; + }, + info: () => { + return; + }, + warn: () => { + return; + }, + error: () => { + return; + } + }; + return { + async flush() { + return []; + }, + withSource: () => { + return logging; + }, + ...logging + }; +}; diff --git a/packages/pulumi-aws/src/apps/api/ApiMigration.ts b/packages/pulumi-aws/src/apps/api/ApiMigration.ts index cb7bd3448b7..344b82038f4 100644 --- a/packages/pulumi-aws/src/apps/api/ApiMigration.ts +++ b/packages/pulumi-aws/src/apps/api/ApiMigration.ts @@ -41,6 +41,7 @@ export const ApiMigration = createAppModule({ COGNITO_REGION: String(process.env.AWS_REGION), COGNITO_USER_POOL_ID: core.cognitoUserPoolId, DB_TABLE: core.primaryDynamodbTableName, + DB_TABLE_LOG: core.logDynamodbTableName, DB_TABLE_ELASTICSEARCH: core.elasticsearchDynamodbTableName, ELASTIC_SEARCH_ENDPOINT: core.elasticsearchDomainEndpoint, ELASTIC_SEARCH_INDEX_PREFIX: process.env.ELASTIC_SEARCH_INDEX_PREFIX, diff --git a/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts b/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts index c9d90f9ee95..63b20930f9b 100644 --- a/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts +++ b/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts @@ -151,6 +151,7 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams = COGNITO_REGION: String(process.env.AWS_REGION), COGNITO_USER_POOL_ID: core.cognitoUserPoolId, DB_TABLE: core.primaryDynamodbTableName, + DB_TABLE_LOG: core.logDynamodbTableName, DB_TABLE_ELASTICSEARCH: core.elasticsearchDynamodbTableName, ELASTIC_SEARCH_ENDPOINT: core.elasticsearchDomainEndpoint, @@ -171,6 +172,7 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams = COGNITO_REGION: String(process.env.AWS_REGION), COGNITO_USER_POOL_ID: core.cognitoUserPoolId, DB_TABLE: core.primaryDynamodbTableName, + DB_TABLE_LOG: core.logDynamodbTableName, S3_BUCKET: core.fileManagerBucketId, WEBINY_LOGS_FORWARD_URL } @@ -181,6 +183,7 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams = COGNITO_REGION: String(process.env.AWS_REGION), COGNITO_USER_POOL_ID: core.cognitoUserPoolId, DB_TABLE: core.primaryDynamodbTableName, + DB_TABLE_LOG: core.logDynamodbTableName, DB_TABLE_ELASTICSEARCH: core.elasticsearchDynamodbTableName, ELASTIC_SEARCH_ENDPOINT: core.elasticsearchDomainEndpoint, @@ -207,7 +210,8 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams = const fileManager = app.addModule(ApiFileManager, { env: { - DB_TABLE: core.primaryDynamodbTableName + DB_TABLE: core.primaryDynamodbTableName, + DB_TABLE_LOG: core.logDynamodbTableName } }); diff --git a/packages/pulumi-aws/src/apps/common/CoreOutput.ts b/packages/pulumi-aws/src/apps/common/CoreOutput.ts index 4175de607be..9fb935334d8 100644 --- a/packages/pulumi-aws/src/apps/common/CoreOutput.ts +++ b/packages/pulumi-aws/src/apps/common/CoreOutput.ts @@ -22,6 +22,10 @@ export const CoreOutput = createAppModule({ primaryDynamodbTableName: output["primaryDynamodbTableName"] as string, primaryDynamodbTableHashKey: output["primaryDynamodbTableHashKey"] as string, primaryDynamodbTableRangeKey: output["primaryDynamodbTableRangeKey"] as string, + logDynamodbTableArn: output["logDynamodbTableArn"] as string, + logDynamodbTableName: output["logDynamodbTableName"] as string, + logDynamodbTableHashKey: output["logDynamodbTableHashKey"] as string, + logDynamodbTableRangeKey: output["logDynamodbTableRangeKey"] as string, cognitoUserPoolId: output["cognitoUserPoolId"] as string, cognitoUserPoolArn: output["cognitoUserPoolArn"] as string, cognitoUserPoolPasswordPolicy: output["cognitoUserPoolPasswordPolicy"] as string, diff --git a/packages/pulumi-aws/src/apps/core/LogDynamo.ts b/packages/pulumi-aws/src/apps/core/LogDynamo.ts new file mode 100644 index 00000000000..a25049c89d5 --- /dev/null +++ b/packages/pulumi-aws/src/apps/core/LogDynamo.ts @@ -0,0 +1,67 @@ +import * as aws from "@pulumi/aws"; +import { createAppModule, PulumiApp, PulumiAppModule } from "@webiny/pulumi"; + +export type LogDynamo = PulumiAppModule; + +export const LogDynamo = createAppModule({ + name: "DynamoDbLog", + config(app: PulumiApp, params: { protect: boolean }) { + return app.addResource(aws.dynamodb.Table, { + name: "webiny-log", + config: { + attributes: [ + { name: "PK", type: "S" }, + { name: "SK", type: "S" }, + { name: "GSI1_PK", type: "S" }, + { name: "GSI1_SK", type: "S" }, + { name: "GSI2_PK", type: "S" }, + { name: "GSI2_SK", type: "S" }, + { name: "GSI3_PK", type: "S" }, + { name: "GSI3_SK", type: "S" }, + { name: "GSI4_PK", type: "S" }, + { name: "GSI4_SK", type: "S" }, + { name: "GSI5_PK", type: "S" }, + { name: "GSI5_SK", type: "S" } + ], + billingMode: "PAY_PER_REQUEST", + hashKey: "PK", + rangeKey: "SK", + globalSecondaryIndexes: [ + { + name: "GSI1", + hashKey: "GSI1_PK", + rangeKey: "GSI1_SK", + projectionType: "ALL" + }, + { + name: "GSI2", + hashKey: "GSI2_PK", + rangeKey: "GSI2_SK", + projectionType: "ALL" + }, + { + name: "GSI3", + hashKey: "GSI3_PK", + rangeKey: "GSI3_SK", + projectionType: "ALL" + }, + { + name: "GSI4", + hashKey: "GSI4_PK", + rangeKey: "GSI4_SK", + projectionType: "ALL" + }, + { + name: "GSI5", + hashKey: "GSI5_PK", + rangeKey: "GSI5_SK", + projectionType: "ALL" + } + ] + }, + opts: { + protect: params.protect + } + }); + } +}); diff --git a/packages/pulumi-aws/src/apps/core/createCorePulumiApp.ts b/packages/pulumi-aws/src/apps/core/createCorePulumiApp.ts index 270f03fae4d..40a2072f192 100644 --- a/packages/pulumi-aws/src/apps/core/createCorePulumiApp.ts +++ b/packages/pulumi-aws/src/apps/core/createCorePulumiApp.ts @@ -14,6 +14,7 @@ import { addServiceManifestTableItem, TableDefinition } from "~/utils/addService import { DEFAULT_PROD_ENV_NAMES } from "~/constants"; import * as random from "@pulumi/random"; import { featureFlags } from "@webiny/feature-flags"; +import { LogDynamo } from "./LogDynamo"; export type CorePulumiApp = ReturnType; @@ -150,6 +151,7 @@ export function createCorePulumiApp(projectAppParams: CreateCorePulumiAppParams // Setup DynamoDB table const dynamoDbTable = app.addModule(CoreDynamo, { protect }); + const logDynamoDbTable = app.addModule(LogDynamo, { protect }); // Setup VPC const vpcEnabled = app.getParam(projectAppParams?.vpc) ?? isProduction; @@ -188,6 +190,10 @@ export function createCorePulumiApp(projectAppParams: CreateCorePulumiAppParams primaryDynamodbTableName: dynamoDbTable.output.name, primaryDynamodbTableHashKey: dynamoDbTable.output.hashKey, primaryDynamodbTableRangeKey: dynamoDbTable.output.rangeKey, + logDynamodbTableArn: logDynamoDbTable.output.arn, + logDynamodbTableName: logDynamoDbTable.output.name, + logDynamodbTableHashKey: logDynamoDbTable.output.hashKey, + logDynamodbTableRangeKey: logDynamoDbTable.output.rangeKey, cognitoUserPoolId: cognito.userPool.output.id, cognitoUserPoolArn: cognito.userPool.output.arn, cognitoUserPoolPasswordPolicy: cognito.userPool.output.passwordPolicy, @@ -203,6 +209,7 @@ export function createCorePulumiApp(projectAppParams: CreateCorePulumiAppParams return { dynamoDbTable, + logDynamoDbTable, vpc, ...cognito, fileManagerBucket, diff --git a/packages/pulumi-aws/src/apps/website/WebsitePrerendering.ts b/packages/pulumi-aws/src/apps/website/WebsitePrerendering.ts index ac93fba2ab6..35a237ef25b 100644 --- a/packages/pulumi-aws/src/apps/website/WebsitePrerendering.ts +++ b/packages/pulumi-aws/src/apps/website/WebsitePrerendering.ts @@ -15,6 +15,7 @@ interface PreRenderingServiceParams { dbTableName: pulumi.Output; dbTableHashKey: pulumi.Output; dbTableRangeKey: pulumi.Output; + logDbTableName: pulumi.Output; appUrl: pulumi.Output; deliveryUrl: pulumi.Output; bucket: pulumi.Output; @@ -104,7 +105,8 @@ function createRenderSubscriber( environment: { variables: getCommonLambdaEnvVariables().apply(value => ({ ...value, - DB_TABLE: params.dbTableName + DB_TABLE: params.dbTableName, + DB_TABLE_LOG: params.logDbTableName })) }, description: "Subscribes to render events on event bus", diff --git a/packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts b/packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts index 103147398a9..43ac60dd9f8 100644 --- a/packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts +++ b/packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts @@ -228,6 +228,7 @@ export const createWebsitePulumiApp = (projectAppParams: CreateWebsitePulumiAppP dbTableName: core.primaryDynamodbTableName, dbTableHashKey: core.primaryDynamodbTableHashKey, dbTableRangeKey: core.primaryDynamodbTableRangeKey, + logDbTableName: core.logDynamodbTableName, appUrl: pulumi.interpolate`https://${appCloudfront.output.domainName}`, deliveryUrl: pulumi.interpolate`https://${deliveryCloudfront.output.domainName}`, bucket: deliveryBucket.bucket.output.bucket, diff --git a/packages/serverless-cms-aws/handlers/common/website/prerendering/flush/src/index.ts b/packages/serverless-cms-aws/handlers/common/website/prerendering/flush/src/index.ts index bd3cf8193c1..c87c2cf64eb 100644 --- a/packages/serverless-cms-aws/handlers/common/website/prerendering/flush/src/index.ts +++ b/packages/serverless-cms-aws/handlers/common/website/prerendering/flush/src/index.ts @@ -8,6 +8,9 @@ const documentClient = getDocumentClient(); export const handler = createHandler({ plugins: [ + createLogger({ + documentClient + }), flushPlugins({ storage: { name: String(process.env.DELIVERY_BUCKET) diff --git a/packages/serverless-cms-aws/handlers/common/website/prerendering/render/src/index.ts b/packages/serverless-cms-aws/handlers/common/website/prerendering/render/src/index.ts index 3365e57321e..9663898308e 100644 --- a/packages/serverless-cms-aws/handlers/common/website/prerendering/render/src/index.ts +++ b/packages/serverless-cms-aws/handlers/common/website/prerendering/render/src/index.ts @@ -3,11 +3,15 @@ import { createHandler } from "@webiny/handler-aws"; import renderPlugins from "@webiny/api-prerendering-service-aws/render/sqsRender"; import renderAwsPlugins from "@webiny/api-prerendering-service-aws/render"; import { createPrerenderingServiceStorageOperations } from "@webiny/api-prerendering-service-so-ddb"; +import { createLogger } from "@webiny/api-log"; const documentClient = getDocumentClient(); export const handler = createHandler({ plugins: [ + createLogger({ + documentClient + }), renderPlugins({ storage: { name: String(process.env.DELIVERY_BUCKET) diff --git a/packages/serverless-cms-aws/handlers/ddb-es/core/dynamoToElastic/src/index.ts b/packages/serverless-cms-aws/handlers/ddb-es/core/dynamoToElastic/src/index.ts index a019d3034ea..4f18abb286c 100644 --- a/packages/serverless-cms-aws/handlers/ddb-es/core/dynamoToElastic/src/index.ts +++ b/packages/serverless-cms-aws/handlers/ddb-es/core/dynamoToElastic/src/index.ts @@ -1,9 +1,22 @@ import { createHandler } from "@webiny/handler-aws"; import elasticsearchClientContextPlugin, { createGzipCompression } from "@webiny/api-elasticsearch"; import { createEventHandler } from "@webiny/api-dynamodb-to-elasticsearch"; +import { getDocumentClient } from "@webiny/aws-sdk/client-dynamodb"; +import { createLogger } from "@webiny/api-log"; + +const documentClient = getDocumentClient(); export const handler = createHandler({ plugins: [ + createLogger({ + documentClient, + getTenant: () => { + return "root"; + }, + getLocale: () => { + return "unknown"; + } + }), elasticsearchClientContextPlugin({ endpoint: `https://${process.env.ELASTIC_SEARCH_ENDPOINT}` }), diff --git a/packages/serverless-cms-aws/package.json b/packages/serverless-cms-aws/package.json index 2c491f5eadb..c55a0146700 100644 --- a/packages/serverless-cms-aws/package.json +++ b/packages/serverless-cms-aws/package.json @@ -32,6 +32,7 @@ "@webiny/api-i18n": "0.0.0", "@webiny/api-i18n-content": "0.0.0", "@webiny/api-i18n-ddb": "0.0.0", + "@webiny/api-log": "0.0.0", "@webiny/api-page-builder": "0.0.0", "@webiny/api-page-builder-aco": "0.0.0", "@webiny/api-page-builder-import-export": "0.0.0", diff --git a/packages/serverless-cms-aws/tsconfig.build.json b/packages/serverless-cms-aws/tsconfig.build.json index 81e05620b36..ecfe9b79703 100644 --- a/packages/serverless-cms-aws/tsconfig.build.json +++ b/packages/serverless-cms-aws/tsconfig.build.json @@ -19,6 +19,7 @@ { "path": "../api-i18n/tsconfig.build.json" }, { "path": "../api-i18n-content/tsconfig.build.json" }, { "path": "../api-i18n-ddb/tsconfig.build.json" }, + { "path": "../api-log/tsconfig.build.json" }, { "path": "../api-page-builder/tsconfig.build.json" }, { "path": "../api-page-builder-aco/tsconfig.build.json" }, { "path": "../api-page-builder-import-export/tsconfig.build.json" }, diff --git a/packages/serverless-cms-aws/tsconfig.json b/packages/serverless-cms-aws/tsconfig.json index c6f8e607e44..567bf6c042f 100644 --- a/packages/serverless-cms-aws/tsconfig.json +++ b/packages/serverless-cms-aws/tsconfig.json @@ -19,6 +19,7 @@ { "path": "../api-i18n" }, { "path": "../api-i18n-content" }, { "path": "../api-i18n-ddb" }, + { "path": "../api-log" }, { "path": "../api-page-builder" }, { "path": "../api-page-builder-aco" }, { "path": "../api-page-builder-import-export" }, @@ -84,6 +85,8 @@ "@webiny/api-i18n-content": ["../api-i18n-content/src"], "@webiny/api-i18n-ddb/*": ["../api-i18n-ddb/src/*"], "@webiny/api-i18n-ddb": ["../api-i18n-ddb/src"], + "@webiny/api-log/*": ["../api-log/src/*"], + "@webiny/api-log": ["../api-log/src"], "@webiny/api-page-builder/*": ["../api-page-builder/src/*"], "@webiny/api-page-builder": ["../api-page-builder/src"], "@webiny/api-page-builder-aco/*": ["../api-page-builder-aco/src/*"], diff --git a/typings/env/index.d.ts b/typings/env/index.d.ts index d01c49c85a4..e1218c7fc9e 100644 --- a/typings/env/index.d.ts +++ b/typings/env/index.d.ts @@ -10,6 +10,7 @@ declare namespace NodeJS { DB_TABLE_HEADLESS_CMS?: string; DB_PAGE_BUILDER?: string; DB_TABLE_PAGE_BUILDER?: string; + DB_TABLE_LOG?: string; ELASTICSEARCH_SHARED_INDEXES?: "true" | "false" | string; WEBINY_VERSION?: string; WEBINY_IS_PRE_529?: "true" | "false"; diff --git a/yarn.lock b/yarn.lock index 03801df5709..8da08f99a6e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12237,6 +12237,7 @@ __metadata: dependencies: "@types/aws-lambda": ^8.10.131 "@webiny/api-elasticsearch": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 "@webiny/error": 0.0.0 @@ -12800,6 +12801,32 @@ __metadata: languageName: unknown linkType: soft +"@webiny/api-log@0.0.0, @webiny/api-log@workspace:packages/api-log": + version: 0.0.0-use.local + resolution: "@webiny/api-log@workspace:packages/api-log" + dependencies: + "@webiny/api": 0.0.0 + "@webiny/api-i18n": 0.0.0 + "@webiny/api-security": 0.0.0 + "@webiny/api-tenancy": 0.0.0 + "@webiny/aws-sdk": 0.0.0 + "@webiny/cli": 0.0.0 + "@webiny/db-dynamodb": 0.0.0 + "@webiny/handler": 0.0.0 + "@webiny/handler-graphql": 0.0.0 + "@webiny/plugins": 0.0.0 + "@webiny/project-utils": 0.0.0 + "@webiny/tasks": 0.0.0 + "@webiny/utils": 0.0.0 + jest: ^29.7.0 + jest-dynalite: ^3.6.1 + rimraf: ^5.0.5 + ttypescript: ^1.5.12 + typescript: 4.9.5 + zod: ^3.22.4 + languageName: unknown + linkType: soft + "@webiny/api-mailer@0.0.0, @webiny/api-mailer@workspace:packages/api-mailer": version: 0.0.0-use.local resolution: "@webiny/api-mailer@workspace:packages/api-mailer" @@ -13054,6 +13081,7 @@ __metadata: resolution: "@webiny/api-prerendering-service-aws@workspace:packages/api-prerendering-service-aws" dependencies: "@webiny/api": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/api-prerendering-service": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 @@ -13096,6 +13124,7 @@ __metadata: "@types/object-hash": ^2.2.1 "@types/puppeteer-core": ^5.4.0 "@webiny/api": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 "@webiny/error": 0.0.0 @@ -13294,6 +13323,7 @@ __metadata: "@webiny/api-headless-cms-tasks": 0.0.0 "@webiny/api-i18n": 0.0.0 "@webiny/api-i18n-content": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/api-page-builder": 0.0.0 "@webiny/api-page-builder-aco": 0.0.0 "@webiny/api-page-builder-import-export": 0.0.0 @@ -15657,6 +15687,7 @@ __metadata: "@pmmmwh/react-refresh-webpack-plugin": ^0.5.3 "@svgr/webpack": ^6.1.1 "@types/webpack-env": 1.16.3 + "@webiny/api-log": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 "@webiny/global-config": 0.0.0 @@ -15908,6 +15939,7 @@ __metadata: "@webiny/api-i18n": 0.0.0 "@webiny/api-i18n-content": 0.0.0 "@webiny/api-i18n-ddb": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/api-page-builder": 0.0.0 "@webiny/api-page-builder-aco": 0.0.0 "@webiny/api-page-builder-import-export": 0.0.0 @@ -16850,6 +16882,7 @@ __metadata: "@webiny/api-i18n": 0.0.0 "@webiny/api-i18n-content": 0.0.0 "@webiny/api-i18n-ddb": 0.0.0 + "@webiny/api-log": 0.0.0 "@webiny/api-page-builder": 0.0.0 "@webiny/api-page-builder-aco": 0.0.0 "@webiny/api-page-builder-import-export": 0.0.0 @@ -39140,7 +39173,7 @@ __metadata: languageName: node linkType: hard -"zod@npm:^3.23.8": +"zod@npm:^3.22.4, zod@npm:^3.23.8": version: 3.23.8 resolution: "zod@npm:3.23.8" checksum: 15949ff82118f59c893dacd9d3c766d02b6fa2e71cf474d5aa888570c469dbf5446ac5ad562bb035bf7ac9650da94f290655c194f4a6de3e766f43febd432c5c