feat: api log (#4366)

.github/workflows/pullRequests.yml

@@ -294,7 +294,8 @@ jobs:
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n
-          --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer
+          --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log
+          --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer
           --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder
           --storage=ddb
           --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder

.github/workflows/pushDev.yml

@@ -208,7 +208,8 @@ jobs:
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n
-          --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer
+          --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log
+          --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer
           --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder
           --storage=ddb
           --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder

.github/workflows/pushNext.yml

@@ -208,7 +208,8 @@ jobs:
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export
           --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n
-          --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer
+          --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log
+          --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer
           --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder
           --storage=ddb
           --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder

.github/workflows/wac/utils/listPackagesWithJestTests.ts

@@ -25,7 +25,7 @@ interface PackageWithTestsWithId extends PackageWithTests {
 // Takes a PackageWithTests object and returns an array of commands, where each
 // command is just running a subset of tests. This is achieved by using the
 // Jest's `--shard` option.
-const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount: number = 6) => {
+const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount = 6) => {
     const commands: PackageWithTests[] = [];
     for (let currentShard = 1; currentShard <= shardsCount; currentShard++) {
         commands.push({ ...pkg, cmd: pkg.cmd + ` --shard=${currentShard}/${shardsCount}` });
@@ -61,6 +61,10 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
         return [{ cmd: "packages/api-tenant-manager --storage=ddb", storage: "ddb" }];
     },
 
+    "api-log": () => {
+        return [{ cmd: "packages/api-log --storage=ddb", storage: "ddb" }];
+    },
+
     "api-file-manager": () => {
         return [
             { cmd: "packages/api-file-manager --storage=ddb", storage: "ddb" },

apps/api/graphql/package.json

@@ -25,6 +25,7 @@
     "@webiny/api-i18n": "0.0.0",
     "@webiny/api-i18n-content": "0.0.0",
     "@webiny/api-i18n-ddb": "0.0.0",
+    "@webiny/api-log": "0.0.0",
     "@webiny/api-page-builder": "0.0.0",
     "@webiny/api-page-builder-aco": "0.0.0",
     "@webiny/api-page-builder-import-export": "0.0.0",

apps/api/graphql/src/index.ts

@@ -43,6 +43,7 @@ import { createCountDynamoDbTask } from "~/plugins/countDynamoDbTask";
 import { createContinuingTask } from "~/plugins/continuingTask";
 import { createWebsockets } from "@webiny/api-websockets";
 import { createRecordLocking } from "@webiny/api-record-locking";
+import { createLogger } from "@webiny/api-log";
 
 import scaffoldsPlugins from "./plugins/scaffolds";
 import { extensions } from "./extensions";
@@ -63,6 +64,9 @@ export const handler = createHandler({
             driver: new DynamoDbDriver({ documentClient })
         }),
         securityPlugins({ documentClient }),
+        createLogger({
+            documentClient
+        }),
         tenantManager(),
         i18nPlugins(),
         i18nDynamoDbStorageOperations(),

apps/api/graphql/tsconfig.json

@@ -127,6 +127,9 @@
     },
     {
       "path": "../../../packages/tasks/tsconfig.build.json"
+    },
+    {
+      "path": "../../../packages/api-log/tsconfig.build.json"
     }
   ],
   "compilerOptions": {
@@ -215,7 +218,9 @@
       "@webiny/tasks/*": ["../../../packages/tasks/src/*"],
       "@webiny/tasks": ["../../../packages/tasks/src"],
       "@webiny/api-websockets/*": ["../../../packages/api-websockets/src/*"],
-      "@webiny/api-websockets": ["../../../packages/api-websockets/src"]
+      "@webiny/api-websockets": ["../../../packages/api-websockets/src"],
+      "@webiny/api-log/*": ["../../../packages/api-log/src/*"],
+      "@webiny/api-log": ["../../../packages/api-log/src"]
     },
     "baseUrl": "."
   }

jest.config.base.js

@@ -91,9 +91,45 @@ module.exports = function ({ path }, presets = []) {
 
 process.env.DB_TABLE = "DynamoDB";
 process.env.DB_TABLE_ELASTICSEARCH = "ElasticsearchStream";
+process.env.DB_TABLE_LOG = "DynamoDBLog";
 process.env.WEBINY_VERSION = version;
 process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
 
+const createGlobalSecondaryIndexesAttributeDefinitions = amount => {
+    const attributes = [];
+
+    for (let current = 1; current <= amount; current++) {
+        attributes.push({ AttributeName: `GSI${current}_PK`, AttributeType: "S" });
+        attributes.push({ AttributeName: `GSI${current}_SK`, AttributeType: "S" });
+    }
+    return attributes;
+};
+
+const createGlobalSecondaryIndexes = options => {
+    if (!options.amount) {
+        return [];
+    }
+    const indexes = [];
+    for (let current = 1; current <= options.amount; current++) {
+        indexes.push({
+            IndexName: `GSI${current}`,
+            KeySchema: [
+                { AttributeName: `GSI${current}_PK`, KeyType: "HASH" },
+                { AttributeName: `GSI${current}_SK`, KeyType: "RANGE" }
+            ],
+            Projection: {
+                ProjectionType: "ALL"
+            },
+            ProvisionedThroughput: {
+                ReadCapacityUnits: 1,
+                WriteCapacityUnits: 1
+            }
+        });
+    }
+
+    return indexes;
+};
+
 const createDynaliteTables = (options = {}) => {
     return {
         tables: [
@@ -106,42 +142,12 @@ const createDynaliteTables = (options = {}) => {
                 AttributeDefinitions: [
                     { AttributeName: "PK", AttributeType: "S" },
                     { AttributeName: "SK", AttributeType: "S" },
-                    { AttributeName: "GSI1_PK", AttributeType: "S" },
-                    { AttributeName: "GSI1_SK", AttributeType: "S" },
-                    { AttributeName: "GSI2_PK", AttributeType: "S" },
-                    { AttributeName: "GSI2_SK", AttributeType: "S" }
+                    ...createGlobalSecondaryIndexesAttributeDefinitions(2)
                 ],
                 ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 },
-                GlobalSecondaryIndexes: [
-                    {
-                        IndexName: "GSI1",
-                        KeySchema: [
-                            { AttributeName: "GSI1_PK", KeyType: "HASH" },
-                            { AttributeName: "GSI1_SK", KeyType: "RANGE" }
-                        ],
-                        Projection: {
-                            ProjectionType: "ALL"
-                        },
-                        ProvisionedThroughput: {
-                            ReadCapacityUnits: 1,
-                            WriteCapacityUnits: 1
-                        }
-                    },
-                    {
-                        IndexName: "GSI2",
-                        KeySchema: [
-                            { AttributeName: "GSI2_PK", KeyType: "HASH" },
-                            { AttributeName: "GSI2_SK", KeyType: "RANGE" }
-                        ],
-                        Projection: {
-                            ProjectionType: "ALL"
-                        },
-                        ProvisionedThroughput: {
-                            ReadCapacityUnits: 1,
-                            WriteCapacityUnits: 1
-                        }
-                    }
-                ],
+                GlobalSecondaryIndexes: createGlobalSecondaryIndexes({
+                    amount: 2
+                }),
                 data: options.data || []
             },
             {
@@ -155,6 +161,23 @@ const createDynaliteTables = (options = {}) => {
                     { AttributeName: "SK", AttributeType: "S" }
                 ],
                 ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 }
+            },
+            {
+                TableName: process.env.DB_TABLE_LOG,
+                KeySchema: [
+                    { AttributeName: "PK", KeyType: "HASH" },
+                    { AttributeName: "SK", KeyType: "RANGE" }
+                ],
+                AttributeDefinitions: [
+                    { AttributeName: "PK", AttributeType: "S" },
+                    { AttributeName: "SK", AttributeType: "S" },
+                    ...createGlobalSecondaryIndexesAttributeDefinitions(5)
+                ],
+                ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 },
+                GlobalSecondaryIndexes: createGlobalSecondaryIndexes({
+                    amount: 5
+                }),
+                data: options.data || []
             }
         ],
         basePort: 8000

packages/api-dynamodb-to-elasticsearch/__tests__/transfer.test.ts

@@ -1,5 +1,7 @@
 import { createEventHandler, OperationType } from "~/index";
 import { createElasticsearchClient } from "@webiny/project-utils/testing/elasticsearch/createClient";
+// @ts-expect-error
+import { createMockApiLog } from "@webiny/project-utils/testing/mockApiLog";
 import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
 import { Context, LambdaContext, Reply, Request } from "@webiny/handler-aws/types";
 import { marshall } from "@webiny/aws-sdk/client-dynamodb";
@@ -13,6 +15,7 @@ describe("transfer data", () => {
 
         const context = {
             elasticsearch,
+            logger: createMockApiLog(),
             plugins: new PluginsContainer()
         } as unknown as ElasticsearchContext & Context;
         /**

packages/api-dynamodb-to-elasticsearch/package.json

@@ -12,6 +12,7 @@
   "author": "Webiny Ltd.",
   "dependencies": {
     "@webiny/api-elasticsearch": "0.0.0",
+    "@webiny/api-log": "0.0.0",
     "@webiny/aws-sdk": "0.0.0",
     "@webiny/error": "0.0.0",
     "@webiny/handler-aws": "0.0.0",

packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts

@@ -1,9 +1,9 @@
 import { getNumberEnvVariable } from "~/helpers/getNumberEnvVariable";
 import { createDynamoDBEventHandler, timerFactory } from "@webiny/handler-aws";
-import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
 import { Decompressor } from "~/Decompressor";
 import { OperationsBuilder } from "~/OperationsBuilder";
 import { executeWithRetry } from "~/executeWithRetry";
+import { Context } from "~/types";
 
 const MAX_PROCESSOR_PERCENT = getNumberEnvVariable(
     "MAX_ES_PROCESSOR",
@@ -20,7 +20,7 @@ const MAX_RUNNING_TIME = 900;
 export const createEventHandler = () => {
     return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {
         const timer = timerFactory(lambdaContext);
-        const context = ctx as unknown as ElasticsearchContext;
+        const context = ctx as unknown as Context;
         if (!context.elasticsearch) {
             console.error("Missing elasticsearch definition on context.");
             return null;

packages/api-dynamodb-to-elasticsearch/src/execute.ts

@@ -5,9 +5,9 @@ import {
     WaitingHealthyClusterAbortedError
 } from "@webiny/api-elasticsearch";
 import { ITimer } from "@webiny/handler-aws";
-import { ApiResponse, ElasticsearchContext } from "@webiny/api-elasticsearch/types";
+import { ApiResponse } from "@webiny/api-elasticsearch/types";
 import { WebinyError } from "@webiny/error";
-import { IOperations } from "./types";
+import { Context, IOperations } from "./types";
 
 export interface BulkOperationsResponseBodyItemIndexError {
     reason?: string;
@@ -30,7 +30,7 @@ export interface IExecuteParams {
     timer: ITimer;
     maxRunningTime: number;
     maxProcessorPercent: number;
-    context: Pick<ElasticsearchContext, "elasticsearch">;
+    context: Context;
     operations: IOperations;
 }
 
@@ -84,6 +84,8 @@ export const execute = (params: IExecuteParams) => {
             maxWaitingTime
         });
 
+        const log = context.logger.withSource("dynamodbToElasticsearch");
+
         try {
             await healthCheck.wait({
                 async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {
@@ -120,6 +122,11 @@ export const execute = (params: IExecuteParams) => {
             });
             checkErrors(res);
         } catch (error) {
+            log.error(error, {
+                tenant: "root",
+                locale: "unknown"
+            });
+
             if (process.env.DEBUG !== "true") {
                 throw error;
             }

packages/api-dynamodb-to-elasticsearch/src/types.ts

@@ -1,5 +1,7 @@
 import { GenericRecord } from "@webiny/cli/types";
 import { DynamoDBRecord } from "@webiny/handler-aws/types";
+import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
+import { Context as LoggerContext } from "@webiny/api-log/types";
 
 export interface IOperationsBuilderBuildParams {
     records: DynamoDBRecord[];
@@ -33,3 +35,5 @@ export interface IOperations {
 export interface IDecompressor {
     decompress(data: GenericRecord): Promise<GenericRecord | null>;
 }
+
+export interface Context extends ElasticsearchContext, Pick<LoggerContext, "logger"> {}

packages/api-dynamodb-to-elasticsearch/tsconfig.build.json

@@ -3,6 +3,7 @@
   "include": ["src"],
   "references": [
     { "path": "../api-elasticsearch/tsconfig.build.json" },
+    { "path": "../api-log/tsconfig.build.json" },
     { "path": "../aws-sdk/tsconfig.build.json" },
     { "path": "../error/tsconfig.build.json" },
     { "path": "../handler-aws/tsconfig.build.json" },

packages/api-dynamodb-to-elasticsearch/tsconfig.json

@@ -3,6 +3,7 @@
   "include": ["src", "__tests__"],
   "references": [
     { "path": "../api-elasticsearch" },
+    { "path": "../api-log" },
     { "path": "../aws-sdk" },
     { "path": "../error" },
     { "path": "../handler-aws" },
@@ -17,6 +18,8 @@
       "~tests/*": ["./__tests__/*"],
       "@webiny/api-elasticsearch/*": ["../api-elasticsearch/src/*"],
       "@webiny/api-elasticsearch": ["../api-elasticsearch/src"],
+      "@webiny/api-log/*": ["../api-log/src/*"],
+      "@webiny/api-log": ["../api-log/src"],
       "@webiny/aws-sdk/*": ["../aws-sdk/src/*"],
       "@webiny/aws-sdk": ["../aws-sdk/src"],
       "@webiny/error/*": ["../error/src/*"],

packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts

@@ -2,6 +2,7 @@ import { ContextPlugin } from "@webiny/api";
 import { HcmsBulkActionsContext } from "~/types";
 import { CmsGraphQLSchemaPlugin, isHeadlessCmsReady } from "@webiny/api-headless-cms";
 import { Response } from "@webiny/handler-graphql";
+import { CMS_MODEL_SINGLETON_TAG } from "@webiny/api-headless-cms/constants";
 
 export interface CreateBulkActionGraphQL {
     name: string;
@@ -16,11 +17,19 @@ export const createBulkActionGraphQL = (config: CreateBulkActionGraphQL) => {
 
         const models = await context.security.withoutAuthorization(async () => {
             const allModels = await context.cms.listModels();
-            return allModels.filter(
-                model =>
-                    !model.isPrivate &&
-                    (!config.modelIds?.length || config.modelIds.includes(model.modelId))
-            );
+            return allModels.filter(model => {
+                if (model.isPrivate) {
+                    return false;
+                }
+                const tags = Array.isArray(model.tags) ? model.tags : [];
+                if (tags.includes(CMS_MODEL_SINGLETON_TAG)) {
+                    return false;
+                }
+                if (config.modelIds?.length) {
+                    return config.modelIds.includes(model.modelId);
+                }
+                return true;
+            });
         });
 
         const plugins: CmsGraphQLSchemaPlugin<HcmsBulkActionsContext>[] = [];

packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts

@@ -1,6 +1,7 @@
 import { ContextPlugin } from "@webiny/api";
 import { HcmsBulkActionsContext } from "~/types";
 import { CmsGraphQLSchemaPlugin, isHeadlessCmsReady } from "@webiny/api-headless-cms";
+import { CMS_MODEL_SINGLETON_TAG } from "@webiny/api-headless-cms/constants";
 
 export const createDefaultGraphQL = () => {
     return new ContextPlugin<HcmsBulkActionsContext>(async context => {
@@ -24,7 +25,16 @@ export const createDefaultGraphQL = () => {
 
         const models = await context.security.withoutAuthorization(async () => {
             const allModels = await context.cms.listModels();
-            return allModels.filter(model => !model.isPrivate);
+            return allModels.filter(model => {
+                if (model.isPrivate) {
+                    return false;
+                }
+                const tags = Array.isArray(model.tags) ? model.tags : [];
+                if (tags.includes(CMS_MODEL_SINGLETON_TAG)) {
+                    return false;
+                }
+                return true;
+            });
         });
 
         const modelPlugins: CmsGraphQLSchemaPlugin<HcmsBulkActionsContext>[] = [];