diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index ea3da24e20a2..c5c1761de8f2 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -13142,30 +13142,6 @@ components: CreateTableRequest: description: Request body for creating a new reference table from a local file or cloud storage. - example: - data: - attributes: - description: this is a cloud table generated via a cloud bucket sync - file_metadata: - access_details: - aws_detail: - aws_account_id: test-account-id - aws_bucket_name: test-bucket - file_path: test_rt.csv - sync_enabled: true - schema: - fields: - - name: name - type: STRING - - name: account_id - type: STRING - primary_keys: - - account_id - source: S3 - table_name: test_reference_table - tags: - - test_tag - type: reference_table properties: data: $ref: '#/components/schemas/CreateTableRequestData' @@ -13387,6 +13363,17 @@ components: CreateUploadRequest: description: Request to create an upload for a file to be ingested into a reference table. + example: + data: + attributes: + headers: + - product_id + - product_name + - price + part_count: 3 + part_size: 10000000 + table_name: my_products_table + type: upload properties: data: $ref: '#/components/schemas/CreateUploadRequestData' @@ -51089,32 +51076,35 @@ components: type: string type: object TableResultV2DataAttributesFileMetadata: - description: Metadata specifying where and how to access the reference table's - data file. - oneOf: - - $ref: '#/components/schemas/TableResultV2DataAttributesFileMetadataCloudStorage' - - $ref: '#/components/schemas/TableResultV2DataAttributesFileMetadataLocalFile' - TableResultV2DataAttributesFileMetadataCloudStorage: additionalProperties: false - description: File metadata for reference tables created by cloud storage. + description: "Metadata specifying where and how to access the reference table's + data file.\n\nFor cloud storage tables (S3/GCS/Azure):\n - sync_enabled and + access_details will always be present\n - error fields (error_message, error_row_count, + error_type) are present only when errors occur\n\nFor local file tables:\n + \ - error fields (error_message, error_row_count) are present only when errors + occur\n - sync_enabled, access_details are never present" properties: access_details: $ref: '#/components/schemas/TableResultV2DataAttributesFileMetadataOneOfAccessDetails' + description: Cloud storage access configuration. Only present for cloud + storage sources (S3, GCS, Azure). error_message: - description: The error message returned from the sync. + description: The error message returned from the last operation (sync for + cloud storage, upload for local file). type: string error_row_count: - description: The number of rows that failed to sync. + description: The number of rows that failed to process. format: int64 type: integer error_type: $ref: '#/components/schemas/TableResultV2DataAttributesFileMetadataCloudStorageErrorType' + description: The type of error that occurred during file processing. Only + applicable for cloud storage sources. sync_enabled: - description: Whether this table is synced automatically. + description: Whether this table is synced automatically from cloud storage. + Only applicable for cloud storage sources. type: boolean - required: - - access_details - title: CloudFileMetadataV2 + title: FileMetadataV2 type: object TableResultV2DataAttributesFileMetadataCloudStorageErrorType: description: The type of error that occurred during file processing. This field @@ -51141,21 +51131,6 @@ components: - STATE_ERROR - OPERATION_ERROR - SYSTEM_ERROR - TableResultV2DataAttributesFileMetadataLocalFile: - additionalProperties: false - description: File metadata for reference tables created by upload. Note that - upload_id is only returned in the immediate create/replace response and is - not available in subsequent GET requests. - properties: - error_message: - description: The error message returned from the creation/update. - type: string - error_row_count: - description: The number of rows that failed to create/update. - format: int64 - type: integer - title: LocalFileMetadataV2 - type: object TableResultV2DataAttributesFileMetadataOneOfAccessDetails: description: Cloud storage access configuration for the reference table data file. @@ -73470,6 +73445,58 @@ paths: requestBody: content: application/json: + examples: + cloud_storage: + summary: Create table from cloud storage (S3) + value: + data: + attributes: + description: Customer reference data synced from S3 + file_metadata: + access_details: + aws_detail: + aws_account_id: '924305315327' + aws_bucket_name: my-data-bucket + file_path: customers.csv + sync_enabled: true + schema: + fields: + - name: customer_id + type: STRING + - name: customer_name + type: STRING + - name: email + type: STRING + primary_keys: + - customer_id + source: S3 + table_name: customer_reference_data + tags: + - team:data-platform + type: reference_table + local_file: + summary: Create table from local file upload + value: + data: + attributes: + description: Product catalog uploaded via local file + file_metadata: + upload_id: 00000000-0000-0000-0000-000000000000 + schema: + fields: + - name: product_id + type: STRING + - name: product_name + type: STRING + - name: price + type: DOUBLE + primary_keys: + - product_id + source: LOCAL_FILE + table_name: product_catalog + tags: + - team:ecommerce + type: reference_table schema: $ref: '#/components/schemas/CreateTableRequest' required: true diff --git a/features/v2/reference_tables.feature b/features/v2/reference_tables.feature index 6d4bc4b97f58..fa3ea42360ab 100644 --- a/features/v2/reference_tables.feature +++ b/features/v2/reference_tables.feature @@ -10,21 +10,21 @@ Feature: Reference Tables @generated @skip @team:DataDog/redapl-experiences Scenario: Create reference table returns "Bad Request" response Given new "CreateReferenceTable" request - And body with value {"data": {"attributes": {"description": "this is a cloud table generated via a cloud bucket sync", "file_metadata": {"access_details": {"aws_detail": {"aws_account_id": "test-account-id", "aws_bucket_name": "test-bucket", "file_path": "test_rt.csv"}}, "sync_enabled": true}, "schema": {"fields": [{"name": "name", "type": "STRING"}, {"name": "account_id", "type": "STRING"}], "primary_keys": ["account_id"]}, "source": "S3", "table_name": "test_reference_table", "tags": ["test_tag"]}, "type": "reference_table"}} + And body with value {"data": {"attributes": {"file_metadata": {"access_details": {"aws_detail": {"aws_account_id": "123456789000", "aws_bucket_name": "example-data-bucket", "file_path": "reference-tables/users.csv"}, "azure_detail": {"azure_client_id": "aaaaaaaa-1111-2222-3333-bbbbbbbbbbbb", "azure_container_name": "reference-data", "azure_storage_account_name": "examplestorageaccount", "azure_tenant_id": "cccccccc-4444-5555-6666-dddddddddddd", "file_path": "tables/users.csv"}, "gcp_detail": {"file_path": "data/reference_tables/users.csv", "gcp_bucket_name": "example-data-bucket", "gcp_project_id": "example-gcp-project-12345", "gcp_service_account_email": "example-service@example-gcp-project-12345.iam.gserviceaccount.com"}}, "sync_enabled": false}, "schema": {"fields": [{"name": "field_1", "type": "STRING"}], "primary_keys": ["field_1"]}, "source": "LOCAL_FILE", "table_name": "table_1", "tags": ["tag_1", "tag_2"]}, "type": "reference_table"}} When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/redapl-experiences Scenario: Create reference table returns "Created" response Given new "CreateReferenceTable" request - And body with value {"data": {"attributes": {"description": "this is a cloud table generated via a cloud bucket sync", "file_metadata": {"access_details": {"aws_detail": {"aws_account_id": "test-account-id", "aws_bucket_name": "test-bucket", "file_path": "test_rt.csv"}}, "sync_enabled": true}, "schema": {"fields": [{"name": "name", "type": "STRING"}, {"name": "account_id", "type": "STRING"}], "primary_keys": ["account_id"]}, "source": "S3", "table_name": "test_reference_table", "tags": ["test_tag"]}, "type": "reference_table"}} + And body with value {"data": {"attributes": {"file_metadata": {"access_details": {"aws_detail": {"aws_account_id": "123456789000", "aws_bucket_name": "example-data-bucket", "file_path": "reference-tables/users.csv"}, "azure_detail": {"azure_client_id": "aaaaaaaa-1111-2222-3333-bbbbbbbbbbbb", "azure_container_name": "reference-data", "azure_storage_account_name": "examplestorageaccount", "azure_tenant_id": "cccccccc-4444-5555-6666-dddddddddddd", "file_path": "tables/users.csv"}, "gcp_detail": {"file_path": "data/reference_tables/users.csv", "gcp_bucket_name": "example-data-bucket", "gcp_project_id": "example-gcp-project-12345", "gcp_service_account_email": "example-service@example-gcp-project-12345.iam.gserviceaccount.com"}}, "sync_enabled": false}, "schema": {"fields": [{"name": "field_1", "type": "STRING"}], "primary_keys": ["field_1"]}, "source": "LOCAL_FILE", "table_name": "table_1", "tags": ["tag_1", "tag_2"]}, "type": "reference_table"}} When the request is sent Then the response status is 201 Created @generated @skip @team:DataDog/redapl-experiences Scenario: Create reference table upload returns "Bad Request" response Given new "CreateReferenceTableUpload" request - And body with value {"data": {"attributes": {"headers": ["field_1", "field_2"], "part_count": 3, "part_size": 10000000, "table_name": ""}, "type": "upload"}} + And body with value {"data": {"attributes": {"headers": ["product_id", "product_name", "price"], "part_count": 3, "part_size": 10000000, "table_name": "my_products_table"}, "type": "upload"}} When the request is sent Then the response status is 400 Bad Request diff --git a/services/reference_tables/src/v2/index.ts b/services/reference_tables/src/v2/index.ts index 4d0285615959..99e4a5266436 100644 --- a/services/reference_tables/src/v2/index.ts +++ b/services/reference_tables/src/v2/index.ts @@ -53,9 +53,7 @@ export { TableResultV2Array } from "./models/TableResultV2Array"; export { TableResultV2Data } from "./models/TableResultV2Data"; export { TableResultV2DataAttributes } from "./models/TableResultV2DataAttributes"; export { TableResultV2DataAttributesFileMetadata } from "./models/TableResultV2DataAttributesFileMetadata"; -export { TableResultV2DataAttributesFileMetadataCloudStorage } from "./models/TableResultV2DataAttributesFileMetadataCloudStorage"; export { TableResultV2DataAttributesFileMetadataCloudStorageErrorType } from "./models/TableResultV2DataAttributesFileMetadataCloudStorageErrorType"; -export { TableResultV2DataAttributesFileMetadataLocalFile } from "./models/TableResultV2DataAttributesFileMetadataLocalFile"; export { TableResultV2DataAttributesFileMetadataOneOfAccessDetails } from "./models/TableResultV2DataAttributesFileMetadataOneOfAccessDetails"; export { TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAwsDetail } from "./models/TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAwsDetail"; export { TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAzureDetail } from "./models/TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAzureDetail"; diff --git a/services/reference_tables/src/v2/models/TableResultV2DataAttributes.ts b/services/reference_tables/src/v2/models/TableResultV2DataAttributes.ts index c4f337929612..694f29bdfc82 100644 --- a/services/reference_tables/src/v2/models/TableResultV2DataAttributes.ts +++ b/services/reference_tables/src/v2/models/TableResultV2DataAttributes.ts @@ -18,6 +18,14 @@ export class TableResultV2DataAttributes { "description"?: string; /** * Metadata specifying where and how to access the reference table's data file. + * + * For cloud storage tables (S3/GCS/Azure): + * - sync_enabled and access_details will always be present + * - error fields (error_message, error_row_count, error_type) are present only when errors occur + * + * For local file tables: + * - error fields (error_message, error_row_count) are present only when errors occur + * - sync_enabled, access_details are never present */ "fileMetadata"?: TableResultV2DataAttributesFileMetadata; /** diff --git a/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadata.ts b/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadata.ts index 1f0c9cc7ba5c..89624a704f54 100644 --- a/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadata.ts +++ b/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadata.ts @@ -1,12 +1,78 @@ -import { UnparsedObject } from "@datadog/datadog-api-client"; +import { AttributeTypeMap } from "@datadog/datadog-api-client"; -import { TableResultV2DataAttributesFileMetadataCloudStorage } from "./TableResultV2DataAttributesFileMetadataCloudStorage"; -import { TableResultV2DataAttributesFileMetadataLocalFile } from "./TableResultV2DataAttributesFileMetadataLocalFile"; +import { TableResultV2DataAttributesFileMetadataCloudStorageErrorType } from "./TableResultV2DataAttributesFileMetadataCloudStorageErrorType"; +import { TableResultV2DataAttributesFileMetadataOneOfAccessDetails } from "./TableResultV2DataAttributesFileMetadataOneOfAccessDetails"; /** * Metadata specifying where and how to access the reference table's data file. + * + * For cloud storage tables (S3/GCS/Azure): + * - sync_enabled and access_details will always be present + * - error fields (error_message, error_row_count, error_type) are present only when errors occur + * + * For local file tables: + * - error fields (error_message, error_row_count) are present only when errors occur + * - sync_enabled, access_details are never present */ -export type TableResultV2DataAttributesFileMetadata = - | TableResultV2DataAttributesFileMetadataCloudStorage - | TableResultV2DataAttributesFileMetadataLocalFile - | UnparsedObject; +export class TableResultV2DataAttributesFileMetadata { + /** + * Cloud storage access configuration for the reference table data file. + */ + "accessDetails"?: TableResultV2DataAttributesFileMetadataOneOfAccessDetails; + /** + * The error message returned from the last operation (sync for cloud storage, upload for local file). + */ + "errorMessage"?: string; + /** + * The number of rows that failed to process. + */ + "errorRowCount"?: number; + /** + * The type of error that occurred during file processing. This field provides high-level error categories for easier troubleshooting and is only present when there are errors. + */ + "errorType"?: TableResultV2DataAttributesFileMetadataCloudStorageErrorType; + /** + * Whether this table is synced automatically from cloud storage. Only applicable for cloud storage sources. + */ + "syncEnabled"?: boolean; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + accessDetails: { + baseName: "access_details", + type: "TableResultV2DataAttributesFileMetadataOneOfAccessDetails", + }, + errorMessage: { + baseName: "error_message", + type: "string", + }, + errorRowCount: { + baseName: "error_row_count", + type: "number", + format: "int64", + }, + errorType: { + baseName: "error_type", + type: "TableResultV2DataAttributesFileMetadataCloudStorageErrorType", + }, + syncEnabled: { + baseName: "sync_enabled", + type: "boolean", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return TableResultV2DataAttributesFileMetadata.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadataCloudStorage.ts b/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadataCloudStorage.ts deleted file mode 100644 index 497029456f44..000000000000 --- a/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadataCloudStorage.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { AttributeTypeMap } from "@datadog/datadog-api-client"; - -import { TableResultV2DataAttributesFileMetadataCloudStorageErrorType } from "./TableResultV2DataAttributesFileMetadataCloudStorageErrorType"; -import { TableResultV2DataAttributesFileMetadataOneOfAccessDetails } from "./TableResultV2DataAttributesFileMetadataOneOfAccessDetails"; - -/** - * File metadata for reference tables created by cloud storage. - */ -export class TableResultV2DataAttributesFileMetadataCloudStorage { - /** - * Cloud storage access configuration for the reference table data file. - */ - "accessDetails": TableResultV2DataAttributesFileMetadataOneOfAccessDetails; - /** - * The error message returned from the sync. - */ - "errorMessage"?: string; - /** - * The number of rows that failed to sync. - */ - "errorRowCount"?: number; - /** - * The type of error that occurred during file processing. This field provides high-level error categories for easier troubleshooting and is only present when there are errors. - */ - "errorType"?: TableResultV2DataAttributesFileMetadataCloudStorageErrorType; - /** - * Whether this table is synced automatically. - */ - "syncEnabled"?: boolean; - /** - * @ignore - */ - "_unparsed"?: boolean; - - /** - * @ignore - */ - static readonly attributeTypeMap: AttributeTypeMap = { - accessDetails: { - baseName: "access_details", - type: "TableResultV2DataAttributesFileMetadataOneOfAccessDetails", - required: true, - }, - errorMessage: { - baseName: "error_message", - type: "string", - }, - errorRowCount: { - baseName: "error_row_count", - type: "number", - format: "int64", - }, - errorType: { - baseName: "error_type", - type: "TableResultV2DataAttributesFileMetadataCloudStorageErrorType", - }, - syncEnabled: { - baseName: "sync_enabled", - type: "boolean", - }, - }; - - /** - * @ignore - */ - static getAttributeTypeMap(): AttributeTypeMap { - return TableResultV2DataAttributesFileMetadataCloudStorage.attributeTypeMap; - } - - public constructor() {} -} diff --git a/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadataLocalFile.ts b/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadataLocalFile.ts deleted file mode 100644 index 9441214486d4..000000000000 --- a/services/reference_tables/src/v2/models/TableResultV2DataAttributesFileMetadataLocalFile.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { AttributeTypeMap } from "@datadog/datadog-api-client"; - -/** - * File metadata for reference tables created by upload. Note that upload_id is only returned in the immediate create/replace response and is not available in subsequent GET requests. - */ -export class TableResultV2DataAttributesFileMetadataLocalFile { - /** - * The error message returned from the creation/update. - */ - "errorMessage"?: string; - /** - * The number of rows that failed to create/update. - */ - "errorRowCount"?: number; - /** - * @ignore - */ - "_unparsed"?: boolean; - - /** - * @ignore - */ - static readonly attributeTypeMap: AttributeTypeMap = { - errorMessage: { - baseName: "error_message", - type: "string", - }, - errorRowCount: { - baseName: "error_row_count", - type: "number", - format: "int64", - }, - }; - - /** - * @ignore - */ - static getAttributeTypeMap(): AttributeTypeMap { - return TableResultV2DataAttributesFileMetadataLocalFile.attributeTypeMap; - } - - public constructor() {} -} diff --git a/services/reference_tables/src/v2/models/TypingInfo.ts b/services/reference_tables/src/v2/models/TypingInfo.ts index 08bf147af292..636efa9248ef 100644 --- a/services/reference_tables/src/v2/models/TypingInfo.ts +++ b/services/reference_tables/src/v2/models/TypingInfo.ts @@ -33,8 +33,7 @@ import { TableResultV2 } from "./TableResultV2"; import { TableResultV2Array } from "./TableResultV2Array"; import { TableResultV2Data } from "./TableResultV2Data"; import { TableResultV2DataAttributes } from "./TableResultV2DataAttributes"; -import { TableResultV2DataAttributesFileMetadataCloudStorage } from "./TableResultV2DataAttributesFileMetadataCloudStorage"; -import { TableResultV2DataAttributesFileMetadataLocalFile } from "./TableResultV2DataAttributesFileMetadataLocalFile"; +import { TableResultV2DataAttributesFileMetadata } from "./TableResultV2DataAttributesFileMetadata"; import { TableResultV2DataAttributesFileMetadataOneOfAccessDetails } from "./TableResultV2DataAttributesFileMetadataOneOfAccessDetails"; import { TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAwsDetail } from "./TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAwsDetail"; import { TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAzureDetail } from "./TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAzureDetail"; @@ -94,10 +93,6 @@ export const TypingInfo: ModelTypingInfo = { "PatchTableRequestDataAttributesFileMetadataCloudStorage", "PatchTableRequestDataAttributesFileMetadataLocalFile", ], - TableResultV2DataAttributesFileMetadata: [ - "TableResultV2DataAttributesFileMetadataCloudStorage", - "TableResultV2DataAttributesFileMetadataLocalFile", - ], }, typeMap: { APIErrorResponse: APIErrorResponse, @@ -149,10 +144,8 @@ export const TypingInfo: ModelTypingInfo = { TableResultV2Array: TableResultV2Array, TableResultV2Data: TableResultV2Data, TableResultV2DataAttributes: TableResultV2DataAttributes, - TableResultV2DataAttributesFileMetadataCloudStorage: - TableResultV2DataAttributesFileMetadataCloudStorage, - TableResultV2DataAttributesFileMetadataLocalFile: - TableResultV2DataAttributesFileMetadataLocalFile, + TableResultV2DataAttributesFileMetadata: + TableResultV2DataAttributesFileMetadata, TableResultV2DataAttributesFileMetadataOneOfAccessDetails: TableResultV2DataAttributesFileMetadataOneOfAccessDetails, TableResultV2DataAttributesFileMetadataOneOfAccessDetailsAwsDetail: