diff --git a/docsfera.json b/docsfera.json index 2653fe1..350903a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4571,6 +4571,66 @@ ] } }, + "/platform/iceberg/namespaces": { + "get": { + "operationId": "PlatformApiController_getIcebergNamespaces", + "summary": "List existing Polaris Iceberg namespaces (CDC destination dropdown)", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/platform/iceberg/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateIcebergTables", + "summary": "Validate CDC Iceberg raw table names against Polaris", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/platform/pipelines/execute": { "post": { "operationId": "PlatformApiController_executePipeline", @@ -10962,11 +11022,20 @@ "iceberg_table_name": { "type": "string" }, + "iceberg_qualify_table_name": { + "type": "string" + }, "columns": { "type": "array", "items": { "$ref": "#/components/schemas/CdcColumnReq" } + }, + "column_exclude_list": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ @@ -10978,6 +11047,9 @@ "properties": { "namespace": { "type": "string" + }, + "qualify_namespace": { + "type": "string" } }, "required": [ diff --git a/package-lock.json b/package-lock.json index 3b8fcd3..c39332e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,9 +1735,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.41.0-cdc-iceberg.1", - "resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz", - "integrity": "sha512-9skEKTRbsgcnACfk6YDg2i9IePdS2FTgH5ngacaxljVv+4+oHsxcWRkIsTbvf/dC6RDAdjd38cU+oa9VGOTCEg==", + "version": "3.41.0-cdc-iceberg.4", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz", + "integrity": "sha512-QcRhM2wHgoRgXbHfpu+ki+Iozbz7qunKD544v2DCdD1fbeuI6kEit/S4napquSzmzP2qcMjANobyAsWabC/f2w==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index faf536f..279b538 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 5f7e330..f7d3aa4 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -82,16 +82,29 @@ export class CdcTableReq { table_schema?: string; @ApiPropertyOptional({ type: [String] }) primary_keys?: string[]; - // Advisory/reserved: the platform currently derives the Iceberg table name itself (create_iceberg_table_name); this value is persisted but not yet consumed on the create/add path. Do not treat as the authoritative table name. + // Per-table raw Iceberg table name override (iceberg destination only). + // Honored on the create/add path: the platform lowercases + sanitizes it + // authoritatively; empty/absent => the platform derives tb____. @ApiPropertyOptional() iceberg_table_name?: string; + // Per-table deduped (qualify) Iceberg table name override (iceberg dest only). + // Empty/absent => the deduped table takes the same name as the raw table. + @ApiPropertyOptional() + iceberg_qualify_table_name?: string; @ApiPropertyOptional({ type: [CdcColumnReq] }) columns?: CdcColumnReq[]; + // Columns the user chose to ignore -> Debezium column.exclude.list. + @ApiPropertyOptional({ type: [String] }) + column_exclude_list?: string[]; } export class IcebergDestinationReq { @ApiProperty() namespace: string; + // Pipeline-wide deduped (qualify) namespace. Absent => the platform derives + // the sibling of `namespace` (cdc_raw -> cdc_dedup). + @ApiPropertyOptional() + qualify_namespace?: string; } export class CdcDestinationReq { diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 38b3a4b..80eef5b 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -198,7 +198,9 @@ export class InputsService { name: t.name, // canonical identity == table_name (in-factory also backfills) primary_keys: t.primary_keys ?? [], iceberg_table_name: t.iceberg_table_name, + iceberg_qualify_table_name: t.iceberg_qualify_table_name, columns: t.columns ?? [], + column_exclude_list: t.column_exclude_list ?? [], })), destination: body.destination, }, diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index cbb9aef..6ec8c45 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -543,6 +543,20 @@ export class PlatformApiController { return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user); } + @Get('iceberg/namespaces') + @ApiOperation({ summary: 'List existing Polaris Iceberg namespaces (CDC destination dropdown)' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getIcebergNamespaces(@User() user: RequestUser) { + return this.platformApiService.proxy('GET', '/iceberg/namespaces', user); + } + + @Post('iceberg/tables/validate') + @ApiOperation({ summary: 'Validate CDC Iceberg raw table names against Polaris' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async validateIcebergTables(@Body() body: any, @User() user: RequestUser) { + return this.platformApiService.proxy('POST', '/iceberg/tables/validate', user, body); + } + @Patch('pipelines/:pipelineId') @ApiOperation({ summary: 'Update pipeline by ID' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) @@ -1079,8 +1093,13 @@ export class PlatformApiController { // Iceberg destination only (protospack CdcTable.iceberg_table_name); // absent for snowflake, back-compat. iceberg_table_name?: string; + // Per-table deduped (qualify) Iceberg table name (protospack + // CdcTable.iceberg_qualify_table_name); absent => same as the raw name. + iceberg_qualify_table_name?: string; // Source column schema for iceberg deduped table pre-create (protospack CdcTable.columns). columns?: { name: string; type: string; is_primary_key: boolean }[]; + // Columns the user chose to ignore -> Debezium column.exclude.list. + column_exclude_list?: string[]; }, @User() user: RequestUser, ) { @@ -1096,7 +1115,9 @@ export class PlatformApiController { primary_keys: body.primary_keys, name: body.table_name, iceberg_table_name: body.iceberg_table_name, + iceberg_qualify_table_name: body.iceberg_qualify_table_name, columns: body.columns ?? [], + column_exclude_list: body.column_exclude_list ?? [], }; this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });