feat(cdc-iceberg): thread qualify namespace/table through maestro

Threads the new CDC→Iceberg fields from the REST DTOs to the gRPC calls:
- CdcTableReq.iceberg_qualify_table_name + IcebergDestinationReq.qualify_namespace
  in input.model.ts
- inputs.service.ts create map forwards iceberg_qualify_table_name
- platform-api.controller.ts addTable body + cdcTable thread iceberg_qualify_table_name

Bumps protospack to v3.41.0-cdc-iceberg.4; docsfera.json regenerated with the new
/platform/iceberg/{namespaces,tables/validate} routes.

Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
Rafael
2026-08-24 09:08:36 -03:00
co-authored by WOZCODE
parent 4457c0ae62
commit 5de033ec19
6 changed files with 114 additions and 6 deletions
+72
View File
@@ -4571,6 +4571,66 @@
]
}
},
"/platform/iceberg/namespaces": {
"get": {
"operationId": "PlatformApiController_getIcebergNamespaces",
"summary": "List existing Polaris Iceberg namespaces (CDC destination dropdown)",
"parameters": [],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/platform/iceberg/tables/validate": {
"post": {
"operationId": "PlatformApiController_validateIcebergTables",
"summary": "Validate CDC Iceberg raw table names against Polaris",
"parameters": [],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/platform/pipelines/execute": {
"post": {
"operationId": "PlatformApiController_executePipeline",
@@ -10962,11 +11022,20 @@
"iceberg_table_name": {
"type": "string"
},
"iceberg_qualify_table_name": {
"type": "string"
},
"columns": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CdcColumnReq"
}
},
"column_exclude_list": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
@@ -10978,6 +11047,9 @@
"properties": {
"namespace": {
"type": "string"
},
"qualify_namespace": {
"type": "string"
}
},
"required": [
+4 -4
View File
@@ -16,7 +16,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.41.0-cdc-iceberg.1",
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz",
"integrity": "sha512-9skEKTRbsgcnACfk6YDg2i9IePdS2FTgH5ngacaxljVv+4+oHsxcWRkIsTbvf/dC6RDAdjd38cU+oa9VGOTCEg==",
"version": "3.41.0-cdc-iceberg.4",
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
"integrity": "sha512-QcRhM2wHgoRgXbHfpu+ki+Iozbz7qunKD544v2DCdD1fbeuI6kEit/S4napquSzmzP2qcMjANobyAsWabC/f2w==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
+1 -1
View File
@@ -34,7 +34,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+14 -1
View File
@@ -82,16 +82,29 @@ export class CdcTableReq {
table_schema?: string;
@ApiPropertyOptional({ type: [String] })
primary_keys?: string[];
// Advisory/reserved: the platform currently derives the Iceberg table name itself (create_iceberg_table_name); this value is persisted but not yet consumed on the create/add path. Do not treat as the authoritative table name.
// Per-table raw Iceberg table name override (iceberg destination only).
// Honored on the create/add path: the platform lowercases + sanitizes it
// authoritatively; empty/absent => the platform derives tb__<hash>__<table>.
@ApiPropertyOptional()
iceberg_table_name?: string;
// Per-table deduped (qualify) Iceberg table name override (iceberg dest only).
// Empty/absent => the deduped table takes the same name as the raw table.
@ApiPropertyOptional()
iceberg_qualify_table_name?: string;
@ApiPropertyOptional({ type: [CdcColumnReq] })
columns?: CdcColumnReq[];
// Columns the user chose to ignore -> Debezium column.exclude.list.
@ApiPropertyOptional({ type: [String] })
column_exclude_list?: string[];
}
export class IcebergDestinationReq {
@ApiProperty()
namespace: string;
// Pipeline-wide deduped (qualify) namespace. Absent => the platform derives
// the sibling of `namespace` (cdc_raw -> cdc_dedup).
@ApiPropertyOptional()
qualify_namespace?: string;
}
export class CdcDestinationReq {
+2
View File
@@ -198,7 +198,9 @@ export class InputsService {
name: t.name, // canonical identity == table_name (in-factory also backfills)
primary_keys: t.primary_keys ?? [],
iceberg_table_name: t.iceberg_table_name,
iceberg_qualify_table_name: t.iceberg_qualify_table_name,
columns: t.columns ?? [],
column_exclude_list: t.column_exclude_list ?? [],
})),
destination: body.destination,
},
@@ -543,6 +543,20 @@ export class PlatformApiController {
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
}
@Get('iceberg/namespaces')
@ApiOperation({ summary: 'List existing Polaris Iceberg namespaces (CDC destination dropdown)' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getIcebergNamespaces(@User() user: RequestUser) {
return this.platformApiService.proxy('GET', '/iceberg/namespaces', user);
}
@Post('iceberg/tables/validate')
@ApiOperation({ summary: 'Validate CDC Iceberg raw table names against Polaris' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async validateIcebergTables(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/iceberg/tables/validate', user, body);
}
@Patch('pipelines/:pipelineId')
@ApiOperation({ summary: 'Update pipeline by ID' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@@ -1079,8 +1093,13 @@ export class PlatformApiController {
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
// absent for snowflake, back-compat.
iceberg_table_name?: string;
// Per-table deduped (qualify) Iceberg table name (protospack
// CdcTable.iceberg_qualify_table_name); absent => same as the raw name.
iceberg_qualify_table_name?: string;
// Source column schema for iceberg deduped table pre-create (protospack CdcTable.columns).
columns?: { name: string; type: string; is_primary_key: boolean }[];
// Columns the user chose to ignore -> Debezium column.exclude.list.
column_exclude_list?: string[];
},
@User() user: RequestUser,
) {
@@ -1096,7 +1115,9 @@ export class PlatformApiController {
primary_keys: body.primary_keys,
name: body.table_name,
iceberg_table_name: body.iceberg_table_name,
iceberg_qualify_table_name: body.iceberg_qualify_table_name,
columns: body.columns ?? [],
column_exclude_list: body.column_exclude_list ?? [],
};
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });