mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-19 04:19:09 +00:00
feat(cdc-iceberg): thread qualify namespace/table through maestro
Threads the new CDC→Iceberg fields from the REST DTOs to the gRPC calls:
- CdcTableReq.iceberg_qualify_table_name + IcebergDestinationReq.qualify_namespace
in input.model.ts
- inputs.service.ts create map forwards iceberg_qualify_table_name
- platform-api.controller.ts addTable body + cdcTable thread iceberg_qualify_table_name
Bumps protospack to v3.41.0-cdc-iceberg.4; docsfera.json regenerated with the new
/platform/iceberg/{namespaces,tables/validate} routes.
Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
@@ -4571,6 +4571,66 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/iceberg/namespaces": {
|
||||
"get": {
|
||||
"operationId": "PlatformApiController_getIcebergNamespaces",
|
||||
"summary": "List existing Polaris Iceberg namespaces (CDC destination dropdown)",
|
||||
"parameters": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Platform API"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/iceberg/tables/validate": {
|
||||
"post": {
|
||||
"operationId": "PlatformApiController_validateIcebergTables",
|
||||
"summary": "Validate CDC Iceberg raw table names against Polaris",
|
||||
"parameters": [],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Platform API"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/pipelines/execute": {
|
||||
"post": {
|
||||
"operationId": "PlatformApiController_executePipeline",
|
||||
@@ -10962,11 +11022,20 @@
|
||||
"iceberg_table_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"iceberg_qualify_table_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"columns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/CdcColumnReq"
|
||||
}
|
||||
},
|
||||
"column_exclude_list": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -10978,6 +11047,9 @@
|
||||
"properties": {
|
||||
"namespace": {
|
||||
"type": "string"
|
||||
},
|
||||
"qualify_namespace": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
Generated
+4
-4
@@ -16,7 +16,7 @@
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz",
|
||||
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -1735,9 +1735,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.41.0-cdc-iceberg.1",
|
||||
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz",
|
||||
"integrity": "sha512-9skEKTRbsgcnACfk6YDg2i9IePdS2FTgH5ngacaxljVv+4+oHsxcWRkIsTbvf/dC6RDAdjd38cU+oa9VGOTCEg==",
|
||||
"version": "3.41.0-cdc-iceberg.4",
|
||||
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
|
||||
"integrity": "sha512-QcRhM2wHgoRgXbHfpu+ki+Iozbz7qunKD544v2DCdD1fbeuI6kEit/S4napquSzmzP2qcMjANobyAsWabC/f2w==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.1.tgz",
|
||||
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
|
||||
@@ -82,16 +82,29 @@ export class CdcTableReq {
|
||||
table_schema?: string;
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
primary_keys?: string[];
|
||||
// Advisory/reserved: the platform currently derives the Iceberg table name itself (create_iceberg_table_name); this value is persisted but not yet consumed on the create/add path. Do not treat as the authoritative table name.
|
||||
// Per-table raw Iceberg table name override (iceberg destination only).
|
||||
// Honored on the create/add path: the platform lowercases + sanitizes it
|
||||
// authoritatively; empty/absent => the platform derives tb__<hash>__<table>.
|
||||
@ApiPropertyOptional()
|
||||
iceberg_table_name?: string;
|
||||
// Per-table deduped (qualify) Iceberg table name override (iceberg dest only).
|
||||
// Empty/absent => the deduped table takes the same name as the raw table.
|
||||
@ApiPropertyOptional()
|
||||
iceberg_qualify_table_name?: string;
|
||||
@ApiPropertyOptional({ type: [CdcColumnReq] })
|
||||
columns?: CdcColumnReq[];
|
||||
// Columns the user chose to ignore -> Debezium column.exclude.list.
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
column_exclude_list?: string[];
|
||||
}
|
||||
|
||||
export class IcebergDestinationReq {
|
||||
@ApiProperty()
|
||||
namespace: string;
|
||||
// Pipeline-wide deduped (qualify) namespace. Absent => the platform derives
|
||||
// the sibling of `namespace` (cdc_raw -> cdc_dedup).
|
||||
@ApiPropertyOptional()
|
||||
qualify_namespace?: string;
|
||||
}
|
||||
|
||||
export class CdcDestinationReq {
|
||||
|
||||
@@ -198,7 +198,9 @@ export class InputsService {
|
||||
name: t.name, // canonical identity == table_name (in-factory also backfills)
|
||||
primary_keys: t.primary_keys ?? [],
|
||||
iceberg_table_name: t.iceberg_table_name,
|
||||
iceberg_qualify_table_name: t.iceberg_qualify_table_name,
|
||||
columns: t.columns ?? [],
|
||||
column_exclude_list: t.column_exclude_list ?? [],
|
||||
})),
|
||||
destination: body.destination,
|
||||
},
|
||||
|
||||
@@ -543,6 +543,20 @@ export class PlatformApiController {
|
||||
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
|
||||
}
|
||||
|
||||
@Get('iceberg/namespaces')
|
||||
@ApiOperation({ summary: 'List existing Polaris Iceberg namespaces (CDC destination dropdown)' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getIcebergNamespaces(@User() user: RequestUser) {
|
||||
return this.platformApiService.proxy('GET', '/iceberg/namespaces', user);
|
||||
}
|
||||
|
||||
@Post('iceberg/tables/validate')
|
||||
@ApiOperation({ summary: 'Validate CDC Iceberg raw table names against Polaris' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async validateIcebergTables(@Body() body: any, @User() user: RequestUser) {
|
||||
return this.platformApiService.proxy('POST', '/iceberg/tables/validate', user, body);
|
||||
}
|
||||
|
||||
@Patch('pipelines/:pipelineId')
|
||||
@ApiOperation({ summary: 'Update pipeline by ID' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
@@ -1079,8 +1093,13 @@ export class PlatformApiController {
|
||||
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
|
||||
// absent for snowflake, back-compat.
|
||||
iceberg_table_name?: string;
|
||||
// Per-table deduped (qualify) Iceberg table name (protospack
|
||||
// CdcTable.iceberg_qualify_table_name); absent => same as the raw name.
|
||||
iceberg_qualify_table_name?: string;
|
||||
// Source column schema for iceberg deduped table pre-create (protospack CdcTable.columns).
|
||||
columns?: { name: string; type: string; is_primary_key: boolean }[];
|
||||
// Columns the user chose to ignore -> Debezium column.exclude.list.
|
||||
column_exclude_list?: string[];
|
||||
},
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
@@ -1096,7 +1115,9 @@ export class PlatformApiController {
|
||||
primary_keys: body.primary_keys,
|
||||
name: body.table_name,
|
||||
iceberg_table_name: body.iceberg_table_name,
|
||||
iceberg_qualify_table_name: body.iceberg_qualify_table_name,
|
||||
columns: body.columns ?? [],
|
||||
column_exclude_list: body.column_exclude_list ?? [],
|
||||
};
|
||||
|
||||
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
|
||||
|
||||
Reference in New Issue
Block a user