mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-15 04:09:08 +00:00
feat(cdc): accept + forward source columns on /inputs/cdc
Adds CdcColumnReq {name, type, is_primary_key} and columns? on
CdcTableReq, forwarded through createCdc and the addTable (Edit
Objects add-table) path so the column schema reaches in-factory for
Iceberg deduped-table pre-create. Bumps protospack-v2 to
3.41.0-cdc-iceberg.1, which adds the matching CdcColumn field
(now required on CdcTable) and regenerates docsfera.json.
Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
@@ -163,6 +163,7 @@ describe('PlatformApiController - addTable', () => {
|
||||
table_name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
name: 'orders',
|
||||
columns: [],
|
||||
},
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
@@ -201,6 +202,37 @@ describe('PlatformApiController - addTable', () => {
|
||||
primary_keys: ['id'],
|
||||
name: 'orders',
|
||||
iceberg_table_name: 'cdc_raw.public__orders',
|
||||
columns: [],
|
||||
},
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
});
|
||||
|
||||
it('addTable: carries columns on the added table through to AddCdcTable', async () => {
|
||||
inputsService.addCdcTable.mockResolvedValue({ input: {} });
|
||||
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_4'], skipped: [] });
|
||||
|
||||
const columnsBody = {
|
||||
...body,
|
||||
columns: [
|
||||
{ name: 'id', type: 'int', is_primary_key: true },
|
||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
||||
],
|
||||
};
|
||||
|
||||
await controller.addTable('pid', 'iid', columnsBody, mockUser);
|
||||
|
||||
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
|
||||
id: 'iid',
|
||||
table: {
|
||||
table_schema: 'public',
|
||||
table_name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
name: 'orders',
|
||||
columns: [
|
||||
{ name: 'id', type: 'int', is_primary_key: true },
|
||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
||||
],
|
||||
},
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
|
||||
@@ -1079,6 +1079,8 @@ export class PlatformApiController {
|
||||
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
|
||||
// absent for snowflake, back-compat.
|
||||
iceberg_table_name?: string;
|
||||
// Source column schema for iceberg deduped table pre-create (protospack CdcTable.columns).
|
||||
columns?: { name: string; type: string; is_primary_key: boolean }[];
|
||||
},
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
@@ -1094,6 +1096,7 @@ export class PlatformApiController {
|
||||
primary_keys: body.primary_keys,
|
||||
name: body.table_name,
|
||||
iceberg_table_name: body.iceberg_table_name,
|
||||
columns: body.columns ?? [],
|
||||
};
|
||||
|
||||
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
|
||||
|
||||
Reference in New Issue
Block a user