mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-19 04:24:49 +00:00
feat(cdc): batch table removal — reconfigure connectors once
Removing N tables via Edit Objects previously looped a single-table
DELETE per table (maestro deleteTable hardcodes job_ids:[one]), so the
Debezium source + Snowflake sink connectors were rewritten/restarted once
per table. platform-api's DELETE /pipeline/:id/jobs already batches (2
connector writes total for any N), but nothing above it used the array.
New maestro DELETE /pipelines/:pipelineId/inputs/:inputId/tables takes
{ table_names: [] }: soft-deletes each in DynamoDB (tracking successes),
resolves all table_names -> job_ids from the platform pipeline in one GET,
then makes ONE DELETE /pipeline/:id/jobs with all job_ids. All-or-nothing:
any failure (a later mark, an unmatched table, or the platform delete)
rolls back only the marks made in this call.
The single-table deleteTable route is kept (unchanged) — nothing else
depends on removing it, and that's a separable cleanup.
Tests: N tables -> one platform DELETE with all job_ids and no per-job
call; rollback on platform failure; rollback + no delete when a later mark
fails; 404 for an unmatched table. 8 controller specs pass; maestro builds.
Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
@@ -992,6 +992,75 @@ export class PlatformApiController {
|
||||
}
|
||||
}
|
||||
|
||||
@Delete('pipelines/:pipelineId/inputs/:inputId/tables')
|
||||
@ApiOperation({ summary: 'Batch-remove tables from an input; reconfigures the CDC connectors once' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||
@UseGuards(PipelineExecutionGuard)
|
||||
async deleteTables(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('inputId') inputId: string,
|
||||
@Body() body: { table_names: string[] },
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const tableNames = body.table_names ?? [];
|
||||
if (tableNames.length === 0) {
|
||||
throw new BadRequestException('table_names must be a non-empty array');
|
||||
}
|
||||
const info = {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
};
|
||||
|
||||
// 1) Soft-delete each table in DynamoDB, tracking which succeeded so a later
|
||||
// failure only rolls back the marks made in THIS call.
|
||||
const marked: string[] = [];
|
||||
const rollback = async () => {
|
||||
for (const name of marked) {
|
||||
try {
|
||||
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: name, info });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('deleteTables: rollback failed', { tableName: name, error: rollbackError.message });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
for (const name of tableNames) {
|
||||
await this.inputsService.markTableDeleted({ input_id: inputId, table_name: name, info });
|
||||
marked.push(name);
|
||||
}
|
||||
|
||||
// 2) Resolve all table_names -> job_ids from the platform pipeline (one GET).
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||
const jobs = platformPipeline?.jobs ?? [];
|
||||
|
||||
const jobIds: string[] = [];
|
||||
for (const name of tableNames) {
|
||||
const job = jobs.find((j: any) => j.input?.table_name === name);
|
||||
if (!job) throw new NotFoundException(`Job for table '${name}' not found in pipeline`);
|
||||
jobIds.push(job.job_id);
|
||||
}
|
||||
|
||||
// 3) Remove them all in ONE platform call so the Debezium source + Snowflake
|
||||
// sink connectors are reconfigured a single time, not once per table.
|
||||
await this.platformApiService.proxy(
|
||||
'DELETE',
|
||||
`/pipeline/${normalizedPipelineId}/jobs`,
|
||||
user,
|
||||
{ job_ids: jobIds, delete_snowflake_tables: false },
|
||||
);
|
||||
this.logger.info('deleteTables: jobs deleted', { jobIds });
|
||||
|
||||
return { table_names: tableNames, deleted: true };
|
||||
} catch (error) {
|
||||
this.logger.error('deleteTables: failed, rolling back this call\'s marks', { tableNames, error: error.message });
|
||||
await rollback();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@Post('pipelines/:pipelineId/inputs/:inputId/tables')
|
||||
@ApiOperation({ summary: 'Add a CDC table to an input and dispatch its platform jobs' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
|
||||
Reference in New Issue
Block a user