feat(cdc): addTable route — DynamoDB-first + rollback

This commit is contained in:
Rafael
2026-08-17 17:01:05 -03:00
parent fab2061efc
commit 55308420ff
7 changed files with 240 additions and 5 deletions
@@ -33,6 +33,7 @@ import { CatalogService } from '../catalog/catalog.service';
import { PackTheMetadata } from '../../utils/PackTheMetadata';
import { ValidationTableDTO } from './platform-api.dto';
import { InputsService } from '../inputs/inputs.service';
import { PipelinesService } from '../pipelinesV2/pipelines.service';
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
@@ -61,6 +62,7 @@ export class PlatformApiController {
private readonly customersService: CustomersService,
private readonly catalogService: CatalogService,
private readonly inputsService: InputsService,
private readonly pipelinesClientService: PipelinesService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
@@ -990,6 +992,72 @@ export class PlatformApiController {
}
}
@Post('pipelines/:pipelineId/inputs/:inputId/tables')
@ApiOperation({ summary: 'Add a CDC table to an input and dispatch its platform jobs' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@UseGuards(PipelineExecutionGuard)
async addTable(
@Param('pipelineId') pipelineId: string,
@Param('inputId') inputId: string,
@Body() body: {
table_name: string;
table_schema: string;
primary_keys: string[];
destinations: {
raw: { table_schema: string; table_name: string };
qualify: { table_schema: string; table_name: string };
};
},
@User() user: RequestUser,
) {
const info = {
customer_id: user.customer_id,
customer: user.customer_name,
user_id: user.user_id,
};
const cdcTable = {
table_schema: body.table_schema,
table_name: body.table_name,
primary_keys: body.primary_keys,
name: body.table_name,
};
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
await this.inputsService.addCdcTable({ id: inputId, table: cdcTable, info });
this.logger.info('addTable: DynamoDB row appended', { inputId, tableName: body.table_name });
try {
const customInfo = {
customer_id: user.customer_id,
user_id: user.user_id,
customer: user.customer_name,
};
const res = await this.pipelinesClientService.addCdcJobs({
pipeline_id: pipelineId,
input_id: inputId,
tables: [{
table_schema: body.table_schema,
table_name: body.table_name,
primary_keys: body.primary_keys,
destinations: body.destinations,
}],
info: customInfo,
});
return res;
} catch (error) {
this.logger.error('addTable: platform AddJobs failed, rolling back the DynamoDB row', { tableName: body.table_name, error: error.message });
try {
await this.inputsService.removeCdcTable({ id: inputId, table_name: body.table_name, info });
} catch (rbErr) {
this.logger.error('addTable: rollback failed', { error: rbErr.message });
}
throw error;
}
}
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
@Get('jobs/jdbc/configs/allowed_datatypes')