mirror of
https://github.com/dadosfera/maestro.git
synced 2026-10-15 18:59:08 +00:00
feat(cdc): addTable route — DynamoDB-first + rollback
This commit is contained in:
@@ -33,6 +33,7 @@ import { CatalogService } from '../catalog/catalog.service';
|
||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||
import { ValidationTableDTO } from './platform-api.dto';
|
||||
import { InputsService } from '../inputs/inputs.service';
|
||||
import { PipelinesService } from '../pipelinesV2/pipelines.service';
|
||||
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
|
||||
|
||||
|
||||
@@ -61,6 +62,7 @@ export class PlatformApiController {
|
||||
private readonly customersService: CustomersService,
|
||||
private readonly catalogService: CatalogService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly pipelinesClientService: PipelinesService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
@@ -990,6 +992,72 @@ export class PlatformApiController {
|
||||
}
|
||||
}
|
||||
|
||||
@Post('pipelines/:pipelineId/inputs/:inputId/tables')
|
||||
@ApiOperation({ summary: 'Add a CDC table to an input and dispatch its platform jobs' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
@UseGuards(PipelineExecutionGuard)
|
||||
async addTable(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('inputId') inputId: string,
|
||||
@Body() body: {
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
primary_keys: string[];
|
||||
destinations: {
|
||||
raw: { table_schema: string; table_name: string };
|
||||
qualify: { table_schema: string; table_name: string };
|
||||
};
|
||||
},
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const info = {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
};
|
||||
|
||||
const cdcTable = {
|
||||
table_schema: body.table_schema,
|
||||
table_name: body.table_name,
|
||||
primary_keys: body.primary_keys,
|
||||
name: body.table_name,
|
||||
};
|
||||
|
||||
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
|
||||
await this.inputsService.addCdcTable({ id: inputId, table: cdcTable, info });
|
||||
this.logger.info('addTable: DynamoDB row appended', { inputId, tableName: body.table_name });
|
||||
|
||||
try {
|
||||
const customInfo = {
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
};
|
||||
|
||||
const res = await this.pipelinesClientService.addCdcJobs({
|
||||
pipeline_id: pipelineId,
|
||||
input_id: inputId,
|
||||
tables: [{
|
||||
table_schema: body.table_schema,
|
||||
table_name: body.table_name,
|
||||
primary_keys: body.primary_keys,
|
||||
destinations: body.destinations,
|
||||
}],
|
||||
info: customInfo,
|
||||
});
|
||||
|
||||
return res;
|
||||
} catch (error) {
|
||||
this.logger.error('addTable: platform AddJobs failed, rolling back the DynamoDB row', { tableName: body.table_name, error: error.message });
|
||||
try {
|
||||
await this.inputsService.removeCdcTable({ id: inputId, table_name: body.table_name, info });
|
||||
} catch (rbErr) {
|
||||
this.logger.error('addTable: rollback failed', { error: rbErr.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||
|
||||
@Get('jobs/jdbc/configs/allowed_datatypes')
|
||||
|
||||
Reference in New Issue
Block a user