feat(cdc): expose CDC routes over REST (on main + beta protospack)

Re-applied onto fresh origin/main. pipelinesV2: live-status/pause/unpause/
restart/reset-state routes (per-route @RequireSomePermission — GET for status,
UPDATE for mutations — matching main's current auth convention). connection-test:
POST /connection-test/cdc-prerequisites. inputs: POST /inputs/cdc -> InputCreateCdc.
Consumes protospack 3.40.0-beta.15-cdc.0 tarball; CI guard added.

Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
Rafael
2026-08-15 19:45:09 -03:00
co-authored by WOZCODE
parent 7051b21d86
commit a597c41070
13 changed files with 790 additions and 6 deletions
@@ -22,6 +22,8 @@ import {
ConnectionTestListTablesRes,
GetTableMetadataRes,
GetTableMetadataReq,
ValidateCdcPrerequisitesReq,
ValidateCdcPrerequisitesRes,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
@@ -125,4 +127,21 @@ export class ConnectionTestController {
user.customer_name,
);
}
@Post('cdc-prerequisites')
@ApiOkResponse({ type: ValidateCdcPrerequisitesRes })
@HttpCode(HttpStatus.OK)
async validateCdcPrerequisites(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: ValidateCdcPrerequisitesReq,
) {
this.logger.info('/connection-test/cdc-prerequisites', {
user: user.user_id,
customer: user.customer_name,
});
return this.connectionTestService.validateCdcPrerequisites(
body,
user.customer_name,
);
}
}
@@ -13,6 +13,8 @@ import {
ConnectionTestPingRes,
GetTableMetadataReq,
GetTableMetadataRes,
ValidateCdcPrerequisitesReq,
ValidateCdcPrerequisitesRes,
} from './dto/connection-test';
import { ConnectionClientService } from '../connection/client.service';
import {
@@ -188,4 +190,18 @@ export class ConnectionTestService {
}),
);
}
async validateCdcPrerequisites(
body: ValidateCdcPrerequisitesReq,
customer_name: string,
): Promise<ValidateCdcPrerequisitesRes> {
const { plugin, connection_id } = body;
return lastValueFrom(
this.connectionTestReadClient.ValidateCdcPrerequisites({
connection_id,
customer_name,
plugin,
}),
);
}
}
@@ -131,3 +131,31 @@ export class GetTableMetadataRes {
@ApiProperty({ type: [TableMetadataDto] })
tables_metadata: TableMetadataDto[];
}
export class CdcCheckDto {
@ApiProperty()
name: string;
@ApiProperty()
expected: string;
@ApiProperty()
actual: string;
@ApiProperty()
passed: boolean;
}
export class ValidateCdcPrerequisitesReq {
@ApiProperty()
@IsString()
plugin: string;
@ApiProperty()
@IsString()
connection_id: string;
}
export class ValidateCdcPrerequisitesRes {
@ApiProperty()
operation_result: boolean;
@ApiProperty({ type: [CdcCheckDto] })
checks: CdcCheckDto[];
}
+20
View File
@@ -65,3 +65,23 @@ export class CreateInputReq extends OmitType(Input, [
'created_at',
'updated_at',
]) {}
export class CdcTableReq {
@ApiProperty()
name: string;
@ApiPropertyOptional()
table_schema?: string;
@ApiPropertyOptional({ type: [String] })
primary_keys?: string[];
}
export class CreateCdcInputReq {
@ApiProperty()
name: string;
@ApiProperty()
plugin: string; // mysql_cdc (v1)
@ApiProperty({ type: [CdcTableReq] })
tables: CdcTableReq[];
@ApiPropertyOptional()
read_only?: boolean;
}
+21
View File
@@ -15,6 +15,7 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import {
CreateCdcInputReq,
CreateInputReq,
GetAvailableEntitiesReq,
GetAvailableEntitiesRes,
@@ -105,6 +106,26 @@ export class InputsController {
return response;
}
@Post('cdc')
@ApiInternalOnlyEndpoint()
@ApiOkResponse({ type: Input })
async createCdc(
@Body() body: CreateCdcInputReq,
@User() user: RequestUser,
) {
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
customer_id: user.customer_id,
};
this.logger.info(`/inputs/cdc - ON CREATE CDC INPUT ROUTE`, {
user: info.user_id,
customer: info.customer,
});
return this.inputService.createCdc({ body, info });
}
@ApiInternalOnlyEndpoint()
@Get()
async findAll(@User() user: RequestUser) {
+25 -1
View File
@@ -15,6 +15,7 @@ import { Input } from '@dadosfera/protospack-v2';
import {
GetAvailableEntitiesRequest,
InputCreateGenericRequest,
InputCreateCdcRequest,
InputCreateS3Request,
InputNewCreateRequest,
InputUpdateResponse,
@@ -22,7 +23,7 @@ import {
TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { CreateInputReq } from './dtos/input.model';
import { CreateCdcInputReq, CreateInputReq } from './dtos/input.model';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
@@ -183,6 +184,29 @@ export class InputsService {
return { input: adjustedInput };
}
async createCdc(data: { body: CreateCdcInputReq; info: Info }) {
const { body, info } = data;
const inputCreateCdcRequest: InputCreateCdcRequest = {
input: {
name: body.name,
plugin: body.plugin,
read_only: body.read_only ?? true,
tables: body.tables.map((t) => ({
table_schema: t.table_schema,
table_name: t.name,
primary_keys: t.primary_keys ?? [],
})),
},
info,
};
const { input } = await lastValueFrom(
this.inputWriteService.InputCreateCdc(inputCreateCdcRequest),
);
return { input };
}
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
}
@@ -510,4 +510,78 @@ export class PipelinesController {
return response;
}
// ---- CDC pipeline operations (Kafka Connect backed) ----
// These operate on an existing pipeline, so they require UPDATE (not CREATE).
@Get(':id/live-status')
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getLiveStatus(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
this.logger.info('PipelinesController - getLiveStatus', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.getLiveStatus(id, metadata);
}
@Post(':id/pause')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pause(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - pause', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.pause(id, metadata);
}
@Post(':id/unpause')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpause(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - unpause', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.unpause(id, metadata);
}
@Post(':id/restart')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async restart(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - restart', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.restart(id, metadata);
}
@Post(':id/jobs/:jobId/reset-state')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async resetJobState(
@Param('id') id: string,
@Param('jobId') jobId: string,
@Body() body: { schedule_minutes?: number },
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - resetJobState', { id, jobId });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.resetJobState(
id,
jobId,
body?.schedule_minutes,
metadata,
);
}
}
@@ -160,6 +160,62 @@ export class PipelinesService implements OnModuleInit {
return findOnePipelineResponse;
}
// CDC lifecycle operations (Kafka Connect backed).
async pause(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Pause');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Pause({ id }, metadata),
);
}
async unpause(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Unpause');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Unpause({ id }, metadata),
);
}
async restart(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Restart');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Restart({ id }, metadata),
);
}
async resetJobState(
id: string,
job_id: string,
schedule_minutes: number | undefined,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - ResetJobState');
return lastValueFrom(
this.pipelineWriteService.PipelineV2ResetJobState(
{ id, job_id, schedule_minutes },
metadata,
),
);
}
async getLiveStatus(
id: string,
metadata,
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
this.logger.info('PipelinesClientService - GetLiveStatus');
return lastValueFrom(
this.pipelineReadService.PipelineV2GetLiveStatus({ id }, metadata),
);
}
async update(
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
metadata,