mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-22 21:49:08 +00:00
feat(cdc): expose CDC routes over REST (on main + beta protospack)
Re-applied onto fresh origin/main. pipelinesV2: live-status/pause/unpause/ restart/reset-state routes (per-route @RequireSomePermission — GET for status, UPDATE for mutations — matching main's current auth convention). connection-test: POST /connection-test/cdc-prerequisites. inputs: POST /inputs/cdc -> InputCreateCdc. Consumes protospack 3.40.0-beta.15-cdc.0 tarball; CI guard added. Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
@@ -22,6 +22,8 @@ import {
|
||||
ConnectionTestListTablesRes,
|
||||
GetTableMetadataRes,
|
||||
GetTableMetadataReq,
|
||||
ValidateCdcPrerequisitesReq,
|
||||
ValidateCdcPrerequisitesRes,
|
||||
} from './dto/connection-test';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
|
||||
@@ -125,4 +127,21 @@ export class ConnectionTestController {
|
||||
user.customer_name,
|
||||
);
|
||||
}
|
||||
|
||||
@Post('cdc-prerequisites')
|
||||
@ApiOkResponse({ type: ValidateCdcPrerequisitesRes })
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async validateCdcPrerequisites(
|
||||
@User() user: RequestUser,
|
||||
@Body(new ValidationPipe()) body: ValidateCdcPrerequisitesReq,
|
||||
) {
|
||||
this.logger.info('/connection-test/cdc-prerequisites', {
|
||||
user: user.user_id,
|
||||
customer: user.customer_name,
|
||||
});
|
||||
return this.connectionTestService.validateCdcPrerequisites(
|
||||
body,
|
||||
user.customer_name,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
ConnectionTestPingRes,
|
||||
GetTableMetadataReq,
|
||||
GetTableMetadataRes,
|
||||
ValidateCdcPrerequisitesReq,
|
||||
ValidateCdcPrerequisitesRes,
|
||||
} from './dto/connection-test';
|
||||
import { ConnectionClientService } from '../connection/client.service';
|
||||
import {
|
||||
@@ -188,4 +190,18 @@ export class ConnectionTestService {
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async validateCdcPrerequisites(
|
||||
body: ValidateCdcPrerequisitesReq,
|
||||
customer_name: string,
|
||||
): Promise<ValidateCdcPrerequisitesRes> {
|
||||
const { plugin, connection_id } = body;
|
||||
return lastValueFrom(
|
||||
this.connectionTestReadClient.ValidateCdcPrerequisites({
|
||||
connection_id,
|
||||
customer_name,
|
||||
plugin,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,3 +131,31 @@ export class GetTableMetadataRes {
|
||||
@ApiProperty({ type: [TableMetadataDto] })
|
||||
tables_metadata: TableMetadataDto[];
|
||||
}
|
||||
|
||||
export class CdcCheckDto {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
expected: string;
|
||||
@ApiProperty()
|
||||
actual: string;
|
||||
@ApiProperty()
|
||||
passed: boolean;
|
||||
}
|
||||
|
||||
export class ValidateCdcPrerequisitesReq {
|
||||
@ApiProperty()
|
||||
@IsString()
|
||||
plugin: string;
|
||||
|
||||
@ApiProperty()
|
||||
@IsString()
|
||||
connection_id: string;
|
||||
}
|
||||
|
||||
export class ValidateCdcPrerequisitesRes {
|
||||
@ApiProperty()
|
||||
operation_result: boolean;
|
||||
@ApiProperty({ type: [CdcCheckDto] })
|
||||
checks: CdcCheckDto[];
|
||||
}
|
||||
|
||||
@@ -65,3 +65,23 @@ export class CreateInputReq extends OmitType(Input, [
|
||||
'created_at',
|
||||
'updated_at',
|
||||
]) {}
|
||||
|
||||
export class CdcTableReq {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiPropertyOptional()
|
||||
table_schema?: string;
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
primary_keys?: string[];
|
||||
}
|
||||
|
||||
export class CreateCdcInputReq {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
plugin: string; // mysql_cdc (v1)
|
||||
@ApiProperty({ type: [CdcTableReq] })
|
||||
tables: CdcTableReq[];
|
||||
@ApiPropertyOptional()
|
||||
read_only?: boolean;
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
|
||||
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
|
||||
import {
|
||||
CreateCdcInputReq,
|
||||
CreateInputReq,
|
||||
GetAvailableEntitiesReq,
|
||||
GetAvailableEntitiesRes,
|
||||
@@ -105,6 +106,26 @@ export class InputsController {
|
||||
return response;
|
||||
}
|
||||
|
||||
@Post('cdc')
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@ApiOkResponse({ type: Input })
|
||||
async createCdc(
|
||||
@Body() body: CreateCdcInputReq,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const info: Info = {
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
customer_id: user.customer_id,
|
||||
};
|
||||
this.logger.info(`/inputs/cdc - ON CREATE CDC INPUT ROUTE`, {
|
||||
user: info.user_id,
|
||||
customer: info.customer,
|
||||
});
|
||||
|
||||
return this.inputService.createCdc({ body, info });
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Get()
|
||||
async findAll(@User() user: RequestUser) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import { Input } from '@dadosfera/protospack-v2';
|
||||
import {
|
||||
GetAvailableEntitiesRequest,
|
||||
InputCreateGenericRequest,
|
||||
InputCreateCdcRequest,
|
||||
InputCreateS3Request,
|
||||
InputNewCreateRequest,
|
||||
InputUpdateResponse,
|
||||
@@ -22,7 +23,7 @@ import {
|
||||
TestConnectionRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { CreateInputReq } from './dtos/input.model';
|
||||
import { CreateCdcInputReq, CreateInputReq } from './dtos/input.model';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
|
||||
@Injectable()
|
||||
@@ -183,6 +184,29 @@ export class InputsService {
|
||||
return { input: adjustedInput };
|
||||
}
|
||||
|
||||
async createCdc(data: { body: CreateCdcInputReq; info: Info }) {
|
||||
const { body, info } = data;
|
||||
|
||||
const inputCreateCdcRequest: InputCreateCdcRequest = {
|
||||
input: {
|
||||
name: body.name,
|
||||
plugin: body.plugin,
|
||||
read_only: body.read_only ?? true,
|
||||
tables: body.tables.map((t) => ({
|
||||
table_schema: t.table_schema,
|
||||
table_name: t.name,
|
||||
primary_keys: t.primary_keys ?? [],
|
||||
})),
|
||||
},
|
||||
info,
|
||||
};
|
||||
|
||||
const { input } = await lastValueFrom(
|
||||
this.inputWriteService.InputCreateCdc(inputCreateCdcRequest),
|
||||
);
|
||||
return { input };
|
||||
}
|
||||
|
||||
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
|
||||
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
|
||||
}
|
||||
|
||||
@@ -510,4 +510,78 @@ export class PipelinesController {
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
// ---- CDC pipeline operations (Kafka Connect backed) ----
|
||||
// These operate on an existing pipeline, so they require UPDATE (not CREATE).
|
||||
|
||||
@Get(':id/live-status')
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getLiveStatus(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
|
||||
this.logger.info('PipelinesController - getLiveStatus', { id });
|
||||
const metadata = PackTheMetadata(user);
|
||||
return this.pipelinesClientService.getLiveStatus(id, metadata);
|
||||
}
|
||||
|
||||
@Post(':id/pause')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async pause(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesController - pause', { id });
|
||||
const metadata = PackTheMetadata(user);
|
||||
return this.pipelinesClientService.pause(id, metadata);
|
||||
}
|
||||
|
||||
@Post(':id/unpause')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async unpause(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesController - unpause', { id });
|
||||
const metadata = PackTheMetadata(user);
|
||||
return this.pipelinesClientService.unpause(id, metadata);
|
||||
}
|
||||
|
||||
@Post(':id/restart')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async restart(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesController - restart', { id });
|
||||
const metadata = PackTheMetadata(user);
|
||||
return this.pipelinesClientService.restart(id, metadata);
|
||||
}
|
||||
|
||||
@Post(':id/jobs/:jobId/reset-state')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async resetJobState(
|
||||
@Param('id') id: string,
|
||||
@Param('jobId') jobId: string,
|
||||
@Body() body: { schedule_minutes?: number },
|
||||
@User() user: RequestUser,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesController - resetJobState', { id, jobId });
|
||||
const metadata = PackTheMetadata(user);
|
||||
return this.pipelinesClientService.resetJobState(
|
||||
id,
|
||||
jobId,
|
||||
body?.schedule_minutes,
|
||||
metadata,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,6 +160,62 @@ export class PipelinesService implements OnModuleInit {
|
||||
return findOnePipelineResponse;
|
||||
}
|
||||
|
||||
// CDC lifecycle operations (Kafka Connect backed).
|
||||
async pause(
|
||||
id: string,
|
||||
metadata,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesClientService - Pause');
|
||||
return lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2Pause({ id }, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async unpause(
|
||||
id: string,
|
||||
metadata,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesClientService - Unpause');
|
||||
return lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2Unpause({ id }, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async restart(
|
||||
id: string,
|
||||
metadata,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesClientService - Restart');
|
||||
return lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2Restart({ id }, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async resetJobState(
|
||||
id: string,
|
||||
job_id: string,
|
||||
schedule_minutes: number | undefined,
|
||||
metadata,
|
||||
): Promise<Messages.PipelineV2OperationResponse> {
|
||||
this.logger.info('PipelinesClientService - ResetJobState');
|
||||
return lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2ResetJobState(
|
||||
{ id, job_id, schedule_minutes },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async getLiveStatus(
|
||||
id: string,
|
||||
metadata,
|
||||
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
|
||||
this.logger.info('PipelinesClientService - GetLiveStatus');
|
||||
return lastValueFrom(
|
||||
this.pipelineReadService.PipelineV2GetLiveStatus({ id }, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async update(
|
||||
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
|
||||
metadata,
|
||||
|
||||
Reference in New Issue
Block a user