gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-08-31 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(cdc): expose CDC routes over REST (on main + beta protospack)

Browse Source 
Re-applied onto fresh origin/main. pipelinesV2: live-status/pause/unpause/
restart/reset-state routes (per-route @RequireSomePermission — GET for status,
UPDATE for mutations — matching main's current auth convention). connection-test:
POST /connection-test/cdc-prerequisites. inputs: POST /inputs/cdc -> InputCreateCdc.
Consumes protospack 3.40.0-beta.15-cdc.0 tarball; CI guard added.

Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
Rafael
2026-08-15 19:45:09 -03:00
co-authored by WOZCODE
parent 7051b21d86
commit a597c41070
13 changed files with 790 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test.yml
+12
View File
@@ -2,9 +2,21 @@ name: Test
on:
pull_request:
branches:
- beta
- main
jobs:
# Blocks a local (file:/tarball/overlay) protospack-v2 dependency from
# reaching staging (beta) or prod (main).
protospack-dep-guard:
if: github.base_ref == 'beta' || github.base_ref == 'main'
runs-on: [self-hosted, prd]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Check protospack-v2 is consumed from the registry
run: node scripts/check-protospack-dep.js
test:
runs-on: [self-hosted, prd]
env:
docsfera.json
+437
View File
@@ -3001,6 +3001,45 @@
]
}
},
"/inputs/cdc": {
"post": {
"operationId": "InputsController_createCdc",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateCdcInputReq"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Input"
}
}
}
},
"201": {
"description": ""
}
},
"tags": [
"Inputs"
],
"security": [
{
"access-token": []
}
]
}
},
"/inputs/{id}": {
"get": {
"operationId": "InputsController_findOne",
@@ -3960,6 +3999,264 @@
]
}
},
"/pipelinesV2/{id}/live-status": {
"get": {
"operationId": "PipelinesController_getLiveStatus",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/pipelinesV2/{id}/pause": {
"post": {
"operationId": "PipelinesController_pause",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/pipelinesV2/{id}/unpause": {
"post": {
"operationId": "PipelinesController_unpause",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/pipelinesV2/{id}/restart": {
"post": {
"operationId": "PipelinesController_restart",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/pipelinesV2/{id}/jobs/{jobId}/reset-state": {
"post": {
"operationId": "PipelinesController_resetJobState",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/transformations": {
"post": {
"operationId": "TransformationsController_create",
@@ -7361,6 +7658,45 @@
]
}
},
"/connection-test/cdc-prerequisites": {
"post": {
"operationId": "ConnectionTestController_validateCdcPrerequisites",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ValidateCdcPrerequisitesReq"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ValidateCdcPrerequisitesRes"
}
}
}
}
},
"tags": [
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/oauth/hubspot": {
"get": {
"operationId": "OauthController_oauthHubspot",
@@ -10504,6 +10840,51 @@
"updated_at"
]
},
"CdcTableReq": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"table_schema": {
"type": "string"
},
"primary_keys": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"name"
]
},
"CreateCdcInputReq": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"plugin": {
"type": "string"
},
"tables": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CdcTableReq"
}
},
"read_only": {
"type": "boolean"
}
},
"required": [
"name",
"plugin",
"tables"
]
},
"ICreatePipelineV2Req": {
"type": "object",
"properties": {
@@ -11730,6 +12111,62 @@
"tables_metadata"
]
},
"ValidateCdcPrerequisitesReq": {
"type": "object",
"properties": {
"plugin": {
"type": "string"
},
"connection_id": {
"type": "string"
}
},
"required": [
"plugin",
"connection_id"
]
},
"CdcCheckDto": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"expected": {
"type": "string"
},
"actual": {
"type": "string"
},
"passed": {
"type": "boolean"
}
},
"required": [
"name",
"expected",
"actual",
"passed"
]
},
"ValidateCdcPrerequisitesRes": {
"type": "object",
"properties": {
"operation_result": {
"type": "boolean"
},
"checks": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CdcCheckDto"
}
}
},
"required": [
"operation_result",
"checks"
]
},
"INote": {
"type": "object",
"properties": {
package-lock.json
Generated
+5 -4
View File
@@ -16,7 +16,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.10",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.40.0-beta.15-cdc.0.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,10 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
"integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
"version": "3.40.0-beta.15-cdc.0",
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.40.0-beta.15-cdc.0.tgz",
"integrity": "sha512-ySucTiWmZ9eiw/uujSY8MFaj2+V/J+42y54P737PlrIXJgTSlLZalhwyKNkHuwo1Az8Uj5gC1J9AN4/sn9mIyg==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
package.json
+1 -1
View File
@@ -34,7 +34,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.10",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.40.0-beta.15-cdc.0.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
scripts/check-protospack-dep.js
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env node
/*
* CI guard: fail if @dadosfera/protospack-v2 is consumed from a LOCAL ref
* (file:/link:/git/relative path/bare tarball) instead of the CodeArtifact
* registry.
*
* Only local consumption is blocked. Versions published to CodeArtifact —
* including alpha/beta/rc prereleases produced by the alpha/beta branches —
* are fine; those resolve to a registry URL in the lockfile. The thing that
* must NOT reach beta (staging) or main (prod) is a dependency wired to a
* local `npm pack` tarball / overlay. Runs in the PR test workflow for PRs
* targeting beta/main and exits non-zero on any local ref.
*/
const fs = require('fs');
const path = require('path');
const PKG = '@dadosfera/protospack-v2';
const root = path.resolve(__dirname, '..');
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
const problems = [];
// A dependency SPEC is local if it's a filesystem path, symlink, git ref, or a
// bare tarball path. A plain semver (incl. prereleases like 3.35.0-beta.1)
// resolves from the registry and is allowed.
function isLocalSpec(spec) {
return /^(file:|link:|git[:+]|\.\.?\/|\/|~\/)/.test(spec) || spec.endsWith('.tgz');
}
const spec =
(pkg.dependencies && pkg.dependencies[PKG]) ||
(pkg.devDependencies && pkg.devDependencies[PKG]);
if (!spec) {
problems.push(`${PKG} is not listed as a dependency at all.`);
} else if (isLocalSpec(spec)) {
problems.push(`${PKG} points at a local path/tarball/git ref: "${spec}".`);
}
// Also catch a lockfile resolved to a LOCAL ref even if package.json looks
// clean. A registry URL (https://.../-/*.tgz) is the normal published
// resolution and is fine — only file: refs and bare local tarball paths
// (no http host) are blocked. Prerelease VERSIONS are not flagged: an
// alpha/beta/rc published to CodeArtifact resolves to a registry URL.
const lockPath = path.join(root, 'package-lock.json');
if (fs.existsSync(lockPath)) {
const lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
const nodes = { ...(lock.packages || {}), ...(lock.dependencies || {}) };
for (const [name, node] of Object.entries(nodes)) {
if (!name.includes('protospack-v2') || !node) continue;
const resolved = node.resolved || '';
const isLocal =
resolved.startsWith('file:') ||
(resolved.endsWith('.tgz') && !/^https?:\/\//.test(resolved));
if (isLocal) {
problems.push(
`package-lock.json resolves ${PKG} to a local ref: "${resolved}".`,
);
}
}
}
if (problems.length) {
console.error('✗ protospack-v2 dependency guard FAILED:');
for (const p of problems) console.error(' - ' + p);
console.error(
'\nMerging to beta/main requires ' +
PKG +
' to come from CodeArtifact, not a local tarball/overlay. Publish ' +
'protospack-v2 (a beta prerelease is fine for the beta branch) and ' +
'repoint this dependency before merging.',
);
process.exit(1);
}
console.log(`✓ ${PKG} is consumed from the registry: "${spec}"`);
src/modules/connection-test/connection-test.controller.ts
+19
View File
@@ -22,6 +22,8 @@ import {
ConnectionTestListTablesRes,
GetTableMetadataRes,
GetTableMetadataReq,
ValidateCdcPrerequisitesReq,
ValidateCdcPrerequisitesRes,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
@@ -125,4 +127,21 @@ export class ConnectionTestController {
user.customer_name,
);
}
@Post('cdc-prerequisites')
@ApiOkResponse({ type: ValidateCdcPrerequisitesRes })
@HttpCode(HttpStatus.OK)
async validateCdcPrerequisites(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: ValidateCdcPrerequisitesReq,
) {
this.logger.info('/connection-test/cdc-prerequisites', {
user: user.user_id,
customer: user.customer_name,
});
return this.connectionTestService.validateCdcPrerequisites(
body,
user.customer_name,
);
}
}
src/modules/connection-test/connection-test.service.ts
+16
View File
@@ -13,6 +13,8 @@ import {
ConnectionTestPingRes,
GetTableMetadataReq,
GetTableMetadataRes,
ValidateCdcPrerequisitesReq,
ValidateCdcPrerequisitesRes,
} from './dto/connection-test';
import { ConnectionClientService } from '../connection/client.service';
import {
@@ -188,4 +190,18 @@ export class ConnectionTestService {
}),
);
}
async validateCdcPrerequisites(
body: ValidateCdcPrerequisitesReq,
customer_name: string,
): Promise<ValidateCdcPrerequisitesRes> {
const { plugin, connection_id } = body;
return lastValueFrom(
this.connectionTestReadClient.ValidateCdcPrerequisites({
connection_id,
customer_name,
plugin,
}),
);
}
}
src/modules/connection-test/dto/connection-test.ts
+28
View File
@@ -131,3 +131,31 @@ export class GetTableMetadataRes {
@ApiProperty({ type: [TableMetadataDto] })
tables_metadata: TableMetadataDto[];
}
export class CdcCheckDto {
@ApiProperty()
name: string;
@ApiProperty()
expected: string;
@ApiProperty()
actual: string;
@ApiProperty()
passed: boolean;
}
export class ValidateCdcPrerequisitesReq {
@ApiProperty()
@IsString()
plugin: string;
@ApiProperty()
@IsString()
connection_id: string;
}
export class ValidateCdcPrerequisitesRes {
@ApiProperty()
operation_result: boolean;
@ApiProperty({ type: [CdcCheckDto] })
checks: CdcCheckDto[];
}
src/modules/inputs/dtos/input.model.ts
+20
View File
@@ -65,3 +65,23 @@ export class CreateInputReq extends OmitType(Input, [
'created_at',
'updated_at',
]) {}
export class CdcTableReq {
@ApiProperty()
name: string;
@ApiPropertyOptional()
table_schema?: string;
@ApiPropertyOptional({ type: [String] })
primary_keys?: string[];
}
export class CreateCdcInputReq {
@ApiProperty()
name: string;
@ApiProperty()
plugin: string; // mysql_cdc (v1)
@ApiProperty({ type: [CdcTableReq] })
tables: CdcTableReq[];
@ApiPropertyOptional()
read_only?: boolean;
}
src/modules/inputs/inputs.controller.ts
+21
View File
@@ -15,6 +15,7 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import {
CreateCdcInputReq,
CreateInputReq,
GetAvailableEntitiesReq,
GetAvailableEntitiesRes,
@@ -105,6 +106,26 @@ export class InputsController {
return response;
}
@Post('cdc')
@ApiInternalOnlyEndpoint()
@ApiOkResponse({ type: Input })
async createCdc(
@Body() body: CreateCdcInputReq,
@User() user: RequestUser,
) {
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
customer_id: user.customer_id,
};
this.logger.info(`/inputs/cdc - ON CREATE CDC INPUT ROUTE`, {
user: info.user_id,
customer: info.customer,
});
return this.inputService.createCdc({ body, info });
}
@ApiInternalOnlyEndpoint()
@Get()
async findAll(@User() user: RequestUser) {
src/modules/inputs/inputs.service.ts
+25 -1
View File
@@ -15,6 +15,7 @@ import { Input } from '@dadosfera/protospack-v2';
import {
GetAvailableEntitiesRequest,
InputCreateGenericRequest,
InputCreateCdcRequest,
InputCreateS3Request,
InputNewCreateRequest,
InputUpdateResponse,
@@ -22,7 +23,7 @@ import {
TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { CreateInputReq } from './dtos/input.model';
import { CreateCdcInputReq, CreateInputReq } from './dtos/input.model';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
@@ -183,6 +184,29 @@ export class InputsService {
return { input: adjustedInput };
}
async createCdc(data: { body: CreateCdcInputReq; info: Info }) {
const { body, info } = data;
const inputCreateCdcRequest: InputCreateCdcRequest = {
input: {
name: body.name,
plugin: body.plugin,
read_only: body.read_only ?? true,
tables: body.tables.map((t) => ({
table_schema: t.table_schema,
table_name: t.name,
primary_keys: t.primary_keys ?? [],
})),
},
info,
};
const { input } = await lastValueFrom(
this.inputWriteService.InputCreateCdc(inputCreateCdcRequest),
);
return { input };
}
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
}
src/modules/pipelinesV2/pipelines.controller.ts
+74
View File
@@ -510,4 +510,78 @@ export class PipelinesController {
return response;
}
// ---- CDC pipeline operations (Kafka Connect backed) ----
// These operate on an existing pipeline, so they require UPDATE (not CREATE).
@Get(':id/live-status')
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getLiveStatus(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
this.logger.info('PipelinesController - getLiveStatus', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.getLiveStatus(id, metadata);
}
@Post(':id/pause')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pause(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - pause', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.pause(id, metadata);
}
@Post(':id/unpause')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpause(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - unpause', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.unpause(id, metadata);
}
@Post(':id/restart')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async restart(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - restart', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.restart(id, metadata);
}
@Post(':id/jobs/:jobId/reset-state')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async resetJobState(
@Param('id') id: string,
@Param('jobId') jobId: string,
@Body() body: { schedule_minutes?: number },
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - resetJobState', { id, jobId });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.resetJobState(
id,
jobId,
body?.schedule_minutes,
metadata,
);
}
}
src/modules/pipelinesV2/pipelines.service.ts
+56
View File
@@ -160,6 +160,62 @@ export class PipelinesService implements OnModuleInit {
return findOnePipelineResponse;
}
// CDC lifecycle operations (Kafka Connect backed).
async pause(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Pause');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Pause({ id }, metadata),
);
}
async unpause(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Unpause');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Unpause({ id }, metadata),
);
}
async restart(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Restart');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Restart({ id }, metadata),
);
}
async resetJobState(
id: string,
job_id: string,
schedule_minutes: number | undefined,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - ResetJobState');
return lastValueFrom(
this.pipelineWriteService.PipelineV2ResetJobState(
{ id, job_id, schedule_minutes },
metadata,
),
);
}
async getLiveStatus(
id: string,
metadata,
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
this.logger.info('PipelinesClientService - GetLiveStatus');
return lastValueFrom(
this.pipelineReadService.PipelineV2GetLiveStatus({ id }, metadata),
);
}
async update(
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
metadata,