Compare commits

..
43 changed files with 425 additions and 2708 deletions
-12
View File
@@ -2,21 +2,9 @@ name: Test
on:
pull_request:
branches:
- beta
- main
jobs:
# Blocks a local (file:/tarball/overlay) protospack-v2 dependency from
# reaching staging (beta) or prod (main).
protospack-dep-guard:
if: github.base_ref == 'beta' || github.base_ref == 'main'
runs-on: [self-hosted, prd]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Check protospack-v2 is consumed from the registry
run: node scripts/check-protospack-dep.js
test:
runs-on: [self-hosted, prd]
env:
+1 -1
View File
@@ -1,5 +1,5 @@
FROM node:20-alpine AS base_image
RUN npm install -g npm@10.8.2
RUN npm install -g npm@latest
FROM base_image AS build_base
WORKDIR /app
+53 -990
View File
File diff suppressed because it is too large Load Diff
+14 -4
View File
@@ -16,7 +16,8 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.8",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1734,10 +1735,19 @@
"winston-log2gelf": "^2.4.0"
}
},
"node_modules/@dadosfera/protospack": {
"version": "2.5.3",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
"license": "ISC",
"dependencies": {
"rxjs": "^7.5.5"
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.41.0-cdc-iceberg.4",
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
"integrity": "sha512-QcRhM2wHgoRgXbHfpu+ki+Iozbz7qunKD544v2DCdD1fbeuI6kEit/S4napquSzmzP2qcMjANobyAsWabC/f2w==",
"version": "3.40.0-beta.8",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz",
"integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
+2 -1
View File
@@ -34,7 +34,8 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-3.41.0-cdc-iceberg.4.tgz",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.8",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
-76
View File
@@ -1,76 +0,0 @@
#!/usr/bin/env node
/*
* CI guard: fail if @dadosfera/protospack-v2 is consumed from a LOCAL ref
* (file:/link:/git/relative path/bare tarball) instead of the CodeArtifact
* registry.
*
* Only local consumption is blocked. Versions published to CodeArtifact —
* including alpha/beta/rc prereleases produced by the alpha/beta branches —
* are fine; those resolve to a registry URL in the lockfile. The thing that
* must NOT reach beta (staging) or main (prod) is a dependency wired to a
* local `npm pack` tarball / overlay. Runs in the PR test workflow for PRs
* targeting beta/main and exits non-zero on any local ref.
*/
const fs = require('fs');
const path = require('path');
const PKG = '@dadosfera/protospack-v2';
const root = path.resolve(__dirname, '..');
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
const problems = [];
// A dependency SPEC is local if it's a filesystem path, symlink, git ref, or a
// bare tarball path. A plain semver (incl. prereleases like 3.35.0-beta.1)
// resolves from the registry and is allowed.
function isLocalSpec(spec) {
return /^(file:|link:|git[:+]|\.\.?\/|\/|~\/)/.test(spec) || spec.endsWith('.tgz');
}
const spec =
(pkg.dependencies && pkg.dependencies[PKG]) ||
(pkg.devDependencies && pkg.devDependencies[PKG]);
if (!spec) {
problems.push(`${PKG} is not listed as a dependency at all.`);
} else if (isLocalSpec(spec)) {
problems.push(`${PKG} points at a local path/tarball/git ref: "${spec}".`);
}
// Also catch a lockfile resolved to a LOCAL ref even if package.json looks
// clean. A registry URL (https://.../-/*.tgz) is the normal published
// resolution and is fine — only file: refs and bare local tarball paths
// (no http host) are blocked. Prerelease VERSIONS are not flagged: an
// alpha/beta/rc published to CodeArtifact resolves to a registry URL.
const lockPath = path.join(root, 'package-lock.json');
if (fs.existsSync(lockPath)) {
const lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
const nodes = { ...(lock.packages || {}), ...(lock.dependencies || {}) };
for (const [name, node] of Object.entries(nodes)) {
if (!name.includes('protospack-v2') || !node) continue;
const resolved = node.resolved || '';
const isLocal =
resolved.startsWith('file:') ||
(resolved.endsWith('.tgz') && !/^https?:\/\//.test(resolved));
if (isLocal) {
problems.push(
`package-lock.json resolves ${PKG} to a local ref: "${resolved}".`,
);
}
}
}
if (problems.length) {
console.error('✗ protospack-v2 dependency guard FAILED:');
for (const p of problems) console.error(' - ' + p);
console.error(
'\nMerging to beta/main requires ' +
PKG +
' to come from CodeArtifact, not a local tarball/overlay. Publish ' +
'protospack-v2 (a beta prerelease is fine for the beta branch) and ' +
'repoint this dependency before merging.',
);
process.exit(1);
}
console.log(`✓ ${PKG} is consumed from the registry: "${spec}"`);
+2
View File
@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
import { NetworkConfigModule } from './modules/network-config/network-config.module';
import { InputsModule } from './modules/inputs/inputs.module';
import { OauthModule } from './modules/oauth/oauth.module';
import { PipelinesModule } from './modules/pipelines/pipelines.module';
import { TransformationsModule } from './modules/transformations/transformations.module';
import { HealthModule } from './modules/health/health.module';
import { CatalogModule } from './modules/catalog/catalog.module';
@@ -59,6 +60,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
PermissionsModule,
TermsOfUseModule,
ConnectionTestModule,
PipelinesModule,
TransformationsModule,
UsersModule,
RolesModule,
-12
View File
@@ -357,16 +357,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Crear y editar atributos en el catálogo',
},
},
CERTIFY: {
seqid: 53,
claim: 'catalog:certify',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Alterar o status de certificação dos Ativos',
'en-us': "Change Assets' certification status",
'es-es': 'Cambiar el estado de certificación de los Activos',
},
},
DELETE: {
seqid: 1,
claim: 'catalog:delete',
@@ -722,8 +712,6 @@ export const DADOSFERA_MODULES_KEYS = {
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
CATALOG: 'catalog',
COLLECT: 'collect',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
@@ -1,71 +0,0 @@
import { BadRequestException } from '@nestjs/common';
import { PipelineExecutionGuard } from './pipeline-execution.guard';
const logger = { info: jest.fn(), error: jest.fn() };
function buildGuard(proxyImpl: jest.Mock) {
const platformApiService: any = { proxy: proxyImpl };
return new PipelineExecutionGuard(
{ logger } as any,
platformApiService,
);
}
function contextWith(pipelineId = 'abc-123') {
return {
switchToHttp: () => ({
getRequest: () => ({ params: { pipelineId }, user: {} }),
}),
} as any;
}
describe('PipelineExecutionGuard', () => {
afterEach(() => jest.clearAllMocks());
it('allows the edit when the pipeline has no run history (empty array)', async () => {
const guard = buildGuard(jest.fn().mockResolvedValue([]));
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
});
it('allows the edit when the last run has no last_status', async () => {
const guard = buildGuard(jest.fn().mockResolvedValue([{}]));
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
});
it('allows the edit when the pipeline is not running', async () => {
const guard = buildGuard(
jest.fn().mockResolvedValue([{ last_status: 'SUCCEEDED' }]),
);
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
});
it('blocks with the is-running message when the pipeline is running', async () => {
const guard = buildGuard(
jest.fn().mockResolvedValue([{ last_status: 'RUNNING' }]),
);
await expect(guard.canActivate(contextWith())).rejects.toThrow(
'Pipeline is running, cannot update input now',
);
});
it('wraps a genuine status-check failure (fail closed)', async () => {
const guard = buildGuard(
jest.fn().mockRejectedValue(new Error('platform down')),
);
await expect(guard.canActivate(contextWith())).rejects.toThrow(
'Error checking pipeline status: platform down',
);
});
it('does not double-wrap the is-running BadRequestException', async () => {
const guard = buildGuard(
jest.fn().mockResolvedValue([{ last_status: 'running' }]),
);
await expect(guard.canActivate(contextWith())).rejects.toBeInstanceOf(
BadRequestException,
);
await expect(guard.canActivate(contextWith())).rejects.not.toThrow(
/Error checking pipeline status/,
);
});
});
+2 -13
View File
@@ -46,16 +46,10 @@ export class PipelineExecutionGuard implements CanActivate {
user,
);
const currentStatus = status?.[status.length - 1];
const currentStatus = status[status.length - 1]
this.logger.info('Pipeline current status response:' + JSON.stringify(currentStatus));
// No run history (e.g. CDC pipelines never record batch runs) means
// nothing is executing — allow the edit rather than crash on .last_status.
if (!currentStatus?.last_status) {
return true;
}
if (currentStatus.last_status.toLowerCase() === 'running') {
this.logger.error('Pipeline is running, cannot update input now');
throw new BadRequestException('Pipeline is running, cannot update input now');
@@ -63,11 +57,6 @@ export class PipelineExecutionGuard implements CanActivate {
return true;
}
} catch (error) {
// Preserve the deliberate is-running rejection; only wrap genuine
// status-check failures (fail closed on those for a destructive gate).
if (error instanceof BadRequestException) {
throw error;
}
this.logger.error('Error in PipelineExecutionGuard: ' + error.message);
throw new BadRequestException('Error checking pipeline status: ' + error.message);
}
-85
View File
@@ -17,7 +17,6 @@ import {
HttpStatus,
Res,
} from '@nestjs/common';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
ApiCreatedResponse,
ApiHeaders,
@@ -47,7 +46,6 @@ import {
IMakeAComment,
IOneDataAsset,
IPreviewResponse,
IUpdateCertificationStatusRequest,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
@@ -85,9 +83,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async searchCatalog(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@@ -127,9 +122,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@@ -173,9 +165,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Get('data-asset')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query;
@@ -234,9 +223,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
@@ -288,9 +274,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id: string,
@@ -402,9 +385,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -436,9 +416,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -470,9 +447,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -504,9 +478,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAsset(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -522,8 +493,6 @@ export class CatalogController {
language,
});
delete (body as any).certification_status;
const result = await this.catalogService.updateOneDataAsset({
body,
data_asset_id,
@@ -537,44 +506,11 @@ export class CatalogController {
return result;
}
@Put('data-asset/:id/certification-status')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAssetCertificationStatus(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') data_asset_id: string,
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
): Promise<IUpdateCertificationStatusRequest> {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
return this.catalogService.updateCertificationStatus({
body,
data_asset_id,
metadata,
});
}
@Post('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetDocs(
@User() user: RequestUser,
@Headers() headers,
@@ -612,9 +548,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@@ -637,9 +570,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/revoke-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async revokeDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@@ -665,9 +595,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async createDataAsset(
@User() user: RequestUser,
@Body() body: ICreateDataAsset,
@@ -692,9 +619,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async commentOnDataAsset(
@Param('id') id: string,
@User() user: RequestUser,
@@ -721,9 +645,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
@@ -745,9 +666,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteComment(
@Param('id') id: string,
@User() user: RequestUser,
@@ -901,9 +819,6 @@ export class CatalogController {
@Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getNimbusDashboards(
@User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest,
+3
View File
@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
-23
View File
@@ -29,7 +29,6 @@ import {
AssetReporter,
BatchRemoveRlsRulesRequest,
CreateDataDocsDTO,
IUpdateCertificationStatusRequest,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
@@ -385,28 +384,6 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] };
}
async updateCertificationStatus(data: {
data_asset_id: string;
body: IUpdateCertificationStatusRequest;
metadata: Metadata;
}) {
const { body, data_asset_id, metadata } = data;
await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{
id: data_asset_id,
changes: JSON.stringify({
certification_status: body.certification_status,
}),
},
metadata,
),
);
return { certification_status: body.certification_status };
}
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
-14
View File
@@ -1,5 +1,4 @@
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
import { IsEnum } from 'class-validator';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export enum DataAssetShareType {
@@ -7,12 +6,6 @@ export enum DataAssetShareType {
public = 'public',
private = 'private',
}
export enum CertificationStatus {
draft = 'draft',
in_review = 'in_review',
approved = 'approved',
deprecated = 'deprecated',
}
export enum OrderEnum {
asc = 'asc',
desc = 'desc',
@@ -212,13 +205,6 @@ export class IUpdateDataRequest {
@ApiPropertyOptional()
docs?: string;
}
export class IUpdateCertificationStatusRequest {
@ApiProperty({ enum: CertificationStatus })
@IsEnum(CertificationStatus)
certification_status: CertificationStatus;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
@@ -22,23 +22,17 @@ import {
ConnectionTestListTablesRes,
GetTableMetadataRes,
GetTableMetadataReq,
ValidateCdcPrerequisitesReq,
ValidateCdcPrerequisitesRes,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
@ApiInternalOnlyController()
@ApiTags('Connection Test')
@Controller('connection-test')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionTestController {
logger: any;
constructor(
@@ -127,21 +121,4 @@ export class ConnectionTestController {
user.customer_name,
);
}
@Post('cdc-prerequisites')
@ApiOkResponse({ type: ValidateCdcPrerequisitesRes })
@HttpCode(HttpStatus.OK)
async validateCdcPrerequisites(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: ValidateCdcPrerequisitesReq,
) {
this.logger.info('/connection-test/cdc-prerequisites', {
user: user.user_id,
customer: user.customer_name,
});
return this.connectionTestService.validateCdcPrerequisites(
body,
user.customer_name,
);
}
}
@@ -13,8 +13,6 @@ import {
ConnectionTestPingRes,
GetTableMetadataReq,
GetTableMetadataRes,
ValidateCdcPrerequisitesReq,
ValidateCdcPrerequisitesRes,
} from './dto/connection-test';
import { ConnectionClientService } from '../connection/client.service';
import {
@@ -190,18 +188,4 @@ export class ConnectionTestService {
}),
);
}
async validateCdcPrerequisites(
body: ValidateCdcPrerequisitesReq,
customer_name: string,
): Promise<ValidateCdcPrerequisitesRes> {
const { plugin, connection_id } = body;
return lastValueFrom(
this.connectionTestReadClient.ValidateCdcPrerequisites({
connection_id,
customer_name,
plugin,
}),
);
}
}
@@ -100,20 +100,11 @@ export class ConnectionTestListTablesReq {
schema: string;
}
export class ConnectionTestListTablesEntry {
@ApiProperty()
table_name: string;
@ApiProperty({ type: [String] })
primary_keys: string[];
}
export class ConnectionTestListTablesRes {
@ApiProperty()
operation_result: boolean;
@ApiProperty()
table_list: string[];
@ApiProperty({ type: [ConnectionTestListTablesEntry] })
tables: ConnectionTestListTablesEntry[];
}
export class GetTableMetadataReq {
@@ -140,31 +131,3 @@ export class GetTableMetadataRes {
@ApiProperty({ type: [TableMetadataDto] })
tables_metadata: TableMetadataDto[];
}
export class CdcCheckDto {
@ApiProperty()
name: string;
@ApiProperty()
expected: string;
@ApiProperty()
actual: string;
@ApiProperty()
passed: boolean;
}
export class ValidateCdcPrerequisitesReq {
@ApiProperty()
@IsString()
plugin: string;
@ApiProperty()
@IsString()
connection_id: string;
}
export class ValidateCdcPrerequisitesRes {
@ApiProperty()
operation_result: boolean;
@ApiProperty({ type: [CdcCheckDto] })
checks: CdcCheckDto[];
}
@@ -16,9 +16,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
} from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
@@ -40,9 +39,6 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@ApiTags('connections')
@Authenticated()
@Controller('connections')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionController {
logger: any;
constructor(
+1 -26
View File
@@ -25,10 +25,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -100,9 +99,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getAllConnectors(
@Language() language: LanguageEnum,
@Query() queries: GetAllDto,
@@ -135,9 +131,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorsTags() {
return await this.connectorClientService.getConnectorsTags();
}
@@ -150,9 +143,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnector(
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@@ -181,9 +171,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorDetails(
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@@ -206,9 +193,6 @@ export class ConnectorController {
@Put('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@ApiConsumes('multipart/form-data')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async updateConnector(
@Param('plugin') plugin: string,
@Body() body: UpdateDto,
@@ -230,9 +214,6 @@ export class ConnectorController {
@Put('/:plugin/add-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async addTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: AddTagDto,
@@ -260,9 +241,6 @@ export class ConnectorController {
@Put('/:plugin/remove-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async removeTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: RemoveTagDto,
@@ -291,9 +269,6 @@ export class ConnectorController {
@Delete('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async deleteConnector(
@Param('plugin') plugin: string,
@Query('version') version: string,
-59
View File
@@ -65,62 +65,3 @@ export class CreateInputReq extends OmitType(Input, [
'created_at',
'updated_at',
]) {}
export class CdcColumnReq {
@ApiProperty()
name: string;
@ApiProperty()
type: string;
@ApiProperty()
is_primary_key: boolean;
}
export class CdcTableReq {
@ApiProperty()
name: string;
@ApiPropertyOptional()
table_schema?: string;
@ApiPropertyOptional({ type: [String] })
primary_keys?: string[];
// Per-table raw Iceberg table name override (iceberg destination only).
// Honored on the create/add path: the platform lowercases + sanitizes it
// authoritatively; empty/absent => the platform derives tb__<hash>__<table>.
@ApiPropertyOptional()
iceberg_table_name?: string;
// Per-table deduped (qualify) Iceberg table name override (iceberg dest only).
// Empty/absent => the deduped table takes the same name as the raw table.
@ApiPropertyOptional()
iceberg_qualify_table_name?: string;
@ApiPropertyOptional({ type: [CdcColumnReq] })
columns?: CdcColumnReq[];
// Columns the user chose to ignore -> Debezium column.exclude.list.
@ApiPropertyOptional({ type: [String] })
column_exclude_list?: string[];
}
export class IcebergDestinationReq {
@ApiProperty()
namespace: string;
// Pipeline-wide deduped (qualify) namespace. Absent => the platform derives
// the sibling of `namespace` (cdc_raw -> cdc_dedup).
@ApiPropertyOptional()
qualify_namespace?: string;
}
export class CdcDestinationReq {
@ApiPropertyOptional({ type: IcebergDestinationReq })
iceberg?: IcebergDestinationReq;
}
export class CreateCdcInputReq {
@ApiProperty()
name: string;
@ApiProperty()
plugin: string; // mysql_cdc (v1)
@ApiProperty({ type: [CdcTableReq] })
tables: CdcTableReq[];
@ApiPropertyOptional()
read_only?: boolean;
@ApiPropertyOptional({ type: CdcDestinationReq })
destination?: CdcDestinationReq;
}
+1 -5
View File
@@ -1,8 +1,4 @@
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
interface Values {
jdbc_user: string;
@@ -1,75 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { InputsController } from './inputs.controller';
import { InputsService } from './inputs.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { CreateCdcInputReq } from './dtos/input.model';
import { RequestUser } from 'src/decorators/user.decorator';
describe('InputsController', () => {
let controller: InputsController;
let inputsService: { createCdc: jest.Mock };
beforeEach(async () => {
inputsService = {
createCdc: jest.fn().mockResolvedValue({ input: {} }),
};
const module: TestingModule = await Test.createTestingModule({
controllers: [InputsController],
providers: [
{
provide: DadosferaLogger,
useValue: { logger: { info: jest.fn() } },
},
{
provide: InputsService,
useValue: inputsService,
},
],
}).compile();
controller = module.get<InputsController>(InputsController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
it('forwards destination.iceberg.namespace to InputsService.createCdc', async () => {
const body: CreateCdcInputReq = {
name: 'my-cdc-input',
plugin: 'mysql_cdc',
tables: [
{
name: 'orders',
table_schema: 'public',
iceberg_table_name: 'orders_iceberg',
},
],
destination: {
iceberg: {
namespace: 'my_namespace',
},
},
};
const user: RequestUser = {
user_id: 'user-1',
customer_id: 'customer-1',
customer_name: 'customer',
} as RequestUser;
await controller.createCdc(body, user);
expect(inputsService.createCdc).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.objectContaining({
destination: {
iceberg: {
namespace: 'my_namespace',
},
},
}),
}),
);
});
});
-21
View File
@@ -15,7 +15,6 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import {
CreateCdcInputReq,
CreateInputReq,
GetAvailableEntitiesReq,
GetAvailableEntitiesRes,
@@ -106,26 +105,6 @@ export class InputsController {
return response;
}
@Post('cdc')
@ApiInternalOnlyEndpoint()
@ApiOkResponse({ type: Input })
async createCdc(
@Body() body: CreateCdcInputReq,
@User() user: RequestUser,
) {
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
customer_id: user.customer_id,
};
this.logger.info(`/inputs/cdc - ON CREATE CDC INPUT ROUTE`, {
user: info.user_id,
customer: info.customer,
});
return this.inputService.createCdc({ body, info });
}
@ApiInternalOnlyEndpoint()
@Get()
async findAll(@User() user: RequestUser) {
-113
View File
@@ -1,113 +0,0 @@
import { of } from 'rxjs';
import { InputsService } from './inputs.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { CreateCdcInputReq } from './dtos/input.model';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
const info = { customer_id: 'cid', user_id: 'u' } as unknown as Info;
describe('InputsService.createCdc', () => {
let service: InputsService;
let inputCreateCdcMock: jest.Mock;
beforeEach(async () => {
inputCreateCdcMock = jest
.fn()
.mockImplementation((req) => of({ input: req.input }));
const grpcClient: any = {
getService: jest.fn().mockReturnValue({
InputCreateCdc: inputCreateCdcMock,
}),
};
service = new InputsService(new DadosferaLogger(), grpcClient);
await service.onModuleInit();
});
it('forwards destination and per-table iceberg_table_name to the gRPC request', async () => {
const body: CreateCdcInputReq = {
name: 'CDC Iceberg Test',
plugin: 'mysql_cdc',
read_only: true,
destination: { iceberg: { namespace: 'cdc_raw' } },
tables: [
{
name: 'orders',
table_schema: 'mydb',
primary_keys: ['id'],
iceberg_table_name: 'cdc_raw.mydb__orders',
},
],
};
await service.createCdc({ body, info });
expect(inputCreateCdcMock).toHaveBeenCalledTimes(1);
const sentRequest = inputCreateCdcMock.mock.calls[0][0];
expect(sentRequest.input).toEqual(
expect.objectContaining({
destination: { iceberg: { namespace: 'cdc_raw' } },
}),
);
expect(sentRequest.input.tables[0]).toEqual(
expect.objectContaining({
iceberg_table_name: 'cdc_raw.mydb__orders',
}),
);
});
it('forwards per-table columns to the gRPC request', async () => {
const body: CreateCdcInputReq = {
name: 'CDC Columns Test',
plugin: 'mysql_cdc',
read_only: true,
tables: [
{
name: 'orders',
table_schema: 'mydb',
primary_keys: ['id'],
columns: [
{ name: 'id', type: 'int', is_primary_key: true },
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
],
},
],
};
await service.createCdc({ body, info });
expect(inputCreateCdcMock).toHaveBeenCalledTimes(1);
const sentRequest = inputCreateCdcMock.mock.calls[0][0];
expect(sentRequest.input.tables[0].columns).toEqual([
{ name: 'id', type: 'int', is_primary_key: true },
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
]);
});
it('back-compat: a body with no destination sends destination undefined, not an error', async () => {
const body: CreateCdcInputReq = {
name: 'CDC Legacy Test',
plugin: 'mysql_cdc',
read_only: true,
tables: [
{
name: 'pedidos',
table_schema: 'cadastros',
primary_keys: ['id'],
},
],
};
const result = await service.createCdc({ body, info });
expect(inputCreateCdcMock).toHaveBeenCalledTimes(1);
const sentRequest = inputCreateCdcMock.mock.calls[0][0];
expect(sentRequest.input.destination).toBeUndefined();
expect(sentRequest.input.tables[0].iceberg_table_name).toBeUndefined();
expect(result.input).toBeDefined();
});
});
+5 -44
View File
@@ -15,7 +15,6 @@ import { Input } from '@dadosfera/protospack-v2';
import {
GetAvailableEntitiesRequest,
InputCreateGenericRequest,
InputCreateCdcRequest,
InputCreateS3Request,
InputNewCreateRequest,
InputUpdateResponse,
@@ -23,8 +22,7 @@ import {
TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { CreateCdcInputReq, CreateInputReq } from './dtos/input.model';
import { Metadata } from '@grpc/grpc-js';
import { CreateInputReq } from './dtos/input.model';
@Injectable()
@@ -75,10 +73,10 @@ export class InputsService {
objectCamelToSnake(createInputResponse);
return createInputResponse;
},
update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
const updateInputResponse = await lastValueFrom(
this.inputWriteService.InputUpdate(updateInputDTO, metadata),
this.inputWriteService.InputUpdate(updateInputDTO),
);
return updateInputResponse;
@@ -184,35 +182,6 @@ export class InputsService {
return { input: adjustedInput };
}
async createCdc(data: { body: CreateCdcInputReq; info: Info }) {
const { body, info } = data;
const inputCreateCdcRequest: InputCreateCdcRequest = {
input: {
name: body.name,
plugin: body.plugin,
read_only: body.read_only ?? true,
tables: body.tables.map((t) => ({
table_schema: t.table_schema,
table_name: t.name,
name: t.name, // canonical identity == table_name (in-factory also backfills)
primary_keys: t.primary_keys ?? [],
iceberg_table_name: t.iceberg_table_name,
iceberg_qualify_table_name: t.iceberg_qualify_table_name,
columns: t.columns ?? [],
column_exclude_list: t.column_exclude_list ?? [],
})),
destination: body.destination,
},
info,
};
const { input } = await lastValueFrom(
this.inputWriteService.InputCreateCdc(inputCreateCdcRequest),
);
return { input };
}
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
}
@@ -237,7 +206,7 @@ export class InputsService {
return findOneInputResponse;
}
async update(id: string, data, info: Info, metadata?: Metadata) {
async update(id: string, data, info: Info) {
// this.validateCron({ ...data, info });
try {
const {
@@ -248,7 +217,7 @@ export class InputsService {
id,
...data,
info,
}, metadata);
});
const updateInputResponse = this.adjustInputPayload(
input,
@@ -327,12 +296,4 @@ export class InputsService {
async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data));
}
async addCdcTable(data: { client_id?: string; id: string; table: any; info: Info }) {
return lastValueFrom(this.inputWriteService.AddCdcTable(data as any));
}
async removeCdcTable(data: { client_id?: string; id: string; table_name: string; info: Info }) {
return lastValueFrom((this.inputWriteService as any).RemoveCdcTable(data));
}
}
+78
View File
@@ -0,0 +1,78 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.pipelineService =
this.grpcClient.getService<PipelinesServiceInterface>(
PipelineServicesNames.PipelineService,
);
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.getPipelineStatus(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.triggerPipeline({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
+36
View File
@@ -0,0 +1,36 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
info: Info;
}
export interface IdRequest {
id: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
id: string;
info: Info;
}
export interface IGetPipelineLogsRequest {
id: string;
details: string;
}
+33
View File
@@ -0,0 +1,33 @@
import {
ClientsProviderAsyncOptions,
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js';
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
export class PipelinesClientConfiguration {
public name = 'PipelinesClientConfiguration';
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: PipelinePackages,
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: PipelineProtoFilePath,
loader: {
keepCase: true,
enums: String,
defaults: false,
},
},
};
providerOptions: ClientsProviderAsyncOptions = {
name: this.name,
...this.config,
};
}
@@ -0,0 +1,72 @@
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelineService: PipelinesService,
) {
this.logger = dadosferaLogger.logger;
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
})
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
this.logger.info(
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.runPipeline({ id, info });
return response;
}
@Get(':id/status')
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.getPipelineStatus(body);
return response;
}
}
+19
View File
@@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesController } from './pipelines.controller';
import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesClientService } from './client.service';
const client = new PipelinesClientConfiguration();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [PipelinesController],
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
exports: [PipelinesService],
})
export class PipelinesModule {}
@@ -0,0 +1,33 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { PipelinesClientService } from './client.service';
import { IIdRequest } from './interfaces';
import { objectCamelToSnake } from 'src/utils/CaseConverter';
@Injectable()
export class PipelinesService {
constructor(private pipelineClient: PipelinesClientService) {}
async getPipelineStatus(data: IIdRequest) {
try {
const pipelineStatusResponse =
await this.pipelineClient.getPipelineStatus(data);
return objectCamelToSnake(pipelineStatusResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async runPipeline({ id, info }: IIdRequest) {
try {
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
id,
info,
});
return objectCamelToSnake(triggerPipelineResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
}
+1 -8
View File
@@ -1,4 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class PipelineInputsDTO {
@ApiProperty()
@@ -31,8 +32,6 @@ export class IPipelineV2 {
tags?: string[];
@ApiPropertyOptional()
properties?: any;
@ApiPropertyOptional()
config?: any;
@ApiProperty()
connector_name: string;
@@ -63,12 +62,6 @@ export interface IIdRequest {
info: Info;
}
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
+14 -97
View File
@@ -25,16 +25,16 @@ import {
} from '@nestjs/swagger';
import {
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
ICompleteUploadCSVFile,
ICreatePipelineCSVFile,
@@ -58,15 +58,14 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelinesClientService: PipelinesService,
private oldPipelinesService: OldPipelineService,
) {
this.logger = dadosferaLogger.logger;
}
@@ -202,7 +201,7 @@ export class PipelinesController {
customer: body.info.customer,
});
const response = await this.pipelinesClientService.getPipelineStatus(body);
const response = await this.oldPipelinesService.getPipelineStatus(body);
return response;
}
@@ -321,6 +320,7 @@ export class PipelinesController {
) {
this.logger.info('PipelinesController - update', { user });
const { customer_id, customer_name, user_id, username } = user;
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
@@ -328,7 +328,13 @@ export class PipelinesController {
pipeline_id: pipelineId
};
const metadata = PackTheMetadata(user);
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const response = await this.pipelinesClientService.updatePipelineInput(
pipelineId,
@@ -362,21 +368,6 @@ export class PipelinesController {
return response;
}
@Patch('/:id/upgrade')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@HttpCode(HttpStatus.NO_CONTENT)
async upgradeConnector(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
) {
this.logger.info('PipelinesController - upgrade connector');
const metadata = PackTheMetadata(user);
await this.pipelinesClientService.upgrade(id, metadata);
}
@Delete(':id')
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
@@ -506,82 +497,8 @@ export class PipelinesController {
},
);
const response = await this.pipelinesClientService.runPipeline({ id, info });
const response = await this.oldPipelinesService.runPipeline({ id, info });
return response;
}
// ---- CDC pipeline operations (Kafka Connect backed) ----
// These operate on an existing pipeline, so they require UPDATE (not CREATE).
@Get(':id/live-status')
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getLiveStatus(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
this.logger.info('PipelinesController - getLiveStatus', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.getLiveStatus(id, metadata);
}
@Post(':id/pause')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pause(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - pause', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.pause(id, metadata);
}
@Post(':id/unpause')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpause(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - unpause', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.unpause(id, metadata);
}
@Post(':id/restart')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async restart(
@Param('id') id: string,
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - restart', { id });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.restart(id, metadata);
}
@Post(':id/jobs/:jobId/reset-state')
@HttpCode(HttpStatus.OK)
@ApiInternalOnlyEndpoint()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async resetJobState(
@Param('id') id: string,
@Param('jobId') jobId: string,
@Body() body: { schedule_minutes?: number },
@User() user: RequestUser,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesController - resetJobState', { id, jobId });
const metadata = PackTheMetadata(user);
return this.pipelinesClientService.resetJobState(
id,
jobId,
body?.schedule_minutes,
metadata,
);
}
}
+4 -2
View File
@@ -1,4 +1,4 @@
import { Module, forwardRef } from '@nestjs/common';
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module';
@@ -20,10 +21,11 @@ const client = new PipelinesClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
ConnectorModule,
InputsModule,
TransformationsModule,
forwardRef(() => PlatformApiModule),
PlatformApiModule,
NimbusServicesModule,
CatalogModule
],
@@ -1,142 +0,0 @@
// These imported modules pull in gRPC client-config / service modules that read
// process.env at load time; mock them (hoisted before imports) so the spec needs
// no runtime env. Each mock severs an entire import subtree while still providing
// a class usable as a value/DI token. Mirrors platform-api.controller.spec.ts.
jest.mock('./pipelines-client', () => ({ PipelinesClientConfiguration: class {} }));
jest.mock('../connector/client.service', () => ({ ConnectorClientService: class {} }));
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
jest.mock('../transformations/transformations.service', () => ({ TransformationsService: class {} }));
jest.mock('../platform-api/platform-api.service', () => ({ PlatformApiService: class {} }));
jest.mock('src/services/nimbus/nimbus.service', () => ({ NimbusService: class {} }));
jest.mock('../catalog/catalog.service', () => ({ CatalogService: class {} }));
import { of } from 'rxjs';
import { PipelinesService } from './pipelines.service';
const logger = {
info: jest.fn(),
error: jest.fn(),
};
const cdcOldInput = { input: { plugin: 'mysql_cdc', tables: [] } };
const batchOldInput = { input: { plugin: 'mysql', type: 'database', tables: [] } };
const updateResponse = {
input: { type: 'database' },
tablesUpdate: [],
dataAssetUpdate: [],
};
const user: any = { customer_modules: [] };
const updateInputDTO: any = { tables: [] };
const info: any = { customer: 'cust' };
const metadata: any = {};
function buildService(oldInput: any) {
const inputsService: any = {
findOne: jest.fn().mockResolvedValue(oldInput),
update: jest.fn().mockResolvedValue(updateResponse),
rollbackUpdate: jest.fn().mockResolvedValue({}),
};
const nimbusService: any = { renameTable: jest.fn().mockResolvedValue({}) };
const service = new PipelinesService(
{ logger } as any, // dadosferaLogger
{} as any, // grpcClient
{} as any, // connectorService
inputsService, // inputsService
{} as any, // transformationsService
{} as any, // platformAPI
nimbusService, // nimbusService
{} as any, // catalogService
);
const updatePlatformJobsSpy = jest
.spyOn(service, 'updatePlatformJobs')
.mockResolvedValue(undefined as any);
return { service, inputsService, updatePlatformJobsSpy };
}
describe('PipelinesService - updatePipelineInput', () => {
afterEach(() => jest.clearAllMocks());
it('CDC input skips updatePlatformJobs', async () => {
const { service, inputsService, updatePlatformJobsSpy } = buildService(cdcOldInput);
const result = await service.updatePipelineInput(
'pipeline-id',
'input-id',
updateInputDTO,
info,
user,
metadata,
);
expect(updatePlatformJobsSpy).not.toHaveBeenCalled();
expect(inputsService.update).toHaveBeenCalled();
expect(result).toBe(updateResponse);
});
it('batch input calls updatePlatformJobs', async () => {
const { service, inputsService, updatePlatformJobsSpy } = buildService(batchOldInput);
await service.updatePipelineInput(
'pipeline-id',
'input-id',
updateInputDTO,
info,
user,
metadata,
);
expect(updatePlatformJobsSpy).toHaveBeenCalled();
expect(inputsService.update).toHaveBeenCalled();
});
});
describe('PipelinesService - create', () => {
afterEach(() => jest.clearAllMocks());
// The body's `config` (carrying CDC destinations) must reach pi-factory as a
// JSON string — the gRPC proto field is a string, so an object would be
// stripped on the wire. Mirrors how `properties` is serialized.
it('serializes the body config into the gRPC create request', async () => {
const { service } = buildService(cdcOldInput);
let captured: any;
(service as any).pipelineWriteService = {
PipelineV2Create: (req: any) => {
captured = req;
// The service does `lastValueFrom(...)`; return a real Observable.
return of({ pipeline: {} });
},
};
const body: any = {
name: 'p',
input_id: 'i',
transformations_ids: [],
tags: [],
properties: { schema: 'cadastros' },
config: {
cron: '@once',
tables: [
{
name: 'pedidos',
destinations: {
raw: { table_schema: 'PUBLIC', table_name: 'pedidos_001' },
},
},
],
},
};
await service.create(body, metadata);
expect(typeof captured.config).toBe('string');
expect(JSON.parse(captured.config).tables[0].destinations.raw.table_name).toBe(
'pedidos_001',
);
});
});
+41 -168
View File
@@ -1,7 +1,6 @@
/* eslint-disable no-async-promise-executor */
import {
BadRequestException,
ConflictException,
HttpException,
HttpStatus,
Inject,
@@ -18,8 +17,8 @@ import { lastValueFrom } from 'rxjs';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest, AddCdcJobsRequest, AddCdcJobsResponse } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
import { InputsService } from '../inputs/inputs.service';
@@ -105,10 +104,6 @@ export class PipelinesService implements OnModuleInit {
transformations_ids: body.transformations_ids,
tags: body.tags,
properties: body.properties && JSON.stringify(body.properties),
// JSON-serialize the create body config so it survives the gRPC wire
// (the proto field is a string). pi-factory's CDC path reads
// config.tables[].destinations to honor a user-supplied raw table name.
config: body.config && JSON.stringify(body.config),
};
const createPipelineResponse = await lastValueFrom(
@@ -164,62 +159,6 @@ export class PipelinesService implements OnModuleInit {
return findOnePipelineResponse;
}
// CDC lifecycle operations (Kafka Connect backed).
async pause(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Pause');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Pause({ id }, metadata),
);
}
async unpause(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Unpause');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Unpause({ id }, metadata),
);
}
async restart(
id: string,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - Restart');
return lastValueFrom(
this.pipelineWriteService.PipelineV2Restart({ id }, metadata),
);
}
async resetJobState(
id: string,
job_id: string,
schedule_minutes: number | undefined,
metadata,
): Promise<Messages.PipelineV2OperationResponse> {
this.logger.info('PipelinesClientService - ResetJobState');
return lastValueFrom(
this.pipelineWriteService.PipelineV2ResetJobState(
{ id, job_id, schedule_minutes },
metadata,
),
);
}
async getLiveStatus(
id: string,
metadata,
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
this.logger.info('PipelinesClientService - GetLiveStatus');
return lastValueFrom(
this.pipelineReadService.PipelineV2GetLiveStatus({ id }, metadata),
);
}
async update(
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
metadata,
@@ -237,15 +176,6 @@ export class PipelinesService implements OnModuleInit {
return updatePipelineResponse;
}
async upgrade(id: string, metadata: Metadata) {
await lastValueFrom(
this.pipelineWriteService.Upgrade(
{ id },
metadata,
),
);
}
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
const { id, metadata, user } = data;
const info = {
@@ -437,15 +367,13 @@ export class PipelinesService implements OnModuleInit {
});
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
const isCdc = !!oldInput.plugin?.endsWith('_cdc');
const pipelineIdFormat = pipelineId.split('-').join('_');
const rollback: RollbackPromise[] = [];
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info,
metadata
info
);
const inputRollback = () => {
@@ -466,53 +394,47 @@ export class PipelinesService implements OnModuleInit {
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
if (user.customer_modules.includes('catalog')) {
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
}
if (!isCdc) {
try {
await this.updatePlatformJobs(
pipelineIdFormat,
updateInputResponse.input.type,
updateInputDTO,
user
);
} catch (error) {
this.logger.error(error);
await this.executeRenameRollback(rollback)
throw new Error("Error Platform API updating jobs");
}
} else {
this.logger.info('CDC input: skipping updatePlatformJobs (batch sync_mode/memory do not apply to CDC jobs)');
try {
await this.updatePlatformJobs(
pipelineIdFormat,
updateInputResponse.input.type,
updateInputDTO,
user
);
} catch (error) {
this.logger.error(error);
await this.executeRenameRollback(rollback)
throw new Error("Error Platform API updating jobs");
}
return updateInputResponse;
@@ -701,53 +623,4 @@ export class PipelinesService implements OnModuleInit {
return assets;
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async addCdcJobs(data: AddCdcJobsRequest): Promise<AddCdcJobsResponse> {
return lastValueFrom(this.pipelineWriteService.AddCdcJobs(data));
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
@@ -1,364 +0,0 @@
// These service modules pull in gRPC client-config modules that read
// process.env at load time; mock them (hoisted before imports) so the spec
// needs no runtime env. Each mock severs an entire import subtree and still
// provides a class usable as a DI token.
jest.mock('../customers/customers.service', () => ({ CustomersService: class {} }));
jest.mock('../catalog/catalog.service', () => ({ CatalogService: class {} }));
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
jest.mock('../pipelinesV2/pipelines.service', () => ({ PipelinesService: class {} }));
import { Test, TestingModule } from '@nestjs/testing';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PlatformApiController } from './platform-api.controller';
import { PlatformApiService } from './platform-api.service';
import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
import { CatalogService } from '../catalog/catalog.service';
import { InputsService } from '../inputs/inputs.service';
import { PipelinesService } from '../pipelinesV2/pipelines.service';
const logger = {
info: (...args) => args,
error: (...args) => args,
};
const mockUser: any = {
customer_id: 'c1',
customer_name: 'cust',
user_id: 'u1',
};
describe('PlatformApiController - deleteTable', () => {
let controller: PlatformApiController;
let platformApiService: { proxy: jest.Mock };
let inputsService: { markTableDeleted: jest.Mock; unmarkTableDeleted: jest.Mock };
beforeEach(async () => {
platformApiService = { proxy: jest.fn() };
inputsService = {
markTableDeleted: jest.fn(),
unmarkTableDeleted: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
controllers: [PlatformApiController],
providers: [
{ provide: PlatformApiService, useValue: platformApiService },
{ provide: ElasticsearchService, useValue: {} },
{ provide: DynamoDBService, useValue: {} },
{ provide: CustomersService, useValue: {} },
{ provide: CatalogService, useValue: {} },
{ provide: InputsService, useValue: inputsService },
{ provide: PipelinesService, useValue: {} },
{ provide: DadosferaLogger, useValue: { logger } },
],
}).compile();
controller = module.get<PlatformApiController>(PlatformApiController);
});
it('CDC removal reconfigures the connector', async () => {
inputsService.markTableDeleted.mockResolvedValue({ is_deleted: true, deleted_at: 't' });
platformApiService.proxy.mockImplementation((method: string, path: string) => {
if (method === 'GET') {
return Promise.resolve({
jobs: [{ job_id: 'p_0', input: { connector: 'cdc', table_name: 'pedidos' } }],
});
}
return Promise.resolve({});
});
await controller.deleteTable('pid', 'iid', { table_name: 'pedidos' }, mockUser);
expect(platformApiService.proxy).toHaveBeenCalledWith(
'DELETE',
'/pipeline/pid/jobs',
mockUser,
{ job_ids: ['p_0'], delete_snowflake_tables: false },
);
const deletedViaJobsRoute = platformApiService.proxy.mock.calls.some(
([method, path]: any[]) => method === 'DELETE' && path === '/jobs/p_0',
);
expect(deletedViaJobsRoute).toBe(false);
});
it('batch removal unchanged', async () => {
inputsService.markTableDeleted.mockResolvedValue({ is_deleted: true, deleted_at: 't' });
platformApiService.proxy.mockImplementation((method: string, path: string) => {
if (method === 'GET') {
return Promise.resolve({
jobs: [{ job_id: 'p_0', input: { connector: 'jdbc', table_name: 'pedidos' } }],
});
}
return Promise.resolve({});
});
await controller.deleteTable('pid', 'iid', { table_name: 'pedidos' }, mockUser);
expect(platformApiService.proxy).toHaveBeenCalledWith(
'DELETE',
'/jobs/p_0',
mockUser,
);
const reconfiguredConnector = platformApiService.proxy.mock.calls.some(
([method, path]: any[]) => method === 'DELETE' && path === '/pipeline/pid/jobs',
);
expect(reconfiguredConnector).toBe(false);
});
});
describe('PlatformApiController - addTable', () => {
let controller: PlatformApiController;
let inputsService: { addCdcTable: jest.Mock; removeCdcTable: jest.Mock };
let pipelinesClientService: { addCdcJobs: jest.Mock };
const body = {
table_name: 'orders',
table_schema: 'public',
primary_keys: ['id'],
destinations: {
raw: { table_schema: 'raw', table_name: 'orders' },
qualify: { table_schema: 'qualify', table_name: 'orders' },
},
};
beforeEach(async () => {
inputsService = {
addCdcTable: jest.fn(),
removeCdcTable: jest.fn(),
};
pipelinesClientService = {
addCdcJobs: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
controllers: [PlatformApiController],
providers: [
{ provide: PlatformApiService, useValue: { proxy: jest.fn() } },
{ provide: ElasticsearchService, useValue: {} },
{ provide: DynamoDBService, useValue: {} },
{ provide: CustomersService, useValue: {} },
{ provide: CatalogService, useValue: {} },
{ provide: InputsService, useValue: inputsService },
{ provide: PipelinesService, useValue: pipelinesClientService },
{ provide: DadosferaLogger, useValue: { logger } },
],
}).compile();
controller = module.get<PlatformApiController>(PlatformApiController);
});
it('addTable: DynamoDB append then platform AddJobs, returns job_ids', async () => {
inputsService.addCdcTable.mockResolvedValue({ input: {} });
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_2'], skipped: [] });
const result = await controller.addTable('pid', 'iid', body, mockUser);
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
id: 'iid',
table: {
table_schema: 'public',
table_name: 'orders',
primary_keys: ['id'],
name: 'orders',
columns: [],
},
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
});
expect(pipelinesClientService.addCdcJobs).toHaveBeenCalledWith({
pipeline_id: 'pid',
input_id: 'iid',
tables: [{
table_schema: 'public',
table_name: 'orders',
primary_keys: ['id'],
destinations: body.destinations,
}],
info: { customer_id: 'c1', user_id: 'u1', customer: 'cust' },
});
expect(result).toEqual({ job_ids: ['p_2'], skipped: [] });
expect(inputsService.removeCdcTable).not.toHaveBeenCalled();
const addCdcTableOrder = inputsService.addCdcTable.mock.invocationCallOrder[0];
const addCdcJobsOrder = pipelinesClientService.addCdcJobs.mock.invocationCallOrder[0];
expect(addCdcTableOrder).toBeLessThan(addCdcJobsOrder);
});
it('addTable: carries iceberg_table_name on the added table through to AddCdcTable', async () => {
inputsService.addCdcTable.mockResolvedValue({ input: {} });
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_3'], skipped: [] });
const icebergBody = { ...body, iceberg_table_name: 'cdc_raw.public__orders' };
await controller.addTable('pid', 'iid', icebergBody, mockUser);
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
id: 'iid',
table: {
table_schema: 'public',
table_name: 'orders',
primary_keys: ['id'],
name: 'orders',
iceberg_table_name: 'cdc_raw.public__orders',
columns: [],
},
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
});
});
it('addTable: carries columns on the added table through to AddCdcTable', async () => {
inputsService.addCdcTable.mockResolvedValue({ input: {} });
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_4'], skipped: [] });
const columnsBody = {
...body,
columns: [
{ name: 'id', type: 'int', is_primary_key: true },
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
],
};
await controller.addTable('pid', 'iid', columnsBody, mockUser);
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
id: 'iid',
table: {
table_schema: 'public',
table_name: 'orders',
primary_keys: ['id'],
name: 'orders',
columns: [
{ name: 'id', type: 'int', is_primary_key: true },
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
],
},
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
});
});
it('addTable: rolls back the DynamoDB row when AddJobs fails', async () => {
inputsService.addCdcTable.mockResolvedValue({ input: {} });
pipelinesClientService.addCdcJobs.mockRejectedValue(new Error('platform down'));
inputsService.removeCdcTable.mockResolvedValue({});
await expect(controller.addTable('pid', 'iid', body, mockUser)).rejects.toThrow('platform down');
expect(inputsService.removeCdcTable).toHaveBeenCalledWith({
id: 'iid',
table_name: 'orders',
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
});
});
});
describe('PlatformApiController - deleteTables (batch)', () => {
let controller: PlatformApiController;
let platformApiService: { proxy: jest.Mock };
let inputsService: { markTableDeleted: jest.Mock; unmarkTableDeleted: jest.Mock };
beforeEach(async () => {
platformApiService = { proxy: jest.fn() };
inputsService = {
markTableDeleted: jest.fn().mockResolvedValue({ is_deleted: true, deleted_at: 't' }),
unmarkTableDeleted: jest.fn().mockResolvedValue({}),
};
const module: TestingModule = await Test.createTestingModule({
controllers: [PlatformApiController],
providers: [
{ provide: PlatformApiService, useValue: platformApiService },
{ provide: ElasticsearchService, useValue: {} },
{ provide: DynamoDBService, useValue: {} },
{ provide: CustomersService, useValue: {} },
{ provide: CatalogService, useValue: {} },
{ provide: InputsService, useValue: inputsService },
{ provide: PipelinesService, useValue: {} },
{ provide: DadosferaLogger, useValue: { logger } },
],
}).compile();
controller = module.get<PlatformApiController>(PlatformApiController);
});
const pipelineWithJobs = () => ({
jobs: [
{ job_id: 'p_0', input: { connector: 'cdc', table_name: 'pedidos' } },
{ job_id: 'p_1', input: { connector: 'cdc', table_name: 'clientes' } },
{ job_id: 'p_2', input: { connector: 'cdc', table_name: 'produtos' } },
],
});
it('removes N tables in ONE platform call (connectors reconfigured once)', async () => {
platformApiService.proxy.mockImplementation((method: string) =>
Promise.resolve(method === 'GET' ? pipelineWithJobs() : {}),
);
await controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'produtos'] }, mockUser);
// one mark per table
expect(inputsService.markTableDeleted).toHaveBeenCalledTimes(2);
// exactly one DELETE to the batch endpoint, with BOTH job_ids
const deleteCalls = platformApiService.proxy.mock.calls.filter(
([m, p]: any[]) => m === 'DELETE' && p === '/pipeline/pid/jobs',
);
expect(deleteCalls).toHaveLength(1);
expect(deleteCalls[0][3]).toEqual({
job_ids: ['p_0', 'p_2'],
delete_snowflake_tables: false,
});
// never the per-job route
const perJob = platformApiService.proxy.mock.calls.some(
([m, p]: any[]) => m === 'DELETE' && String(p).startsWith('/jobs/'),
);
expect(perJob).toBe(false);
});
it('rolls back only this call\'s marks when the platform delete fails', async () => {
platformApiService.proxy.mockImplementation((method: string) => {
if (method === 'GET') return Promise.resolve(pipelineWithJobs());
return Promise.reject(new Error('platform boom'));
});
await expect(
controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'clientes'] }, mockUser),
).rejects.toThrow();
// both marks rolled back, nothing else
expect(inputsService.unmarkTableDeleted).toHaveBeenCalledTimes(2);
const unmarked = inputsService.unmarkTableDeleted.mock.calls.map((c: any[]) => c[0].table_name).sort();
expect(unmarked).toEqual(['clientes', 'pedidos']);
});
it('rolls back the marks made so far if a later mark fails (atomic)', async () => {
// second mark fails → first must be rolled back, no platform delete attempted
inputsService.markTableDeleted
.mockResolvedValueOnce({ is_deleted: true, deleted_at: 't' })
.mockRejectedValueOnce(new Error('dynamo boom'));
await expect(
controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'clientes'] }, mockUser),
).rejects.toThrow();
expect(inputsService.unmarkTableDeleted).toHaveBeenCalledTimes(1);
expect(inputsService.unmarkTableDeleted.mock.calls[0][0].table_name).toBe('pedidos');
// never reached the platform delete
const attemptedDelete = platformApiService.proxy.mock.calls.some(([m]: any[]) => m === 'DELETE');
expect(attemptedDelete).toBe(false);
});
it('404s when a requested table has no matching job', async () => {
platformApiService.proxy.mockImplementation((method: string) =>
Promise.resolve(method === 'GET' ? pipelineWithJobs() : {}),
);
await expect(
controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'ghost'] }, mockUser),
).rejects.toThrow();
// the successful mark (pedidos) must be rolled back
expect(inputsService.unmarkTableDeleted).toHaveBeenCalled();
});
});
@@ -20,11 +20,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
} from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
@@ -33,7 +32,6 @@ import { CatalogService } from '../catalog/catalog.service';
import { PackTheMetadata } from '../../utils/PackTheMetadata';
import { ValidationTableDTO } from './platform-api.dto';
import { InputsService } from '../inputs/inputs.service';
import { PipelinesService } from '../pipelinesV2/pipelines.service';
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
@@ -51,7 +49,6 @@ type RenameTablesBody = {
@ApiTags('Platform API')
@Controller('platform')
@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
export class PlatformApiController {
private logger: any;
@@ -62,7 +59,6 @@ export class PlatformApiController {
private readonly customersService: CustomersService,
private readonly catalogService: CatalogService,
private readonly inputsService: InputsService,
private readonly pipelinesClientService: PipelinesService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
@@ -543,20 +539,6 @@ export class PlatformApiController {
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
}
@Get('iceberg/namespaces')
@ApiOperation({ summary: 'List existing Polaris Iceberg namespaces (CDC destination dropdown)' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getIcebergNamespaces(@User() user: RequestUser) {
return this.platformApiService.proxy('GET', '/iceberg/namespaces', user);
}
@Post('iceberg/tables/validate')
@ApiOperation({ summary: 'Validate CDC Iceberg raw table names against Polaris' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async validateIcebergTables(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/iceberg/tables/validate', user, body);
}
@Patch('pipelines/:pipelineId')
@ApiOperation({ summary: 'Update pipeline by ID' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@@ -980,18 +962,7 @@ export class PlatformApiController {
if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`);
this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id });
if (job.input?.connector === 'cdc') {
// CDC: reconfigure the Debezium/Kafka-Connect connector (stop
// replicating this table); keep the landed Snowflake data.
await this.platformApiService.proxy(
'DELETE',
`/pipeline/${normalizedPipelineId}/jobs`,
user,
{ job_ids: [job.job_id], delete_snowflake_tables: false },
);
} else {
await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
}
await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
this.logger.info('deleteTable: job deleted', { jobId: job.job_id });
return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at };
@@ -1006,155 +977,6 @@ export class PlatformApiController {
}
}
@Delete('pipelines/:pipelineId/inputs/:inputId/tables')
@ApiOperation({ summary: 'Batch-remove tables from an input; reconfigures the CDC connectors once' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
@UseGuards(PipelineExecutionGuard)
async deleteTables(
@Param('pipelineId') pipelineId: string,
@Param('inputId') inputId: string,
@Body() body: { table_names: string[] },
@User() user: RequestUser,
) {
const tableNames = body.table_names ?? [];
if (tableNames.length === 0) {
throw new BadRequestException('table_names must be a non-empty array');
}
const info = {
customer_id: user.customer_id,
customer: user.customer_name,
user_id: user.user_id,
};
// 1) Soft-delete each table in DynamoDB, tracking which succeeded so a later
// failure only rolls back the marks made in THIS call.
const marked: string[] = [];
const rollback = async () => {
for (const name of marked) {
try {
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: name, info });
} catch (rollbackError) {
this.logger.error('deleteTables: rollback failed', { tableName: name, error: rollbackError.message });
}
}
};
try {
for (const name of tableNames) {
await this.inputsService.markTableDeleted({ input_id: inputId, table_name: name, info });
marked.push(name);
}
// 2) Resolve all table_names -> job_ids from the platform pipeline (one GET).
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
const jobs = platformPipeline?.jobs ?? [];
const jobIds: string[] = [];
for (const name of tableNames) {
const job = jobs.find((j: any) => j.input?.table_name === name);
if (!job) throw new NotFoundException(`Job for table '${name}' not found in pipeline`);
jobIds.push(job.job_id);
}
// 3) Remove them all in ONE platform call so the Debezium source + Snowflake
// sink connectors are reconfigured a single time, not once per table.
await this.platformApiService.proxy(
'DELETE',
`/pipeline/${normalizedPipelineId}/jobs`,
user,
{ job_ids: jobIds, delete_snowflake_tables: false },
);
this.logger.info('deleteTables: jobs deleted', { jobIds });
return { table_names: tableNames, deleted: true };
} catch (error) {
this.logger.error('deleteTables: failed, rolling back this call\'s marks', { tableNames, error: error.message });
await rollback();
throw error;
}
}
@Post('pipelines/:pipelineId/inputs/:inputId/tables')
@ApiOperation({ summary: 'Add a CDC table to an input and dispatch its platform jobs' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@UseGuards(PipelineExecutionGuard)
async addTable(
@Param('pipelineId') pipelineId: string,
@Param('inputId') inputId: string,
@Body() body: {
table_name: string;
table_schema: string;
primary_keys: string[];
destinations: {
raw: { table_schema: string; table_name: string };
qualify: { table_schema: string; table_name: string };
};
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
// absent for snowflake, back-compat.
iceberg_table_name?: string;
// Per-table deduped (qualify) Iceberg table name (protospack
// CdcTable.iceberg_qualify_table_name); absent => same as the raw name.
iceberg_qualify_table_name?: string;
// Source column schema for iceberg deduped table pre-create (protospack CdcTable.columns).
columns?: { name: string; type: string; is_primary_key: boolean }[];
// Columns the user chose to ignore -> Debezium column.exclude.list.
column_exclude_list?: string[];
},
@User() user: RequestUser,
) {
const info = {
customer_id: user.customer_id,
customer: user.customer_name,
user_id: user.user_id,
};
const cdcTable = {
table_schema: body.table_schema,
table_name: body.table_name,
primary_keys: body.primary_keys,
name: body.table_name,
iceberg_table_name: body.iceberg_table_name,
iceberg_qualify_table_name: body.iceberg_qualify_table_name,
columns: body.columns ?? [],
column_exclude_list: body.column_exclude_list ?? [],
};
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
await this.inputsService.addCdcTable({ id: inputId, table: cdcTable, info });
this.logger.info('addTable: DynamoDB row appended', { inputId, tableName: body.table_name });
try {
const customInfo = {
customer_id: user.customer_id,
user_id: user.user_id,
customer: user.customer_name,
};
const res = await this.pipelinesClientService.addCdcJobs({
pipeline_id: pipelineId,
input_id: inputId,
tables: [{
table_schema: body.table_schema,
table_name: body.table_name,
primary_keys: body.primary_keys,
destinations: body.destinations,
}],
info: customInfo,
});
return res;
} catch (error) {
this.logger.error('addTable: platform AddJobs failed, rolling back the DynamoDB row', { tableName: body.table_name, error: error.message });
try {
await this.inputsService.removeCdcTable({ id: inputId, table_name: body.table_name, info });
} catch (rbErr) {
this.logger.error('addTable: rollback failed', { error: rbErr.message });
}
throw error;
}
}
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
@Get('jobs/jdbc/configs/allowed_datatypes')
@@ -1,4 +1,4 @@
import { Module, forwardRef } from '@nestjs/common';
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -9,17 +9,9 @@ import { DynamoDBModule } from '../../services/dynamodb';
import { CustomersModule } from '../customers/customers.module';
import { CatalogModule } from '../catalog/catalog.module';
import { InputsModule } from '../inputs/inputs.module';
import { PipelinesV2Module } from '../pipelinesV2/pipelines.module';
@Module({
imports: [
ElasticsearchModule,
DynamoDBModule,
CustomersModule,
CatalogModule,
InputsModule,
forwardRef(() => PipelinesV2Module),
],
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule],
controllers: [PlatformApiController],
providers: [PlatformApiService, DadosferaLogger],
exports: [PlatformApiService],
@@ -1,7 +1,6 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteController } from './release_note.controller';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteController', () => {
let controller: ReleaseNoteController;
@@ -9,7 +8,7 @@ describe('ReleaseNoteController', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [ReleaseNoteController],
providers: [ReleaseNoteService, DadosferaLogger],
providers: [ReleaseNoteService],
}).compile();
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
@@ -1,13 +1,12 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteService', () => {
let service: ReleaseNoteService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ReleaseNoteService, DadosferaLogger],
providers: [ReleaseNoteService],
}).compile();
service = module.get<ReleaseNoteService>(ReleaseNoteService);
+2 -5
View File
@@ -1,8 +1,5 @@
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreateTransformationsRequest {
transformations: Transformation[];
info: Info;
-1
View File
@@ -9,7 +9,6 @@ interface IMetadata {
details?: string;
sensitive?: string;
roles?: string[];
customer_modules?: string[];
is_data_manager?: boolean;
access_token?: string;
host?: string;