From 0d30c1cf834bd3f69d3b3b482c03dea8196c3a47 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Sat, 13 Jun 2026 21:07:14 -0300 Subject: [PATCH 1/6] FEAT: remove deprecated protospack lib --- docsfera.json | 135 ++++++------------ package-lock.json | 18 +-- package.json | 3 +- src/app.module.ts | 2 - src/modules/catalog/catalog.module.ts | 3 - src/modules/inputs/dtos/old_interfaces.ts | 6 +- src/modules/pipelines/client.service.ts | 78 ---------- src/modules/pipelines/interfaces.d.ts | 36 ----- src/modules/pipelines/pipelines-client.ts | 33 ----- src/modules/pipelines/pipelines.controller.ts | 72 ---------- src/modules/pipelines/pipelines.module.ts | 19 --- src/modules/pipelines/pipelines.service.ts | 33 ----- src/modules/pipelinesV2/interfaces.ts | 7 +- .../pipelinesV2/pipelines.controller.ts | 8 +- src/modules/pipelinesV2/pipelines.module.ts | 2 - src/modules/pipelinesV2/pipelines.service.ts | 48 ++++++- src/modules/transformations/interfaces.d.ts | 7 +- 17 files changed, 115 insertions(+), 395 deletions(-) delete mode 100644 src/modules/pipelines/client.service.ts delete mode 100644 src/modules/pipelines/interfaces.d.ts delete mode 100644 src/modules/pipelines/pipelines-client.ts delete mode 100644 src/modules/pipelines/pipelines.controller.ts delete mode 100644 src/modules/pipelines/pipelines.module.ts delete mode 100644 src/modules/pipelines/pipelines.service.ts diff --git a/docsfera.json b/docsfera.json index c68e491..eaaf00e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,14 +3343,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -3874,88 +3867,6 @@ ] } }, - "/pipelines/start/{id}": { - "post": { - "operationId": "PipelinesController_activate", - "summary": "", - "deprecated": true, - "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Pipelines" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/pipelines/{id}/status": { - "get": { - "operationId": "PipelinesController_getPipelineStatus", - "summary": "", - "deprecated": true, - "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Pipelines" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/transformations": { "post": { "operationId": "TransformationsController_create", @@ -4659,6 +4570,50 @@ ] } }, + "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunJobs", + "summary": "Get pipeline run jobs", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", diff --git a/package-lock.json b/package-lock.json index f26f5fe..28b3128 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,8 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.9", + "@dadosfera/protospack-v2": "3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,19 +1734,10 @@ "winston-log2gelf": "^2.4.0" } }, - "node_modules/@dadosfera/protospack": { - "version": "2.5.3", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz", - "integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==", - "license": "ISC", - "dependencies": { - "rxjs": "^7.5.5" - } - }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.9", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz", - "integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==", + "version": "3.40.0-beta.10", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", + "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 878d46a..dbd506b 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.9", + "@dadosfera/protospack-v2": "3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index 4db914b..b0bc6ae 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -17,7 +17,6 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test. import { NetworkConfigModule } from './modules/network-config/network-config.module'; import { InputsModule } from './modules/inputs/inputs.module'; import { OauthModule } from './modules/oauth/oauth.module'; -import { PipelinesModule } from './modules/pipelines/pipelines.module'; import { TransformationsModule } from './modules/transformations/transformations.module'; import { HealthModule } from './modules/health/health.module'; import { CatalogModule } from './modules/catalog/catalog.module'; @@ -60,7 +59,6 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module'; PermissionsModule, TermsOfUseModule, ConnectionTestModule, - PipelinesModule, TransformationsModule, UsersModule, RolesModule, diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index 8ce1579..982ef09 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -5,20 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { CatalogController } from './catalog.controller'; import { CatalogClientConfiguration } from './catalog-client'; import { ClientsModule } from '@nestjs/microservices'; -import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { UsersModule } from '../users/users.module'; import { RolesModule } from '../roles/roles.module'; import { CustomersModule } from '../customers/customers.module'; import { ShareModule } from './share/share.module'; import { CatalogService } from './catalog.service'; -import { MixpanelModule } from '../mixpanel/mixpanel.module'; const client = new CatalogClientConfiguration(); @Module({ imports: [ ClientsModule.register([client.providerOptions]), - OldPipelineModule, UsersModule, RolesModule, CustomersModule, diff --git a/src/modules/inputs/dtos/old_interfaces.ts b/src/modules/inputs/dtos/old_interfaces.ts index 21c8cff..e02a567 100644 --- a/src/modules/inputs/dtos/old_interfaces.ts +++ b/src/modules/inputs/dtos/old_interfaces.ts @@ -1,4 +1,8 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} interface Values { jdbc_user: string; diff --git a/src/modules/pipelines/client.service.ts b/src/modules/pipelines/client.service.ts deleted file mode 100644 index 1dc395e..0000000 --- a/src/modules/pipelines/client.service.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { ConflictException, Inject, OnModuleInit } from '@nestjs/common'; -import { ClientGrpc } from '@nestjs/microservices'; -import { - PipelineServicesNames, - PipelinesServiceInterface, -} from '@dadosfera/protospack'; -import { lastValueFrom } from 'rxjs'; - -import { IIdRequest } from './interfaces'; - -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { PipelinesClientConfiguration } from './pipelines-client'; - -export class PipelinesClientService implements OnModuleInit { - private pipelineService: PipelinesServiceInterface; - logger: DadosferaLogger; - - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - @Inject(PipelinesClientConfiguration.name) - private readonly grpcClient: ClientGrpc, - ) { - this.logger = dadosferaLogger.logger; - } - - onModuleInit() { - this.pipelineService = - this.grpcClient.getService( - PipelineServicesNames.PipelineService, - ); - } - - async getPipelineStatus(data) { - this.logger.info('PipelinesClientService - GetPipelineStatus'); - - const statusPipelineResponse = await lastValueFrom( - this.pipelineService.getPipelineStatus(data), - ) - .then((res) => { - const statusArray = - res.status?.sort((a, b) => { - if (a.id < b.id) { - return 1; - } else { - return -1; - } - }) || []; - return { status: statusArray }; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - this.logger.info('Done'); - - return statusPipelineResponse; - } - - async runPipeline({ id, info }: IIdRequest) { - this.logger.info('PipelinesClientService - RunPipeline'); - const statusPipelineResponse = await lastValueFrom( - this.pipelineService.triggerPipeline({ id, info }), - ).catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - if (statusPipelineResponse.status == false) { - throw new ConflictException( - 'This pipeline is not ready yet to execute, Try again later!', - ); - } - - this.logger.info('Done'); - return statusPipelineResponse; - } -} diff --git a/src/modules/pipelines/interfaces.d.ts b/src/modules/pipelines/interfaces.d.ts deleted file mode 100644 index 9a9dbc5..0000000 --- a/src/modules/pipelines/interfaces.d.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; - -export interface ICreatePipelineDto { - input: IdRequest; - transformations: IdRequest[]; - output: IdRequest; - tags: string[]; - name: string; - description: string; - info: Info; -} - -export interface IdRequest { - id: string; -} - -export interface IIdRequest { - id: string; - info: Info; -} - -export interface IUpdatePipelineRequest { - input: IdRequest; - transformations: IdRequest[]; - output: IdRequest; - tags: string[]; - name: string; - description: string; - id: string; - info: Info; -} - -export interface IGetPipelineLogsRequest { - id: string; - details: string; -} diff --git a/src/modules/pipelines/pipelines-client.ts b/src/modules/pipelines/pipelines-client.ts deleted file mode 100644 index 2dcf630..0000000 --- a/src/modules/pipelines/pipelines-client.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { - ClientsProviderAsyncOptions, - GrpcOptions, - Transport, -} from '@nestjs/microservices'; -import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack'; -import { credentials } from '@grpc/grpc-js'; - -const isLocalConnection = - process.env.PIFACTORY_URL.startsWith('pi-factory:') || - process.env.PIFACTORY_URL.includes('0.0.0.0'); - -export class PipelinesClientConfiguration { - public name = 'PipelinesClientConfiguration'; - private config: GrpcOptions = { - transport: Transport.GRPC, - options: { - url: process.env.PIFACTORY_URL, - package: PipelinePackages, - credentials: isLocalConnection ? undefined : credentials.createSsl(), - protoPath: PipelineProtoFilePath, - loader: { - keepCase: true, - enums: String, - defaults: false, - }, - }, - }; - providerOptions: ClientsProviderAsyncOptions = { - name: this.name, - ...this.config, - }; -} diff --git a/src/modules/pipelines/pipelines.controller.ts b/src/modules/pipelines/pipelines.controller.ts deleted file mode 100644 index 6054ff6..0000000 --- a/src/modules/pipelines/pipelines.controller.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common'; -import { ApiOperation, ApiTags } from '@nestjs/swagger'; -import { - AuthenticateCondition, - Authenticated, - RequireSomePermission, -} from 'src/decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; -import { PipelinesService } from './pipelines.service'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; - -@ApiInternalOnlyController() -@ApiTags('Pipelines') -@Controller('pipelines') -@Authenticated() -export class PipelinesController { - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private pipelineService: PipelinesService, - ) { - this.logger = dadosferaLogger.logger; - } - - @Post('start/:id') - @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - @ApiOperation({ - deprecated: true, - description: - 'This method is deprecated. Please use route /pipelinesV2/start/:id instead', - }) - async activate(@Param('id') id: string, @Body() body) { - const { info } = body; - - this.logger.info( - process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`, - { - user: body.info.user_id, - customer: body.info.customer, - }, - ); - - const response = await this.pipelineService.runPipeline({ id, info }); - - return response; - } - - @Get(':id/status') - @ApiOperation({ - deprecated: true, - description: - 'This method is deprecated. Please use route /pipelinesV2/:id/status instead', - }) - @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus(@Body() body, @Param('id') id: string) { - body.id = id; - - this.logger.info( - process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, - { - user: body.info.user_id, - customer: body.info.customer, - }, - ); - - const response = await this.pipelineService.getPipelineStatus(body); - - return response; - } -} diff --git a/src/modules/pipelines/pipelines.module.ts b/src/modules/pipelines/pipelines.module.ts deleted file mode 100644 index 192dcab..0000000 --- a/src/modules/pipelines/pipelines.module.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ClientsModule } from '@nestjs/microservices'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; - -import { PipelinesController } from './pipelines.controller'; -import { PipelinesService } from './pipelines.service'; - -import { PipelinesClientConfiguration } from './pipelines-client'; -import { PipelinesClientService } from './client.service'; - -const client = new PipelinesClientConfiguration(); - -@Module({ - imports: [ClientsModule.register([client.providerOptions])], - controllers: [PipelinesController], - providers: [PipelinesService, PipelinesClientService, DadosferaLogger], - exports: [PipelinesService], -}) -export class PipelinesModule {} diff --git a/src/modules/pipelines/pipelines.service.ts b/src/modules/pipelines/pipelines.service.ts deleted file mode 100644 index afc8103..0000000 --- a/src/modules/pipelines/pipelines.service.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; -import { PipelinesClientService } from './client.service'; -import { IIdRequest } from './interfaces'; -import { objectCamelToSnake } from 'src/utils/CaseConverter'; - -@Injectable() -export class PipelinesService { - constructor(private pipelineClient: PipelinesClientService) {} - - async getPipelineStatus(data: IIdRequest) { - try { - const pipelineStatusResponse = - await this.pipelineClient.getPipelineStatus(data); - - return objectCamelToSnake(pipelineStatusResponse); - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async runPipeline({ id, info }: IIdRequest) { - try { - const triggerPipelineResponse = await this.pipelineClient.runPipeline({ - id, - info, - }); - - return objectCamelToSnake(triggerPipelineResponse); - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } -} diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 252b94c..635ae80 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -1,5 +1,4 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; export class PipelineInputsDTO { @ApiProperty() @@ -62,6 +61,12 @@ export interface IIdRequest { info: Info; } +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} + export interface IUpdatePipelineRequest { input: IdRequest; transformations: IdRequest[]; diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index e90af52..70fab15 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -15,7 +15,6 @@ import { HttpException, BadRequestException, UseGuards, - Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -35,7 +34,6 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { ICompleteUploadCSVFile, ICreatePipelineCSVFile, @@ -64,9 +62,7 @@ export class PipelinesController { constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, - private pipelinesClientService: PipelinesService, - private oldPipelinesService: OldPipelineService, ) { this.logger = dadosferaLogger.logger; } @@ -202,7 +198,7 @@ export class PipelinesController { customer: body.info.customer, }); - const response = await this.oldPipelinesService.getPipelineStatus(body); + const response = await this.pipelinesClientService.getPipelineStatus(body); return response; } @@ -513,7 +509,7 @@ export class PipelinesController { }, ); - const response = await this.oldPipelinesService.runPipeline({ id, info }); + const response = await this.pipelinesClientService.runPipeline({ id, info }); return response; } diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 24562c9..b4f24af 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -7,7 +7,6 @@ import { PipelinesService } from './pipelines.service'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; @@ -21,7 +20,6 @@ const client = new PipelinesClientConfiguration(); @Module({ imports: [ ClientsModule.register([client.providerOptions]), - OldPipelineModule, ConnectorModule, InputsModule, TransformationsModule, diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 53d7711..2a8db84 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,6 +1,7 @@ /* eslint-disable no-async-promise-executor */ import { BadRequestException, + ConflictException, HttpException, HttpStatus, Inject, @@ -17,7 +18,7 @@ import { lastValueFrom } from 'rxjs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; +import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { Metadata } from '@grpc/grpc-js'; import { ConnectorClientService } from '../connector/client.service'; @@ -632,4 +633,49 @@ export class PipelinesService implements OnModuleInit { return assets; } + async getPipelineStatus(data) { + this.logger.info('PipelinesClientService - GetPipelineStatus'); + + const statusPipelineResponse = await lastValueFrom( + this.pipelineReadService.PipelineV2GetPipelineV2Status(data), + ) + .then((res) => { + const statusArray = + res.status?.sort((a, b) => { + if (a.id < b.id) { + return 1; + } else { + return -1; + } + }) || []; + return { status: statusArray }; + }) + .catch((err) => { + this.logger.error(err.message); + throw new Error(err); + }); + this.logger.info('Done'); + + return statusPipelineResponse; + } + + async runPipeline({ id, info }: IIdRequest) { + this.logger.info('PipelinesClientService - RunPipeline'); + const statusPipelineResponse = await lastValueFrom( + this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }), + ).catch((err) => { + this.logger.error(err.message); + throw new Error(err); + }); + + if (statusPipelineResponse.status == false) { + throw new ConflictException( + 'This pipeline is not ready yet to execute, Try again later!', + ); + } + + this.logger.info('Done'); + return statusPipelineResponse; + } + } diff --git a/src/modules/transformations/interfaces.d.ts b/src/modules/transformations/interfaces.d.ts index 0e7ab3c..20ba822 100644 --- a/src/modules/transformations/interfaces.d.ts +++ b/src/modules/transformations/interfaces.d.ts @@ -1,5 +1,8 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; - +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} export interface ICreateTransformationsRequest { transformations: Transformation[]; info: Info; From a444f5e5ecdb8ad25741ac496b1575e851507cb6 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 13 Jul 2026 08:33:35 -0300 Subject: [PATCH 2/6] FEAT: add documentation status enum and property to data asset --- docsfera.json | 9 +++++++++ src/modules/catalog/dtos/index.ts | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/docsfera.json b/docsfera.json index eaaf00e..be0e972 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11001,6 +11001,15 @@ }, "docs": { "type": "string" + }, + "documentation_status": { + "type": "string", + "enum": [ + "draft", + "in_review", + "approved", + "deprecated" + ] } }, "required": [ diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index f35e6e0..09a57b6 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -6,6 +6,12 @@ export enum DataAssetShareType { public = 'public', private = 'private', } +export enum DocumentationStatus { + draft = 'draft', + in_review = 'in_review', + approved = 'approved', + deprecated = 'deprecated', +} export enum OrderEnum { asc = 'asc', desc = 'desc', @@ -204,6 +210,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; + @ApiPropertyOptional({ enum: DocumentationStatus }) + documentation_status?: DocumentationStatus; } export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() From 4a5f8f679a19194f827944d8af85b0f24e39e876 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 16 Jul 2026 09:08:11 -0300 Subject: [PATCH 3/6] FEAT: rename documentation_status to certification_status in docs and update package.json for protospack versioning --- docsfera.json | 2 +- package-lock.json | 3 +-- package.json | 2 +- src/modules/catalog/dtos/index.ts | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docsfera.json b/docsfera.json index be0e972..57b9a6e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11002,7 +11002,7 @@ "docs": { "type": "string" }, - "documentation_status": { + "certification_status": { "type": "string", "enum": [ "draft", diff --git a/package-lock.json b/package-lock.json index 28b3128..dcb4ea9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1738,7 +1738,6 @@ "version": "3.40.0-beta.10", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", - "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index dbd506b..fdbc61e 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 09a57b6..dab295a 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -6,7 +6,7 @@ export enum DataAssetShareType { public = 'public', private = 'private', } -export enum DocumentationStatus { +export enum CertificationStatus { draft = 'draft', in_review = 'in_review', approved = 'approved', @@ -210,8 +210,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; - @ApiPropertyOptional({ enum: DocumentationStatus }) - documentation_status?: DocumentationStatus; + @ApiPropertyOptional({ enum: CertificationStatus }) + certification_status?: CertificationStatus; } export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() From 9b3894f9e49e853bc2929452410501136e3c0e15 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 16 Jul 2026 17:05:10 -0300 Subject: [PATCH 4/6] FIX: pin npm version to 10.8.2 in Dockerfile for consistency --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 607f833..97b68de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM node:20-alpine AS base_image -RUN npm install -g npm@latest +RUN npm install -g npm@10.8.2 FROM base_image AS build_base WORKDIR /app From 3fd586753e99276ea095b8402e45bec238257158 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 17 Jul 2026 09:55:17 -0300 Subject: [PATCH 5/6] FEAT: add endpoint and logic to update data asset certification status --- docsfera.json | 86 ++++++++++++++++++++--- src/authentication/permissions.enum.ts | 10 +++ src/modules/catalog/catalog.controller.ts | 31 ++++++++ src/modules/catalog/catalog.service.ts | 23 ++++++ src/modules/catalog/dtos/index.ts | 10 ++- 5 files changed, 149 insertions(+), 11 deletions(-) diff --git a/docsfera.json b/docsfera.json index 57b9a6e..5373202 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5940,6 +5940,66 @@ ] } }, + "/catalog/data-asset/{id}/certification-status": { + "put": { + "operationId": "CatalogController_updateDataAssetCertificationStatus", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IUpdateCertificationStatusRequest" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IUpdateCertificationStatusRequest" + } + } + } + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset/{id}/manage-permissions": { "put": { "operationId": "CatalogController_manageDataAssetPermissions", @@ -11001,15 +11061,6 @@ }, "docs": { "type": "string" - }, - "certification_status": { - "type": "string", - "enum": [ - "draft", - "in_review", - "approved", - "deprecated" - ] } }, "required": [ @@ -11029,6 +11080,23 @@ "data_asset" ] }, + "IUpdateCertificationStatusRequest": { + "type": "object", + "properties": { + "certification_status": { + "type": "string", + "enum": [ + "draft", + "in_review", + "approved", + "deprecated" + ] + } + }, + "required": [ + "certification_status" + ] + }, "ICreateDataAsset": { "type": "object", "properties": { diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index ca5e9ac..82ee026 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -357,6 +357,16 @@ export const PERMISSIONS_GROUPS = { 'es-es': 'Crear y editar atributos en el catálogo', }, }, + CERTIFY: { + seqid: 53, + claim: 'catalog:certify', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Alterar o status de certificação dos Ativos', + 'en-us': "Change Assets' certification status", + 'es-es': 'Cambiar el estado de certificación de los Activos', + }, + }, DELETE: { seqid: 1, claim: 'catalog:delete', diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index f097115..308a082 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -17,6 +17,7 @@ import { HttpStatus, Res, } from '@nestjs/common'; +import { ValidationPipe } from '../../pipes/object-validation.pipe'; import { ApiCreatedResponse, ApiHeaders, @@ -46,6 +47,7 @@ import { IMakeAComment, IOneDataAsset, IPreviewResponse, + IUpdateCertificationStatusRequest, IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, @@ -493,6 +495,8 @@ export class CatalogController { language, }); + delete (body as any).certification_status; + const result = await this.catalogService.updateOneDataAsset({ body, data_asset_id, @@ -506,6 +510,33 @@ export class CatalogController { return result; } + @Put('data-asset/:id/certification-status') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async updateDataAssetCertificationStatus( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') data_asset_id: string, + @Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest, + ): Promise { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + language, + }); + + return this.catalogService.updateCertificationStatus({ + body, + data_asset_id, + metadata, + }); + } + @Post('data-asset/:id/docs') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 638e7b4..f0fced8 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -29,6 +29,7 @@ import { AssetReporter, BatchRemoveRlsRulesRequest, CreateDataDocsDTO, + IUpdateCertificationStatusRequest, IUpdateDataRequest, TriggerCatalogReq, } from './dtos'; @@ -384,6 +385,28 @@ class CatalogService implements OnModuleInit { return { data_asset: asset[0] }; } + async updateCertificationStatus(data: { + data_asset_id: string; + body: IUpdateCertificationStatusRequest; + metadata: Metadata; + }) { + const { body, data_asset_id, metadata } = data; + + await lastValueFrom( + this.catalogWriteService.UpdateDataAsset( + { + id: data_asset_id, + changes: JSON.stringify({ + certification_status: body.certification_status, + }), + }, + metadata, + ), + ); + + return { certification_status: body.certification_status }; + } + async updateOneDataAsset(data: { data_asset_id: string; customer_id: string; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index dab295a..770b01c 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -1,4 +1,5 @@ import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger'; +import { IsEnum } from 'class-validator'; import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; export enum DataAssetShareType { @@ -210,9 +211,14 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; - @ApiPropertyOptional({ enum: CertificationStatus }) - certification_status?: CertificationStatus; } + +export class IUpdateCertificationStatusRequest { + @ApiProperty({ enum: CertificationStatus }) + @IsEnum(CertificationStatus) + certification_status: CertificationStatus; +} + export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() display_name: string; From 47ad527d38ec329865f91e06c393d963175caea5 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Fri, 7 Aug 2026 12:18:37 -0300 Subject: [PATCH 6/6] FIX: skip nimbus update when customer haven't catalog module --- docsfera.json | 44 -------------- src/modules/inputs/inputs.service.ts | 9 +-- .../pipelinesV2/pipelines.controller.ts | 9 +-- src/modules/pipelinesV2/pipelines.service.ts | 59 ++++++++++--------- src/utils/PackTheMetadata.ts | 1 + 5 files changed, 38 insertions(+), 84 deletions(-) diff --git a/docsfera.json b/docsfera.json index 5373202..bdf641a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4570,50 +4570,6 @@ ] } }, - "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunJobs", - "summary": "Get pipeline run jobs", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 96df891..99a6753 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -23,6 +23,7 @@ import { } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { CreateInputReq } from './dtos/input.model'; +import { Metadata } from '@grpc/grpc-js'; @Injectable() @@ -73,10 +74,10 @@ export class InputsService { objectCamelToSnake(createInputResponse); return createInputResponse; }, - update: async (updateInputDTO: UpdateInputRequest): Promise => { + update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise => { this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO)); const updateInputResponse = await lastValueFrom( - this.inputWriteService.InputUpdate(updateInputDTO), + this.inputWriteService.InputUpdate(updateInputDTO, metadata), ); return updateInputResponse; @@ -206,7 +207,7 @@ export class InputsService { return findOneInputResponse; } - async update(id: string, data, info: Info) { + async update(id: string, data, info: Info, metadata?: Metadata) { // this.validateCron({ ...data, info }); try { const { @@ -217,7 +218,7 @@ export class InputsService { id, ...data, info, - }); + }, metadata); const updateInputResponse = this.adjustInputPayload( input, diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 70fab15..ea64d8c 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -317,7 +317,6 @@ export class PipelinesController { ) { this.logger.info('PipelinesController - update', { user }); - const { customer_id, customer_name, user_id, username } = user; const info: Info = { user_id: user.user_id, customer: user.customer_name, @@ -325,13 +324,7 @@ export class PipelinesController { pipeline_id: pipelineId }; - const metadata = PackTheMetadata({ - customer_id, - customer_name, - user_id, - username, - language, - }); + const metadata = PackTheMetadata(user); const response = await this.pipelinesClientService.updatePipelineInput( pipelineId, diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 2a8db84..41d9354 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -383,7 +383,8 @@ export class PipelinesService implements OnModuleInit { const updateInputResponse = await this.inputsService.update( inputId, updateInputDTO, - info + info, + metadata ); const inputRollback = () => { @@ -404,34 +405,36 @@ export class PipelinesService implements OnModuleInit { const nimbusUpdates = updateInputResponse?.tablesUpdate || []; - nimbusUpdates.forEach(update => { - const nimbusRollback = () => { - return this.nimbusService.renameTable( - info.customer, - update.database, - { - table_name: update.table_name, - table_schema: update.table_schema - }, - { - table_name: update.old_table_name, - table_schema: update.old_table_schema - } - ); + if (user.customer_modules.includes('catalog')) { + nimbusUpdates.forEach(update => { + const nimbusRollback = () => { + return this.nimbusService.renameTable( + info.customer, + update.database, + { + table_name: update.table_name, + table_schema: update.table_schema + }, + { + table_name: update.old_table_name, + table_schema: update.old_table_schema + } + ); + } + rollback.push(nimbusRollback); + }); + + try { + await this.updateNimbus(info.customer, nimbusUpdates); + } catch (error) { + this.logger.error(error); + if (error instanceof AxiosError) { + this.logger.error(JSON.stringify(error.response.data)); + } + await this.executeRenameRollback(rollback); + + throw new Error("Error Nimbus updating tables"); } - rollback.push(nimbusRollback); - }); - - try { - await this.updateNimbus(info.customer, nimbusUpdates); - } catch (error) { - this.logger.error(error); - if (error instanceof AxiosError) { - this.logger.error(JSON.stringify(error.response.data)); - } - await this.executeRenameRollback(rollback); - - throw new Error("Error Nimbus updating tables"); } try { diff --git a/src/utils/PackTheMetadata.ts b/src/utils/PackTheMetadata.ts index 18958fe..c584daa 100644 --- a/src/utils/PackTheMetadata.ts +++ b/src/utils/PackTheMetadata.ts @@ -9,6 +9,7 @@ interface IMetadata { details?: string; sensitive?: string; roles?: string[]; + customer_modules?: string[]; is_data_manager?: boolean; access_token?: string; host?: string;