From 6182705410d7b32feef8c28d964679196d0b3443 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 11 May 2026 12:52:07 -0300 Subject: [PATCH 01/37] FEAT: pipeline upgrade route --- docsfera.json | 45 ++++++++++++++++++- package-lock.json | 8 ++-- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 16 +++++++ src/modules/pipelinesV2/pipelines.service.ts | 9 ++++ 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/docsfera.json b/docsfera.json index 7ec6259..00b43df 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3580,7 +3580,10 @@ "content": { "application/json": { "schema": { - "type": "string" + "type": "array", + "items": { + "type": "object" + } } } } @@ -3644,6 +3647,46 @@ ] } }, + "/pipelinesV2/{id}/upgrade": { + "patch": { + "operationId": "PipelinesController_upgradeConnector", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "" + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2/init-upload": { "post": { "operationId": "PipelinesController_initUploadFile", diff --git a/package-lock.json b/package-lock.json index a5470e6..f26f5fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.8", + "@dadosfera/protospack-v2": "3.40.0-beta.9", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.8", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz", - "integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==", + "version": "3.40.0-beta.9", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz", + "integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index ab6d629..878d46a 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.8", + "@dadosfera/protospack-v2": "3.40.0-beta.9", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index c424f51..e90af52 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -15,6 +15,7 @@ import { HttpException, BadRequestException, UseGuards, + Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -368,6 +369,21 @@ export class PipelinesController { return response; } + @Patch('/:id/upgrade') + @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + @HttpCode(HttpStatus.NO_CONTENT) + async upgradeConnector( + @Language() language: LanguageEnum, + @Param('id') id: string, + @User() user: RequestUser + ) { + this.logger.info('PipelinesController - upgrade connector'); + + const metadata = PackTheMetadata(user); + + await this.pipelinesClientService.upgrade(id, metadata); + } + @Delete(':id') @ApiNoContentResponse() @HttpCode(HttpStatus.NO_CONTENT) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index b07330f..53d7711 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -176,6 +176,15 @@ export class PipelinesService implements OnModuleInit { return updatePipelineResponse; } + async upgrade(id: string, metadata: Metadata) { + await lastValueFrom( + this.pipelineWriteService.Upgrade( + { id }, + metadata, + ), + ); + } + async remove(data: { id: string; metadata: Metadata; user: RequestUser }) { const { id, metadata, user } = data; const info = { From b38b8f51e2ef2efc18c7e6aeee559985151d026d Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Wed, 20 May 2026 15:34:49 -0300 Subject: [PATCH 02/37] FEAT: Add endpoint to fetch pipeline run jobs --- .../platform-api/platform-api.controller.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 16d27b3..62ca9f5 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -832,6 +832,23 @@ export class PlatformApiController { ); } + @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getPipelineRunJobs( + @Param('pipelineId') pipelineId: string, + @Param('runId') runId: string, + @User() user: RequestUser, + ) { + const normalizedPipelineId = this.normalizePipelineId(pipelineId); + const normalizedRunId = this.normalizePipelineId(runId); + + return this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`, + user, + ); + } + // ==================== JOBS - COLUMN EDITING ROUTES ==================== @Put('jobs/:jobId/input') From 985170d7ae56787e32767aaaad5b9e9f9eef0a50 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:12:35 -0300 Subject: [PATCH 03/37] chore: exposure from route pipeline run jobs to maestro --- src/modules/platform-api/platform-api.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 62ca9f5..f3abaf5 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -833,6 +833,7 @@ export class PlatformApiController { } @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') + @ApiOperation({ summary: 'Get pipeline run jobs' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getPipelineRunJobs( @Param('pipelineId') pipelineId: string, From 94fbdb222603a0e664d37d88a2b4e0586cab25be Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:21:30 -0300 Subject: [PATCH 04/37] fix: validate workflow --- .github/workflows/validate-k8s.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 8a67a9c..85e48a9 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -36,6 +36,13 @@ jobs: with: version: 'v3.9.0' + - name: Install Helm Diff plugin + run: | + if ! helm plugin list | grep -q '^diff'; then + helm plugin install https://github.com/databus23/helm-diff + fi + helm diff version + - name: Determine DNS_HOST based on environment id: set_dns env: From 2414fcf21e1390470d44aafaa89c1bc4ff954903 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:23:45 -0300 Subject: [PATCH 05/37] fix: validate workflow --- .github/workflows/validate-k8s.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 85e48a9..139fc6a 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -37,11 +37,7 @@ jobs: version: 'v3.9.0' - name: Install Helm Diff plugin - run: | - if ! helm plugin list | grep -q '^diff'; then - helm plugin install https://github.com/databus23/helm-diff - fi - helm diff version + run: helm plugin install https://github.com/databus23/helm-diff || true - name: Determine DNS_HOST based on environment id: set_dns From 21a82b64f371ef4c5fb8fa5d10a1346afb07c429 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:31:45 -0300 Subject: [PATCH 06/37] fix: validate workflow --- .github/workflows/validate-k8s.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 139fc6a..d2e3487 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -37,7 +37,15 @@ jobs: version: 'v3.9.0' - name: Install Helm Diff plugin - run: helm plugin install https://github.com/databus23/helm-diff || true + env: + HELM_PLUGINS: /home/runner/.local/share/helm/plugins + run: | + mkdir -p "$HELM_PLUGINS" + if ! helm plugin list | grep -q '^diff'; then + helm plugin install https://github.com/databus23/helm-diff + fi + helm plugin list + helm diff version - name: Determine DNS_HOST based on environment id: set_dns @@ -105,4 +113,5 @@ jobs: - name: Run Helmfile Diff env: ENV: ${{ needs.extract_environment.outputs.environment }} + HELM_PLUGINS: /home/runner/.local/share/helm/plugins run: helmfile -f deploy/helmfiles/${ENV}.yaml diff From 9e49abb40da88094978661b7d23b34ee63dcba2b Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:37:47 -0300 Subject: [PATCH 07/37] fix: validate workflow --- .github/workflows/validate-k8s.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index d2e3487..d3f7861 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -36,17 +36,6 @@ jobs: with: version: 'v3.9.0' - - name: Install Helm Diff plugin - env: - HELM_PLUGINS: /home/runner/.local/share/helm/plugins - run: | - mkdir -p "$HELM_PLUGINS" - if ! helm plugin list | grep -q '^diff'; then - helm plugin install https://github.com/databus23/helm-diff - fi - helm plugin list - helm diff version - - name: Determine DNS_HOST based on environment id: set_dns env: @@ -82,6 +71,9 @@ jobs: sudo mv helmfile /usr/local/bin/ helmfile --version + - name: Install Helm Diff plugin + run: helm plugin install https://github.com/databus23/helm-diff || true + - name: Debug Helm env run: | helm env From ccd4159c59bbe7d53066906a7d954676fbc5f835 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:43:37 -0300 Subject: [PATCH 08/37] fix: validate workflow --- .github/workflows/validate-k8s.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index d3f7861..ae6d383 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -72,7 +72,9 @@ jobs: helmfile --version - name: Install Helm Diff plugin - run: helm plugin install https://github.com/databus23/helm-diff || true + run: | + helm plugin install https://github.com/databus23/helm-diff --version v3.9.3 + helm diff version - name: Debug Helm env run: | From b4cc8151d7cdd0208933dd8a9de3012759def593 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 25 May 2026 14:01:00 -0300 Subject: [PATCH 09/37] FIX: release note endpoint --- docsfera.json | 28 ++++++++++- src/app.module.ts | 3 ++ .../release_note/dto/release_note.dto.ts | 14 ++++++ .../release_note.controller.spec.ts | 20 ++++++++ .../release_note/release_note.controller.ts | 26 +++++++++++ .../release_note/release_note.module.ts | 10 ++++ .../release_note/release_note.service.spec.ts | 18 ++++++++ .../release_note/release_note.service.ts | 46 +++++++++++++++++++ 8 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/modules/release_note/dto/release_note.dto.ts create mode 100644 src/modules/release_note/release_note.controller.spec.ts create mode 100644 src/modules/release_note/release_note.controller.ts create mode 100644 src/modules/release_note/release_note.module.ts create mode 100644 src/modules/release_note/release_note.service.spec.ts create mode 100644 src/modules/release_note/release_note.service.ts diff --git a/docsfera.json b/docsfera.json index 7ec6259..4c1d57e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3580,7 +3580,10 @@ "content": { "application/json": { "schema": { - "type": "string" + "type": "array", + "items": { + "type": "object" + } } } } @@ -8593,6 +8596,29 @@ "Health" ] } + }, + "/release_note": { + "get": { + "operationId": "ReleaseNoteController_getLatestReleaseNote", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } } }, "info": { diff --git a/src/app.module.ts b/src/app.module.ts index 0865ad2..4db914b 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -35,6 +35,8 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod import { ApiKeyModule } from './modules/api-key/api-key.module'; import { PlatformApiModule } from './modules/platform-api/platform-api.module'; import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module'; +import { ReleaseNoteModule } from './modules/release_note/release_note.module'; + @Module({ providers: [ @@ -79,6 +81,7 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor StorageExplorerModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, + ReleaseNoteModule, ], }) export class AppModule {} diff --git a/src/modules/release_note/dto/release_note.dto.ts b/src/modules/release_note/dto/release_note.dto.ts new file mode 100644 index 0000000..22cc745 --- /dev/null +++ b/src/modules/release_note/dto/release_note.dto.ts @@ -0,0 +1,14 @@ +export type ReleaseNoteDTO = { + id: string; + date: string; + tag: string; + title: string; + visible: boolean; + expiryDate: string; + content: string; + showEmojis: boolean; + image?: string; + link?: string; + linkText?: string; + }; + \ No newline at end of file diff --git a/src/modules/release_note/release_note.controller.spec.ts b/src/modules/release_note/release_note.controller.spec.ts new file mode 100644 index 0000000..921eca1 --- /dev/null +++ b/src/modules/release_note/release_note.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReleaseNoteController } from './release_note.controller'; +import { ReleaseNoteService } from './release_note.service'; + +describe('ReleaseNoteController', () => { + let controller: ReleaseNoteController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [ReleaseNoteController], + providers: [ReleaseNoteService], + }).compile(); + + controller = module.get(ReleaseNoteController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/src/modules/release_note/release_note.controller.ts b/src/modules/release_note/release_note.controller.ts new file mode 100644 index 0000000..fde44bd --- /dev/null +++ b/src/modules/release_note/release_note.controller.ts @@ -0,0 +1,26 @@ +import { Controller, Get, Inject } from '@nestjs/common'; +import { ReleaseNoteService } from './release_note.service'; +import { Authenticated } from 'src/decorators/authentication.decorator'; +import { Language } from 'src/decorators/language.decorator'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +@Controller('release_note') +@Authenticated() +export class ReleaseNoteController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private readonly releaseNoteService: ReleaseNoteService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Get() + async getLatestReleaseNote(@Language() language: LanguageEnum) { + this.logger.info(`Fetching latest release note for language: ${language}`); + return await this.releaseNoteService.getLatestReleaseNote(language); + } +} diff --git a/src/modules/release_note/release_note.module.ts b/src/modules/release_note/release_note.module.ts new file mode 100644 index 0000000..60cb55c --- /dev/null +++ b/src/modules/release_note/release_note.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { ReleaseNoteService } from './release_note.service'; +import { ReleaseNoteController } from './release_note.controller'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +@Module({ + controllers: [ReleaseNoteController], + providers: [ReleaseNoteService, DadosferaLogger] +}) +export class ReleaseNoteModule {} diff --git a/src/modules/release_note/release_note.service.spec.ts b/src/modules/release_note/release_note.service.spec.ts new file mode 100644 index 0000000..72add9d --- /dev/null +++ b/src/modules/release_note/release_note.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReleaseNoteService } from './release_note.service'; + +describe('ReleaseNoteService', () => { + let service: ReleaseNoteService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ReleaseNoteService], + }).compile(); + + service = module.get(ReleaseNoteService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/modules/release_note/release_note.service.ts b/src/modules/release_note/release_note.service.ts new file mode 100644 index 0000000..776d6e0 --- /dev/null +++ b/src/modules/release_note/release_note.service.ts @@ -0,0 +1,46 @@ +import { Inject, Injectable } from '@nestjs/common'; +import axios, { AxiosInstance } from 'axios'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { ReleaseNoteDTO } from './dto/release_note.dto'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +@Injectable() +export class ReleaseNoteService { + client: AxiosInstance; + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + this.client = axios.create({ + baseURL: process.env.FIREBASE_BASE_URL, + }); + } + + async getLatestReleaseNote(lang: LanguageEnum) { + try { + const lng = lang.split('-'); + const language = lng[0] + '-' + lng[1].toUpperCase(); + + const endpoint = `/release_note/${language}.json`; + const { + data, + status, + config + } = await this.client.get(endpoint) + this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`); + this.logger.info(`Request URL: ${config.baseURL}/${config.url}`); + + return data; + } catch (error) { + this.logger.error(`Error fetching release note: ${error.message}`); + + if (axios.isAxiosError(error)) { + this.logger.error(`Axios error details: ${error.toJSON()}`); + } + } + + } +} From 65ab16236f7ed19a745a02cbaeb3e86ca04921dc Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 25 May 2026 14:09:16 -0300 Subject: [PATCH 10/37] FEAT: update deployment to include firebase base url --- deploy/helm-chart/templates/deployment.yaml | 2 ++ deploy/helm-chart/values-stg.yaml | 1 + deploy/helm-chart/values.yaml | 1 + 3 files changed, 4 insertions(+) diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 68b8e1d..518fd7b 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -113,6 +113,8 @@ spec: value: {{ .Values.maestro.platform_api_url }} - name: STORAGE_EXPLORER_API_URL value: {{ .Values.maestro.storage_explorer_api_url | quote }} + - name: FIREBASE_BASE_URL + value: {{ .Values.maestro.firebase_base_url }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 2c1737d..70d2170 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -10,6 +10,7 @@ maestro: redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" + firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg hostname: maestro.stg.dadosfera.ai diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 4e431f5..5042b24 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -55,6 +55,7 @@ maestro: redis_database: "0" redis_tls: "true" cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf" + firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd autoscaling: enabled: false minReplicas: 1 From 06d505c50ab088617d86ec505c27dab9d98adea9 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Sat, 13 Jun 2026 21:07:14 -0300 Subject: [PATCH 11/37] FEAT: remove deprecated protospack lib --- docsfera.json | 135 ++++++------------ package-lock.json | 18 +-- package.json | 3 +- src/app.module.ts | 2 - src/modules/catalog/catalog.module.ts | 3 - src/modules/inputs/dtos/old_interfaces.ts | 6 +- src/modules/pipelines/client.service.ts | 78 ---------- src/modules/pipelines/interfaces.d.ts | 36 ----- src/modules/pipelines/pipelines-client.ts | 33 ----- src/modules/pipelines/pipelines.controller.ts | 72 ---------- src/modules/pipelines/pipelines.module.ts | 19 --- src/modules/pipelines/pipelines.service.ts | 33 ----- src/modules/pipelinesV2/interfaces.ts | 7 +- .../pipelinesV2/pipelines.controller.ts | 8 +- src/modules/pipelinesV2/pipelines.module.ts | 2 - src/modules/pipelinesV2/pipelines.service.ts | 48 ++++++- src/modules/transformations/interfaces.d.ts | 7 +- 17 files changed, 115 insertions(+), 395 deletions(-) delete mode 100644 src/modules/pipelines/client.service.ts delete mode 100644 src/modules/pipelines/interfaces.d.ts delete mode 100644 src/modules/pipelines/pipelines-client.ts delete mode 100644 src/modules/pipelines/pipelines.controller.ts delete mode 100644 src/modules/pipelines/pipelines.module.ts delete mode 100644 src/modules/pipelines/pipelines.service.ts diff --git a/docsfera.json b/docsfera.json index c68e491..eaaf00e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,14 +3343,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -3874,88 +3867,6 @@ ] } }, - "/pipelines/start/{id}": { - "post": { - "operationId": "PipelinesController_activate", - "summary": "", - "deprecated": true, - "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Pipelines" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/pipelines/{id}/status": { - "get": { - "operationId": "PipelinesController_getPipelineStatus", - "summary": "", - "deprecated": true, - "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Pipelines" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/transformations": { "post": { "operationId": "TransformationsController_create", @@ -4659,6 +4570,50 @@ ] } }, + "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunJobs", + "summary": "Get pipeline run jobs", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", diff --git a/package-lock.json b/package-lock.json index f26f5fe..28b3128 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,8 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.9", + "@dadosfera/protospack-v2": "3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,19 +1734,10 @@ "winston-log2gelf": "^2.4.0" } }, - "node_modules/@dadosfera/protospack": { - "version": "2.5.3", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz", - "integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==", - "license": "ISC", - "dependencies": { - "rxjs": "^7.5.5" - } - }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.9", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz", - "integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==", + "version": "3.40.0-beta.10", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", + "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 878d46a..dbd506b 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.9", + "@dadosfera/protospack-v2": "3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index 4db914b..b0bc6ae 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -17,7 +17,6 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test. import { NetworkConfigModule } from './modules/network-config/network-config.module'; import { InputsModule } from './modules/inputs/inputs.module'; import { OauthModule } from './modules/oauth/oauth.module'; -import { PipelinesModule } from './modules/pipelines/pipelines.module'; import { TransformationsModule } from './modules/transformations/transformations.module'; import { HealthModule } from './modules/health/health.module'; import { CatalogModule } from './modules/catalog/catalog.module'; @@ -60,7 +59,6 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module'; PermissionsModule, TermsOfUseModule, ConnectionTestModule, - PipelinesModule, TransformationsModule, UsersModule, RolesModule, diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index 8ce1579..982ef09 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -5,20 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { CatalogController } from './catalog.controller'; import { CatalogClientConfiguration } from './catalog-client'; import { ClientsModule } from '@nestjs/microservices'; -import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { UsersModule } from '../users/users.module'; import { RolesModule } from '../roles/roles.module'; import { CustomersModule } from '../customers/customers.module'; import { ShareModule } from './share/share.module'; import { CatalogService } from './catalog.service'; -import { MixpanelModule } from '../mixpanel/mixpanel.module'; const client = new CatalogClientConfiguration(); @Module({ imports: [ ClientsModule.register([client.providerOptions]), - OldPipelineModule, UsersModule, RolesModule, CustomersModule, diff --git a/src/modules/inputs/dtos/old_interfaces.ts b/src/modules/inputs/dtos/old_interfaces.ts index 21c8cff..e02a567 100644 --- a/src/modules/inputs/dtos/old_interfaces.ts +++ b/src/modules/inputs/dtos/old_interfaces.ts @@ -1,4 +1,8 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} interface Values { jdbc_user: string; diff --git a/src/modules/pipelines/client.service.ts b/src/modules/pipelines/client.service.ts deleted file mode 100644 index 1dc395e..0000000 --- a/src/modules/pipelines/client.service.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { ConflictException, Inject, OnModuleInit } from '@nestjs/common'; -import { ClientGrpc } from '@nestjs/microservices'; -import { - PipelineServicesNames, - PipelinesServiceInterface, -} from '@dadosfera/protospack'; -import { lastValueFrom } from 'rxjs'; - -import { IIdRequest } from './interfaces'; - -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { PipelinesClientConfiguration } from './pipelines-client'; - -export class PipelinesClientService implements OnModuleInit { - private pipelineService: PipelinesServiceInterface; - logger: DadosferaLogger; - - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - @Inject(PipelinesClientConfiguration.name) - private readonly grpcClient: ClientGrpc, - ) { - this.logger = dadosferaLogger.logger; - } - - onModuleInit() { - this.pipelineService = - this.grpcClient.getService( - PipelineServicesNames.PipelineService, - ); - } - - async getPipelineStatus(data) { - this.logger.info('PipelinesClientService - GetPipelineStatus'); - - const statusPipelineResponse = await lastValueFrom( - this.pipelineService.getPipelineStatus(data), - ) - .then((res) => { - const statusArray = - res.status?.sort((a, b) => { - if (a.id < b.id) { - return 1; - } else { - return -1; - } - }) || []; - return { status: statusArray }; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - this.logger.info('Done'); - - return statusPipelineResponse; - } - - async runPipeline({ id, info }: IIdRequest) { - this.logger.info('PipelinesClientService - RunPipeline'); - const statusPipelineResponse = await lastValueFrom( - this.pipelineService.triggerPipeline({ id, info }), - ).catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - if (statusPipelineResponse.status == false) { - throw new ConflictException( - 'This pipeline is not ready yet to execute, Try again later!', - ); - } - - this.logger.info('Done'); - return statusPipelineResponse; - } -} diff --git a/src/modules/pipelines/interfaces.d.ts b/src/modules/pipelines/interfaces.d.ts deleted file mode 100644 index 9a9dbc5..0000000 --- a/src/modules/pipelines/interfaces.d.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; - -export interface ICreatePipelineDto { - input: IdRequest; - transformations: IdRequest[]; - output: IdRequest; - tags: string[]; - name: string; - description: string; - info: Info; -} - -export interface IdRequest { - id: string; -} - -export interface IIdRequest { - id: string; - info: Info; -} - -export interface IUpdatePipelineRequest { - input: IdRequest; - transformations: IdRequest[]; - output: IdRequest; - tags: string[]; - name: string; - description: string; - id: string; - info: Info; -} - -export interface IGetPipelineLogsRequest { - id: string; - details: string; -} diff --git a/src/modules/pipelines/pipelines-client.ts b/src/modules/pipelines/pipelines-client.ts deleted file mode 100644 index 2dcf630..0000000 --- a/src/modules/pipelines/pipelines-client.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { - ClientsProviderAsyncOptions, - GrpcOptions, - Transport, -} from '@nestjs/microservices'; -import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack'; -import { credentials } from '@grpc/grpc-js'; - -const isLocalConnection = - process.env.PIFACTORY_URL.startsWith('pi-factory:') || - process.env.PIFACTORY_URL.includes('0.0.0.0'); - -export class PipelinesClientConfiguration { - public name = 'PipelinesClientConfiguration'; - private config: GrpcOptions = { - transport: Transport.GRPC, - options: { - url: process.env.PIFACTORY_URL, - package: PipelinePackages, - credentials: isLocalConnection ? undefined : credentials.createSsl(), - protoPath: PipelineProtoFilePath, - loader: { - keepCase: true, - enums: String, - defaults: false, - }, - }, - }; - providerOptions: ClientsProviderAsyncOptions = { - name: this.name, - ...this.config, - }; -} diff --git a/src/modules/pipelines/pipelines.controller.ts b/src/modules/pipelines/pipelines.controller.ts deleted file mode 100644 index 6054ff6..0000000 --- a/src/modules/pipelines/pipelines.controller.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common'; -import { ApiOperation, ApiTags } from '@nestjs/swagger'; -import { - AuthenticateCondition, - Authenticated, - RequireSomePermission, -} from 'src/decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; -import { PipelinesService } from './pipelines.service'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; - -@ApiInternalOnlyController() -@ApiTags('Pipelines') -@Controller('pipelines') -@Authenticated() -export class PipelinesController { - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private pipelineService: PipelinesService, - ) { - this.logger = dadosferaLogger.logger; - } - - @Post('start/:id') - @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - @ApiOperation({ - deprecated: true, - description: - 'This method is deprecated. Please use route /pipelinesV2/start/:id instead', - }) - async activate(@Param('id') id: string, @Body() body) { - const { info } = body; - - this.logger.info( - process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`, - { - user: body.info.user_id, - customer: body.info.customer, - }, - ); - - const response = await this.pipelineService.runPipeline({ id, info }); - - return response; - } - - @Get(':id/status') - @ApiOperation({ - deprecated: true, - description: - 'This method is deprecated. Please use route /pipelinesV2/:id/status instead', - }) - @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus(@Body() body, @Param('id') id: string) { - body.id = id; - - this.logger.info( - process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, - { - user: body.info.user_id, - customer: body.info.customer, - }, - ); - - const response = await this.pipelineService.getPipelineStatus(body); - - return response; - } -} diff --git a/src/modules/pipelines/pipelines.module.ts b/src/modules/pipelines/pipelines.module.ts deleted file mode 100644 index 192dcab..0000000 --- a/src/modules/pipelines/pipelines.module.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ClientsModule } from '@nestjs/microservices'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; - -import { PipelinesController } from './pipelines.controller'; -import { PipelinesService } from './pipelines.service'; - -import { PipelinesClientConfiguration } from './pipelines-client'; -import { PipelinesClientService } from './client.service'; - -const client = new PipelinesClientConfiguration(); - -@Module({ - imports: [ClientsModule.register([client.providerOptions])], - controllers: [PipelinesController], - providers: [PipelinesService, PipelinesClientService, DadosferaLogger], - exports: [PipelinesService], -}) -export class PipelinesModule {} diff --git a/src/modules/pipelines/pipelines.service.ts b/src/modules/pipelines/pipelines.service.ts deleted file mode 100644 index afc8103..0000000 --- a/src/modules/pipelines/pipelines.service.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; -import { PipelinesClientService } from './client.service'; -import { IIdRequest } from './interfaces'; -import { objectCamelToSnake } from 'src/utils/CaseConverter'; - -@Injectable() -export class PipelinesService { - constructor(private pipelineClient: PipelinesClientService) {} - - async getPipelineStatus(data: IIdRequest) { - try { - const pipelineStatusResponse = - await this.pipelineClient.getPipelineStatus(data); - - return objectCamelToSnake(pipelineStatusResponse); - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async runPipeline({ id, info }: IIdRequest) { - try { - const triggerPipelineResponse = await this.pipelineClient.runPipeline({ - id, - info, - }); - - return objectCamelToSnake(triggerPipelineResponse); - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } -} diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 252b94c..635ae80 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -1,5 +1,4 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; export class PipelineInputsDTO { @ApiProperty() @@ -62,6 +61,12 @@ export interface IIdRequest { info: Info; } +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} + export interface IUpdatePipelineRequest { input: IdRequest; transformations: IdRequest[]; diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index e90af52..70fab15 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -15,7 +15,6 @@ import { HttpException, BadRequestException, UseGuards, - Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -35,7 +34,6 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { ICompleteUploadCSVFile, ICreatePipelineCSVFile, @@ -64,9 +62,7 @@ export class PipelinesController { constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, - private pipelinesClientService: PipelinesService, - private oldPipelinesService: OldPipelineService, ) { this.logger = dadosferaLogger.logger; } @@ -202,7 +198,7 @@ export class PipelinesController { customer: body.info.customer, }); - const response = await this.oldPipelinesService.getPipelineStatus(body); + const response = await this.pipelinesClientService.getPipelineStatus(body); return response; } @@ -513,7 +509,7 @@ export class PipelinesController { }, ); - const response = await this.oldPipelinesService.runPipeline({ id, info }); + const response = await this.pipelinesClientService.runPipeline({ id, info }); return response; } diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 24562c9..b4f24af 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -7,7 +7,6 @@ import { PipelinesService } from './pipelines.service'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; @@ -21,7 +20,6 @@ const client = new PipelinesClientConfiguration(); @Module({ imports: [ ClientsModule.register([client.providerOptions]), - OldPipelineModule, ConnectorModule, InputsModule, TransformationsModule, diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 53d7711..2a8db84 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,6 +1,7 @@ /* eslint-disable no-async-promise-executor */ import { BadRequestException, + ConflictException, HttpException, HttpStatus, Inject, @@ -17,7 +18,7 @@ import { lastValueFrom } from 'rxjs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; +import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { Metadata } from '@grpc/grpc-js'; import { ConnectorClientService } from '../connector/client.service'; @@ -632,4 +633,49 @@ export class PipelinesService implements OnModuleInit { return assets; } + async getPipelineStatus(data) { + this.logger.info('PipelinesClientService - GetPipelineStatus'); + + const statusPipelineResponse = await lastValueFrom( + this.pipelineReadService.PipelineV2GetPipelineV2Status(data), + ) + .then((res) => { + const statusArray = + res.status?.sort((a, b) => { + if (a.id < b.id) { + return 1; + } else { + return -1; + } + }) || []; + return { status: statusArray }; + }) + .catch((err) => { + this.logger.error(err.message); + throw new Error(err); + }); + this.logger.info('Done'); + + return statusPipelineResponse; + } + + async runPipeline({ id, info }: IIdRequest) { + this.logger.info('PipelinesClientService - RunPipeline'); + const statusPipelineResponse = await lastValueFrom( + this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }), + ).catch((err) => { + this.logger.error(err.message); + throw new Error(err); + }); + + if (statusPipelineResponse.status == false) { + throw new ConflictException( + 'This pipeline is not ready yet to execute, Try again later!', + ); + } + + this.logger.info('Done'); + return statusPipelineResponse; + } + } diff --git a/src/modules/transformations/interfaces.d.ts b/src/modules/transformations/interfaces.d.ts index 0e7ab3c..20ba822 100644 --- a/src/modules/transformations/interfaces.d.ts +++ b/src/modules/transformations/interfaces.d.ts @@ -1,5 +1,8 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; - +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} export interface ICreateTransformationsRequest { transformations: Transformation[]; info: Info; From 5c775779928a4c767f8417ccc84bc2fa1bd5feba Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Fri, 19 Jun 2026 17:04:42 -0300 Subject: [PATCH 12/37] feat: add endpoint to retrieve pipeline run jobs --- docsfera.json | 58 ++++++++++++++++++- .../platform-api/platform-api.controller.ts | 20 ++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/docsfera.json b/docsfera.json index 00b43df..aae1a29 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4659,6 +4659,62 @@ ] } }, + "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunJobs", + "summary": "Get pipeline run jobs", + "description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "DB-backed job runs for the selected pipeline run.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", @@ -11662,4 +11718,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index f3abaf5..4263842 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -14,7 +14,7 @@ import { NotFoundException, UseGuards, } from '@nestjs/common'; -import { ApiTags, ApiOperation } from '@nestjs/swagger'; +import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { @@ -833,7 +833,23 @@ export class PlatformApiController { } @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') - @ApiOperation({ summary: 'Get pipeline run jobs' }) + @ApiOperation({ + summary: 'Get pipeline run jobs', + description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.', + }) + @ApiOkResponse({ + description: 'DB-backed job runs for the selected pipeline run.', + schema: { + type: 'object', + properties: { + jobs: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['jobs'], + }, + }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getPipelineRunJobs( @Param('pipelineId') pipelineId: string, From 63efff6adf764b010a67422dfe05dabf98e916ef Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Mon, 22 Jun 2026 20:25:16 -0300 Subject: [PATCH 13/37] FEAT: enhance pipeline run jobs endpoint with error handling and response structure --- docsfera.json | 78 +++++------- .../pipelinesV2/pipelines.controller.ts | 116 +++++++++++++++++- .../platform-api/platform-api.controller.ts | 40 ++++-- 3 files changed, 170 insertions(+), 64 deletions(-) diff --git a/docsfera.json b/docsfera.json index f6130d2..f6515c0 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,7 +3343,14 @@ ], "responses": { "200": { - "description": "" + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } } }, "tags": [ @@ -4570,50 +4577,6 @@ ] } }, - "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunJobs", - "summary": "Get pipeline run jobs", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { "get": { "operationId": "PlatformApiController_getPipelineRunJobs", @@ -8647,10 +8610,33 @@ "Health" ] } + }, + "/release_note": { + "get": { + "operationId": "ReleaseNoteController_getLatestReleaseNote", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } } }, "info": { - "title": "Maestro", + "title": "Maestro - feat/pipeline-run-jobs", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 70fab15..8e9a0e4 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -49,9 +49,23 @@ import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard'; +import { PlatformApiService } from '../platform-api/platform-api.service'; type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; +type PlatformPipelineJob = { job_id?: string; input?: Record }; +type PlatformPipeline = { + pipeline_id?: string; + created_at?: string; + description?: string; + name?: string; + last_status?: string; + status?: string; + cron?: string; + jobs?: PlatformPipelineJob[]; + properties?: Record; + user_id?: string; +}; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -63,10 +77,70 @@ export class PipelinesController { @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, private pipelinesClientService: PipelinesService, + private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } + private normalizePipelineId(id: string): string { + return id?.replace(/-/g, '_') || ''; + } + + private buildPlatformPipelineFallback( + id: string, + platformPipeline: PlatformPipeline, + ): Messages.PipelineV2FindOneResponse { + const jobs = platformPipeline.jobs || []; + const firstInput = jobs.find((job) => job.input)?.input || {}; + const plugin = firstInput.plugin || firstInput.connector; + + const tables = jobs.map((job) => ({ + ...job.input, + name: + job.input?.table_name || + job.input?.source_prefix || + job.input?.stream || + job.job_id || + '', + job_id: job.job_id, + })); + + return { + pipeline: { + id, + created_at: platformPipeline.created_at, + description: platformPipeline.description, + name: platformPipeline.name, + transformations: [], + status: platformPipeline.status || platformPipeline.last_status || '', + input: { + ...firstInput, + plugin, + category: firstInput.category || (firstInput.connector === 's3' ? 'file' : 'database'), + cron: platformPipeline.cron, + tables, + input_id: firstInput.input_id || null, + }, + properties: platformPipeline.properties || {}, + username: platformPipeline.user_id, + } as any, + }; + } + + private async getPipelineFromPlatformApi( + id: string, + user: RequestUser, + ): Promise { + const normalizedId = this.normalizePipelineId(id); + const platformPipeline = await this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedId}`, + user, + ); + + return this.buildPlatformPipelineFallback(id, platformPipeline); + } + @Get('monitoring-dashboard') @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getMonitoringDashboard(@User() user: RequestUser) { @@ -189,18 +263,38 @@ export class PipelinesController { @Get(':id/status') @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus(@Body() body, @Param('id') id: string) { - - body.id = id; + async getPipelineStatus( + @Param('id') id: string, + @User() user: RequestUser, + ) { + const body = { + id, + info: { + customer_id: user.customer_id, + user_id: user.user_id, + customer: user.customer_name, + }, + }; this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, { user: body.info.user_id, customer: body.info.customer, }); - const response = await this.pipelinesClientService.getPipelineStatus(body); + try { + return await this.pipelinesClientService.getPipelineStatus(body); + } catch (error) { + this.logger.warn('PipelinesController - getPipelineStatus fallback to platform-api', { + id, + error: error.message, + }); - return response; + return this.platformApiService.proxy( + 'GET', + `/pipeline/${this.normalizePipelineId(id)}/pipeline_run`, + user, + ); + } } @Get('/:id') @@ -222,7 +316,17 @@ export class PipelinesController { language, }); - const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); + let pipelineRes: Messages.PipelineV2FindOneResponse; + try { + pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); + } catch (error) { + this.logger.warn('PipelinesController - findOne fallback to platform-api', { + id, + error: error.message, + }); + + return this.getPipelineFromPlatformApi(id, user); + } const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); const input_id = parsed.input_id; diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 4263842..ba6ddd1 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -72,6 +72,10 @@ export class PlatformApiController { return id?.replace(/-/g, '_') || ''; } + private decodePathParam(value: string): string { + return value ? decodeURIComponent(value) : ''; + } + /** * Denormalize ID back to UUID format (replace _ with -). * Used when we receive a normalized ID but need the original UUID. @@ -778,10 +782,10 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}`, user, ); } @@ -794,10 +798,10 @@ export class PlatformApiController { @User() user: RequestUser, @Query() query: Record, ) { - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/pipeline_run/${normalizedRunId}/logs`, + `/pipeline/pipeline_run/${decodedRunId}/logs`, user, undefined, query, @@ -813,7 +817,7 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); const status = await this.platformApiService.proxy( 'GET', @@ -827,7 +831,7 @@ export class PlatformApiController { return this.platformApiService.proxy( 'POST', - `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/cancel`, user, ); } @@ -857,13 +861,25 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); - return this.platformApiService.proxy( - 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`, - user, - ); + try { + return await this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, + user, + ); + } catch (error) { + if (error instanceof HttpException && error.getStatus() === 404) { + this.logger.warn('Platform API pipeline run jobs not found; returning empty jobs list', { + pipelineId: normalizedPipelineId, + runId: decodedRunId, + }); + return { jobs: [] }; + } + + throw error; + } } // ==================== JOBS - COLUMN EDITING ROUTES ==================== From 17363e74f414fbb9c05bf0298be5bb76fedf4688 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Tue, 23 Jun 2026 11:54:59 -0300 Subject: [PATCH 14/37] FEAT: simplify pipeline run jobs handling and normalize run ID usage --- docsfera.json | 9 +- .../pipelinesV2/pipelines.controller.ts | 116 +----------------- .../platform-api/platform-api.controller.ts | 34 ++--- 3 files changed, 18 insertions(+), 141 deletions(-) diff --git a/docsfera.json b/docsfera.json index f6515c0..0d70456 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,14 +3343,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 8e9a0e4..70fab15 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -49,23 +49,9 @@ import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard'; -import { PlatformApiService } from '../platform-api/platform-api.service'; type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; -type PlatformPipelineJob = { job_id?: string; input?: Record }; -type PlatformPipeline = { - pipeline_id?: string; - created_at?: string; - description?: string; - name?: string; - last_status?: string; - status?: string; - cron?: string; - jobs?: PlatformPipelineJob[]; - properties?: Record; - user_id?: string; -}; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -77,70 +63,10 @@ export class PipelinesController { @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, private pipelinesClientService: PipelinesService, - private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } - private normalizePipelineId(id: string): string { - return id?.replace(/-/g, '_') || ''; - } - - private buildPlatformPipelineFallback( - id: string, - platformPipeline: PlatformPipeline, - ): Messages.PipelineV2FindOneResponse { - const jobs = platformPipeline.jobs || []; - const firstInput = jobs.find((job) => job.input)?.input || {}; - const plugin = firstInput.plugin || firstInput.connector; - - const tables = jobs.map((job) => ({ - ...job.input, - name: - job.input?.table_name || - job.input?.source_prefix || - job.input?.stream || - job.job_id || - '', - job_id: job.job_id, - })); - - return { - pipeline: { - id, - created_at: platformPipeline.created_at, - description: platformPipeline.description, - name: platformPipeline.name, - transformations: [], - status: platformPipeline.status || platformPipeline.last_status || '', - input: { - ...firstInput, - plugin, - category: firstInput.category || (firstInput.connector === 's3' ? 'file' : 'database'), - cron: platformPipeline.cron, - tables, - input_id: firstInput.input_id || null, - }, - properties: platformPipeline.properties || {}, - username: platformPipeline.user_id, - } as any, - }; - } - - private async getPipelineFromPlatformApi( - id: string, - user: RequestUser, - ): Promise { - const normalizedId = this.normalizePipelineId(id); - const platformPipeline = await this.platformApiService.proxy( - 'GET', - `/pipeline/${normalizedId}`, - user, - ); - - return this.buildPlatformPipelineFallback(id, platformPipeline); - } - @Get('monitoring-dashboard') @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getMonitoringDashboard(@User() user: RequestUser) { @@ -263,38 +189,18 @@ export class PipelinesController { @Get(':id/status') @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus( - @Param('id') id: string, - @User() user: RequestUser, - ) { - const body = { - id, - info: { - customer_id: user.customer_id, - user_id: user.user_id, - customer: user.customer_name, - }, - }; + async getPipelineStatus(@Body() body, @Param('id') id: string) { + + body.id = id; this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, { user: body.info.user_id, customer: body.info.customer, }); - try { - return await this.pipelinesClientService.getPipelineStatus(body); - } catch (error) { - this.logger.warn('PipelinesController - getPipelineStatus fallback to platform-api', { - id, - error: error.message, - }); + const response = await this.pipelinesClientService.getPipelineStatus(body); - return this.platformApiService.proxy( - 'GET', - `/pipeline/${this.normalizePipelineId(id)}/pipeline_run`, - user, - ); - } + return response; } @Get('/:id') @@ -316,17 +222,7 @@ export class PipelinesController { language, }); - let pipelineRes: Messages.PipelineV2FindOneResponse; - try { - pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); - } catch (error) { - this.logger.warn('PipelinesController - findOne fallback to platform-api', { - id, - error: error.message, - }); - - return this.getPipelineFromPlatformApi(id, user); - } + const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); const input_id = parsed.input_id; diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index ba6ddd1..e8de5ca 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -782,10 +782,10 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const decodedRunId = this.decodePathParam(runId); + const normalizedRunId = this.normalizePipelineId(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`, user, ); } @@ -798,10 +798,10 @@ export class PlatformApiController { @User() user: RequestUser, @Query() query: Record, ) { - const decodedRunId = this.decodePathParam(runId); + const normalizedRunId = this.normalizePipelineId(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/pipeline_run/${decodedRunId}/logs`, + `/pipeline/pipeline_run/${normalizedRunId}/logs`, user, undefined, query, @@ -817,7 +817,7 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const decodedRunId = this.decodePathParam(runId); + const normalizedRunId = this.normalizePipelineId(runId); const status = await this.platformApiService.proxy( 'GET', @@ -831,7 +831,7 @@ export class PlatformApiController { return this.platformApiService.proxy( 'POST', - `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/cancel`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`, user, ); } @@ -863,23 +863,11 @@ export class PlatformApiController { const normalizedPipelineId = this.normalizePipelineId(pipelineId); const decodedRunId = this.decodePathParam(runId); - try { - return await this.platformApiService.proxy( - 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, - user, - ); - } catch (error) { - if (error instanceof HttpException && error.getStatus() === 404) { - this.logger.warn('Platform API pipeline run jobs not found; returning empty jobs list', { - pipelineId: normalizedPipelineId, - runId: decodedRunId, - }); - return { jobs: [] }; - } - - throw error; - } + return this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, + user, + ); } // ==================== JOBS - COLUMN EDITING ROUTES ==================== From 011032e3d4c323350409b0a69f18c7976c06b315 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Tue, 23 Jun 2026 11:58:28 -0300 Subject: [PATCH 15/37] FEAT: update API title in docsfera.json to reflect project name --- docsfera.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 0d70456..ae9f040 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8629,7 +8629,7 @@ } }, "info": { - "title": "Maestro - feat/pipeline-run-jobs", + "title": "Maestro", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} From e03900811b3b5e89cd2f989993d84882c2b95e37 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 13 Jul 2026 08:33:35 -0300 Subject: [PATCH 16/37] FEAT: add documentation status enum and property to data asset --- docsfera.json | 9 +++++++++ src/modules/catalog/dtos/index.ts | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/docsfera.json b/docsfera.json index c68e491..0a384a1 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11046,6 +11046,15 @@ }, "docs": { "type": "string" + }, + "documentation_status": { + "type": "string", + "enum": [ + "draft", + "in_review", + "approved", + "deprecated" + ] } }, "required": [ diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index f35e6e0..09a57b6 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -6,6 +6,12 @@ export enum DataAssetShareType { public = 'public', private = 'private', } +export enum DocumentationStatus { + draft = 'draft', + in_review = 'in_review', + approved = 'approved', + deprecated = 'deprecated', +} export enum OrderEnum { asc = 'asc', desc = 'desc', @@ -204,6 +210,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; + @ApiPropertyOptional({ enum: DocumentationStatus }) + documentation_status?: DocumentationStatus; } export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() From 2e181e70af972b4a25618dbb86fbed960a4c49c2 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 16 Jul 2026 09:08:11 -0300 Subject: [PATCH 17/37] FEAT: rename documentation_status to certification_status in docs and update package.json for protospack versioning --- docsfera.json | 2 +- package-lock.json | 3 +-- package.json | 2 +- src/modules/catalog/dtos/index.ts | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docsfera.json b/docsfera.json index c57e7eb..715ab9a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11014,7 +11014,7 @@ "docs": { "type": "string" }, - "documentation_status": { + "certification_status": { "type": "string", "enum": [ "draft", diff --git a/package-lock.json b/package-lock.json index 28b3128..dcb4ea9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1738,7 +1738,6 @@ "version": "3.40.0-beta.10", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", - "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index dbd506b..fdbc61e 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 09a57b6..dab295a 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -6,7 +6,7 @@ export enum DataAssetShareType { public = 'public', private = 'private', } -export enum DocumentationStatus { +export enum CertificationStatus { draft = 'draft', in_review = 'in_review', approved = 'approved', @@ -210,8 +210,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; - @ApiPropertyOptional({ enum: DocumentationStatus }) - documentation_status?: DocumentationStatus; + @ApiPropertyOptional({ enum: CertificationStatus }) + certification_status?: CertificationStatus; } export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() From 8a9d6c2f9c9573a4fd68441364c894e6aed76d1b Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 16 Jul 2026 17:05:10 -0300 Subject: [PATCH 18/37] FIX: pin npm version to 10.8.2 in Dockerfile for consistency --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 607f833..97b68de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM node:20-alpine AS base_image -RUN npm install -g npm@latest +RUN npm install -g npm@10.8.2 FROM base_image AS build_base WORKDIR /app From 8eb7fd0169cd608176ba5bf327481a377356a6ff Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 17 Jul 2026 09:55:17 -0300 Subject: [PATCH 19/37] FEAT: add endpoint and logic to update data asset certification status --- docsfera.json | 86 ++++++++++++++++++++--- src/authentication/permissions.enum.ts | 10 +++ src/modules/catalog/catalog.controller.ts | 31 ++++++++ src/modules/catalog/catalog.service.ts | 23 ++++++ src/modules/catalog/dtos/index.ts | 10 ++- 5 files changed, 149 insertions(+), 11 deletions(-) diff --git a/docsfera.json b/docsfera.json index 715ab9a..e0aea4c 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5952,6 +5952,66 @@ ] } }, + "/catalog/data-asset/{id}/certification-status": { + "put": { + "operationId": "CatalogController_updateDataAssetCertificationStatus", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IUpdateCertificationStatusRequest" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IUpdateCertificationStatusRequest" + } + } + } + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset/{id}/manage-permissions": { "put": { "operationId": "CatalogController_manageDataAssetPermissions", @@ -11013,15 +11073,6 @@ }, "docs": { "type": "string" - }, - "certification_status": { - "type": "string", - "enum": [ - "draft", - "in_review", - "approved", - "deprecated" - ] } }, "required": [ @@ -11041,6 +11092,23 @@ "data_asset" ] }, + "IUpdateCertificationStatusRequest": { + "type": "object", + "properties": { + "certification_status": { + "type": "string", + "enum": [ + "draft", + "in_review", + "approved", + "deprecated" + ] + } + }, + "required": [ + "certification_status" + ] + }, "ICreateDataAsset": { "type": "object", "properties": { diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index ca5e9ac..82ee026 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -357,6 +357,16 @@ export const PERMISSIONS_GROUPS = { 'es-es': 'Crear y editar atributos en el catálogo', }, }, + CERTIFY: { + seqid: 53, + claim: 'catalog:certify', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Alterar o status de certificação dos Ativos', + 'en-us': "Change Assets' certification status", + 'es-es': 'Cambiar el estado de certificación de los Activos', + }, + }, DELETE: { seqid: 1, claim: 'catalog:delete', diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index f097115..308a082 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -17,6 +17,7 @@ import { HttpStatus, Res, } from '@nestjs/common'; +import { ValidationPipe } from '../../pipes/object-validation.pipe'; import { ApiCreatedResponse, ApiHeaders, @@ -46,6 +47,7 @@ import { IMakeAComment, IOneDataAsset, IPreviewResponse, + IUpdateCertificationStatusRequest, IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, @@ -493,6 +495,8 @@ export class CatalogController { language, }); + delete (body as any).certification_status; + const result = await this.catalogService.updateOneDataAsset({ body, data_asset_id, @@ -506,6 +510,33 @@ export class CatalogController { return result; } + @Put('data-asset/:id/certification-status') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async updateDataAssetCertificationStatus( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') data_asset_id: string, + @Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest, + ): Promise { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + language, + }); + + return this.catalogService.updateCertificationStatus({ + body, + data_asset_id, + metadata, + }); + } + @Post('data-asset/:id/docs') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 638e7b4..f0fced8 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -29,6 +29,7 @@ import { AssetReporter, BatchRemoveRlsRulesRequest, CreateDataDocsDTO, + IUpdateCertificationStatusRequest, IUpdateDataRequest, TriggerCatalogReq, } from './dtos'; @@ -384,6 +385,28 @@ class CatalogService implements OnModuleInit { return { data_asset: asset[0] }; } + async updateCertificationStatus(data: { + data_asset_id: string; + body: IUpdateCertificationStatusRequest; + metadata: Metadata; + }) { + const { body, data_asset_id, metadata } = data; + + await lastValueFrom( + this.catalogWriteService.UpdateDataAsset( + { + id: data_asset_id, + changes: JSON.stringify({ + certification_status: body.certification_status, + }), + }, + metadata, + ), + ); + + return { certification_status: body.certification_status }; + } + async updateOneDataAsset(data: { data_asset_id: string; customer_id: string; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index dab295a..770b01c 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -1,4 +1,5 @@ import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger'; +import { IsEnum } from 'class-validator'; import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; export enum DataAssetShareType { @@ -210,9 +211,14 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; - @ApiPropertyOptional({ enum: CertificationStatus }) - certification_status?: CertificationStatus; } + +export class IUpdateCertificationStatusRequest { + @ApiProperty({ enum: CertificationStatus }) + @IsEnum(CertificationStatus) + certification_status: CertificationStatus; +} + export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() display_name: string; From e1cfc6e1a8bbf5ae8c7ed8aa9a894dc468828f1d Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 21 Jul 2026 11:21:22 -0300 Subject: [PATCH 20/37] FEAT: add custom properties endpoints and DTOs for catalog management --- docsfera.json | 169 ++++++++++++++++++++++ package-lock.json | 9 +- package.json | 2 +- src/modules/catalog/catalog.controller.ts | 63 +++++++- src/modules/catalog/catalog.service.ts | 20 +++ src/modules/catalog/dtos/index.ts | 27 +++- 6 files changed, 282 insertions(+), 8 deletions(-) diff --git a/docsfera.json b/docsfera.json index e0aea4c..4fd0289 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5538,6 +5538,148 @@ ] } }, + "/catalog/custom-properties": { + "get": { + "operationId": "CatalogController_getCustomPropertyDefinitions", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "post": { + "operationId": "CatalogController_upsertCustomPropertyDefinition", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomPropertyDefinitionDto" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/catalog/custom-properties/{key}": { + "delete": { + "operationId": "CatalogController_deleteCustomPropertyDefinition", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "key", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset/{id}": { "get": { "operationId": "CatalogController_getDataAsset", @@ -10851,6 +10993,30 @@ "total" ] }, + "CustomPropertyDefinitionDto": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "text", + "number", + "boolean", + "date" + ] + } + }, + "required": [ + "name", + "type" + ] + }, "IColumnMetadata": { "type": "object", "properties": { @@ -11073,6 +11239,9 @@ }, "docs": { "type": "string" + }, + "custom_properties": { + "type": "object" } }, "required": [ diff --git a/package-lock.json b/package-lock.json index dcb4ea9..eb4a4c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.12", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,9 +1735,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.10", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", - "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", + "version": "3.40.0-beta.12", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.12.tgz", + "integrity": "sha512-nmam/ZJK5ecZlrh/yrMw+C0cHy3bH4NXthKt5csePX9Y/Sgri7tPwPScz9QHQh2Ux2eLflhIcFI6lW2jvOrGNQ==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index fdbc61e..346950d 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.12", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 308a082..f8a9c95 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -37,6 +37,7 @@ import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { BatchRemoveRlsRulesRequest, + CustomPropertyDefinitionDto, GetDatasetCatalogTaskRes, ICatalogAllRequest, ICatalogAllResponse, @@ -271,6 +272,66 @@ export class CatalogController { } + @Get('custom-properties') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getCustomPropertyDefinitions(@User() user: RequestUser) { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + }); + + return this.catalogService.getCustomPropertyDefinitions(metadata); + } + + @Post('custom-properties') + @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) + async upsertCustomPropertyDefinition( + @User() user: RequestUser, + @Body() definition: CustomPropertyDefinitionDto, + ) { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + }); + + return this.catalogService.upsertCustomPropertyDefinition( + { + definition: { + key: definition.key || '', + name: definition.name, + type: definition.type, + }, + }, + metadata, + ); + } + + @Delete('custom-properties/:key') + @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) + async deleteCustomPropertyDefinition( + @User() user: RequestUser, + @Param('key') key: string, + ) { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + }); + + return this.catalogService.deleteCustomPropertyDefinition({ key }, metadata); + } + @Get('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, @@ -1002,4 +1063,4 @@ export class CatalogController { this.logger.error(error.message); } } -} \ No newline at end of file +} diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index f0fced8..6f38e41 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -35,9 +35,11 @@ import { } from './dtos'; import { AddRlsRuleRequest, + DeleteCustomPropertyDefinitionRequest, GetNimbusDashboardsRequest, GetRlsRulesRequest, PiiMetadata, + UpsertCustomPropertyDefinitionRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; @@ -120,6 +122,24 @@ class CatalogService implements OnModuleInit { } } + async getCustomPropertyDefinitions(metadata: Metadata) { + return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata)); + } + + async upsertCustomPropertyDefinition( + data: UpsertCustomPropertyDefinitionRequest, + metadata: Metadata, + ) { + return lastValueFrom(this.catalogWriteService.UpsertCustomPropertyDefinition(data, metadata)); + } + + async deleteCustomPropertyDefinition( + data: DeleteCustomPropertyDefinitionRequest, + metadata: Metadata, + ) { + return lastValueFrom(this.catalogWriteService.DeleteCustomPropertyDefinition(data, metadata)); + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 770b01c..aa87f21 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -1,5 +1,10 @@ import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger'; -import { IsEnum } from 'class-validator'; +import { + IsEnum, + IsNotEmpty, + IsOptional, + IsString, +} from 'class-validator'; import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; export enum DataAssetShareType { @@ -198,6 +203,22 @@ export class IData { day_opening: number; } +export type CustomPropertyType = 'text' | 'number' | 'boolean' | 'date'; + +export class CustomPropertyDefinitionDto { + @ApiPropertyOptional() + @IsOptional() + @IsString() + key?: string; + @ApiProperty() + @IsString() + @IsNotEmpty() + name: string; + @ApiProperty({ enum: ['text', 'number', 'boolean', 'date'] }) + @IsEnum(['text', 'number', 'boolean', 'date']) + type: CustomPropertyType; +} + export class IUpdateDataRequest { @ApiProperty() name: string; @@ -211,6 +232,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; + @ApiPropertyOptional() + custom_properties?: Record; } export class IUpdateCertificationStatusRequest { @@ -372,4 +395,4 @@ export type CreateDataDocsDTO = { docs: string; asset_type: string; -} \ No newline at end of file +} From a93fbfbbd9ff0e7143f62c5303d5cf480b2164df Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 21 Jul 2026 16:32:38 -0300 Subject: [PATCH 21/37] FEAT: remove custom properties endpoints and DTOs from catalog management --- docsfera.json | 124 ---------------------- package-lock.json | 8 +- package.json | 2 +- src/modules/catalog/catalog.controller.ts | 44 -------- src/modules/catalog/catalog.service.ts | 16 --- src/modules/catalog/dtos/index.ts | 17 +-- 6 files changed, 6 insertions(+), 205 deletions(-) diff --git a/docsfera.json b/docsfera.json index 4fd0289..08320c6 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5578,106 +5578,6 @@ "access-token": [] } ] - }, - "post": { - "operationId": "CatalogController_upsertCustomPropertyDefinition", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomPropertyDefinitionDto" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/catalog/custom-properties/{key}": { - "delete": { - "operationId": "CatalogController_deleteCustomPropertyDefinition", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - }, - { - "name": "key", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] } }, "/catalog/data-asset/{id}": { @@ -10993,30 +10893,6 @@ "total" ] }, - "CustomPropertyDefinitionDto": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "text", - "number", - "boolean", - "date" - ] - } - }, - "required": [ - "name", - "type" - ] - }, "IColumnMetadata": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index eb4a4c9..0ce8dc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.12", + "@dadosfera/protospack-v2": "^3.40.0-beta.13", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,9 +1735,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.12", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.12.tgz", - "integrity": "sha512-nmam/ZJK5ecZlrh/yrMw+C0cHy3bH4NXthKt5csePX9Y/Sgri7tPwPScz9QHQh2Ux2eLflhIcFI6lW2jvOrGNQ==", + "version": "3.40.0-beta.13", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.13.tgz", + "integrity": "sha512-XYrq29fG0yddQAid6wnuD1n2X9NsXQiHpItngfLL0bXZZBP0ufgiotiEKmOsDtb3Z+apzGJVK9dJ9YduvmVzlQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 346950d..7d0c1dd 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.12", + "@dadosfera/protospack-v2": "^3.40.0-beta.13", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index f8a9c95..e5d7d00 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -37,7 +37,6 @@ import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { BatchRemoveRlsRulesRequest, - CustomPropertyDefinitionDto, GetDatasetCatalogTaskRes, ICatalogAllRequest, ICatalogAllResponse, @@ -289,49 +288,6 @@ export class CatalogController { return this.catalogService.getCustomPropertyDefinitions(metadata); } - @Post('custom-properties') - @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) - async upsertCustomPropertyDefinition( - @User() user: RequestUser, - @Body() definition: CustomPropertyDefinitionDto, - ) { - const { customer_id, customer_name, user_id, username } = user; - const metadata = PackTheMetadata({ - customer_id, - customer_name, - user_id, - username, - }); - - return this.catalogService.upsertCustomPropertyDefinition( - { - definition: { - key: definition.key || '', - name: definition.name, - type: definition.type, - }, - }, - metadata, - ); - } - - @Delete('custom-properties/:key') - @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) - async deleteCustomPropertyDefinition( - @User() user: RequestUser, - @Param('key') key: string, - ) { - const { customer_id, customer_name, user_id, username } = user; - const metadata = PackTheMetadata({ - customer_id, - customer_name, - user_id, - username, - }); - - return this.catalogService.deleteCustomPropertyDefinition({ key }, metadata); - } - @Get('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 6f38e41..e6e9d18 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -35,11 +35,9 @@ import { } from './dtos'; import { AddRlsRuleRequest, - DeleteCustomPropertyDefinitionRequest, GetNimbusDashboardsRequest, GetRlsRulesRequest, PiiMetadata, - UpsertCustomPropertyDefinitionRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; @@ -126,20 +124,6 @@ class CatalogService implements OnModuleInit { return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata)); } - async upsertCustomPropertyDefinition( - data: UpsertCustomPropertyDefinitionRequest, - metadata: Metadata, - ) { - return lastValueFrom(this.catalogWriteService.UpsertCustomPropertyDefinition(data, metadata)); - } - - async deleteCustomPropertyDefinition( - data: DeleteCustomPropertyDefinitionRequest, - metadata: Metadata, - ) { - return lastValueFrom(this.catalogWriteService.DeleteCustomPropertyDefinition(data, metadata)); - } - async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index aa87f21..00c641b 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -203,21 +203,6 @@ export class IData { day_opening: number; } -export type CustomPropertyType = 'text' | 'number' | 'boolean' | 'date'; - -export class CustomPropertyDefinitionDto { - @ApiPropertyOptional() - @IsOptional() - @IsString() - key?: string; - @ApiProperty() - @IsString() - @IsNotEmpty() - name: string; - @ApiProperty({ enum: ['text', 'number', 'boolean', 'date'] }) - @IsEnum(['text', 'number', 'boolean', 'date']) - type: CustomPropertyType; -} export class IUpdateDataRequest { @ApiProperty() @@ -233,7 +218,7 @@ export class IUpdateDataRequest { @ApiPropertyOptional() docs?: string; @ApiPropertyOptional() - custom_properties?: Record; + custom_properties?: Record; } export class IUpdateCertificationStatusRequest { From cffda86eab35de3680877c7200ba2f8025c20bac Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 22 Jul 2026 10:51:53 -0300 Subject: [PATCH 22/37] FEAT: add CustomPropertyDto and update IUpdateDataRequest to use custom properties array --- docsfera.json | 30 +++++++++++++++++++++++++++++- src/modules/catalog/dtos/index.ts | 20 ++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docsfera.json b/docsfera.json index 08320c6..d758028 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11087,6 +11087,31 @@ "docs" ] }, + "CustomPropertyDto": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "text", + "number", + "date", + "boolean" + ] + } + }, + "required": [ + "key", + "value", + "type" + ] + }, "IUpdateDataRequest": { "type": "object", "properties": { @@ -11117,7 +11142,10 @@ "type": "string" }, "custom_properties": { - "type": "object" + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomPropertyDto" + } } }, "required": [ diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 00c641b..aa2ee07 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -204,6 +204,22 @@ export class IData { } +export enum CustomPropertyType { + TEXT = 'text', + NUMBER = 'number', + DATE = 'date', + BOOLEAN = 'boolean', +} + +export class CustomPropertyDto { + @ApiProperty() + key: string; + @ApiProperty() + value: string; + @ApiProperty({ enum: CustomPropertyType }) + type: CustomPropertyType; +} + export class IUpdateDataRequest { @ApiProperty() name: string; @@ -217,8 +233,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; - @ApiPropertyOptional() - custom_properties?: Record; + @ApiPropertyOptional({ type: [CustomPropertyDto] }) + custom_properties?: CustomPropertyDto[]; } export class IUpdateCertificationStatusRequest { From af3b11ad543c976dc0c96a0c23bbbabd207915de Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 28 Jul 2026 07:42:18 -0300 Subject: [PATCH 23/37] FEAT: add color and emoji properties to CustomPropertyDto --- docsfera.json | 8 ++++++++ package-lock.json | 9 ++++----- package.json | 2 +- src/modules/catalog/dtos/index.ts | 4 ++++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docsfera.json b/docsfera.json index d758028..fa2886f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11104,6 +11104,14 @@ "date", "boolean" ] + }, + "color": { + "type": "string", + "description": "nb-status name used to color the property badge" + }, + "emoji": { + "type": "string", + "description": "Emoji shown before the property badge" } }, "required": [ diff --git a/package-lock.json b/package-lock.json index 0ce8dc9..cf5d6e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.13", + "@dadosfera/protospack-v2": "^3.40.0-beta.14", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,10 +1735,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.13", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.13.tgz", - "integrity": "sha512-XYrq29fG0yddQAid6wnuD1n2X9NsXQiHpItngfLL0bXZZBP0ufgiotiEKmOsDtb3Z+apzGJVK9dJ9YduvmVzlQ==", - "license": "ISC", + "version": "3.40.0-beta.14", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz", + "integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 7d0c1dd..2c10052 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.13", + "@dadosfera/protospack-v2": "^3.40.0-beta.14", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index aa2ee07..4334de7 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -218,6 +218,10 @@ export class CustomPropertyDto { value: string; @ApiProperty({ enum: CustomPropertyType }) type: CustomPropertyType; + @ApiPropertyOptional() + color?: string; + @ApiPropertyOptional() + emoji?: string; } export class IUpdateDataRequest { From ad86a6a698bee3031348e6d21ad48f5c1e750ec6 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 28 Jul 2026 12:01:50 -0300 Subject: [PATCH 24/37] FEAT: simplify color and emoji property definitions in docsfera.json --- docsfera.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docsfera.json b/docsfera.json index fa2886f..a1185a4 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11106,12 +11106,10 @@ ] }, "color": { - "type": "string", - "description": "nb-status name used to color the property badge" + "type": "string" }, "emoji": { - "type": "string", - "description": "Emoji shown before the property badge" + "type": "string" } }, "required": [ From 84d64424caf8e24aaf61924c16080f7c70ed0de3 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 30 Jul 2026 10:17:28 -0300 Subject: [PATCH 25/37] feat: read connection metadata from catalog cache --- deploy/helm-chart/templates/deployment.yaml | 2 + deploy/helm-chart/values-stg.yaml | 1 + .../connection-test.controller.ts | 6 +- .../connection-test/connection-test.module.ts | 7 +- .../connection-test.service.spec.ts | 109 ++++++++++++++++++ .../connection-test.service.ts | 68 ++++++----- .../connection-test/dto/connection-test.ts | 2 + .../connections-api/connections-api.config.ts | 11 ++ .../connections-api/connections-api.module.ts | 10 ++ .../connections-api.service.ts | 87 ++++++++++++++ 10 files changed, 273 insertions(+), 30 deletions(-) create mode 100644 src/modules/connection-test/connection-test.service.spec.ts create mode 100644 src/modules/connections-api/connections-api.config.ts create mode 100644 src/modules/connections-api/connections-api.module.ts create mode 100644 src/modules/connections-api/connections-api.service.ts diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 518fd7b..2cdfb2c 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -111,6 +111,8 @@ spec: value: "{{ .Values.maestro.redis_tls }}" - name: PLATFORM_API_URL value: {{ .Values.maestro.platform_api_url }} + - name: CONNECTIONS_API_URL + value: {{ .Values.maestro.connections_api_url | default "" | quote }} - name: STORAGE_EXPLORER_API_URL value: {{ .Values.maestro.storage_explorer_api_url | quote }} - name: FIREBASE_BASE_URL diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 70d2170..9515b1c 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,6 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com + connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg diff --git a/src/modules/connection-test/connection-test.controller.ts b/src/modules/connection-test/connection-test.controller.ts index f197358..ac3d03f 100644 --- a/src/modules/connection-test/connection-test.controller.ts +++ b/src/modules/connection-test/connection-test.controller.ts @@ -84,7 +84,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListSchemas( body, - user.customer_name, + user, ); } @@ -101,7 +101,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListTables( body, - user.customer_name, + user, ); } @@ -118,7 +118,7 @@ export class ConnectionTestController { }); return this.connectionTestService.getTableMetadata( body, - user.customer_name, + user, ); } } diff --git a/src/modules/connection-test/connection-test.module.ts b/src/modules/connection-test/connection-test.module.ts index 1eb4099..b162429 100644 --- a/src/modules/connection-test/connection-test.module.ts +++ b/src/modules/connection-test/connection-test.module.ts @@ -5,10 +5,15 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { ClientsModule } from '@nestjs/microservices'; import { ConnectionTestClientConfiguration } from './connection-test-client.config'; import { ConnectionModule } from '../connection/connection.module'; +import { ConnectionsApiModule } from '../connections-api/connections-api.module'; const client = new ConnectionTestClientConfiguration(); @Module({ controllers: [ConnectionTestController], providers: [ConnectionTestService, DadosferaLogger], - imports: [ClientsModule.register([client.providerOptions]), ConnectionModule], + imports: [ + ClientsModule.register([client.providerOptions]), + ConnectionModule, + ConnectionsApiModule, + ], }) export class ConnectionTestModule {} diff --git a/src/modules/connection-test/connection-test.service.spec.ts b/src/modules/connection-test/connection-test.service.spec.ts new file mode 100644 index 0000000..a4f71a5 --- /dev/null +++ b/src/modules/connection-test/connection-test.service.spec.ts @@ -0,0 +1,109 @@ +import { ConnectionTestService } from './connection-test.service'; +import { RequestUser } from 'src/decorators/user.decorator'; + +describe('ConnectionTestService catalog cache', () => { + const user: RequestUser = { + user_id: 'user-id', + username: 'user@example.com', + permissions: [], + customer_id: 'customer-id', + customer_name: 'customer-name', + customer_tier: 'standard', + access_token: 'token', + customer_modules: [], + roles: [], + }; + const grpcClient = { getService: jest.fn().mockReturnValue({}) }; + const connectionsService = {}; + const connectionsApiService = { proxy: jest.fn() }; + let service: ConnectionTestService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new ConnectionTestService( + grpcClient as any, + connectionsService as any, + connectionsApiService as any, + ); + }); + + it('keeps the existing schemas response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }], + }); + + await expect( + service.connectionTestListSchemas( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + schema_list: ['analytics', 'public'], + }); + }); + + it('keeps the existing tables response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + tables: [{ table_name: 'customers' }, { table_name: 'orders' }], + }); + + await expect( + service.connectionTestListTables( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + table_list: ['customers', 'orders'], + }); + }); + + it('maps cached columns to the existing table metadata contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + columns: [ + { + column_name: 'id', + data_type: 'bigint', + is_primary_key: true, + }, + ], + }); + + await expect( + service.getTableMetadata( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + table_list: ['customers'], + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + tables_metadata: [ + { + table_name: 'customers', + columns: [ + { + name: 'id', + type: 'bigint', + is_primary_key: true, + }, + ], + references: [], + }, + ], + }); + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'GET', + '/connection_catalog/config-id/schemas/public/tables/customers/columns', + user, + ); + }); +}); diff --git a/src/modules/connection-test/connection-test.service.ts b/src/modules/connection-test/connection-test.service.ts index 3b92e12..af5c10b 100644 --- a/src/modules/connection-test/connection-test.service.ts +++ b/src/modules/connection-test/connection-test.service.ts @@ -21,6 +21,7 @@ import { } from '../connection/dtos/connection'; import { RequestUser } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { ConnectionsApiService } from '../connections-api/connections-api.service'; @Injectable() export class ConnectionTestService { @@ -28,6 +29,7 @@ export class ConnectionTestService { constructor( @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc, private connectionsService: ConnectionClientService, + private connectionsApiService: ConnectionsApiService, ) { this.connectionTestReadClient = grpcClient.getService( @@ -147,45 +149,59 @@ export class ConnectionTestService { } async connectionTestListSchemas( body: ConnectionTestListSchemasReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin } = body; - return lastValueFrom( - this.connectionTestReadClient.ListSchemas({ - connection_id, - customer_name, - plugin, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`, + user, ); + return { + operation_result: true, + schema_list: result.schemas.map((schema) => schema.schema_name), + }; } + async connectionTestListTables( body: ConnectionTestListTablesReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin, schema } = body; - return lastValueFrom( - this.connectionTestReadClient.ListTables({ - connection_id, - customer_name, - plugin, - schema, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}/tables`, + user, ); + return { + operation_result: true, + table_list: result.tables.map((table) => table.table_name), + }; } async getTableMetadata( body: GetTableMetadataReq, - customer_name: string, + user: RequestUser, ): Promise { - const { schema, plugin, table_list, connection_id } = body; - return lastValueFrom( - this.connectionTestReadClient.GetTableMetadata({ - connection_id, - customer_name, - plugin, - schema, - table_list, + const tables_metadata = await Promise.all( + body.table_list.map(async (table_name) => { + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}` + + `/tables/${encodeURIComponent(table_name)}/columns`, + user, + ); + return { + table_name, + columns: result.columns.map((column) => ({ + name: column.column_name, + type: column.data_type, + is_primary_key: column.is_primary_key, + })), + references: [], + }; }), ); + return { operation_result: true, tables_metadata }; } } diff --git a/src/modules/connection-test/dto/connection-test.ts b/src/modules/connection-test/dto/connection-test.ts index b6bb4c4..44120fc 100644 --- a/src/modules/connection-test/dto/connection-test.ts +++ b/src/modules/connection-test/dto/connection-test.ts @@ -7,6 +7,8 @@ export class ColumnDto { name: string; @ApiProperty() type: string; + @ApiProperty() + is_primary_key: boolean; } export class TableMetadataDto { @ApiProperty() diff --git a/src/modules/connections-api/connections-api.config.ts b/src/modules/connections-api/connections-api.config.ts new file mode 100644 index 0000000..0e6e450 --- /dev/null +++ b/src/modules/connections-api/connections-api.config.ts @@ -0,0 +1,11 @@ +export const CONNECTIONS_API_CONFIG = { + getUrl: (): string => { + const url = process.env.CONNECTIONS_API_URL; + if (!url) { + throw new Error('CONNECTIONS_API_URL environment variable is not set'); + } + return url; + }, + region: process.env.AWS_REGION || 'us-east-1', + timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10), +}; diff --git a/src/modules/connections-api/connections-api.module.ts b/src/modules/connections-api/connections-api.module.ts new file mode 100644 index 0000000..1835186 --- /dev/null +++ b/src/modules/connections-api/connections-api.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { ConnectionsApiService } from './connections-api.service'; + +@Module({ + providers: [ConnectionsApiService, DadosferaLogger], + exports: [ConnectionsApiService], +}) +export class ConnectionsApiModule {} diff --git a/src/modules/connections-api/connections-api.service.ts b/src/modules/connections-api/connections-api.service.ts new file mode 100644 index 0000000..81e5c41 --- /dev/null +++ b/src/modules/connections-api/connections-api.service.ts @@ -0,0 +1,87 @@ +import { Injectable, Inject, HttpException } from '@nestjs/common'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import axios, { AxiosResponse, Method } from 'axios'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { RequestUser } from '../../decorators/user.decorator'; +import { CONNECTIONS_API_CONFIG } from './connections-api.config'; + +@Injectable() +export class ConnectionsApiService { + private signer: SignatureV4; + private logger: any; + + constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) { + this.logger = dadosferaLogger.logger; + this.signer = new SignatureV4({ + service: 'execute-api', + region: CONNECTIONS_API_CONFIG.region, + credentials: defaultProvider(), + sha256: Sha256, + }); + } + + async proxy( + method: string, + path: string, + user: RequestUser, + ): Promise { + const baseUrl = CONNECTIONS_API_CONFIG.getUrl(); + const url = new URL(`${baseUrl}${path}`); + const headers: Record = { + host: url.hostname, + 'content-type': 'application/json', + customer_name: user.customer_name || '', + customer_id: user.customer_id || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + 'x-customer-id': user.customer_id || '', + }; + const requestToSign = { + method: method.toUpperCase(), + protocol: url.protocol, + hostname: url.hostname, + port: url.port ? parseInt(url.port, 10) : undefined, + path: url.pathname, + headers, + }; + + try { + const signedRequest = await this.signer.sign(requestToSign); + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers: signedRequest.headers as Record, + timeout: CONNECTIONS_API_CONFIG.timeout, + validateStatus: () => true, + }); + + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + return response.data; + } catch (error) { + this.logger.error('Connections API proxy error', { + error: error.message, + path, + method: method.toUpperCase(), + }); + if (error instanceof HttpException) { + throw error; + } + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + if (error.code === 'ECONNREFUSED') { + throw new HttpException('Connections API service unavailable', 503); + } + if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') { + throw new HttpException('Connections API request timeout', 504); + } + throw new HttpException('Internal server error', 500); + } + } +} From 0eafa67e6fb7ad3525804b99cae48c0148e23c5d Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Fri, 31 Jul 2026 09:56:05 -0300 Subject: [PATCH 26/37] FIX: trigger cache connections deployment From 9d0f449eeb0472fdebdb30356f95e09674bb69e8 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Fri, 31 Jul 2026 16:33:15 -0300 Subject: [PATCH 27/37] feat(connection-test): refresh connection catalog cache --- .../connection-test.controller.ts | 34 +++++++ .../connection-test/connection-test.module.ts | 2 + .../connection-test.service.spec.ts | 96 +++++++++++++++++++ .../connection-test.service.ts | 85 +++++++++++++++- .../connection-test/dto/connection-test.ts | 36 ++++++- .../connections-api.service.ts | 14 ++- 6 files changed, 264 insertions(+), 3 deletions(-) diff --git a/src/modules/connection-test/connection-test.controller.ts b/src/modules/connection-test/connection-test.controller.ts index ac3d03f..288d008 100644 --- a/src/modules/connection-test/connection-test.controller.ts +++ b/src/modules/connection-test/connection-test.controller.ts @@ -22,6 +22,9 @@ import { ConnectionTestListTablesRes, GetTableMetadataRes, GetTableMetadataReq, + RefreshCatalogReq, + RefreshCatalogRes, + RefreshCatalogStatusReq, } from './dto/connection-test'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Authenticated } from 'src/decorators/authentication.decorator'; @@ -121,4 +124,35 @@ export class ConnectionTestController { user, ); } + + @Post('refresh-catalog') + @ApiOkResponse({ type: RefreshCatalogRes }) + @HttpCode(HttpStatus.ACCEPTED) + async refreshCatalog( + @User() user: RequestUser, + @Body(new ValidationPipe()) body: RefreshCatalogReq, + ) { + this.logger.info('/connection-test/refresh-catalog', { + user: user.user_id, + customer: user.customer_name, + connection: body.connection_id, + }); + return this.connectionTestService.refreshCatalog(body, user); + } + + @Post('refresh-catalog/status') + @ApiOkResponse({ type: RefreshCatalogRes }) + @HttpCode(HttpStatus.OK) + async refreshCatalogStatus( + @User() user: RequestUser, + @Body(new ValidationPipe()) body: RefreshCatalogStatusReq, + ) { + this.logger.info('/connection-test/refresh-catalog/status', { + user: user.user_id, + customer: user.customer_name, + connection: body.connection_id, + session: body.session_id, + }); + return this.connectionTestService.refreshCatalogStatus(body, user); + } } diff --git a/src/modules/connection-test/connection-test.module.ts b/src/modules/connection-test/connection-test.module.ts index b162429..c184391 100644 --- a/src/modules/connection-test/connection-test.module.ts +++ b/src/modules/connection-test/connection-test.module.ts @@ -6,6 +6,7 @@ import { ClientsModule } from '@nestjs/microservices'; import { ConnectionTestClientConfiguration } from './connection-test-client.config'; import { ConnectionModule } from '../connection/connection.module'; import { ConnectionsApiModule } from '../connections-api/connections-api.module'; +import { PlatformApiModule } from '../platform-api/platform-api.module'; const client = new ConnectionTestClientConfiguration(); @Module({ controllers: [ConnectionTestController], @@ -14,6 +15,7 @@ const client = new ConnectionTestClientConfiguration(); ClientsModule.register([client.providerOptions]), ConnectionModule, ConnectionsApiModule, + PlatformApiModule, ], }) export class ConnectionTestModule {} diff --git a/src/modules/connection-test/connection-test.service.spec.ts b/src/modules/connection-test/connection-test.service.spec.ts index a4f71a5..aed103f 100644 --- a/src/modules/connection-test/connection-test.service.spec.ts +++ b/src/modules/connection-test/connection-test.service.spec.ts @@ -16,6 +16,7 @@ describe('ConnectionTestService catalog cache', () => { const grpcClient = { getService: jest.fn().mockReturnValue({}) }; const connectionsService = {}; const connectionsApiService = { proxy: jest.fn() }; + const platformApiService = { proxy: jest.fn() }; let service: ConnectionTestService; beforeEach(() => { @@ -24,6 +25,7 @@ describe('ConnectionTestService catalog cache', () => { grpcClient as any, connectionsService as any, connectionsApiService as any, + platformApiService as any, ); }); @@ -106,4 +108,98 @@ describe('ConnectionTestService catalog cache', () => { user, ); }); + + it('submits a catalog refresh without holding the request open', async () => { + platformApiService.proxy.mockResolvedValue({ + session_id: 'session-id', + date: '20260731', + }); + + await expect( + service.refreshCatalog( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + status: 'PENDING', + session_id: 'session-id', + date: '20260731', + }); + + expect(platformApiService.proxy).toHaveBeenCalledWith( + 'POST', + '/connection_test', + user, + { + customer_id: user.customer_name, + plugin: 'postgresql', + task: { + task_type: 'refresh_catalog', + connection: { + provider: 'connection_manager', + config_id: 'config-id', + }, + }, + }, + ); + }); + + it('keeps polling without changing the catalog pointer while pending', async () => { + platformApiService.proxy.mockResolvedValue({ status: 'PENDING' }); + + await expect( + service.refreshCatalogStatus( + { + connection_id: 'config-id', + plugin: 'postgresql', + session_id: 'session-id', + date: '20260731', + }, + user, + ), + ).resolves.toEqual({ + operation_result: false, + status: 'PENDING', + session_id: 'session-id', + date: '20260731', + }); + + expect(connectionsApiService.proxy).not.toHaveBeenCalled(); + }); + + it('publishes the catalog pointer after the refresh finishes', async () => { + platformApiService.proxy.mockResolvedValue({ status: 'DONE' }); + connectionsApiService.proxy.mockResolvedValue({ + last_catalog_refresh_status: 'SUCCESS', + }); + + await expect( + service.refreshCatalogStatus( + { + connection_id: 'config/id', + plugin: 'postgresql', + session_id: 'session-id', + date: '20260731', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + status: 'DONE', + session_id: 'session-id', + date: '20260731', + }); + + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'PUT', + '/connection_config/config%2Fid/catalog_metadata', + user, + { + last_catalog_refresh_status: 'SUCCESS', + last_catalog_connection_test_date: '20260731', + last_catalog_connection_test_session_id: 'session-id', + }, + ); + }); }); diff --git a/src/modules/connection-test/connection-test.service.ts b/src/modules/connection-test/connection-test.service.ts index af5c10b..fe4930d 100644 --- a/src/modules/connection-test/connection-test.service.ts +++ b/src/modules/connection-test/connection-test.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { ConnectionTest } from '@dadosfera/protospack-v2'; import { lastValueFrom } from 'rxjs'; @@ -13,6 +13,9 @@ import { ConnectionTestPingRes, GetTableMetadataReq, GetTableMetadataRes, + RefreshCatalogReq, + RefreshCatalogRes, + RefreshCatalogStatusReq, } from './dto/connection-test'; import { ConnectionClientService } from '../connection/client.service'; import { @@ -22,6 +25,7 @@ import { import { RequestUser } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { ConnectionsApiService } from '../connections-api/connections-api.service'; +import { PlatformApiService } from '../platform-api/platform-api.service'; @Injectable() export class ConnectionTestService { @@ -30,6 +34,7 @@ export class ConnectionTestService { @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc, private connectionsService: ConnectionClientService, private connectionsApiService: ConnectionsApiService, + private platformApiService: PlatformApiService, ) { this.connectionTestReadClient = grpcClient.getService( @@ -204,4 +209,82 @@ export class ConnectionTestService { ); return { operation_result: true, tables_metadata }; } + + async refreshCatalog( + body: RefreshCatalogReq, + user: RequestUser, + ): Promise { + const task = await this.platformApiService.proxy( + 'POST', + '/connection_test', + user, + { + customer_id: user.customer_name, + plugin: body.plugin, + task: { + task_type: 'refresh_catalog', + connection: { + provider: 'connection_manager', + config_id: body.connection_id, + }, + }, + }, + ); + + if (!task.session_id || !task.date) { + throw new HttpException( + 'Platform API did not return a catalog refresh task identifier', + HttpStatus.BAD_GATEWAY, + ); + } + + return { + operation_result: true, + status: 'PENDING', + session_id: task.session_id, + date: task.date, + }; + } + + async refreshCatalogStatus( + body: RefreshCatalogStatusReq, + user: RequestUser, + ): Promise { + const result = await this.platformApiService.proxy( + 'POST', + '/connection_test/status', + user, + { + session_id: body.session_id, + date: body.date, + }, + ); + + if (result.status === 'DONE') { + await this.connectionsApiService.proxy( + 'PUT', + `/connection_config/${encodeURIComponent( + body.connection_id, + )}/catalog_metadata`, + user, + { + last_catalog_refresh_status: 'SUCCESS', + last_catalog_connection_test_date: body.date, + last_catalog_connection_test_session_id: body.session_id, + }, + ); + } else if (result.status === 'ERROR' || result.status === 'EXPIRED') { + throw new HttpException( + `Catalog refresh finished with status ${result.status}`, + HttpStatus.BAD_GATEWAY, + ); + } + + return { + operation_result: result.status === 'DONE', + status: result.status, + session_id: body.session_id, + date: body.date, + }; + } } diff --git a/src/modules/connection-test/dto/connection-test.ts b/src/modules/connection-test/dto/connection-test.ts index 44120fc..3496d4c 100644 --- a/src/modules/connection-test/dto/connection-test.ts +++ b/src/modules/connection-test/dto/connection-test.ts @@ -1,5 +1,5 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { IsString, IsOptional } from 'class-validator'; +import { IsIn, IsString, IsOptional } from 'class-validator'; import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection'; import { CreateConnectionDto } from 'src/modules/connection/dtos/connection'; export class ColumnDto { @@ -133,3 +133,37 @@ export class GetTableMetadataRes { @ApiProperty({ type: [TableMetadataDto] }) tables_metadata: TableMetadataDto[]; } + +export class RefreshCatalogReq { + @ApiProperty() + @IsString() + connection_id: string; + + @ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] }) + @IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver']) + plugin: string; +} + +export class RefreshCatalogStatusReq extends RefreshCatalogReq { + @ApiProperty() + @IsString() + session_id: string; + + @ApiProperty() + @IsString() + date: string; +} + +export class RefreshCatalogRes { + @ApiProperty() + operation_result: boolean; + + @ApiProperty() + status: string; + + @ApiProperty() + session_id: string; + + @ApiProperty() + date: string; +} diff --git a/src/modules/connections-api/connections-api.service.ts b/src/modules/connections-api/connections-api.service.ts index 81e5c41..ae2dfd4 100644 --- a/src/modules/connections-api/connections-api.service.ts +++ b/src/modules/connections-api/connections-api.service.ts @@ -27,9 +27,19 @@ export class ConnectionsApiService { method: string, path: string, user: RequestUser, + body?: any, + query?: Record, ): Promise { const baseUrl = CONNECTIONS_API_CONFIG.getUrl(); const url = new URL(`${baseUrl}${path}`); + + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } const headers: Record = { host: url.hostname, 'content-type': 'application/json', @@ -45,8 +55,9 @@ export class ConnectionsApiService { protocol: url.protocol, hostname: url.hostname, port: url.port ? parseInt(url.port, 10) : undefined, - path: url.pathname, + path: url.pathname + url.search, headers, + body: body ? JSON.stringify(body) : undefined, }; try { @@ -55,6 +66,7 @@ export class ConnectionsApiService { method: method as Method, url: url.href, headers: signedRequest.headers as Record, + data: body, timeout: CONNECTIONS_API_CONFIG.timeout, validateStatus: () => true, }); From 51044a23b3f04d2e502b9efaeccf694661d1ed13 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Mon, 3 Aug 2026 09:19:27 -0300 Subject: [PATCH 28/37] FIX: trigger cache connections rollout From 47ad527d38ec329865f91e06c393d963175caea5 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Fri, 7 Aug 2026 12:18:37 -0300 Subject: [PATCH 29/37] FIX: skip nimbus update when customer haven't catalog module --- docsfera.json | 44 -------------- src/modules/inputs/inputs.service.ts | 9 +-- .../pipelinesV2/pipelines.controller.ts | 9 +-- src/modules/pipelinesV2/pipelines.service.ts | 59 ++++++++++--------- src/utils/PackTheMetadata.ts | 1 + 5 files changed, 38 insertions(+), 84 deletions(-) diff --git a/docsfera.json b/docsfera.json index 5373202..bdf641a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4570,50 +4570,6 @@ ] } }, - "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunJobs", - "summary": "Get pipeline run jobs", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 96df891..99a6753 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -23,6 +23,7 @@ import { } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { CreateInputReq } from './dtos/input.model'; +import { Metadata } from '@grpc/grpc-js'; @Injectable() @@ -73,10 +74,10 @@ export class InputsService { objectCamelToSnake(createInputResponse); return createInputResponse; }, - update: async (updateInputDTO: UpdateInputRequest): Promise => { + update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise => { this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO)); const updateInputResponse = await lastValueFrom( - this.inputWriteService.InputUpdate(updateInputDTO), + this.inputWriteService.InputUpdate(updateInputDTO, metadata), ); return updateInputResponse; @@ -206,7 +207,7 @@ export class InputsService { return findOneInputResponse; } - async update(id: string, data, info: Info) { + async update(id: string, data, info: Info, metadata?: Metadata) { // this.validateCron({ ...data, info }); try { const { @@ -217,7 +218,7 @@ export class InputsService { id, ...data, info, - }); + }, metadata); const updateInputResponse = this.adjustInputPayload( input, diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 70fab15..ea64d8c 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -317,7 +317,6 @@ export class PipelinesController { ) { this.logger.info('PipelinesController - update', { user }); - const { customer_id, customer_name, user_id, username } = user; const info: Info = { user_id: user.user_id, customer: user.customer_name, @@ -325,13 +324,7 @@ export class PipelinesController { pipeline_id: pipelineId }; - const metadata = PackTheMetadata({ - customer_id, - customer_name, - user_id, - username, - language, - }); + const metadata = PackTheMetadata(user); const response = await this.pipelinesClientService.updatePipelineInput( pipelineId, diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 2a8db84..41d9354 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -383,7 +383,8 @@ export class PipelinesService implements OnModuleInit { const updateInputResponse = await this.inputsService.update( inputId, updateInputDTO, - info + info, + metadata ); const inputRollback = () => { @@ -404,34 +405,36 @@ export class PipelinesService implements OnModuleInit { const nimbusUpdates = updateInputResponse?.tablesUpdate || []; - nimbusUpdates.forEach(update => { - const nimbusRollback = () => { - return this.nimbusService.renameTable( - info.customer, - update.database, - { - table_name: update.table_name, - table_schema: update.table_schema - }, - { - table_name: update.old_table_name, - table_schema: update.old_table_schema - } - ); + if (user.customer_modules.includes('catalog')) { + nimbusUpdates.forEach(update => { + const nimbusRollback = () => { + return this.nimbusService.renameTable( + info.customer, + update.database, + { + table_name: update.table_name, + table_schema: update.table_schema + }, + { + table_name: update.old_table_name, + table_schema: update.old_table_schema + } + ); + } + rollback.push(nimbusRollback); + }); + + try { + await this.updateNimbus(info.customer, nimbusUpdates); + } catch (error) { + this.logger.error(error); + if (error instanceof AxiosError) { + this.logger.error(JSON.stringify(error.response.data)); + } + await this.executeRenameRollback(rollback); + + throw new Error("Error Nimbus updating tables"); } - rollback.push(nimbusRollback); - }); - - try { - await this.updateNimbus(info.customer, nimbusUpdates); - } catch (error) { - this.logger.error(error); - if (error instanceof AxiosError) { - this.logger.error(JSON.stringify(error.response.data)); - } - await this.executeRenameRollback(rollback); - - throw new Error("Error Nimbus updating tables"); } try { diff --git a/src/utils/PackTheMetadata.ts b/src/utils/PackTheMetadata.ts index 18958fe..c584daa 100644 --- a/src/utils/PackTheMetadata.ts +++ b/src/utils/PackTheMetadata.ts @@ -9,6 +9,7 @@ interface IMetadata { details?: string; sensitive?: string; roles?: string[]; + customer_modules?: string[]; is_data_manager?: boolean; access_token?: string; host?: string; From 31dda867d12b5c6babd95c6f80936e5b4f27a125 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Wed, 12 Aug 2026 09:57:23 -0300 Subject: [PATCH 30/37] FIX: require collect module in endpoints --- docsfera.json | 234 ++++++++++++++++++ src/authentication/permissions.enum.ts | 2 + src/modules/catalog/catalog.controller.ts | 54 ++++ .../connection-test.controller.ts | 6 +- .../connection/connection.controller.ts | 6 +- src/modules/connector/connector.controller.ts | 27 +- .../pipelinesV2/pipelines.controller.ts | 6 +- .../platform-api/platform-api.controller.ts | 4 +- 8 files changed, 334 insertions(+), 5 deletions(-) diff --git a/docsfera.json b/docsfera.json index bdf641a..b3a85b5 100644 --- a/docsfera.json +++ b/docsfera.json @@ -900,6 +900,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -949,6 +952,9 @@ "connections" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -997,6 +1003,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -1033,6 +1042,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -1066,6 +1078,9 @@ "connections" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -2403,6 +2418,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2486,6 +2504,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2525,6 +2546,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2564,6 +2588,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2601,6 +2628,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2640,6 +2670,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2681,6 +2714,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -2722,6 +2758,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -3071,6 +3110,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3120,6 +3162,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3190,6 +3235,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3230,6 +3278,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3270,6 +3321,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3310,6 +3364,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3350,6 +3407,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3397,6 +3457,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3442,6 +3505,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3490,6 +3556,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3528,6 +3597,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3586,6 +3658,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3634,6 +3709,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3674,6 +3752,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3716,6 +3797,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3765,6 +3849,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3814,6 +3901,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3861,6 +3951,9 @@ "PipelinesV2" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4031,6 +4124,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4056,6 +4152,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4092,6 +4191,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4126,6 +4228,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4160,6 +4265,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4187,6 +4295,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4214,6 +4325,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4241,6 +4355,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4277,6 +4394,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4313,6 +4433,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4340,6 +4463,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4367,6 +4493,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4404,6 +4533,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4440,6 +4572,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4484,6 +4619,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4520,6 +4658,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4564,6 +4705,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4600,6 +4744,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4634,6 +4781,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4670,6 +4820,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4706,6 +4859,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4743,6 +4899,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4770,6 +4929,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4797,6 +4959,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -5208,6 +5373,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5251,6 +5419,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5360,6 +5531,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5392,6 +5566,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -5434,6 +5611,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5527,6 +5707,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5585,6 +5768,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5633,6 +5819,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5726,6 +5915,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5776,6 +5968,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5834,6 +6029,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5890,6 +6088,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5950,6 +6151,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5990,6 +6194,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -6030,6 +6237,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -6090,6 +6300,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -6148,6 +6361,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -6524,6 +6740,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -6977,6 +7196,9 @@ "Connection Test" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -7013,6 +7235,9 @@ "Connection Test" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -7049,6 +7274,9 @@ "Connection Test" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -7085,6 +7313,9 @@ "Connection Test" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -7121,6 +7352,9 @@ "Connection Test" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 82ee026..1cb5045 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -722,6 +722,8 @@ export const DADOSFERA_MODULES_KEYS = { PII: 'pii', EMBED: 'embedded-analytics', EMBED_ASSIGNED: 'embed-assigned', + CATALOG: 'catalog', + COLLECT: 'collect', } export const DADOSFERA_MODULES: Array = [ diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 308a082..ef8ab78 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -85,6 +85,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async searchCatalog( @User() user: RequestUser, @Query() query: ICatalogAllRequest, @@ -124,6 +127,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async dowloadAsserts( @User() user: RequestUser, @Query() query: ICatalogAllRequest, @@ -167,6 +173,9 @@ export class CatalogController { @ApiInternalOnlyEndpoint() @Get('data-asset') + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async findByPipelineAndObject(@User() user: RequestUser, @Query() query) { const { username, user_id, customer_id, customer_name, permissions } = user; const { pipeline, object } = query; @@ -225,6 +234,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async findAllTags(@Body() body) { this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, { user: body.info.user_id, @@ -276,6 +288,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async getDataAsset( @User() user: RequestUser, @Param('id') id: string, @@ -387,6 +402,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async getDataAssetColumnsMetadata( @User() user: RequestUser, @Language() language: LanguageEnum, @@ -418,6 +436,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async getDataAssetPreview( @User() user: RequestUser, @Language() language: LanguageEnum, @@ -449,6 +470,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async getDataAssetDocs( @User() user: RequestUser, @Language() language: LanguageEnum, @@ -480,6 +504,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async updateDataAsset( @User() user: RequestUser, @Language() language: LanguageEnum, @@ -515,6 +542,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async updateDataAssetCertificationStatus( @User() user: RequestUser, @Language() language: LanguageEnum, @@ -542,6 +572,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async manageDataAssetDocs( @User() user: RequestUser, @Headers() headers, @@ -579,6 +612,9 @@ export class CatalogController { @ApiInternalOnlyEndpoint() @Put('data-asset/:id/manage-permissions') + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async manageDataAssetPermissions( @Param('id') id: string, @User() user: RequestUser, @@ -601,6 +637,9 @@ export class CatalogController { @ApiInternalOnlyEndpoint() @Put('data-asset/:id/revoke-permissions') + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async revokeDataAssetPermissions( @Param('id') id: string, @User() user: RequestUser, @@ -626,6 +665,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async createDataAsset( @User() user: RequestUser, @Body() body: ICreateDataAsset, @@ -650,6 +692,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async commentOnDataAsset( @Param('id') id: string, @User() user: RequestUser, @@ -676,6 +721,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.DELETE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ @@ -697,6 +745,9 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async deleteComment( @Param('id') id: string, @User() user: RequestUser, @@ -850,6 +901,9 @@ export class CatalogController { @Get('nimbus-dashboards') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) + @RequireModule( + DADOSFERA_MODULES_KEYS.CATALOG + ) async getNimbusDashboards( @User() user: RequestUser, @Body() body: GetNimbusDashboardsRequest, diff --git a/src/modules/connection-test/connection-test.controller.ts b/src/modules/connection-test/connection-test.controller.ts index f197358..029da68 100644 --- a/src/modules/connection-test/connection-test.controller.ts +++ b/src/modules/connection-test/connection-test.controller.ts @@ -24,15 +24,19 @@ import { GetTableMetadataReq, } from './dto/connection-test'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { Authenticated } from 'src/decorators/authentication.decorator'; +import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; +import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum'; @ApiInternalOnlyController() @ApiTags('Connection Test') @Controller('connection-test') @UseFilters(new GrpcToHttpExceptionFilter()) @Authenticated() +@RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT +) export class ConnectionTestController { logger: any; constructor( diff --git a/src/modules/connection/connection.controller.ts b/src/modules/connection/connection.controller.ts index 194e9aa..b1987dc 100644 --- a/src/modules/connection/connection.controller.ts +++ b/src/modules/connection/connection.controller.ts @@ -16,8 +16,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Authenticated, RequireAllPermissions, + RequireModule, } from 'src/decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { ValidationPipe } from '../../pipes/object-validation.pipe'; import { @@ -39,6 +40,9 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions; @ApiTags('connections') @Authenticated() @Controller('connections') +@RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT +) export class ConnectionController { logger: any; constructor( diff --git a/src/modules/connector/connector.controller.ts b/src/modules/connector/connector.controller.ts index c755ae8..858ef8c 100644 --- a/src/modules/connector/connector.controller.ts +++ b/src/modules/connector/connector.controller.ts @@ -25,9 +25,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Authenticated, RequireAllPermissions, + RequireModule, RequireSomePermission, } from 'src/decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; @@ -99,6 +100,9 @@ export class ConnectorController { PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async getAllConnectors( @Language() language: LanguageEnum, @Query() queries: GetAllDto, @@ -131,6 +135,9 @@ export class ConnectorController { PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async getConnectorsTags() { return await this.connectorClientService.getConnectorsTags(); } @@ -143,6 +150,9 @@ export class ConnectorController { PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async getConnector( @Language() language: LanguageEnum, @Param('plugin') plugin: string, @@ -171,6 +181,9 @@ export class ConnectorController { PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, ) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async getConnectorDetails( @Language() language: LanguageEnum, @Param('plugin') plugin: string, @@ -193,6 +206,9 @@ export class ConnectorController { @Put('/:plugin') @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE) @ApiConsumes('multipart/form-data') + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async updateConnector( @Param('plugin') plugin: string, @Body() body: UpdateDto, @@ -214,6 +230,9 @@ export class ConnectorController { @Put('/:plugin/add-tag') @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async addTagOnConnector( @Param('plugin') plugin: string, @Body() body: AddTagDto, @@ -241,6 +260,9 @@ export class ConnectorController { @Put('/:plugin/remove-tag') @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async removeTagOnConnector( @Param('plugin') plugin: string, @Body() body: RemoveTagDto, @@ -269,6 +291,9 @@ export class ConnectorController { @Delete('/:plugin') @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE) + @RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT + ) async deleteConnector( @Param('plugin') plugin: string, @Query('version') version: string, diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index ea64d8c..c24c090 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -25,9 +25,10 @@ import { } from '@nestjs/swagger'; import { RequireAllPermissions, + RequireModule, RequireSomePermission, } from 'src/decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { PipelinesService } from './pipelines.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; @@ -57,6 +58,9 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @UseFilters(new GrpcToHttpExceptionFilter()) @Controller('pipelinesV2') +@RequireModule( + DADOSFERA_MODULES_KEYS.COLLECT +) export class PipelinesController { logger: DadosferaLogger; constructor( diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 16d27b3..503d117 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -20,10 +20,11 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Authenticated, RequireAllPermissions, + RequireModule, } from '../../decorators/authentication.decorator'; import { User, RequestUser } from '../../decorators/user.decorator'; import { PlatformApiService } from './platform-api.service'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; @@ -49,6 +50,7 @@ type RenameTablesBody = { @ApiTags('Platform API') @Controller('platform') +@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT) export class PlatformApiController { private logger: any; From 9c5748503137f209ac7c78d5771797721fc76ba4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 24 Aug 2026 15:39:40 -0300 Subject: [PATCH 31/37] feat(auth): pure helper deriving Orchest identity from permissions --- src/modules/auth/orchest-identity.spec.ts | 36 +++++++++++++++++++++++ src/modules/auth/orchest-identity.ts | 27 +++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 src/modules/auth/orchest-identity.spec.ts create mode 100644 src/modules/auth/orchest-identity.ts diff --git a/src/modules/auth/orchest-identity.spec.ts b/src/modules/auth/orchest-identity.spec.ts new file mode 100644 index 0000000..c25ceaf --- /dev/null +++ b/src/modules/auth/orchest-identity.spec.ts @@ -0,0 +1,36 @@ +import { deriveOrchestIdentity } from './orchest-identity'; + +describe('deriveOrchestIdentity', () => { + it('passes permissions through verbatim', () => { + expect(deriveOrchestIdentity(['intelligence:open']).permissions).toEqual([ + 'intelligence:open', + ]); + }); + + it('marks super-admin from users:admin', () => { + expect(deriveOrchestIdentity(['users:admin']).roles).toContain('super-admin'); + }); + + it('omits super-admin when users:admin absent', () => { + expect(deriveOrchestIdentity(['intelligence:open']).roles).not.toContain( + 'super-admin', + ); + }); + + it('maps intelligence:open and process:open to module keys', () => { + const { modules } = deriveOrchestIdentity(['intelligence:open', 'process:open']); + expect(modules.sort()).toEqual(['intelligence', 'process']); + }); + + it('ignores non-module permissions in modules', () => { + expect(deriveOrchestIdentity(['users:admin']).modules).toEqual([]); + }); + + it('handles undefined/empty permissions', () => { + expect(deriveOrchestIdentity(undefined)).toEqual({ + permissions: [], + roles: [], + modules: [], + }); + }); +}); diff --git a/src/modules/auth/orchest-identity.ts b/src/modules/auth/orchest-identity.ts new file mode 100644 index 0000000..8060e34 --- /dev/null +++ b/src/modules/auth/orchest-identity.ts @@ -0,0 +1,27 @@ +// Claim → semantic strings. Seqid knowledge stays here, beside the enum +// that owns it (users:admin=34, intelligence:open=31, process:open=43). +const ADMIN_CLAIM = 'users:admin'; +const MODULE_CLAIMS: Record = { + 'intelligence:open': 'intelligence', + 'process:open': 'process', +}; + +export type OrchestIdentityFields = { + permissions: string[]; + roles: string[]; + modules: string[]; +}; + +export function deriveOrchestIdentity( + permissions: string[] | undefined, +): OrchestIdentityFields { + const perms = permissions ?? []; + const roles: string[] = []; + if (perms.includes(ADMIN_CLAIM)) { + roles.push('super-admin'); + } + const modules = Object.entries(MODULE_CLAIMS) + .filter(([claim]) => perms.includes(claim)) + .map(([, key]) => key); + return { permissions: perms, roles, modules }; +} From a16fefe691daf7cf5583a575e6d17fa795a42c69 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 24 Aug 2026 15:45:09 -0300 Subject: [PATCH 32/37] feat(auth): return permissions/roles/modules from /auth/me (all branches) --- src/modules/auth/auth.controller.ts | 5 +++- src/modules/auth/auth.service.ts | 2 ++ src/modules/auth/dtos/login.ts | 5 +++- .../auth/orchest-identity.integration.spec.ts | 27 +++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 src/modules/auth/orchest-identity.integration.spec.ts diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 40eaad9..e8725ea 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -494,7 +494,10 @@ export class AuthController { id: api_key.customer_id, name: api_key.customer_name, tier: api_key.customer_tier, - } + }, + permissions: [], + roles: [], + modules: [], }; return res.status(200).json(userDto); diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 8539d0e..e0f6ba8 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -35,6 +35,7 @@ import { BulkEditResponse, UserDTO } from './dtos/login'; import jwt, { JwtPayload } from 'jsonwebtoken'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { Request, Response } from 'express'; +import { deriveOrchestIdentity } from './orchest-identity'; type AuthSession = { accessToken?: string; @@ -447,6 +448,7 @@ export class AuthClientService implements OnModuleInit { name: payload.customer_name, tier: payload.customer_tier, }, + ...deriveOrchestIdentity(payload.permissions), }; return userDto; diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 2e22363..8532cb1 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -152,5 +152,8 @@ export type UserDTO = { id: string, name: string, tier: string, - } + }, + permissions: string[], + roles: string[], + modules: string[], } diff --git a/src/modules/auth/orchest-identity.integration.spec.ts b/src/modules/auth/orchest-identity.integration.spec.ts new file mode 100644 index 0000000..bfe5a29 --- /dev/null +++ b/src/modules/auth/orchest-identity.integration.spec.ts @@ -0,0 +1,27 @@ +import { deriveOrchestIdentity } from './orchest-identity'; + +// The three getMe branches must all yield the three fields. This test +// pins the SHAPE contract without loading the DUC gRPC client. +describe('/auth/me field contract', () => { + it('cookie/refresh path derives from payload permissions', () => { + const enriched = { + id: 'u', name: 'n', email: 'e', + customer: { id: 'c', name: 'cust', tier: 't' }, + ...deriveOrchestIdentity(['users:admin', 'intelligence:open']), + }; + expect(enriched.roles).toContain('super-admin'); + expect(enriched.modules).toContain('intelligence'); + expect(enriched.permissions).toHaveLength(2); + }); + + it('api-key branch is empty for all three fields', () => { + const apiKeyDto = { + id: 'u', name: 'n', email: 'n', + customer: { id: 'c', name: 'cust', tier: 't' }, + permissions: [] as string[], roles: [] as string[], modules: [] as string[], + }; + expect(apiKeyDto.permissions).toEqual([]); + expect(apiKeyDto.roles).toEqual([]); + expect(apiKeyDto.modules).toEqual([]); + }); +}); From a5a685ee3fd5ddaf34075c9da7c5b25327d4f1d8 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 24 Aug 2026 16:03:20 -0300 Subject: [PATCH 33/37] fix(auth): derive Orchest identity from numeric seqids (JWT carries seqids not claim strings) The JWT `permissions` claim is an array of numeric seqids at runtime (see authentication.guard.ts / authentication.decorator.ts), not claim strings. deriveOrchestIdentity previously matched claim strings against this numeric array, so roles[]/modules[] were always empty for every real user. - deriveOrchestIdentity now takes number[] | undefined and matches seqids sourced from PERMISSIONS_GROUPS (permissions.enum.ts) instead of hand-copied literals. - permissions is translated back to claim strings via a full seqid->claim catalog built once from PERMISSIONS_GROUPS; unknown seqids are dropped (auth-server ignores permissions[] in v1). - auth.controller.ts's api-key branch literal is now annotated `: UserDTO` so tsc enforces the three fields there. - Both spec files re-fixtured with numeric seqid inputs, including a mixed admin+module case and an exact claim-string translation assertion. Co-Authored-By: WOZCODE --- src/modules/auth/auth.controller.ts | 3 +- .../auth/orchest-identity.integration.spec.ts | 10 ++-- src/modules/auth/orchest-identity.spec.ts | 32 ++++++----- src/modules/auth/orchest-identity.ts | 53 ++++++++++++++----- 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index e8725ea..e88cc0f 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -37,6 +37,7 @@ import { RequireAllPermissions, } from 'src/decorators/authentication.decorator'; import { AuthClientService } from './auth.service'; +import { UserDTO } from './dtos/login'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter'; import { RequestUser, User } from 'src/decorators/user.decorator'; @@ -486,7 +487,7 @@ export class AuthController { this.logger.info('Authenticating via X-Api-key header'); const { api_key } = await this.apiKeyService.get(apiKey); - const userDto = { + const userDto: UserDTO = { id: api_key.user_id, name: api_key.username, email: api_key.username, diff --git a/src/modules/auth/orchest-identity.integration.spec.ts b/src/modules/auth/orchest-identity.integration.spec.ts index bfe5a29..ab694ef 100644 --- a/src/modules/auth/orchest-identity.integration.spec.ts +++ b/src/modules/auth/orchest-identity.integration.spec.ts @@ -1,17 +1,19 @@ import { deriveOrchestIdentity } from './orchest-identity'; // The three getMe branches must all yield the three fields. This test -// pins the SHAPE contract without loading the DUC gRPC client. +// pins the SHAPE contract without loading the DUC gRPC client. The JWT +// permissions claim carries numeric seqids at runtime (34 = users:admin, +// 31 = intelligence:open), so fixtures here use numeric seqid arrays. describe('/auth/me field contract', () => { - it('cookie/refresh path derives from payload permissions', () => { + it('cookie/refresh path derives from payload permissions (numeric seqids)', () => { const enriched = { id: 'u', name: 'n', email: 'e', customer: { id: 'c', name: 'cust', tier: 't' }, - ...deriveOrchestIdentity(['users:admin', 'intelligence:open']), + ...deriveOrchestIdentity([34, 31]), }; expect(enriched.roles).toContain('super-admin'); expect(enriched.modules).toContain('intelligence'); - expect(enriched.permissions).toHaveLength(2); + expect(enriched.permissions).toEqual(['users:admin', 'intelligence:open']); }); it('api-key branch is empty for all three fields', () => { diff --git a/src/modules/auth/orchest-identity.spec.ts b/src/modules/auth/orchest-identity.spec.ts index c25ceaf..dbaa3fe 100644 --- a/src/modules/auth/orchest-identity.spec.ts +++ b/src/modules/auth/orchest-identity.spec.ts @@ -1,29 +1,37 @@ import { deriveOrchestIdentity } from './orchest-identity'; +// JWT permissions claim carries numeric seqids (not claim strings) — see +// src/decorators/authentication.decorator.ts and +// src/authentication/authentication.guard.ts. 34 = users:admin, +// 31 = intelligence:open, 43 = process:open. describe('deriveOrchestIdentity', () => { - it('passes permissions through verbatim', () => { - expect(deriveOrchestIdentity(['intelligence:open']).permissions).toEqual([ + it('translates seqids to claim strings in permissions', () => { + expect(deriveOrchestIdentity([31]).permissions).toEqual([ 'intelligence:open', ]); }); - it('marks super-admin from users:admin', () => { - expect(deriveOrchestIdentity(['users:admin']).roles).toContain('super-admin'); + it('marks super-admin from the users:admin seqid (34)', () => { + expect(deriveOrchestIdentity([34]).roles).toContain('super-admin'); }); - it('omits super-admin when users:admin absent', () => { - expect(deriveOrchestIdentity(['intelligence:open']).roles).not.toContain( - 'super-admin', - ); + it('omits super-admin when the admin seqid is absent', () => { + expect(deriveOrchestIdentity([31]).roles).not.toContain('super-admin'); }); - it('maps intelligence:open and process:open to module keys', () => { - const { modules } = deriveOrchestIdentity(['intelligence:open', 'process:open']); + it('maps intelligence and process seqids to module keys', () => { + const { modules } = deriveOrchestIdentity([31, 43]); expect(modules.sort()).toEqual(['intelligence', 'process']); }); - it('ignores non-module permissions in modules', () => { - expect(deriveOrchestIdentity(['users:admin']).modules).toEqual([]); + it('ignores non-module seqids in modules', () => { + expect(deriveOrchestIdentity([34]).modules).toEqual([]); + }); + + it('handles a mixed real-shape seqid array (admin + intelligence)', () => { + const { roles, modules } = deriveOrchestIdentity([34, 31]); + expect(roles).toEqual(['super-admin']); + expect(modules).toEqual(['intelligence']); }); it('handles undefined/empty permissions', () => { diff --git a/src/modules/auth/orchest-identity.ts b/src/modules/auth/orchest-identity.ts index 8060e34..38d90eb 100644 --- a/src/modules/auth/orchest-identity.ts +++ b/src/modules/auth/orchest-identity.ts @@ -1,11 +1,29 @@ -// Claim → semantic strings. Seqid knowledge stays here, beside the enum -// that owns it (users:admin=34, intelligence:open=31, process:open=43). -const ADMIN_CLAIM = 'users:admin'; -const MODULE_CLAIMS: Record = { - 'intelligence:open': 'intelligence', - 'process:open': 'process', +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; + +// The JWT `permissions` claim is an array of numeric seqids (not claim +// strings) — see src/authentication/authentication.guard.ts and +// src/decorators/authentication.decorator.ts. Seqids are sourced from +// PERMISSIONS_GROUPS (the enum that owns them), not hand-copied here. +const ADMIN_SEQID = PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid; + +const MODULE_SEQID_TO_KEY: Record = { + [PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid]: + 'intelligence', + [PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid]: 'process', }; +// Full seqid -> claim catalog, built once from every group/permission in +// PERMISSIONS_GROUPS, so the `permissions` field can be translated back to +// honest claim strings for UserDTO. +const SEQID_TO_CLAIM: Record = Object.values( + PERMISSIONS_GROUPS, +).reduce((catalog, group) => { + for (const permission of Object.values(group.permissions)) { + catalog[permission.seqid] = permission.claim; + } + return catalog; +}, {} as Record); + export type OrchestIdentityFields = { permissions: string[]; roles: string[]; @@ -13,15 +31,24 @@ export type OrchestIdentityFields = { }; export function deriveOrchestIdentity( - permissions: string[] | undefined, + permissions: number[] | undefined, ): OrchestIdentityFields { - const perms = permissions ?? []; + const seqids = permissions ?? []; + const roles: string[] = []; - if (perms.includes(ADMIN_CLAIM)) { + if (seqids.includes(ADMIN_SEQID)) { roles.push('super-admin'); } - const modules = Object.entries(MODULE_CLAIMS) - .filter(([claim]) => perms.includes(claim)) - .map(([, key]) => key); - return { permissions: perms, roles, modules }; + + const modules = seqids + .filter((seqid) => seqid in MODULE_SEQID_TO_KEY) + .map((seqid) => MODULE_SEQID_TO_KEY[seqid]); + + // Unknown seqids (not in the catalog) are dropped: auth-server ignores + // permissions[] in v1, so completeness of this translation isn't required. + const translatedPermissions = seqids + .filter((seqid) => seqid in SEQID_TO_CLAIM) + .map((seqid) => SEQID_TO_CLAIM[seqid]); + + return { permissions: translatedPermissions, roles, modules }; } From cd21fd0b7bbaed25114b4759c71d6b73ceae99c1 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 24 Aug 2026 16:30:33 -0300 Subject: [PATCH 34/37] refactor(auth): /auth/me returns permissions only (drop roles/modules) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep Maestro a pure identity provider: /auth/me exposes the user's permission claim strings and nothing consumer-specific. Consumers derive whatever meaning they need (roles, module access, groups) from the claim vocabulary — claims are already namespaced group:action. - UserDTO: drop roles[]/modules[], keep permissions[]. - Helper shrinks to a generic seqid->claim translation (orchest-identity.ts -> permission-claims.ts, translateSeqidsToClaims). - api-key branch: permissions: [] only. The roles/modules derivation moves entirely to the consumer (Orchest's auth-server adapter). Co-Authored-By: WOZCODE --- src/modules/auth/auth.controller.ts | 2 - src/modules/auth/auth.service.ts | 4 +- src/modules/auth/dtos/login.ts | 2 - .../auth/orchest-identity.integration.spec.ts | 29 ---------- src/modules/auth/orchest-identity.spec.ts | 44 --------------- src/modules/auth/orchest-identity.ts | 54 ------------------- src/modules/auth/permission-claims.spec.ts | 30 +++++++++++ src/modules/auth/permission-claims.ts | 29 ++++++++++ 8 files changed, 61 insertions(+), 133 deletions(-) delete mode 100644 src/modules/auth/orchest-identity.integration.spec.ts delete mode 100644 src/modules/auth/orchest-identity.spec.ts delete mode 100644 src/modules/auth/orchest-identity.ts create mode 100644 src/modules/auth/permission-claims.spec.ts create mode 100644 src/modules/auth/permission-claims.ts diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index e88cc0f..5eb2b0f 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -497,8 +497,6 @@ export class AuthController { tier: api_key.customer_tier, }, permissions: [], - roles: [], - modules: [], }; return res.status(200).json(userDto); diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index e0f6ba8..a7d9868 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -35,7 +35,7 @@ import { BulkEditResponse, UserDTO } from './dtos/login'; import jwt, { JwtPayload } from 'jsonwebtoken'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { Request, Response } from 'express'; -import { deriveOrchestIdentity } from './orchest-identity'; +import { translateSeqidsToClaims } from './permission-claims'; type AuthSession = { accessToken?: string; @@ -448,7 +448,7 @@ export class AuthClientService implements OnModuleInit { name: payload.customer_name, tier: payload.customer_tier, }, - ...deriveOrchestIdentity(payload.permissions), + permissions: translateSeqidsToClaims(payload.permissions), }; return userDto; diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 8532cb1..3317904 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -154,6 +154,4 @@ export type UserDTO = { tier: string, }, permissions: string[], - roles: string[], - modules: string[], } diff --git a/src/modules/auth/orchest-identity.integration.spec.ts b/src/modules/auth/orchest-identity.integration.spec.ts deleted file mode 100644 index ab694ef..0000000 --- a/src/modules/auth/orchest-identity.integration.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { deriveOrchestIdentity } from './orchest-identity'; - -// The three getMe branches must all yield the three fields. This test -// pins the SHAPE contract without loading the DUC gRPC client. The JWT -// permissions claim carries numeric seqids at runtime (34 = users:admin, -// 31 = intelligence:open), so fixtures here use numeric seqid arrays. -describe('/auth/me field contract', () => { - it('cookie/refresh path derives from payload permissions (numeric seqids)', () => { - const enriched = { - id: 'u', name: 'n', email: 'e', - customer: { id: 'c', name: 'cust', tier: 't' }, - ...deriveOrchestIdentity([34, 31]), - }; - expect(enriched.roles).toContain('super-admin'); - expect(enriched.modules).toContain('intelligence'); - expect(enriched.permissions).toEqual(['users:admin', 'intelligence:open']); - }); - - it('api-key branch is empty for all three fields', () => { - const apiKeyDto = { - id: 'u', name: 'n', email: 'n', - customer: { id: 'c', name: 'cust', tier: 't' }, - permissions: [] as string[], roles: [] as string[], modules: [] as string[], - }; - expect(apiKeyDto.permissions).toEqual([]); - expect(apiKeyDto.roles).toEqual([]); - expect(apiKeyDto.modules).toEqual([]); - }); -}); diff --git a/src/modules/auth/orchest-identity.spec.ts b/src/modules/auth/orchest-identity.spec.ts deleted file mode 100644 index dbaa3fe..0000000 --- a/src/modules/auth/orchest-identity.spec.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { deriveOrchestIdentity } from './orchest-identity'; - -// JWT permissions claim carries numeric seqids (not claim strings) — see -// src/decorators/authentication.decorator.ts and -// src/authentication/authentication.guard.ts. 34 = users:admin, -// 31 = intelligence:open, 43 = process:open. -describe('deriveOrchestIdentity', () => { - it('translates seqids to claim strings in permissions', () => { - expect(deriveOrchestIdentity([31]).permissions).toEqual([ - 'intelligence:open', - ]); - }); - - it('marks super-admin from the users:admin seqid (34)', () => { - expect(deriveOrchestIdentity([34]).roles).toContain('super-admin'); - }); - - it('omits super-admin when the admin seqid is absent', () => { - expect(deriveOrchestIdentity([31]).roles).not.toContain('super-admin'); - }); - - it('maps intelligence and process seqids to module keys', () => { - const { modules } = deriveOrchestIdentity([31, 43]); - expect(modules.sort()).toEqual(['intelligence', 'process']); - }); - - it('ignores non-module seqids in modules', () => { - expect(deriveOrchestIdentity([34]).modules).toEqual([]); - }); - - it('handles a mixed real-shape seqid array (admin + intelligence)', () => { - const { roles, modules } = deriveOrchestIdentity([34, 31]); - expect(roles).toEqual(['super-admin']); - expect(modules).toEqual(['intelligence']); - }); - - it('handles undefined/empty permissions', () => { - expect(deriveOrchestIdentity(undefined)).toEqual({ - permissions: [], - roles: [], - modules: [], - }); - }); -}); diff --git a/src/modules/auth/orchest-identity.ts b/src/modules/auth/orchest-identity.ts deleted file mode 100644 index 38d90eb..0000000 --- a/src/modules/auth/orchest-identity.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; - -// The JWT `permissions` claim is an array of numeric seqids (not claim -// strings) — see src/authentication/authentication.guard.ts and -// src/decorators/authentication.decorator.ts. Seqids are sourced from -// PERMISSIONS_GROUPS (the enum that owns them), not hand-copied here. -const ADMIN_SEQID = PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid; - -const MODULE_SEQID_TO_KEY: Record = { - [PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid]: - 'intelligence', - [PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid]: 'process', -}; - -// Full seqid -> claim catalog, built once from every group/permission in -// PERMISSIONS_GROUPS, so the `permissions` field can be translated back to -// honest claim strings for UserDTO. -const SEQID_TO_CLAIM: Record = Object.values( - PERMISSIONS_GROUPS, -).reduce((catalog, group) => { - for (const permission of Object.values(group.permissions)) { - catalog[permission.seqid] = permission.claim; - } - return catalog; -}, {} as Record); - -export type OrchestIdentityFields = { - permissions: string[]; - roles: string[]; - modules: string[]; -}; - -export function deriveOrchestIdentity( - permissions: number[] | undefined, -): OrchestIdentityFields { - const seqids = permissions ?? []; - - const roles: string[] = []; - if (seqids.includes(ADMIN_SEQID)) { - roles.push('super-admin'); - } - - const modules = seqids - .filter((seqid) => seqid in MODULE_SEQID_TO_KEY) - .map((seqid) => MODULE_SEQID_TO_KEY[seqid]); - - // Unknown seqids (not in the catalog) are dropped: auth-server ignores - // permissions[] in v1, so completeness of this translation isn't required. - const translatedPermissions = seqids - .filter((seqid) => seqid in SEQID_TO_CLAIM) - .map((seqid) => SEQID_TO_CLAIM[seqid]); - - return { permissions: translatedPermissions, roles, modules }; -} diff --git a/src/modules/auth/permission-claims.spec.ts b/src/modules/auth/permission-claims.spec.ts new file mode 100644 index 0000000..c528c5b --- /dev/null +++ b/src/modules/auth/permission-claims.spec.ts @@ -0,0 +1,30 @@ +import { translateSeqidsToClaims } from './permission-claims'; + +// The JWT permissions claim carries numeric seqids at runtime +// (34 = users:admin, 31 = intelligence:open, 43 = process:open), so +// fixtures here use numeric seqid arrays. +describe('translateSeqidsToClaims', () => { + it('translates a single seqid to its claim string', () => { + expect(translateSeqidsToClaims([31])).toEqual(['intelligence:open']); + }); + + it('translates multiple seqids, preserving order', () => { + expect(translateSeqidsToClaims([34, 31, 43])).toEqual([ + 'users:admin', + 'intelligence:open', + 'process:open', + ]); + }); + + it('drops unknown seqids not present in the catalog', () => { + expect(translateSeqidsToClaims([34, 999999])).toEqual(['users:admin']); + }); + + it('returns an empty array for undefined permissions', () => { + expect(translateSeqidsToClaims(undefined)).toEqual([]); + }); + + it('returns an empty array for empty permissions', () => { + expect(translateSeqidsToClaims([])).toEqual([]); + }); +}); diff --git a/src/modules/auth/permission-claims.ts b/src/modules/auth/permission-claims.ts new file mode 100644 index 0000000..470631e --- /dev/null +++ b/src/modules/auth/permission-claims.ts @@ -0,0 +1,29 @@ +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; + +// The JWT `permissions` claim is an array of numeric seqids (not claim +// strings) — see src/authentication/authentication.guard.ts and +// src/decorators/authentication.decorator.ts. This module translates those +// seqids into their human-readable claim strings so /auth/me can expose a +// self-describing `permissions` array; consumers derive whatever meaning +// they need (roles, module access, groups) from the claim vocabulary. +// +// Full seqid -> claim catalog, built once from every group/permission in +// PERMISSIONS_GROUPS. Generic: no consumer-specific knowledge lives here. +const SEQID_TO_CLAIM: Record = Object.values( + PERMISSIONS_GROUPS, +).reduce((catalog, group) => { + for (const permission of Object.values(group.permissions)) { + catalog[permission.seqid] = permission.claim; + } + return catalog; +}, {} as Record); + +// Translate the JWT's numeric permission seqids into claim strings. +// Unknown seqids (not present in the catalog) are dropped. +export function translateSeqidsToClaims( + permissions: number[] | undefined, +): string[] { + return (permissions ?? []) + .filter((seqid) => seqid in SEQID_TO_CLAIM) + .map((seqid) => SEQID_TO_CLAIM[seqid]); +} From cac36f2c604867d66b95cc74ad21e845434c623b Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 24 Aug 2026 16:56:23 -0300 Subject: [PATCH 35/37] refactor(auth): /auth/me returns raw permission seqids Return payload.permissions verbatim (numeric seqids) instead of translating them to claim strings. Consumers own the seqid->meaning mapping. Drops permission-claims.ts entirely; UserDTO.permissions is now number[]. Co-Authored-By: WOZCODE --- src/modules/auth/auth.service.ts | 5 ++-- src/modules/auth/dtos/login.ts | 2 +- src/modules/auth/permission-claims.spec.ts | 30 ---------------------- src/modules/auth/permission-claims.ts | 29 --------------------- 4 files changed, 4 insertions(+), 62 deletions(-) delete mode 100644 src/modules/auth/permission-claims.spec.ts delete mode 100644 src/modules/auth/permission-claims.ts diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index a7d9868..c82aeef 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -35,7 +35,6 @@ import { BulkEditResponse, UserDTO } from './dtos/login'; import jwt, { JwtPayload } from 'jsonwebtoken'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { Request, Response } from 'express'; -import { translateSeqidsToClaims } from './permission-claims'; type AuthSession = { accessToken?: string; @@ -448,7 +447,9 @@ export class AuthClientService implements OnModuleInit { name: payload.customer_name, tier: payload.customer_tier, }, - permissions: translateSeqidsToClaims(payload.permissions), + // Raw permission seqids from the JWT. Consumers own the seqid->meaning + // mapping (e.g. Orchest's auth-server); Maestro reports them as-is. + permissions: payload.permissions ?? [], }; return userDto; diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 3317904..1ea22f0 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -153,5 +153,5 @@ export type UserDTO = { name: string, tier: string, }, - permissions: string[], + permissions: number[], } diff --git a/src/modules/auth/permission-claims.spec.ts b/src/modules/auth/permission-claims.spec.ts deleted file mode 100644 index c528c5b..0000000 --- a/src/modules/auth/permission-claims.spec.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { translateSeqidsToClaims } from './permission-claims'; - -// The JWT permissions claim carries numeric seqids at runtime -// (34 = users:admin, 31 = intelligence:open, 43 = process:open), so -// fixtures here use numeric seqid arrays. -describe('translateSeqidsToClaims', () => { - it('translates a single seqid to its claim string', () => { - expect(translateSeqidsToClaims([31])).toEqual(['intelligence:open']); - }); - - it('translates multiple seqids, preserving order', () => { - expect(translateSeqidsToClaims([34, 31, 43])).toEqual([ - 'users:admin', - 'intelligence:open', - 'process:open', - ]); - }); - - it('drops unknown seqids not present in the catalog', () => { - expect(translateSeqidsToClaims([34, 999999])).toEqual(['users:admin']); - }); - - it('returns an empty array for undefined permissions', () => { - expect(translateSeqidsToClaims(undefined)).toEqual([]); - }); - - it('returns an empty array for empty permissions', () => { - expect(translateSeqidsToClaims([])).toEqual([]); - }); -}); diff --git a/src/modules/auth/permission-claims.ts b/src/modules/auth/permission-claims.ts deleted file mode 100644 index 470631e..0000000 --- a/src/modules/auth/permission-claims.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; - -// The JWT `permissions` claim is an array of numeric seqids (not claim -// strings) — see src/authentication/authentication.guard.ts and -// src/decorators/authentication.decorator.ts. This module translates those -// seqids into their human-readable claim strings so /auth/me can expose a -// self-describing `permissions` array; consumers derive whatever meaning -// they need (roles, module access, groups) from the claim vocabulary. -// -// Full seqid -> claim catalog, built once from every group/permission in -// PERMISSIONS_GROUPS. Generic: no consumer-specific knowledge lives here. -const SEQID_TO_CLAIM: Record = Object.values( - PERMISSIONS_GROUPS, -).reduce((catalog, group) => { - for (const permission of Object.values(group.permissions)) { - catalog[permission.seqid] = permission.claim; - } - return catalog; -}, {} as Record); - -// Translate the JWT's numeric permission seqids into claim strings. -// Unknown seqids (not present in the catalog) are dropped. -export function translateSeqidsToClaims( - permissions: number[] | undefined, -): string[] { - return (permissions ?? []) - .filter((seqid) => seqid in SEQID_TO_CLAIM) - .map((seqid) => SEQID_TO_CLAIM[seqid]); -} From bf0314f5b12b0a26f375edddada4c1f6183ec42a Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 24 Aug 2026 19:18:59 -0300 Subject: [PATCH 36/37] FIX: trigger release for /auth/me permission seqids (PR #513) PR #513 merged to beta but no semantic-release ran: its commits used conventional-commits prefixes (feat(auth):, fix(auth):) which the .releaserc.json eslint preset does not recognise, so commit-analyzer found no release-worthy change. This empty FIX: commit matches the eslint preset's releaseRules (tag FIX -> patch) to cut a beta release that includes the /auth/me permission-seqids change, so stg can deploy it. Co-Authored-By: WOZCODE From 81c446354b3b1843805b65336b424483ef7d1e94 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Thu, 27 Aug 2026 14:33:30 -0300 Subject: [PATCH 37/37] FIX: mock process.env and dadosfera logger --- jest.config.ts | 1 + jest.setup.ts | 3 +++ .../permissions/permissions.controller.spec.ts | 4 ++-- .../release_note/release_note.controller.spec.ts | 13 ++++++++++++- .../release_note/release_note.service.spec.ts | 13 ++++++++++++- 5 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 jest.setup.ts diff --git a/jest.config.ts b/jest.config.ts index 9442e04..9aea4aa 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -11,6 +11,7 @@ const config: Config.InitialOptions = { '/node_modules/', '.*\\.module\\.[jt]s$', ], + setupFiles: ['/jest.setup.ts'], // moduleDirectories: ['node_modules'], // default is already 'node_modules' // rootDir: '.', //No need // testEnvironment: 'node', //Defaults to 'node' diff --git a/jest.setup.ts b/jest.setup.ts new file mode 100644 index 0000000..c2d29e3 --- /dev/null +++ b/jest.setup.ts @@ -0,0 +1,3 @@ +process.env.DUC_URL="duc:50051" +process.env.INFACTORY_URL="in-factory:50052" +process.env.PIFACTORY_URL="pi-factory:50053" \ No newline at end of file diff --git a/src/modules/permissions/permissions.controller.spec.ts b/src/modules/permissions/permissions.controller.spec.ts index 7ab875b..1bc3004 100644 --- a/src/modules/permissions/permissions.controller.spec.ts +++ b/src/modules/permissions/permissions.controller.spec.ts @@ -1,6 +1,6 @@ import { ClientsModule } from '@nestjs/microservices'; import { Test, TestingModule } from '@nestjs/testing'; -// import { DucClient } from 'src/clients/duc/client.config'; +// import { DucClient } from '../duc/client.config' import { PermissionsController } from './permissions.controller'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PermissionsService } from './permissions.service'; @@ -22,7 +22,7 @@ describe.skip('PermissionsController', () => { provide: DadosferaLogger, useValue: { logger }, }, - PermissionsService, + PermissionsService ], }).compile(); diff --git a/src/modules/release_note/release_note.controller.spec.ts b/src/modules/release_note/release_note.controller.spec.ts index 921eca1..af307e7 100644 --- a/src/modules/release_note/release_note.controller.spec.ts +++ b/src/modules/release_note/release_note.controller.spec.ts @@ -1,14 +1,25 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ReleaseNoteController } from './release_note.controller'; import { ReleaseNoteService } from './release_note.service'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; describe('ReleaseNoteController', () => { let controller: ReleaseNoteController; + const logger = { + info: (...args) => args, + error: (...args) => args, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [ReleaseNoteController], - providers: [ReleaseNoteService], + providers: [ + { + provide: DadosferaLogger, + useValue: { logger }, + }, + ReleaseNoteService + ], }).compile(); controller = module.get(ReleaseNoteController); diff --git a/src/modules/release_note/release_note.service.spec.ts b/src/modules/release_note/release_note.service.spec.ts index 72add9d..fa919ae 100644 --- a/src/modules/release_note/release_note.service.spec.ts +++ b/src/modules/release_note/release_note.service.spec.ts @@ -1,12 +1,23 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ReleaseNoteService } from './release_note.service'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; describe('ReleaseNoteService', () => { let service: ReleaseNoteService; + const logger = { + info: (...args) => args, + error: (...args) => args, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - providers: [ReleaseNoteService], + providers: [ + { + provide: DadosferaLogger, + useValue: { logger }, + }, + ReleaseNoteService + ], }).compile(); service = module.get(ReleaseNoteService);