From 6182705410d7b32feef8c28d964679196d0b3443 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 11 May 2026 12:52:07 -0300 Subject: [PATCH 01/15] FEAT: pipeline upgrade route --- docsfera.json | 45 ++++++++++++++++++- package-lock.json | 8 ++-- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 16 +++++++ src/modules/pipelinesV2/pipelines.service.ts | 9 ++++ 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/docsfera.json b/docsfera.json index 7ec6259..00b43df 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3580,7 +3580,10 @@ "content": { "application/json": { "schema": { - "type": "string" + "type": "array", + "items": { + "type": "object" + } } } } @@ -3644,6 +3647,46 @@ ] } }, + "/pipelinesV2/{id}/upgrade": { + "patch": { + "operationId": "PipelinesController_upgradeConnector", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "" + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2/init-upload": { "post": { "operationId": "PipelinesController_initUploadFile", diff --git a/package-lock.json b/package-lock.json index a5470e6..f26f5fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.8", + "@dadosfera/protospack-v2": "3.40.0-beta.9", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.8", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz", - "integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==", + "version": "3.40.0-beta.9", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz", + "integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index ab6d629..878d46a 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.8", + "@dadosfera/protospack-v2": "3.40.0-beta.9", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index c424f51..e90af52 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -15,6 +15,7 @@ import { HttpException, BadRequestException, UseGuards, + Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -368,6 +369,21 @@ export class PipelinesController { return response; } + @Patch('/:id/upgrade') + @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + @HttpCode(HttpStatus.NO_CONTENT) + async upgradeConnector( + @Language() language: LanguageEnum, + @Param('id') id: string, + @User() user: RequestUser + ) { + this.logger.info('PipelinesController - upgrade connector'); + + const metadata = PackTheMetadata(user); + + await this.pipelinesClientService.upgrade(id, metadata); + } + @Delete(':id') @ApiNoContentResponse() @HttpCode(HttpStatus.NO_CONTENT) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index b07330f..53d7711 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -176,6 +176,15 @@ export class PipelinesService implements OnModuleInit { return updatePipelineResponse; } + async upgrade(id: string, metadata: Metadata) { + await lastValueFrom( + this.pipelineWriteService.Upgrade( + { id }, + metadata, + ), + ); + } + async remove(data: { id: string; metadata: Metadata; user: RequestUser }) { const { id, metadata, user } = data; const info = { From b38b8f51e2ef2efc18c7e6aeee559985151d026d Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Wed, 20 May 2026 15:34:49 -0300 Subject: [PATCH 02/15] FEAT: Add endpoint to fetch pipeline run jobs --- .../platform-api/platform-api.controller.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 16d27b3..62ca9f5 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -832,6 +832,23 @@ export class PlatformApiController { ); } + @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getPipelineRunJobs( + @Param('pipelineId') pipelineId: string, + @Param('runId') runId: string, + @User() user: RequestUser, + ) { + const normalizedPipelineId = this.normalizePipelineId(pipelineId); + const normalizedRunId = this.normalizePipelineId(runId); + + return this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`, + user, + ); + } + // ==================== JOBS - COLUMN EDITING ROUTES ==================== @Put('jobs/:jobId/input') From 985170d7ae56787e32767aaaad5b9e9f9eef0a50 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:12:35 -0300 Subject: [PATCH 03/15] chore: exposure from route pipeline run jobs to maestro --- src/modules/platform-api/platform-api.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 62ca9f5..f3abaf5 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -833,6 +833,7 @@ export class PlatformApiController { } @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') + @ApiOperation({ summary: 'Get pipeline run jobs' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getPipelineRunJobs( @Param('pipelineId') pipelineId: string, From 94fbdb222603a0e664d37d88a2b4e0586cab25be Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:21:30 -0300 Subject: [PATCH 04/15] fix: validate workflow --- .github/workflows/validate-k8s.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 8a67a9c..85e48a9 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -36,6 +36,13 @@ jobs: with: version: 'v3.9.0' + - name: Install Helm Diff plugin + run: | + if ! helm plugin list | grep -q '^diff'; then + helm plugin install https://github.com/databus23/helm-diff + fi + helm diff version + - name: Determine DNS_HOST based on environment id: set_dns env: From 2414fcf21e1390470d44aafaa89c1bc4ff954903 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:23:45 -0300 Subject: [PATCH 05/15] fix: validate workflow --- .github/workflows/validate-k8s.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 85e48a9..139fc6a 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -37,11 +37,7 @@ jobs: version: 'v3.9.0' - name: Install Helm Diff plugin - run: | - if ! helm plugin list | grep -q '^diff'; then - helm plugin install https://github.com/databus23/helm-diff - fi - helm diff version + run: helm plugin install https://github.com/databus23/helm-diff || true - name: Determine DNS_HOST based on environment id: set_dns From 21a82b64f371ef4c5fb8fa5d10a1346afb07c429 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:31:45 -0300 Subject: [PATCH 06/15] fix: validate workflow --- .github/workflows/validate-k8s.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 139fc6a..d2e3487 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -37,7 +37,15 @@ jobs: version: 'v3.9.0' - name: Install Helm Diff plugin - run: helm plugin install https://github.com/databus23/helm-diff || true + env: + HELM_PLUGINS: /home/runner/.local/share/helm/plugins + run: | + mkdir -p "$HELM_PLUGINS" + if ! helm plugin list | grep -q '^diff'; then + helm plugin install https://github.com/databus23/helm-diff + fi + helm plugin list + helm diff version - name: Determine DNS_HOST based on environment id: set_dns @@ -105,4 +113,5 @@ jobs: - name: Run Helmfile Diff env: ENV: ${{ needs.extract_environment.outputs.environment }} + HELM_PLUGINS: /home/runner/.local/share/helm/plugins run: helmfile -f deploy/helmfiles/${ENV}.yaml diff From 9e49abb40da88094978661b7d23b34ee63dcba2b Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:37:47 -0300 Subject: [PATCH 07/15] fix: validate workflow --- .github/workflows/validate-k8s.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index d2e3487..d3f7861 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -36,17 +36,6 @@ jobs: with: version: 'v3.9.0' - - name: Install Helm Diff plugin - env: - HELM_PLUGINS: /home/runner/.local/share/helm/plugins - run: | - mkdir -p "$HELM_PLUGINS" - if ! helm plugin list | grep -q '^diff'; then - helm plugin install https://github.com/databus23/helm-diff - fi - helm plugin list - helm diff version - - name: Determine DNS_HOST based on environment id: set_dns env: @@ -82,6 +71,9 @@ jobs: sudo mv helmfile /usr/local/bin/ helmfile --version + - name: Install Helm Diff plugin + run: helm plugin install https://github.com/databus23/helm-diff || true + - name: Debug Helm env run: | helm env From ccd4159c59bbe7d53066906a7d954676fbc5f835 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 21 May 2026 16:43:37 -0300 Subject: [PATCH 08/15] fix: validate workflow --- .github/workflows/validate-k8s.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index d3f7861..ae6d383 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -72,7 +72,9 @@ jobs: helmfile --version - name: Install Helm Diff plugin - run: helm plugin install https://github.com/databus23/helm-diff || true + run: | + helm plugin install https://github.com/databus23/helm-diff --version v3.9.3 + helm diff version - name: Debug Helm env run: | From b4cc8151d7cdd0208933dd8a9de3012759def593 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 25 May 2026 14:01:00 -0300 Subject: [PATCH 09/15] FIX: release note endpoint --- docsfera.json | 28 ++++++++++- src/app.module.ts | 3 ++ .../release_note/dto/release_note.dto.ts | 14 ++++++ .../release_note.controller.spec.ts | 20 ++++++++ .../release_note/release_note.controller.ts | 26 +++++++++++ .../release_note/release_note.module.ts | 10 ++++ .../release_note/release_note.service.spec.ts | 18 ++++++++ .../release_note/release_note.service.ts | 46 +++++++++++++++++++ 8 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/modules/release_note/dto/release_note.dto.ts create mode 100644 src/modules/release_note/release_note.controller.spec.ts create mode 100644 src/modules/release_note/release_note.controller.ts create mode 100644 src/modules/release_note/release_note.module.ts create mode 100644 src/modules/release_note/release_note.service.spec.ts create mode 100644 src/modules/release_note/release_note.service.ts diff --git a/docsfera.json b/docsfera.json index 7ec6259..4c1d57e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3580,7 +3580,10 @@ "content": { "application/json": { "schema": { - "type": "string" + "type": "array", + "items": { + "type": "object" + } } } } @@ -8593,6 +8596,29 @@ "Health" ] } + }, + "/release_note": { + "get": { + "operationId": "ReleaseNoteController_getLatestReleaseNote", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } } }, "info": { diff --git a/src/app.module.ts b/src/app.module.ts index 0865ad2..4db914b 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -35,6 +35,8 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod import { ApiKeyModule } from './modules/api-key/api-key.module'; import { PlatformApiModule } from './modules/platform-api/platform-api.module'; import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module'; +import { ReleaseNoteModule } from './modules/release_note/release_note.module'; + @Module({ providers: [ @@ -79,6 +81,7 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor StorageExplorerModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, + ReleaseNoteModule, ], }) export class AppModule {} diff --git a/src/modules/release_note/dto/release_note.dto.ts b/src/modules/release_note/dto/release_note.dto.ts new file mode 100644 index 0000000..22cc745 --- /dev/null +++ b/src/modules/release_note/dto/release_note.dto.ts @@ -0,0 +1,14 @@ +export type ReleaseNoteDTO = { + id: string; + date: string; + tag: string; + title: string; + visible: boolean; + expiryDate: string; + content: string; + showEmojis: boolean; + image?: string; + link?: string; + linkText?: string; + }; + \ No newline at end of file diff --git a/src/modules/release_note/release_note.controller.spec.ts b/src/modules/release_note/release_note.controller.spec.ts new file mode 100644 index 0000000..921eca1 --- /dev/null +++ b/src/modules/release_note/release_note.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReleaseNoteController } from './release_note.controller'; +import { ReleaseNoteService } from './release_note.service'; + +describe('ReleaseNoteController', () => { + let controller: ReleaseNoteController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [ReleaseNoteController], + providers: [ReleaseNoteService], + }).compile(); + + controller = module.get(ReleaseNoteController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/src/modules/release_note/release_note.controller.ts b/src/modules/release_note/release_note.controller.ts new file mode 100644 index 0000000..fde44bd --- /dev/null +++ b/src/modules/release_note/release_note.controller.ts @@ -0,0 +1,26 @@ +import { Controller, Get, Inject } from '@nestjs/common'; +import { ReleaseNoteService } from './release_note.service'; +import { Authenticated } from 'src/decorators/authentication.decorator'; +import { Language } from 'src/decorators/language.decorator'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +@Controller('release_note') +@Authenticated() +export class ReleaseNoteController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private readonly releaseNoteService: ReleaseNoteService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Get() + async getLatestReleaseNote(@Language() language: LanguageEnum) { + this.logger.info(`Fetching latest release note for language: ${language}`); + return await this.releaseNoteService.getLatestReleaseNote(language); + } +} diff --git a/src/modules/release_note/release_note.module.ts b/src/modules/release_note/release_note.module.ts new file mode 100644 index 0000000..60cb55c --- /dev/null +++ b/src/modules/release_note/release_note.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { ReleaseNoteService } from './release_note.service'; +import { ReleaseNoteController } from './release_note.controller'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +@Module({ + controllers: [ReleaseNoteController], + providers: [ReleaseNoteService, DadosferaLogger] +}) +export class ReleaseNoteModule {} diff --git a/src/modules/release_note/release_note.service.spec.ts b/src/modules/release_note/release_note.service.spec.ts new file mode 100644 index 0000000..72add9d --- /dev/null +++ b/src/modules/release_note/release_note.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReleaseNoteService } from './release_note.service'; + +describe('ReleaseNoteService', () => { + let service: ReleaseNoteService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ReleaseNoteService], + }).compile(); + + service = module.get(ReleaseNoteService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/modules/release_note/release_note.service.ts b/src/modules/release_note/release_note.service.ts new file mode 100644 index 0000000..776d6e0 --- /dev/null +++ b/src/modules/release_note/release_note.service.ts @@ -0,0 +1,46 @@ +import { Inject, Injectable } from '@nestjs/common'; +import axios, { AxiosInstance } from 'axios'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { ReleaseNoteDTO } from './dto/release_note.dto'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +@Injectable() +export class ReleaseNoteService { + client: AxiosInstance; + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + this.client = axios.create({ + baseURL: process.env.FIREBASE_BASE_URL, + }); + } + + async getLatestReleaseNote(lang: LanguageEnum) { + try { + const lng = lang.split('-'); + const language = lng[0] + '-' + lng[1].toUpperCase(); + + const endpoint = `/release_note/${language}.json`; + const { + data, + status, + config + } = await this.client.get(endpoint) + this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`); + this.logger.info(`Request URL: ${config.baseURL}/${config.url}`); + + return data; + } catch (error) { + this.logger.error(`Error fetching release note: ${error.message}`); + + if (axios.isAxiosError(error)) { + this.logger.error(`Axios error details: ${error.toJSON()}`); + } + } + + } +} From 65ab16236f7ed19a745a02cbaeb3e86ca04921dc Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 25 May 2026 14:09:16 -0300 Subject: [PATCH 10/15] FEAT: update deployment to include firebase base url --- deploy/helm-chart/templates/deployment.yaml | 2 ++ deploy/helm-chart/values-stg.yaml | 1 + deploy/helm-chart/values.yaml | 1 + 3 files changed, 4 insertions(+) diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 68b8e1d..518fd7b 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -113,6 +113,8 @@ spec: value: {{ .Values.maestro.platform_api_url }} - name: STORAGE_EXPLORER_API_URL value: {{ .Values.maestro.storage_explorer_api_url | quote }} + - name: FIREBASE_BASE_URL + value: {{ .Values.maestro.firebase_base_url }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 2c1737d..70d2170 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -10,6 +10,7 @@ maestro: redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" + firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg hostname: maestro.stg.dadosfera.ai diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 4e431f5..5042b24 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -55,6 +55,7 @@ maestro: redis_database: "0" redis_tls: "true" cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf" + firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd autoscaling: enabled: false minReplicas: 1 From 06d505c50ab088617d86ec505c27dab9d98adea9 Mon Sep 17 00:00:00 2001 From: marcosrodrigues-dadosfera Date: Sat, 13 Jun 2026 21:07:14 -0300 Subject: [PATCH 11/15] FEAT: remove deprecated protospack lib --- docsfera.json | 135 ++++++------------ package-lock.json | 18 +-- package.json | 3 +- src/app.module.ts | 2 - src/modules/catalog/catalog.module.ts | 3 - src/modules/inputs/dtos/old_interfaces.ts | 6 +- src/modules/pipelines/client.service.ts | 78 ---------- src/modules/pipelines/interfaces.d.ts | 36 ----- src/modules/pipelines/pipelines-client.ts | 33 ----- src/modules/pipelines/pipelines.controller.ts | 72 ---------- src/modules/pipelines/pipelines.module.ts | 19 --- src/modules/pipelines/pipelines.service.ts | 33 ----- src/modules/pipelinesV2/interfaces.ts | 7 +- .../pipelinesV2/pipelines.controller.ts | 8 +- src/modules/pipelinesV2/pipelines.module.ts | 2 - src/modules/pipelinesV2/pipelines.service.ts | 48 ++++++- src/modules/transformations/interfaces.d.ts | 7 +- 17 files changed, 115 insertions(+), 395 deletions(-) delete mode 100644 src/modules/pipelines/client.service.ts delete mode 100644 src/modules/pipelines/interfaces.d.ts delete mode 100644 src/modules/pipelines/pipelines-client.ts delete mode 100644 src/modules/pipelines/pipelines.controller.ts delete mode 100644 src/modules/pipelines/pipelines.module.ts delete mode 100644 src/modules/pipelines/pipelines.service.ts diff --git a/docsfera.json b/docsfera.json index c68e491..eaaf00e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,14 +3343,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -3874,88 +3867,6 @@ ] } }, - "/pipelines/start/{id}": { - "post": { - "operationId": "PipelinesController_activate", - "summary": "", - "deprecated": true, - "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Pipelines" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/pipelines/{id}/status": { - "get": { - "operationId": "PipelinesController_getPipelineStatus", - "summary": "", - "deprecated": true, - "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Pipelines" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/transformations": { "post": { "operationId": "TransformationsController_create", @@ -4659,6 +4570,50 @@ ] } }, + "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunJobs", + "summary": "Get pipeline run jobs", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", diff --git a/package-lock.json b/package-lock.json index f26f5fe..28b3128 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,8 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.9", + "@dadosfera/protospack-v2": "3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,19 +1734,10 @@ "winston-log2gelf": "^2.4.0" } }, - "node_modules/@dadosfera/protospack": { - "version": "2.5.3", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz", - "integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==", - "license": "ISC", - "dependencies": { - "rxjs": "^7.5.5" - } - }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.9", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz", - "integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==", + "version": "3.40.0-beta.10", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", + "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 878d46a..dbd506b 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.9", + "@dadosfera/protospack-v2": "3.40.0-beta.10", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index 4db914b..b0bc6ae 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -17,7 +17,6 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test. import { NetworkConfigModule } from './modules/network-config/network-config.module'; import { InputsModule } from './modules/inputs/inputs.module'; import { OauthModule } from './modules/oauth/oauth.module'; -import { PipelinesModule } from './modules/pipelines/pipelines.module'; import { TransformationsModule } from './modules/transformations/transformations.module'; import { HealthModule } from './modules/health/health.module'; import { CatalogModule } from './modules/catalog/catalog.module'; @@ -60,7 +59,6 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module'; PermissionsModule, TermsOfUseModule, ConnectionTestModule, - PipelinesModule, TransformationsModule, UsersModule, RolesModule, diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index 8ce1579..982ef09 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -5,20 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { CatalogController } from './catalog.controller'; import { CatalogClientConfiguration } from './catalog-client'; import { ClientsModule } from '@nestjs/microservices'; -import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { UsersModule } from '../users/users.module'; import { RolesModule } from '../roles/roles.module'; import { CustomersModule } from '../customers/customers.module'; import { ShareModule } from './share/share.module'; import { CatalogService } from './catalog.service'; -import { MixpanelModule } from '../mixpanel/mixpanel.module'; const client = new CatalogClientConfiguration(); @Module({ imports: [ ClientsModule.register([client.providerOptions]), - OldPipelineModule, UsersModule, RolesModule, CustomersModule, diff --git a/src/modules/inputs/dtos/old_interfaces.ts b/src/modules/inputs/dtos/old_interfaces.ts index 21c8cff..e02a567 100644 --- a/src/modules/inputs/dtos/old_interfaces.ts +++ b/src/modules/inputs/dtos/old_interfaces.ts @@ -1,4 +1,8 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} interface Values { jdbc_user: string; diff --git a/src/modules/pipelines/client.service.ts b/src/modules/pipelines/client.service.ts deleted file mode 100644 index 1dc395e..0000000 --- a/src/modules/pipelines/client.service.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { ConflictException, Inject, OnModuleInit } from '@nestjs/common'; -import { ClientGrpc } from '@nestjs/microservices'; -import { - PipelineServicesNames, - PipelinesServiceInterface, -} from '@dadosfera/protospack'; -import { lastValueFrom } from 'rxjs'; - -import { IIdRequest } from './interfaces'; - -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { PipelinesClientConfiguration } from './pipelines-client'; - -export class PipelinesClientService implements OnModuleInit { - private pipelineService: PipelinesServiceInterface; - logger: DadosferaLogger; - - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - @Inject(PipelinesClientConfiguration.name) - private readonly grpcClient: ClientGrpc, - ) { - this.logger = dadosferaLogger.logger; - } - - onModuleInit() { - this.pipelineService = - this.grpcClient.getService( - PipelineServicesNames.PipelineService, - ); - } - - async getPipelineStatus(data) { - this.logger.info('PipelinesClientService - GetPipelineStatus'); - - const statusPipelineResponse = await lastValueFrom( - this.pipelineService.getPipelineStatus(data), - ) - .then((res) => { - const statusArray = - res.status?.sort((a, b) => { - if (a.id < b.id) { - return 1; - } else { - return -1; - } - }) || []; - return { status: statusArray }; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - this.logger.info('Done'); - - return statusPipelineResponse; - } - - async runPipeline({ id, info }: IIdRequest) { - this.logger.info('PipelinesClientService - RunPipeline'); - const statusPipelineResponse = await lastValueFrom( - this.pipelineService.triggerPipeline({ id, info }), - ).catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - if (statusPipelineResponse.status == false) { - throw new ConflictException( - 'This pipeline is not ready yet to execute, Try again later!', - ); - } - - this.logger.info('Done'); - return statusPipelineResponse; - } -} diff --git a/src/modules/pipelines/interfaces.d.ts b/src/modules/pipelines/interfaces.d.ts deleted file mode 100644 index 9a9dbc5..0000000 --- a/src/modules/pipelines/interfaces.d.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; - -export interface ICreatePipelineDto { - input: IdRequest; - transformations: IdRequest[]; - output: IdRequest; - tags: string[]; - name: string; - description: string; - info: Info; -} - -export interface IdRequest { - id: string; -} - -export interface IIdRequest { - id: string; - info: Info; -} - -export interface IUpdatePipelineRequest { - input: IdRequest; - transformations: IdRequest[]; - output: IdRequest; - tags: string[]; - name: string; - description: string; - id: string; - info: Info; -} - -export interface IGetPipelineLogsRequest { - id: string; - details: string; -} diff --git a/src/modules/pipelines/pipelines-client.ts b/src/modules/pipelines/pipelines-client.ts deleted file mode 100644 index 2dcf630..0000000 --- a/src/modules/pipelines/pipelines-client.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { - ClientsProviderAsyncOptions, - GrpcOptions, - Transport, -} from '@nestjs/microservices'; -import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack'; -import { credentials } from '@grpc/grpc-js'; - -const isLocalConnection = - process.env.PIFACTORY_URL.startsWith('pi-factory:') || - process.env.PIFACTORY_URL.includes('0.0.0.0'); - -export class PipelinesClientConfiguration { - public name = 'PipelinesClientConfiguration'; - private config: GrpcOptions = { - transport: Transport.GRPC, - options: { - url: process.env.PIFACTORY_URL, - package: PipelinePackages, - credentials: isLocalConnection ? undefined : credentials.createSsl(), - protoPath: PipelineProtoFilePath, - loader: { - keepCase: true, - enums: String, - defaults: false, - }, - }, - }; - providerOptions: ClientsProviderAsyncOptions = { - name: this.name, - ...this.config, - }; -} diff --git a/src/modules/pipelines/pipelines.controller.ts b/src/modules/pipelines/pipelines.controller.ts deleted file mode 100644 index 6054ff6..0000000 --- a/src/modules/pipelines/pipelines.controller.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common'; -import { ApiOperation, ApiTags } from '@nestjs/swagger'; -import { - AuthenticateCondition, - Authenticated, - RequireSomePermission, -} from 'src/decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; -import { PipelinesService } from './pipelines.service'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; - -@ApiInternalOnlyController() -@ApiTags('Pipelines') -@Controller('pipelines') -@Authenticated() -export class PipelinesController { - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private pipelineService: PipelinesService, - ) { - this.logger = dadosferaLogger.logger; - } - - @Post('start/:id') - @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - @ApiOperation({ - deprecated: true, - description: - 'This method is deprecated. Please use route /pipelinesV2/start/:id instead', - }) - async activate(@Param('id') id: string, @Body() body) { - const { info } = body; - - this.logger.info( - process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`, - { - user: body.info.user_id, - customer: body.info.customer, - }, - ); - - const response = await this.pipelineService.runPipeline({ id, info }); - - return response; - } - - @Get(':id/status') - @ApiOperation({ - deprecated: true, - description: - 'This method is deprecated. Please use route /pipelinesV2/:id/status instead', - }) - @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus(@Body() body, @Param('id') id: string) { - body.id = id; - - this.logger.info( - process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, - { - user: body.info.user_id, - customer: body.info.customer, - }, - ); - - const response = await this.pipelineService.getPipelineStatus(body); - - return response; - } -} diff --git a/src/modules/pipelines/pipelines.module.ts b/src/modules/pipelines/pipelines.module.ts deleted file mode 100644 index 192dcab..0000000 --- a/src/modules/pipelines/pipelines.module.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ClientsModule } from '@nestjs/microservices'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; - -import { PipelinesController } from './pipelines.controller'; -import { PipelinesService } from './pipelines.service'; - -import { PipelinesClientConfiguration } from './pipelines-client'; -import { PipelinesClientService } from './client.service'; - -const client = new PipelinesClientConfiguration(); - -@Module({ - imports: [ClientsModule.register([client.providerOptions])], - controllers: [PipelinesController], - providers: [PipelinesService, PipelinesClientService, DadosferaLogger], - exports: [PipelinesService], -}) -export class PipelinesModule {} diff --git a/src/modules/pipelines/pipelines.service.ts b/src/modules/pipelines/pipelines.service.ts deleted file mode 100644 index afc8103..0000000 --- a/src/modules/pipelines/pipelines.service.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; -import { PipelinesClientService } from './client.service'; -import { IIdRequest } from './interfaces'; -import { objectCamelToSnake } from 'src/utils/CaseConverter'; - -@Injectable() -export class PipelinesService { - constructor(private pipelineClient: PipelinesClientService) {} - - async getPipelineStatus(data: IIdRequest) { - try { - const pipelineStatusResponse = - await this.pipelineClient.getPipelineStatus(data); - - return objectCamelToSnake(pipelineStatusResponse); - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async runPipeline({ id, info }: IIdRequest) { - try { - const triggerPipelineResponse = await this.pipelineClient.runPipeline({ - id, - info, - }); - - return objectCamelToSnake(triggerPipelineResponse); - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } -} diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 252b94c..635ae80 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -1,5 +1,4 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; export class PipelineInputsDTO { @ApiProperty() @@ -62,6 +61,12 @@ export interface IIdRequest { info: Info; } +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} + export interface IUpdatePipelineRequest { input: IdRequest; transformations: IdRequest[]; diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index e90af52..70fab15 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -15,7 +15,6 @@ import { HttpException, BadRequestException, UseGuards, - Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -35,7 +34,6 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { ICompleteUploadCSVFile, ICreatePipelineCSVFile, @@ -64,9 +62,7 @@ export class PipelinesController { constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, - private pipelinesClientService: PipelinesService, - private oldPipelinesService: OldPipelineService, ) { this.logger = dadosferaLogger.logger; } @@ -202,7 +198,7 @@ export class PipelinesController { customer: body.info.customer, }); - const response = await this.oldPipelinesService.getPipelineStatus(body); + const response = await this.pipelinesClientService.getPipelineStatus(body); return response; } @@ -513,7 +509,7 @@ export class PipelinesController { }, ); - const response = await this.oldPipelinesService.runPipeline({ id, info }); + const response = await this.pipelinesClientService.runPipeline({ id, info }); return response; } diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 24562c9..b4f24af 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -7,7 +7,6 @@ import { PipelinesService } from './pipelines.service'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; @@ -21,7 +20,6 @@ const client = new PipelinesClientConfiguration(); @Module({ imports: [ ClientsModule.register([client.providerOptions]), - OldPipelineModule, ConnectorModule, InputsModule, TransformationsModule, diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 53d7711..2a8db84 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,6 +1,7 @@ /* eslint-disable no-async-promise-executor */ import { BadRequestException, + ConflictException, HttpException, HttpStatus, Inject, @@ -17,7 +18,7 @@ import { lastValueFrom } from 'rxjs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; +import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { Metadata } from '@grpc/grpc-js'; import { ConnectorClientService } from '../connector/client.service'; @@ -632,4 +633,49 @@ export class PipelinesService implements OnModuleInit { return assets; } + async getPipelineStatus(data) { + this.logger.info('PipelinesClientService - GetPipelineStatus'); + + const statusPipelineResponse = await lastValueFrom( + this.pipelineReadService.PipelineV2GetPipelineV2Status(data), + ) + .then((res) => { + const statusArray = + res.status?.sort((a, b) => { + if (a.id < b.id) { + return 1; + } else { + return -1; + } + }) || []; + return { status: statusArray }; + }) + .catch((err) => { + this.logger.error(err.message); + throw new Error(err); + }); + this.logger.info('Done'); + + return statusPipelineResponse; + } + + async runPipeline({ id, info }: IIdRequest) { + this.logger.info('PipelinesClientService - RunPipeline'); + const statusPipelineResponse = await lastValueFrom( + this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }), + ).catch((err) => { + this.logger.error(err.message); + throw new Error(err); + }); + + if (statusPipelineResponse.status == false) { + throw new ConflictException( + 'This pipeline is not ready yet to execute, Try again later!', + ); + } + + this.logger.info('Done'); + return statusPipelineResponse; + } + } diff --git a/src/modules/transformations/interfaces.d.ts b/src/modules/transformations/interfaces.d.ts index 0e7ab3c..20ba822 100644 --- a/src/modules/transformations/interfaces.d.ts +++ b/src/modules/transformations/interfaces.d.ts @@ -1,5 +1,8 @@ -import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; - +export interface Info { + user_id: string; + customer_id: string; + customer: string; +} export interface ICreateTransformationsRequest { transformations: Transformation[]; info: Info; From 5c775779928a4c767f8417ccc84bc2fa1bd5feba Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Fri, 19 Jun 2026 17:04:42 -0300 Subject: [PATCH 12/15] feat: add endpoint to retrieve pipeline run jobs --- docsfera.json | 58 ++++++++++++++++++- .../platform-api/platform-api.controller.ts | 20 ++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/docsfera.json b/docsfera.json index 00b43df..aae1a29 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4659,6 +4659,62 @@ ] } }, + "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunJobs", + "summary": "Get pipeline run jobs", + "description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "DB-backed job runs for the selected pipeline run.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", @@ -11662,4 +11718,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index f3abaf5..4263842 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -14,7 +14,7 @@ import { NotFoundException, UseGuards, } from '@nestjs/common'; -import { ApiTags, ApiOperation } from '@nestjs/swagger'; +import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { @@ -833,7 +833,23 @@ export class PlatformApiController { } @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') - @ApiOperation({ summary: 'Get pipeline run jobs' }) + @ApiOperation({ + summary: 'Get pipeline run jobs', + description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.', + }) + @ApiOkResponse({ + description: 'DB-backed job runs for the selected pipeline run.', + schema: { + type: 'object', + properties: { + jobs: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['jobs'], + }, + }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getPipelineRunJobs( @Param('pipelineId') pipelineId: string, From 63efff6adf764b010a67422dfe05dabf98e916ef Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Mon, 22 Jun 2026 20:25:16 -0300 Subject: [PATCH 13/15] FEAT: enhance pipeline run jobs endpoint with error handling and response structure --- docsfera.json | 78 +++++------- .../pipelinesV2/pipelines.controller.ts | 116 +++++++++++++++++- .../platform-api/platform-api.controller.ts | 40 ++++-- 3 files changed, 170 insertions(+), 64 deletions(-) diff --git a/docsfera.json b/docsfera.json index f6130d2..f6515c0 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,7 +3343,14 @@ ], "responses": { "200": { - "description": "" + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } } }, "tags": [ @@ -4570,50 +4577,6 @@ ] } }, - "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunJobs", - "summary": "Get pipeline run jobs", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { "get": { "operationId": "PlatformApiController_getPipelineRunJobs", @@ -8647,10 +8610,33 @@ "Health" ] } + }, + "/release_note": { + "get": { + "operationId": "ReleaseNoteController_getLatestReleaseNote", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } } }, "info": { - "title": "Maestro", + "title": "Maestro - feat/pipeline-run-jobs", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 70fab15..8e9a0e4 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -49,9 +49,23 @@ import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard'; +import { PlatformApiService } from '../platform-api/platform-api.service'; type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; +type PlatformPipelineJob = { job_id?: string; input?: Record }; +type PlatformPipeline = { + pipeline_id?: string; + created_at?: string; + description?: string; + name?: string; + last_status?: string; + status?: string; + cron?: string; + jobs?: PlatformPipelineJob[]; + properties?: Record; + user_id?: string; +}; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -63,10 +77,70 @@ export class PipelinesController { @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, private pipelinesClientService: PipelinesService, + private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } + private normalizePipelineId(id: string): string { + return id?.replace(/-/g, '_') || ''; + } + + private buildPlatformPipelineFallback( + id: string, + platformPipeline: PlatformPipeline, + ): Messages.PipelineV2FindOneResponse { + const jobs = platformPipeline.jobs || []; + const firstInput = jobs.find((job) => job.input)?.input || {}; + const plugin = firstInput.plugin || firstInput.connector; + + const tables = jobs.map((job) => ({ + ...job.input, + name: + job.input?.table_name || + job.input?.source_prefix || + job.input?.stream || + job.job_id || + '', + job_id: job.job_id, + })); + + return { + pipeline: { + id, + created_at: platformPipeline.created_at, + description: platformPipeline.description, + name: platformPipeline.name, + transformations: [], + status: platformPipeline.status || platformPipeline.last_status || '', + input: { + ...firstInput, + plugin, + category: firstInput.category || (firstInput.connector === 's3' ? 'file' : 'database'), + cron: platformPipeline.cron, + tables, + input_id: firstInput.input_id || null, + }, + properties: platformPipeline.properties || {}, + username: platformPipeline.user_id, + } as any, + }; + } + + private async getPipelineFromPlatformApi( + id: string, + user: RequestUser, + ): Promise { + const normalizedId = this.normalizePipelineId(id); + const platformPipeline = await this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedId}`, + user, + ); + + return this.buildPlatformPipelineFallback(id, platformPipeline); + } + @Get('monitoring-dashboard') @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getMonitoringDashboard(@User() user: RequestUser) { @@ -189,18 +263,38 @@ export class PipelinesController { @Get(':id/status') @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus(@Body() body, @Param('id') id: string) { - - body.id = id; + async getPipelineStatus( + @Param('id') id: string, + @User() user: RequestUser, + ) { + const body = { + id, + info: { + customer_id: user.customer_id, + user_id: user.user_id, + customer: user.customer_name, + }, + }; this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, { user: body.info.user_id, customer: body.info.customer, }); - const response = await this.pipelinesClientService.getPipelineStatus(body); + try { + return await this.pipelinesClientService.getPipelineStatus(body); + } catch (error) { + this.logger.warn('PipelinesController - getPipelineStatus fallback to platform-api', { + id, + error: error.message, + }); - return response; + return this.platformApiService.proxy( + 'GET', + `/pipeline/${this.normalizePipelineId(id)}/pipeline_run`, + user, + ); + } } @Get('/:id') @@ -222,7 +316,17 @@ export class PipelinesController { language, }); - const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); + let pipelineRes: Messages.PipelineV2FindOneResponse; + try { + pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); + } catch (error) { + this.logger.warn('PipelinesController - findOne fallback to platform-api', { + id, + error: error.message, + }); + + return this.getPipelineFromPlatformApi(id, user); + } const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); const input_id = parsed.input_id; diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 4263842..ba6ddd1 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -72,6 +72,10 @@ export class PlatformApiController { return id?.replace(/-/g, '_') || ''; } + private decodePathParam(value: string): string { + return value ? decodeURIComponent(value) : ''; + } + /** * Denormalize ID back to UUID format (replace _ with -). * Used when we receive a normalized ID but need the original UUID. @@ -778,10 +782,10 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}`, user, ); } @@ -794,10 +798,10 @@ export class PlatformApiController { @User() user: RequestUser, @Query() query: Record, ) { - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/pipeline_run/${normalizedRunId}/logs`, + `/pipeline/pipeline_run/${decodedRunId}/logs`, user, undefined, query, @@ -813,7 +817,7 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); const status = await this.platformApiService.proxy( 'GET', @@ -827,7 +831,7 @@ export class PlatformApiController { return this.platformApiService.proxy( 'POST', - `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/cancel`, user, ); } @@ -857,13 +861,25 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const normalizedRunId = this.normalizePipelineId(runId); + const decodedRunId = this.decodePathParam(runId); - return this.platformApiService.proxy( - 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`, - user, - ); + try { + return await this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, + user, + ); + } catch (error) { + if (error instanceof HttpException && error.getStatus() === 404) { + this.logger.warn('Platform API pipeline run jobs not found; returning empty jobs list', { + pipelineId: normalizedPipelineId, + runId: decodedRunId, + }); + return { jobs: [] }; + } + + throw error; + } } // ==================== JOBS - COLUMN EDITING ROUTES ==================== From 17363e74f414fbb9c05bf0298be5bb76fedf4688 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Tue, 23 Jun 2026 11:54:59 -0300 Subject: [PATCH 14/15] FEAT: simplify pipeline run jobs handling and normalize run ID usage --- docsfera.json | 9 +- .../pipelinesV2/pipelines.controller.ts | 116 +----------------- .../platform-api/platform-api.controller.ts | 34 ++--- 3 files changed, 18 insertions(+), 141 deletions(-) diff --git a/docsfera.json b/docsfera.json index f6515c0..0d70456 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3343,14 +3343,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 8e9a0e4..70fab15 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -49,23 +49,9 @@ import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard'; -import { PlatformApiService } from '../platform-api/platform-api.service'; type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; -type PlatformPipelineJob = { job_id?: string; input?: Record }; -type PlatformPipeline = { - pipeline_id?: string; - created_at?: string; - description?: string; - name?: string; - last_status?: string; - status?: string; - cron?: string; - jobs?: PlatformPipelineJob[]; - properties?: Record; - user_id?: string; -}; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -77,70 +63,10 @@ export class PipelinesController { @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, private pipelinesClientService: PipelinesService, - private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } - private normalizePipelineId(id: string): string { - return id?.replace(/-/g, '_') || ''; - } - - private buildPlatformPipelineFallback( - id: string, - platformPipeline: PlatformPipeline, - ): Messages.PipelineV2FindOneResponse { - const jobs = platformPipeline.jobs || []; - const firstInput = jobs.find((job) => job.input)?.input || {}; - const plugin = firstInput.plugin || firstInput.connector; - - const tables = jobs.map((job) => ({ - ...job.input, - name: - job.input?.table_name || - job.input?.source_prefix || - job.input?.stream || - job.job_id || - '', - job_id: job.job_id, - })); - - return { - pipeline: { - id, - created_at: platformPipeline.created_at, - description: platformPipeline.description, - name: platformPipeline.name, - transformations: [], - status: platformPipeline.status || platformPipeline.last_status || '', - input: { - ...firstInput, - plugin, - category: firstInput.category || (firstInput.connector === 's3' ? 'file' : 'database'), - cron: platformPipeline.cron, - tables, - input_id: firstInput.input_id || null, - }, - properties: platformPipeline.properties || {}, - username: platformPipeline.user_id, - } as any, - }; - } - - private async getPipelineFromPlatformApi( - id: string, - user: RequestUser, - ): Promise { - const normalizedId = this.normalizePipelineId(id); - const platformPipeline = await this.platformApiService.proxy( - 'GET', - `/pipeline/${normalizedId}`, - user, - ); - - return this.buildPlatformPipelineFallback(id, platformPipeline); - } - @Get('monitoring-dashboard') @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getMonitoringDashboard(@User() user: RequestUser) { @@ -263,38 +189,18 @@ export class PipelinesController { @Get(':id/status') @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getPipelineStatus( - @Param('id') id: string, - @User() user: RequestUser, - ) { - const body = { - id, - info: { - customer_id: user.customer_id, - user_id: user.user_id, - customer: user.customer_name, - }, - }; + async getPipelineStatus(@Body() body, @Param('id') id: string) { + + body.id = id; this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, { user: body.info.user_id, customer: body.info.customer, }); - try { - return await this.pipelinesClientService.getPipelineStatus(body); - } catch (error) { - this.logger.warn('PipelinesController - getPipelineStatus fallback to platform-api', { - id, - error: error.message, - }); + const response = await this.pipelinesClientService.getPipelineStatus(body); - return this.platformApiService.proxy( - 'GET', - `/pipeline/${this.normalizePipelineId(id)}/pipeline_run`, - user, - ); - } + return response; } @Get('/:id') @@ -316,17 +222,7 @@ export class PipelinesController { language, }); - let pipelineRes: Messages.PipelineV2FindOneResponse; - try { - pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); - } catch (error) { - this.logger.warn('PipelinesController - findOne fallback to platform-api', { - id, - error: error.message, - }); - - return this.getPipelineFromPlatformApi(id, user); - } + const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); const input_id = parsed.input_id; diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index ba6ddd1..e8de5ca 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -782,10 +782,10 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const decodedRunId = this.decodePathParam(runId); + const normalizedRunId = this.normalizePipelineId(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`, user, ); } @@ -798,10 +798,10 @@ export class PlatformApiController { @User() user: RequestUser, @Query() query: Record, ) { - const decodedRunId = this.decodePathParam(runId); + const normalizedRunId = this.normalizePipelineId(runId); return this.platformApiService.proxy( 'GET', - `/pipeline/pipeline_run/${decodedRunId}/logs`, + `/pipeline/pipeline_run/${normalizedRunId}/logs`, user, undefined, query, @@ -817,7 +817,7 @@ export class PlatformApiController { @User() user: RequestUser, ) { const normalizedPipelineId = this.normalizePipelineId(pipelineId); - const decodedRunId = this.decodePathParam(runId); + const normalizedRunId = this.normalizePipelineId(runId); const status = await this.platformApiService.proxy( 'GET', @@ -831,7 +831,7 @@ export class PlatformApiController { return this.platformApiService.proxy( 'POST', - `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/cancel`, + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`, user, ); } @@ -863,23 +863,11 @@ export class PlatformApiController { const normalizedPipelineId = this.normalizePipelineId(pipelineId); const decodedRunId = this.decodePathParam(runId); - try { - return await this.platformApiService.proxy( - 'GET', - `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, - user, - ); - } catch (error) { - if (error instanceof HttpException && error.getStatus() === 404) { - this.logger.warn('Platform API pipeline run jobs not found; returning empty jobs list', { - pipelineId: normalizedPipelineId, - runId: decodedRunId, - }); - return { jobs: [] }; - } - - throw error; - } + return this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, + user, + ); } // ==================== JOBS - COLUMN EDITING ROUTES ==================== From 011032e3d4c323350409b0a69f18c7976c06b315 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Tue, 23 Jun 2026 11:58:28 -0300 Subject: [PATCH 15/15] FEAT: update API title in docsfera.json to reflect project name --- docsfera.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 0d70456..ae9f040 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8629,7 +8629,7 @@ } }, "info": { - "title": "Maestro - feat/pipeline-run-jobs", + "title": "Maestro", "description": "This is the Maestro API", "version": "1.0.0", "contact": {}