From b5d933eaf3438e6a9acdec055b04610194000de5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 4 Feb 2026 16:43:49 -0300 Subject: [PATCH 01/27] FEAT: qualify contract --- package-lock.json | 12 ++++-------- package.json | 2 +- src/modules/inputs/dtos/input.model.ts | 7 +++++++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4d90eb7..69f7636 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -95,6 +95,7 @@ "node": "18.17" } }, + "@dadosfera/protospack-v2@3.38.0-beta.27": {}, "node_modules/@angular-devkit/core": { "version": "16.0.1", "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-16.0.1.tgz", @@ -1744,13 +1745,8 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.26", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", - "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", - "dependencies": { - "@grpc/grpc-js": "^1.9.3", - "rxjs": "^7.5.5" - } + "resolved": "@dadosfera/protospack-v2@3.38.0-beta.27", + "link": true }, "node_modules/@elastic/ecs-helpers": { "version": "2.1.1", diff --git a/package.json b/package.json index 4fd10bd..cb4bbf4 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 7b40d1e..3b34a75 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -14,6 +14,13 @@ export class TableColumns { @ApiProperty() references: Column[]; @ApiProperty() + identifier_columns: string[]; + @ApiProperty() + destination: Record<'raw' | 'qualify', { + table_name: string; + table_schema: string; + }> | null; + @ApiProperty() type: string; } export class AvailableEntity { From 0e169a3cbc6aec4b86d6ee17128c16b14ed6d0c0 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 4 Feb 2026 16:50:20 -0300 Subject: [PATCH 02/27] FIX: protospack version --- docsfera.json | 41 +++++++++++------------------------------ package-lock.json | 13 +++++++++---- package.json | 2 +- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/docsfera.json b/docsfera.json index d13bcd6..6515e6e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8440,36 +8440,6 @@ ] } }, - "/storage-explorer/storage/link-to-table": { - "post": { - "operationId": "StorageExplorerController_linkFileToTable", - "summary": "Link an existing file in storage to a table", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/storage-explorer/storage/metadata": { "get": { "operationId": "StorageExplorerController_getFileMetadata", @@ -10054,6 +10024,15 @@ "type": "string" } }, + "identifier_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "destination": { + "type": "object" + }, "type": { "type": "string" } @@ -10062,6 +10041,8 @@ "name", "columns", "references", + "identifier_columns", + "destination", "type" ] }, diff --git a/package-lock.json b/package-lock.json index 69f7636..7d2079b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -95,7 +95,6 @@ "node": "18.17" } }, - "@dadosfera/protospack-v2@3.38.0-beta.27": {}, "node_modules/@angular-devkit/core": { "version": "16.0.1", "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-16.0.1.tgz", @@ -1745,8 +1744,14 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "resolved": "@dadosfera/protospack-v2@3.38.0-beta.27", - "link": true + "version": "3.38.0-beta.27", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz", + "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==", + "license": "ISC", + "dependencies": { + "@grpc/grpc-js": "^1.9.3", + "rxjs": "^7.5.5" + } }, "node_modules/@elastic/ecs-helpers": { "version": "2.1.1", diff --git a/package.json b/package.json index cb4bbf4..0fd3006 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 66309c7bbe103fca4fbf42a5f03120b220f37eab Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 5 Feb 2026 10:29:26 -0300 Subject: [PATCH 03/27] FIX: platform routes --- docsfera.json | 78 +++++++++++++++++++ .../platform-api/platform-api.controller.ts | 37 +++++++++ src/modules/platform-api/platform-api.dto.ts | 9 +++ 3 files changed, 124 insertions(+) create mode 100644 src/modules/platform-api/platform-api.dto.ts diff --git a/docsfera.json b/docsfera.json index 6515e6e..6225efb 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7275,6 +7275,70 @@ ] } }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get available schemas", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate Table and Schema", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationTableDTO" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/pipeline/{pipelineId}/pipeline_run": { "get": { "operationId": "PlatformApiController_getPipelineRuns", @@ -11451,6 +11515,20 @@ "required": [ "providers" ] + }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] } } } diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index e0d59f2..e761e96 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -25,6 +25,7 @@ import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; +import { ValidationTableDTO } from './platform-api.dto'; @ApiTags('Platform API') @Controller('platform') @@ -720,6 +721,42 @@ export class PlatformApiController { ); } + // ==================== PIPELINE VALIDATION ==================== + + @Get('pipelines/catalog/schemas') + @ApiOperation({ summary: 'Get available schemas' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getAvailableSchemas( + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'GET', + `/catalog/schemas`, + user, + undefined, + query, + ); + } + + @Post('pipelines/catalog/tables/validate') + @ApiOperation({ summary: 'Validate Table and Schema' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async validateTableAndSchema( + @Body() payload: ValidationTableDTO, + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'POST', + `/catalog/tables/validate`, + user, + payload, + query, + ); + } + + // ==================== PIPELINE RUN ROUTES ==================== @Get('pipeline/:pipelineId/pipeline_run') diff --git a/src/modules/platform-api/platform-api.dto.ts b/src/modules/platform-api/platform-api.dto.ts new file mode 100644 index 0000000..3f6b89f --- /dev/null +++ b/src/modules/platform-api/platform-api.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class ValidationTableDTO { + @ApiProperty() + tables: Array<{ + table_name: string; + table_schema: string; + }> +} \ No newline at end of file From 30a41ba1445a4952228eb3788004f70546b138be Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Feb 2026 17:48:05 -0300 Subject: [PATCH 04/27] FEAT: qualify route update --- docsfera.json | 2583 +++++++++-------- src/modules/inputs/inputs.service.ts | 2 +- src/modules/pipelinesV2/interfaces.ts | 36 + .../pipelinesV2/pipelines.controller.ts | 45 + src/modules/pipelinesV2/pipelines.module.ts | 2 + src/modules/pipelinesV2/pipelines.service.ts | 140 +- 6 files changed, 1542 insertions(+), 1266 deletions(-) diff --git a/docsfera.json b/docsfera.json index 6225efb..0ae93e6 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3548,6 +3548,61 @@ ] } }, + "/pipelinesV2/{pipelineId}/inputs/{id}": { + "patch": { + "operationId": "PipelinesController_updatePipelineInput", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2/init-upload": { "post": { "operationId": "PipelinesController_initUploadFile", @@ -3960,6 +4015,1195 @@ ] } }, + "/platform/pipeline": { + "post": { + "operationId": "PlatformApiController_createPipeline", + "summary": "Create a new pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines": { + "get": { + "operationId": "PlatformApiController_getPipelines", + "summary": "List all pipelines for customer", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}": { + "get": { + "operationId": "PlatformApiController_getPipeline", + "summary": "Get pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_updatePipeline", + "summary": "Update pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "PlatformApiController_deletePipeline", + "summary": "Delete pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/execute": { + "post": { + "operationId": "PlatformApiController_executePipeline", + "summary": "Execute a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pause": { + "post": { + "operationId": "PlatformApiController_pausePipeline", + "summary": "Pause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/unpause": { + "post": { + "operationId": "PlatformApiController_unpausePipeline", + "summary": "Unpause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/memory": { + "put": { + "operationId": "PlatformApiController_updatePipelineMemory", + "summary": "Update pipeline memory configuration", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/metadata": { + "put": { + "operationId": "PlatformApiController_updatePipelineMetadata", + "summary": "Update pipeline metadata", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/metadata": { + "get": { + "operationId": "PlatformApiController_getPipelinesMetadata", + "summary": "Get all pipelines metadata", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get available schemas", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate Table and Schema", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationTableDTO" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run": { + "get": { + "operationId": "PlatformApiController_getPipelineRuns", + "summary": "Get pipeline runs for a pipeline", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { + "get": { + "operationId": "PlatformApiController_getPipelineRun", + "summary": "Get specific pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pipeline_run/{runId}/logs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunLogs", + "summary": "Get pipeline run logs", + "parameters": [ + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/input": { + "put": { + "operationId": "PlatformApiController_updateJobInput", + "summary": "Update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_patchJobInput", + "summary": "Partial update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/memory": { + "put": { + "operationId": "PlatformApiController_updateJobMemory", + "summary": "Update job memory configuration", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/reset-state": { + "post": { + "operationId": "PlatformApiController_resetJobState", + "summary": "Reset job state", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}": { + "get": { + "operationId": "PlatformApiController_getJdbcJob", + "summary": "Get JDBC job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateJdbcSyncMode", + "summary": "Update JDBC job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/configs/allowed_datatypes": { + "get": { + "operationId": "PlatformApiController_getJdbcAllowedDatatypes", + "summary": "Get allowed datatypes for JDBC", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}": { + "get": { + "operationId": "PlatformApiController_getSingerJob", + "summary": "Get Singer job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateSingerSyncMode", + "summary": "Update Singer job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/s3/{jobId}": { + "get": { + "operationId": "PlatformApiController_getS3Job", + "summary": "Get S3 job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/health": { + "get": { + "operationId": "PlatformApiController_healthCheck", + "summary": "Platform API health check", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/links": { + "get": { + "operationId": "CustomersController_getCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinksResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_setCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinkRequest" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/token": { + "get": { + "operationId": "CustomersController_getCustomerToken", + "parameters": [ + { + "name": "exp", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/monitoring-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMonitoringDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/logs-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/organization-info": { + "get": { + "operationId": "CustomersController_getOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_updateOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information updated" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog": { "get": { "operationId": "CatalogController_searchCatalog", @@ -5371,301 +6615,6 @@ ] } }, - "/customers/{id}/mfa": { - "post": { - "operationId": "CustomersController_enableMfaEnforce", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EnforceMfa" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/links": { - "get": { - "operationId": "CustomersController_getCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinksResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_setCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinkRequest" - } - } - } - }, - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/token": { - "get": { - "operationId": "CustomersController_getCustomerToken", - "parameters": [ - { - "name": "exp", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/monitoring-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMonitoringDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/logs-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/access-dashboard": { - "get": { - "operationId": "CustomersController_getAccessDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/organization-info": { - "get": { - "operationId": "CustomersController_getOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_updateOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information updated" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/catalog/data-asset/share/{id}": { "get": { "operationId": "ShareController_getShareDataAsset", @@ -6937,900 +7886,6 @@ } } }, - "/platform/pipeline": { - "post": { - "operationId": "PlatformApiController_createPipeline", - "summary": "Create a new pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines": { - "get": { - "operationId": "PlatformApiController_getPipelines", - "summary": "List all pipelines for customer", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}": { - "get": { - "operationId": "PlatformApiController_getPipeline", - "summary": "Get pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_updatePipeline", - "summary": "Update pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "delete": { - "operationId": "PlatformApiController_deletePipeline", - "summary": "Delete pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/execute": { - "post": { - "operationId": "PlatformApiController_executePipeline", - "summary": "Execute a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pause": { - "post": { - "operationId": "PlatformApiController_pausePipeline", - "summary": "Pause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/unpause": { - "post": { - "operationId": "PlatformApiController_unpausePipeline", - "summary": "Unpause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/memory": { - "put": { - "operationId": "PlatformApiController_updatePipelineMemory", - "summary": "Update pipeline memory configuration", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/metadata": { - "put": { - "operationId": "PlatformApiController_updatePipelineMetadata", - "summary": "Update pipeline metadata", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/metadata": { - "get": { - "operationId": "PlatformApiController_getPipelinesMetadata", - "summary": "Get all pipelines metadata", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/schemas": { - "get": { - "operationId": "PlatformApiController_getAvailableSchemas", - "summary": "Get available schemas", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables/validate": { - "post": { - "operationId": "PlatformApiController_validateTableAndSchema", - "summary": "Validate Table and Schema", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ValidationTableDTO" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run": { - "get": { - "operationId": "PlatformApiController_getPipelineRuns", - "summary": "Get pipeline runs for a pipeline", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { - "get": { - "operationId": "PlatformApiController_getPipelineRun", - "summary": "Get specific pipeline run", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pipeline_run/{runId}/logs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunLogs", - "summary": "Get pipeline run logs", - "parameters": [ - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/input": { - "put": { - "operationId": "PlatformApiController_updateJobInput", - "summary": "Update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_patchJobInput", - "summary": "Partial update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/memory": { - "put": { - "operationId": "PlatformApiController_updateJobMemory", - "summary": "Update job memory configuration", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/reset-state": { - "post": { - "operationId": "PlatformApiController_resetJobState", - "summary": "Reset job state", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}": { - "get": { - "operationId": "PlatformApiController_getJdbcJob", - "summary": "Get JDBC job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateJdbcSyncMode", - "summary": "Update JDBC job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/configs/allowed_datatypes": { - "get": { - "operationId": "PlatformApiController_getJdbcAllowedDatatypes", - "summary": "Get allowed datatypes for JDBC", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}": { - "get": { - "operationId": "PlatformApiController_getSingerJob", - "summary": "Get Singer job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateSingerSyncMode", - "summary": "Update Singer job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/s3/{jobId}": { - "get": { - "operationId": "PlatformApiController_getS3Job", - "summary": "Get S3 job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/health": { - "get": { - "operationId": "PlatformApiController_healthCheck", - "summary": "Platform API health check", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -10386,6 +10441,81 @@ "engine" ] }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] + }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLink": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, "IRole": { "type": "object", "properties": { @@ -10975,67 +11105,6 @@ "updated_by" ] }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLink": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, "CreateShareMetadataDto": { "type": "object", "properties": { @@ -11515,20 +11584,6 @@ "required": [ "providers" ] - }, - "ValidationTableDTO": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tables" - ] } } } diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 94dd27a..c71ecc7 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -200,7 +200,7 @@ export class InputsService { } async update(id: string, data, info: Info) { - this.validateCron({ ...data, info }); + // this.validateCron({ ...data, info }); try { const updateInputResponse: any = await this.OLD_inputClient.update({ id, diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index db9a201..252b94c 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -1,6 +1,15 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; +export class PipelineInputsDTO { + @ApiProperty() + tables: Array<{ + name: string, + type: string, + + }> +} + export class IPipelineV2 { @ApiProperty() id: string; @@ -123,3 +132,30 @@ export class PipelineFindAllReq { @ApiPropertyOptional() type?: string | undefined; } + +export interface UpdateTableDTO { + name: string; + type: string; + columns: string[]; + destinations: { + raw: { + table_schema: string; + table_name: string; + }; + qualify: { + table_schema: string; + table_name: string; + }; + }; + identifier_columns: string[]; + reference_column: { + name: string; + type: string; + }; + memory: number; +} + +export interface UpdatePlatformInputRequest { + cron: string; + tables: Array; +} diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 4f5ff1d..11e7b10 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -43,11 +43,15 @@ import { IPipelineV2, IInitUploadCSVFile, PipelineFindAllReq, + UpdatePlatformInputRequest, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; +import { TableColumns } from '../inputs/dtos/input.model'; +import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -223,6 +227,7 @@ export class PipelinesController { .then((res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} let tables = JSON.parse(res.pipeline.config.tables); + const input_id = tables?.input_id; if (tables?.tables) tables = tables.tables; Object.assign(res.pipeline, { transformations: res.pipeline.transformations @@ -231,6 +236,7 @@ export class PipelinesController { config: { cron: res.pipeline.config.cron, tables, + input_id }, properties: res.pipeline.properties ? JSON.parse(res.pipeline.properties) @@ -277,6 +283,45 @@ export class PipelinesController { return response; } + @Patch('/:pipelineId/inputs/:id') + @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async updatePipelineInput( + @Language() language: LanguageEnum, + @Body() pipelineInputDTO: UpdatePlatformInputRequest, + @Param('id') inputId: string, + @Param('pipelineId') pipelineId: string, + @User() user: RequestUser, + ) { + this.logger.info('PipelinesController - update', { user }); + + + const { customer_id, customer_name, user_id, username } = user; + const info: Info = { + user_id: user.user_id, + customer: user.customer_name, + customer_id: user.customer_id, + }; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + language, + }); + + const response = await this.pipelinesClientService.updatePipelineInput( + pipelineId, + inputId, + pipelineInputDTO, + info, + user, + metadata, + ); + + this.logger.info('PipelinesController - update: OK', { user }); + return response; + } + @ApiInternalOnlyEndpoint() @Put('/:id') @ApiOperation({ diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 7bc7cc2..9550aeb 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; +import { PlatformApiModule } from '../platform-api/platform-api.module'; const client = new PipelinesClientConfiguration(); @@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration(); ConnectorModule, InputsModule, TransformationsModule, + PlatformApiModule ], controllers: [PipelinesController], providers: [PipelinesService, DadosferaLogger], diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index f10b8ee..e23ce23 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-async-promise-executor */ import { BadRequestException, HttpException, @@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { ICreatePipelineV2Req } from './interfaces'; +import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { Metadata } from '@grpc/grpc-js'; import { ConnectorClientService } from '../connector/client.service'; @@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; +import { PlatformApiService } from '../platform-api/platform-api.service'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; @@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit { private readonly connectorService: ConnectorClientService, private readonly inputsService: InputsService, private readonly transformationsService: TransformationsService, + private readonly platformAPI: PlatformApiService ) { this.logger = dadosferaLogger.logger; } @@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit { const findOnePipelineResponse = await lastValueFrom( this.pipelineReadService.PipelineV2FindOne(data, metadata), ); + console.log('pipeline find one response', findOnePipelineResponse); this.logger.info('Done'); return findOnePipelineResponse; @@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit { return res; } + + async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) { + this.logger.info('InputClientService - Update'); + + this.logger.info('Update Dynamo Reference'); + const pipelineIdFormat = pipelineId.split('-').join('_'); + const updateInputResponse = await this.inputsService.update( + inputId, + updateInputDTO, + info + ) + + const requests = []; + + this.logger.info('Dynamo Response', updateInputResponse); + + for (const [index, table] of updateInputDTO.tables.entries()) { + const id = `${pipelineIdFormat}_${index}`; + this.logger.info('Updating input reference for table', table.name); + const body = {} + if (table.columns) { + body['column_include_list'] = table.columns; + } + + if (table.reference_column) { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + } + + if (table.identifier_columns) { + body['primary_keys'] = table.identifier_columns; + } + + this.logger.info('Request body', body); + const updateCollumns = this.platformAPI.proxy( + 'PATCH', + `/jobs/${id}/input`, + user, + body + ) + requests.push(updateCollumns); + + if (table.memory) { + this.logger.info('Updating memory allocation for table', table.name); + const updateMemory = this.platformAPI.proxy( + 'PUT', + `/jobs/${id}/memory`, + user, + { + amount: table.memory + } + ) + requests.push(updateMemory); + } + + if (table.type) { + const updateSyncMode = this.updatePipelineSyncMode(table, id, user); + requests.push(updateSyncMode); + } + } + + this.logger.info('Create Platform Request for each JOB'); + + if (updateInputDTO.cron) { + const crnUpdatedRequest = new Promise(async (resolve, reject) => { + const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user); + + if (response.error) { + this.logger.error('Error updating pipeline cron', response.error); + return reject(new ErrorBuilder(response.error)); + } + this.logger.error('Pipeline cron updated successfully', response); + return resolve(response); + }); + requests.push(crnUpdatedRequest); + } + + this.logger.info('Executing all request for the platform api'); + + const results = await Promise.allSettled(requests); + this.logger.info('Platform api response', results); + + return updateInputResponse; + + } + + private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) { + const body = { + target_load_type: table.type + } + + if (table.type === 'incremental_with_qualify') { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + body['primary_keys'] = table.identifier_columns; + } + + if (table.type === 'incremental') { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + } + + this.logger.info('Updating pipeline sync mode', { + pipelineId, + body + }); + + return this.platformAPI.proxy( + "POST", + `/jobs/jdbc/${pipelineId}/sync-mode`, + user, + body + ) + } + + private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) { + try { + const response = await this.platformAPI.proxy( + 'PATCH', + `/pipeline/${pipelineId}`, + user, + { + cron + } + ); + + return response + } catch (error) { + return { + error: error.message + } + } + } } From 30eec733b2592e75aa16466f1c8f407b4c834cf7 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 13 Feb 2026 10:01:46 -0300 Subject: [PATCH 05/27] FEAT: update proto --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d2079b..89fbba9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.28", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.27", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz", - "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==", + "version": "3.38.0-beta.28", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz", + "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 0fd3006..bbd9c19 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.28", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 72ed637640a2048c651a74f0718be6a28ab2d50b Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 16 Feb 2026 18:11:49 -0300 Subject: [PATCH 06/27] FIX: update storage port --- deploy/helm-chart/values-stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index a9aa543..2c1737d 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,7 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com - storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api" + storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" hostname: maestro.stg.dadosfera.ai From cf8eed35a36b05489526bd1f099cc9bcdeaa9339 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Feb 2026 15:32:05 -0300 Subject: [PATCH 07/27] FIX: storage url --- src/modules/storage-explorer/storage-explorer.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts index 02d919f..8f4e4b0 100644 --- a/src/modules/storage-explorer/storage-explorer.service.ts +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -113,7 +113,7 @@ export class StorageExplorerService { } // Get customer-specific storage-explorer URL - const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name); const url = new URL(`${baseUrl}${path}`); // Add query params From 38a9e21f5f737e2e93859ed7beff7dd4c4be7c90 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 5 Mar 2026 17:29:12 -0300 Subject: [PATCH 08/27] feat: add rename-tables endpoint with catalog sync and rollback Add POST /platform/jobs/:jobId/rename-tables that renames Snowflake tables via platform-api and syncs the rename to Elasticsearch and Nimbus (table-metadata, column-metadata, data-preview). If catalog sync fails, all completed catalog steps are rolled back in reverse order and the Snowflake rename is reverted. - Support any connector type (jdbc, singer, s3) via getJobByAnyConnectorType - Resolve old table names from output_config (raw/qualify) - Skip qualify sync when output_config.qualify has no table_name - Add findDataAssetByPipelineAndTable and updateDataAsset to ElasticsearchService - Add renameTableOnNimbus, renameColumnMetadataOnNimbus, renameDataPreviewOnNimbus to CatalogService - Add upstream error logging to PlatformApiService Co-Authored-By: Claude Opus 4.6 --- src/modules/catalog/catalog.service.ts | 58 +++++ .../platform-api/platform-api.controller.ts | 207 ++++++++++++++++++ .../platform-api/platform-api.module.ts | 3 +- .../platform-api/platform-api.service.ts | 6 + .../elasticsearch/elasticsearch.service.ts | 78 +++++++ 5 files changed, 351 insertions(+), 1 deletion(-) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 89dcf84..ebb888c 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit { } } + async renameTableOnNimbus( + nimbusUrl: string, + nimbusId: number, + changes: { table_name?: string; table_schema?: string; display_name?: string }, + ): Promise { + const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`; + this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes }); + await axios.patch(endpoint, changes); + } + + async renameColumnMetadataOnNimbus( + nimbusUrl: string, + databaseName: string, + oldTableName: string, + oldTableSchema: string, + newTableName: string, + newTableSchema: string, + ): Promise { + const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`; + this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint }); + const { data: columns } = await axios.get(listEndpoint); + + const filtered = Array.isArray(columns) ? columns : []; + + for (const column of filtered) { + const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`; + await axios.patch(patchEndpoint, { + table_name: newTableName, + table_schema: newTableSchema, + }); + } + this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`); + } + + async renameDataPreviewOnNimbus( + nimbusUrl: string, + databaseName: string, + oldTableName: string, + oldTableSchema: string, + newTableName: string, + newTableSchema: string, + ): Promise { + const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`; + this.logger.info(`Fetching data-preview records to rename`, { listEndpoint }); + const { data: previews } = await axios.get(listEndpoint); + + const filtered = Array.isArray(previews) ? previews : []; + + for (const preview of filtered) { + const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`; + await axios.patch(patchEndpoint, { + table_name: newTableName, + table_schema: newTableSchema, + }); + } + this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`); + } + async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 978f464..319dcd4 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -10,6 +10,7 @@ import { Query, Inject, BadRequestException, + HttpException, } from '@nestjs/common'; import { ApiTags, ApiOperation } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; @@ -25,6 +26,8 @@ import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; +import { CatalogService } from '../catalog/catalog.service'; +import { PackTheMetadata } from '../../utils/PackTheMetadata'; type ValidateTablesDTO = { @@ -34,6 +37,11 @@ type ValidateTablesDTO = { }> } +type RenameTablesBody = { + raw?: { table_name: string; table_schema: string }; + qualify?: { table_name: string; table_schema: string }; +} + @ApiTags('Platform API') @Controller('platform') export class PlatformApiController { @@ -44,6 +52,7 @@ export class PlatformApiController { private readonly elasticsearchService: ElasticsearchService, private readonly dynamoDBService: DynamoDBService, private readonly customersService: CustomersService, + private readonly catalogService: CatalogService, @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, ) { this.logger = dadosferaLogger.logger; @@ -75,6 +84,23 @@ export class PlatformApiController { return jobId?.replace(/-/g, '_') || ''; } + private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise { + const connectorTypes = ['jdbc', 'singer', 's3']; + for (const type of connectorTypes) { + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${type}/${normalizedJobId}`, + user, + ); + return job; + } catch (error) { + // Continue to next connector type + } + } + throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404); + } + /** * Extract the pipeline ID (base UUID) from a job ID. * Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1"). @@ -968,6 +994,187 @@ export class PlatformApiController { return result; } + @Post('jobs/:jobId/rename-tables') + @ApiOperation({ summary: 'Rename job output tables and sync to catalog' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async renameJobTables( + @Param('jobId') jobId: string, + @Body() body: RenameTablesBody, + @User() user: RequestUser, + ) { + const normalizedJobId = this.normalizeJobId(jobId); + + const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user); + + const result = await this.platformApiService.proxy( + 'POST', + `/jobs/${normalizedJobId}/rename-tables`, + user, + body, + ); + + try { + await this.syncTableRenameToCatalog(jobId, body, currentJob, user); + } catch (error) { + this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message }); + + const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {}); + if (reverseBody) { + try { + await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody); + this.logger.info('Snowflake rename rolled back', { jobId }); + } catch (rollbackError) { + this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message }); + } + } + + throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500); + } + + return result; + } + + private buildSnowflakeRollbackBody( + body: RenameTablesBody, + outputConfig: any, + ): RenameTablesBody | null { + const reverse: RenameTablesBody = {}; + + if (body.raw) { + const nested = outputConfig.raw; + const oldTableName = nested?.table_name || outputConfig.table_name; + const oldTableSchema = nested?.table_schema || 'PUBLIC'; + if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema }; + } + + if (body.qualify) { + const nested = outputConfig.qualify; + if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' }; + } + + return Object.keys(reverse).length > 0 ? reverse : null; + } + + /** + * Sync table rename to Elasticsearch and Nimbus. + * + * For each target (raw, qualify): + * 1. Resolve old table name from output_config + * 2. Find the ES data asset by pipeline + table + schema + * 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview + * 4. If any step fails, rollback all completed steps for that target + */ + private async syncTableRenameToCatalog( + jobId: string, + body: RenameTablesBody, + currentJob: any, + user: RequestUser, + ): Promise { + const pipelineId = this.extractPipelineIdFromJobId(jobId); + const outputConfig = currentJob.output_config || {}; + const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } }); + const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`; + + const targets = this.buildRenameTargets(body, outputConfig); + + for (const { key, oldTableName, oldTableSchema, newValues } of targets) { + const rollbackSteps: Array<() => Promise> = []; + + try { + const dataAsset = await this.elasticsearchService.findDataAssetByPipelineAndTable( + user.customer_name, pipelineId, oldTableName, oldTableSchema, + ); + + if (!dataAsset) { + this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema }); + continue; + } + + const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset; + const oldValues = { table_name: oldTableName, table_schema: oldTableSchema }; + + // ES update + const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name }; + await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields); + rollbackSteps.push(() => this.elasticsearchService.updateDataAsset( + user.customer_name, esAssetId, + { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName }, + )); + + // Nimbus table-metadata + if (nimbusId) { + await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, newValues); + rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, oldValues)); + } + + // Nimbus column-metadata + await this.catalogService.renameColumnMetadataOnNimbus( + nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + ); + rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus( + nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + )); + + // Nimbus data-preview + await this.catalogService.renameDataPreviewOnNimbus( + nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + ); + rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus( + nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + )); + + this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name }); + } catch (error) { + this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message }); + await this.executeRollback(rollbackSteps, key, jobId); + throw error; + } + } + } + + private buildRenameTargets( + body: RenameTablesBody, + outputConfig: any, + ): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> { + const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' }; + const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = []; + + for (const key of ['raw', 'qualify'] as const) { + if (!body[key]) continue; + + const nested = outputConfig[key]; + + // qualify: only sync if output_config.qualify already exists + if (key === 'qualify' && !nested?.table_name) continue; + + const oldTableName = nested?.table_name || outputConfig.table_name; + if (!oldTableName) continue; + + targets.push({ + key, + oldTableName, + oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key], + newValues: body[key], + }); + } + + return targets; + } + + private async executeRollback( + steps: Array<() => Promise>, + targetKey: string, + jobId: string, + ): Promise { + for (const rollback of steps.reverse()) { + try { + await rollback(); + } catch (error) { + this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message }); + } + } + } + @Get('jobs/jdbc/configs/allowed_datatypes') @ApiOperation({ summary: 'Get allowed datatypes for JDBC' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index c2b31e4..b78e680 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -7,9 +7,10 @@ import { PlatformApiService } from './platform-api.service'; import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; +import { CatalogModule } from '../catalog/catalog.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/modules/platform-api/platform-api.service.ts b/src/modules/platform-api/platform-api.service.ts index 97797a3..f204678 100644 --- a/src/modules/platform-api/platform-api.service.ts +++ b/src/modules/platform-api/platform-api.service.ts @@ -89,6 +89,12 @@ export class PlatformApiService { // Propagate non-2xx responses as HttpExceptions if (response.status >= 400) { + this.logger.error('Platform API upstream error', { + status: response.status, + data: response.data, + path, + method: method.toUpperCase(), + }); throw new HttpException(response.data, response.status); } diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index e2c1857..8fff83e 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -358,6 +358,84 @@ export class ElasticsearchService { } } + private getDataAssetIndex(customerName: string): string { + return `${customerName}_data_assets_catalog`; + } + + async findDataAssetByPipelineAndTable( + customerName: string, + pipelineId: string, + tableName: string, + tableSchema: string, + ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Searching data asset', { + index, + pipelineId, + tableName, + tableSchema, + }); + + try { + const response = await this.client.post(`/${index}/_search`, { + query: { + bool: { + must: [ + { term: { 'pipeline_id.keyword': pipelineId } }, + { term: { 'table_name.keyword': tableName } }, + { term: { 'table_schema.keyword': tableSchema } }, + ], + }, + }, + size: 1, + }); + + const hits = response.data.hits?.hits || []; + if (hits.length === 0) { + this.logger.warn('Elasticsearch: Data asset not found', { pipelineId, tableName, tableSchema, index }); + return null; + } + + return { ...hits[0]._source, _es_id: hits[0]._id }; + } catch (error) { + this.handleError('findDataAssetByPipelineAndTable', error, { pipelineId, tableName, tableSchema, index }); + throw error; + } + } + + async updateDataAsset( + customerName: string, + assetId: string, + updates: Record, + ): Promise { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Updating data asset', { + index, + assetId, + fields: Object.keys(updates), + }); + + try { + const response = await this.client.post( + `/${index}/_update/${assetId}`, + { doc: updates }, + { params: { refresh: 'wait_for' } }, + ); + + this.logger.info('Elasticsearch: Data asset updated', { + assetId, + result: response.data.result, + }); + + return response.data; + } catch (error) { + this.handleError('updateDataAsset', error, { assetId, index }); + throw error; + } + } + private handleError( operation: string, error: any, From 61a4f724ef7cadf2a3f20015b7bb172fb36d607f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Mar 2026 14:43:11 -0300 Subject: [PATCH 09/27] feat: list headers --- src/modules/auth/auth.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 7c3e073..5eb25f5 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -478,6 +478,7 @@ export class AuthController { @Get('me') async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') + this.logger.info(req.headers); // Check for API key header first const apiKey = req.get('X-Api-key'); From 93f452ae0515f7f23cf0e757258bd49531bb8d50 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Mar 2026 15:19:18 -0300 Subject: [PATCH 10/27] FIX: ghost commit --- src/main.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.ts b/src/main.ts index 1085fb3..85e460a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) { ); } bootstrap(); + From b8bdc5beeaef3107798b0e3ac9d0d9475424d35c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Mar 2026 16:00:39 -0300 Subject: [PATCH 11/27] FIX: stringify headers --- src/modules/auth/auth.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 5eb25f5..bcf1ff8 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -478,7 +478,7 @@ export class AuthController { @Get('me') async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') - this.logger.info(req.headers); + this.logger.info(JSON.stringify(req.headers)); // Check for API key header first const apiKey = req.get('X-Api-key'); From 99c2a9ecf5856901a2dfcbb71a819eca57c00501 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 12 Mar 2026 18:02:49 -0300 Subject: [PATCH 12/27] UPDATE: force deployment of maestro --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 028ca81..4432ce9 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ # Maestro + Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. ```mermaid From ff3999a6aa21f7557347acc9b1d4df7fbd08c27f Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 13 Mar 2026 17:35:56 -0300 Subject: [PATCH 13/27] FEAT: update customer links handling and DTOs for improved structure sidebar. update protospack-v2 --- docsfera.json | 118 +++++++++++++++++++-- package-lock.json | 8 +- package.json | 2 +- src/modules/customers/customers.service.ts | 18 ++-- src/modules/customers/dtos/customers.ts | 53 +++++++-- src/modules/inputs/dtos/input.model.ts | 6 +- src/modules/inputs/inputs.service.ts | 5 + 7 files changed, 175 insertions(+), 35 deletions(-) diff --git a/docsfera.json b/docsfera.json index 801658d..0acfbbe 100644 --- a/docsfera.json +++ b/docsfera.json @@ -10100,6 +10100,21 @@ "entities" ] }, + "Column": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "name", + "type" + ] + }, "TableColumns": { "type": "object", "properties": { @@ -10115,17 +10130,25 @@ "references": { "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/Column" } }, "type": { "type": "string" + }, + "identifier_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "reference_column": { + "$ref": "#/components/schemas/Column" } }, "required": [ "name", "columns", - "references", "type" ] }, @@ -11005,7 +11028,7 @@ "enabled" ] }, - "CustomerLink": { + "CustomerLinkItem": { "type": "object", "properties": { "href": { @@ -11027,28 +11050,101 @@ "description" ] }, - "CustomerLinksResponse": { + "CustomerSidebarLinkItem": { "type": "object", "properties": { - "links": { + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "link": { + "type": "string" + }, + "icon": { + "type": "string" + } + }, + "required": [ + "type", + "title", + "link" + ] + }, + "CustomerSidebarMenuItem": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "items": { "type": "array", "items": { - "$ref": "#/components/schemas/CustomerLink" + "$ref": "#/components/schemas/CustomerSidebarLinkItem" } } }, "required": [ - "links" + "type", + "title", + "items" ] }, + "CustomerSidebarSection": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + } + } + }, + "required": [ + "title", + "items" + ] + }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, "CustomerLinkRequest": { "type": "object", "properties": { "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } + "$ref": "#/components/schemas/CustomerLinksConfig" } }, "required": [ diff --git a/package-lock.json b/package-lock.json index 4d90eb7..615ef97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "^3.38.0-beta.29", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.26", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", - "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", + "version": "3.38.0-beta.29", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.29.tgz", + "integrity": "sha512-1TUHKoSi2h/Pt+YEno9KOj8Zv2uAjT4qPS3xK26Tk2Wxg/OTlbxevPc+iwy8EI6YsT24WsbZfkb5JfKRlvKFHA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 4fd10bd..9a13d42 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "^3.38.0-beta.29", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 602943b..bcbd11d 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -9,12 +9,12 @@ import { } from '@nestjs/common'; import { firstValueFrom, lastValueFrom } from 'rxjs'; -import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { DucClient } from '../duc/client.config'; import { ClientGrpc } from '@nestjs/microservices'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; -import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; +import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { CustomerLinksConfig } from './dtos/customers'; import ErrorCodes from 'src/utils/errorCodes'; import jwt from 'jsonwebtoken'; import { @@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit { ) } - async getLinks(customerId: string) { + async getLinks(customerId: string): Promise { try { const result = await lastValueFrom( - this.customerService.CustomerFindOneById({ id: customerId }), + this.customerService.CustomerGetLinks({ id: customerId }), ); - return result.customer?.links || []; + return (result.links as unknown as CustomerLinksConfig) || null; } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) throw new HttpException(err.details, HttpStatus.NOT_FOUND); @@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit { } } - async setLinks(customerId: string, links: Link[]) { + async setLinks(customerId: string, links: CustomerLinksConfig) { if (!customerId || !links) { throw new HttpException(null, HttpStatus.BAD_REQUEST); } try { return await firstValueFrom( - this.customerService.CustomerUpdate({ + this.customerService.CustomerSetLinks({ id: customerId, - links, - } as CustomerUpdateRequest), + links: links as unknown as CustomerSetLinksRequest['links'], + }), ); } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index db56ee3..7299b23 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -1,7 +1,6 @@ -import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; -export class CustomerLink implements Link { +export class CustomerLinkItem { @ApiProperty() href: string; @ApiProperty() @@ -9,15 +8,51 @@ export class CustomerLink implements Link { @ApiProperty() description: string; @ApiPropertyOptional() - iconSrc: string; + iconSrc?: string; } + +export class CustomerSidebarLinkItem { + @ApiProperty() + type: 'link'; + @ApiProperty() + title: string; + @ApiProperty() + link: string; + @ApiPropertyOptional() + icon?: string; +} + +export class CustomerSidebarMenuItem { + @ApiProperty() + type: 'menu'; + @ApiProperty() + title: string; + @ApiPropertyOptional() + icon?: string; + @ApiProperty({ type: [CustomerSidebarLinkItem] }) + items: CustomerSidebarLinkItem[]; +} + +export class CustomerSidebarSection { + @ApiProperty() + title: string; + @ApiProperty({ type: [CustomerSidebarMenuItem] }) + items: CustomerSidebarMenuItem[]; +} + +export class CustomerLinksConfig { + @ApiPropertyOptional({ type: [CustomerLinkItem] }) + home?: CustomerLinkItem[]; + @ApiPropertyOptional({ type: [CustomerSidebarSection] }) + sidebar?: CustomerSidebarSection[]; +} + export class CustomerLinkRequest { - @ApiProperty({ type: [CustomerLink] }) - links: CustomerLink[]; + @ApiProperty({ type: CustomerLinksConfig }) + links: CustomerLinksConfig; } export class CustomerLinksResponse { - @ApiProperty({ type: [CustomerLink] }) - links: CustomerLink[]; -} - + @ApiPropertyOptional({ type: CustomerLinksConfig }) + links?: CustomerLinksConfig; +} \ No newline at end of file diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 7b40d1e..ef558be 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -11,10 +11,14 @@ export class TableColumns { name: string; @ApiProperty() columns: string[]; - @ApiProperty() + @ApiPropertyOptional({ type: [Column] }) references: Column[]; @ApiProperty() type: string; + @ApiPropertyOptional({ type: [String] }) + identifier_columns?: string[]; + @ApiPropertyOptional({ type: Column }) + reference_column?: Column; } export class AvailableEntity { @ApiProperty() diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 94dd27a..d258215 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -164,6 +164,11 @@ export class InputsService { const inputCreateGenericRequest: InputCreateGenericRequest = { input: { ...body, + tables: (body.tables || []).map((table) => ({ + ...table, + identifier_columns: table.identifier_columns || [], + reference_column: table.reference_column || table.references?.[0], + })), }, info, }; From 12c61d9b5dd7c0bbe033ba864d9d6ce8120273c8 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Mar 2026 18:08:39 -0300 Subject: [PATCH 14/27] FIX: rename-tables proxy path and ES lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix proxy path: /jobs/jdbc/:jobId/rename-tables → /jobs/:jobId/rename-tables - Remove pipeline_id from ES data asset lookup, search by table_name + table_schema only Co-Authored-By: Claude Opus 4.6 --- src/modules/platform-api/platform-api.controller.ts | 4 ++-- src/services/elasticsearch/elasticsearch.service.ts | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 97b0a08..31bf1c3 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -1066,8 +1066,8 @@ export class PlatformApiController { const rollbackSteps: Array<() => Promise> = []; try { - const dataAsset = await this.elasticsearchService.findDataAssetByPipelineAndTable( - user.customer_name, pipelineId, oldTableName, oldTableSchema, + const dataAsset = await this.elasticsearchService.findDataAssetByTable( + user.customer_name, oldTableName, oldTableSchema, ); if (!dataAsset) { diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index 8fff83e..8e2dfe7 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -362,9 +362,8 @@ export class ElasticsearchService { return `${customerName}_data_assets_catalog`; } - async findDataAssetByPipelineAndTable( + async findDataAssetByTable( customerName: string, - pipelineId: string, tableName: string, tableSchema: string, ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { @@ -372,7 +371,6 @@ export class ElasticsearchService { this.logger.info('Elasticsearch: Searching data asset', { index, - pipelineId, tableName, tableSchema, }); @@ -382,7 +380,6 @@ export class ElasticsearchService { query: { bool: { must: [ - { term: { 'pipeline_id.keyword': pipelineId } }, { term: { 'table_name.keyword': tableName } }, { term: { 'table_schema.keyword': tableSchema } }, ], @@ -393,13 +390,13 @@ export class ElasticsearchService { const hits = response.data.hits?.hits || []; if (hits.length === 0) { - this.logger.warn('Elasticsearch: Data asset not found', { pipelineId, tableName, tableSchema, index }); + this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index }); return null; } return { ...hits[0]._source, _es_id: hits[0]._id }; } catch (error) { - this.handleError('findDataAssetByPipelineAndTable', error, { pipelineId, tableName, tableSchema, index }); + this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index }); throw error; } } From e3099aa2b2ec2a57906764b7b1ea4bd572a417eb Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 16 Mar 2026 10:17:56 -0300 Subject: [PATCH 15/27] FIX: uppercase table_name and table_schema in ES lookup Co-Authored-By: Claude Opus 4.6 --- src/services/elasticsearch/elasticsearch.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index 8e2dfe7..ab2791a 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -380,8 +380,8 @@ export class ElasticsearchService { query: { bool: { must: [ - { term: { 'table_name.keyword': tableName } }, - { term: { 'table_schema.keyword': tableSchema } }, + { term: { 'table_name.keyword': tableName.toUpperCase() } }, + { term: { 'table_schema.keyword': tableSchema.toUpperCase() } }, ], }, }, From 051fb6e4dd7680cdb6716147409afabae3015250 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 16 Mar 2026 10:39:55 -0300 Subject: [PATCH 16/27] FIX: uppercase table_name and table_schema in Nimbus rename calls Co-Authored-By: Claude Opus 4.6 --- .../platform-api/platform-api.controller.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 31bf1c3..044f64b 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -1086,26 +1086,31 @@ export class PlatformApiController { { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName }, )); + const newTableNameUpper = newValues.table_name.toUpperCase(); + const newTableSchemaUpper = newValues.table_schema.toUpperCase(); + const oldTableNameUpper = oldTableName.toUpperCase(); + const oldTableSchemaUpper = oldTableSchema.toUpperCase(); + // Nimbus table-metadata if (nimbusId) { - await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, newValues); - rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, oldValues)); + await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper }); + rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper })); } // Nimbus column-metadata await this.catalogService.renameColumnMetadataOnNimbus( - nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, ); rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus( - nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, )); // Nimbus data-preview await this.catalogService.renameDataPreviewOnNimbus( - nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, ); rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus( - nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, )); this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name }); From 9ef4c51ba1878b4eeb16178b87b65aa0758e3dc3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:16:15 -0300 Subject: [PATCH 17/27] FEAT: update CustomerSidebarSection to support both menu and link items --- docsfera.json | 56 ++++------------------ src/modules/customers/customers.service.ts | 4 +- src/modules/customers/dtos/customers.ts | 12 ++++- 3 files changed, 20 insertions(+), 52 deletions(-) diff --git a/docsfera.json b/docsfera.json index ef933ea..5cdfb82 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11166,53 +11166,6 @@ "description" ] }, - "CustomerSidebarLinkItem": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "title": { - "type": "string" - }, - "link": { - "type": "string" - }, - "icon": { - "type": "string" - } - }, - "required": [ - "type", - "title", - "link" - ] - }, - "CustomerSidebarMenuItem": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "title": { - "type": "string" - }, - "icon": { - "type": "string" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - } - }, - "required": [ - "type", - "title", - "items" - ] - }, "CustomerSidebarSection": { "type": "object", "properties": { @@ -11222,7 +11175,14 @@ "items": { "type": "array", "items": { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] } } }, diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index bcbd11d..f0a161c 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -72,7 +72,7 @@ export class CustomersService implements OnModuleInit { const result = await lastValueFrom( this.customerService.CustomerGetLinks({ id: customerId }), ); - return (result.links as unknown as CustomerLinksConfig) || null; + return (result.links as CustomerLinksConfig) || null; } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) throw new HttpException(err.details, HttpStatus.NOT_FOUND); @@ -89,7 +89,7 @@ export class CustomersService implements OnModuleInit { return await firstValueFrom( this.customerService.CustomerSetLinks({ id: customerId, - links: links as unknown as CustomerSetLinksRequest['links'], + links: links as CustomerSetLinksRequest['links'], }), ); } catch (err) { diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index 7299b23..479bace 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -36,8 +36,16 @@ export class CustomerSidebarMenuItem { export class CustomerSidebarSection { @ApiProperty() title: string; - @ApiProperty({ type: [CustomerSidebarMenuItem] }) - items: CustomerSidebarMenuItem[]; + @ApiProperty({ + type: 'array', + items: { + oneOf: [ + { $ref: '#/components/schemas/CustomerSidebarMenuItem' }, + { $ref: '#/components/schemas/CustomerSidebarLinkItem' }, + ], + }, + }) + items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[]; } export class CustomerLinksConfig { From 8b0bf84d3464aa659190b70c02c6053932d3cc11 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:34:12 -0300 Subject: [PATCH 18/27] FIX: resolve merge conflict in package.json for protospack-v2 version Co-Authored-By: Claude Sonnet 4.6 --- package.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package.json b/package.json index 5b727d6..9a13d42 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", -<<<<<<< feat/sidebar-menu "@dadosfera/protospack-v2": "^3.38.0-beta.29", -======= - "@dadosfera/protospack-v2": "3.38.0-beta.28", ->>>>>>> beta "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 6523f707e330037af72af3ebd02ad089ece48cac Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:37:08 -0300 Subject: [PATCH 19/27] FIX: remove duplicate identifier_columns declaration in TableColumns DTO Co-Authored-By: Claude Sonnet 4.6 --- src/modules/inputs/dtos/input.model.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 504e39a..c026f8d 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -14,8 +14,6 @@ export class TableColumns { @ApiPropertyOptional({ type: [Column] }) references: Column[]; @ApiProperty() - identifier_columns: string[]; - @ApiProperty() destination: Record<'raw' | 'qualify', { table_name: string; table_schema: string; From bf19d29a1dd00aa5827f12e333602f7143574ca3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:38:35 -0300 Subject: [PATCH 20/27] FIX: docsfera.json --- docsfera.json | 195 ++++++++++++++++++++------------------------------ 1 file changed, 78 insertions(+), 117 deletions(-) diff --git a/docsfera.json b/docsfera.json index 5cdfb82..b0bb337 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4747,6 +4747,42 @@ ] } }, + "/platform/jobs/{jobId}/rename-tables": { + "post": { + "operationId": "PlatformApiController_renameJobTables", + "summary": "Rename job output tables and sync to catalog", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/jdbc/configs/allowed_datatypes": { "get": { "operationId": "PlatformApiController_getJdbcAllowedDatatypes", @@ -10158,12 +10194,6 @@ "$ref": "#/components/schemas/Column" } }, - "identifier_columns": { - "type": "array", - "items": { - "type": "string" - } - }, "destination": { "type": "object" }, @@ -10183,12 +10213,7 @@ "required": [ "name", "columns", -<<<<<<< feat/sidebar-menu -======= - "references", - "identifier_columns", "destination", ->>>>>>> beta "type" ] }, @@ -10493,7 +10518,7 @@ "enabled" ] }, - "CustomerLink": { + "CustomerLinkItem": { "type": "object", "properties": { "href": { @@ -10515,28 +10540,61 @@ "description" ] }, - "CustomerLinksResponse": { + "CustomerSidebarSection": { "type": "object", "properties": { - "links": { + "title": { + "type": "string" + }, + "items": { "type": "array", "items": { - "$ref": "#/components/schemas/CustomerLink" + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] } } }, "required": [ - "links" + "title", + "items" ] }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, "CustomerLinkRequest": { "type": "object", "properties": { "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } + "$ref": "#/components/schemas/CustomerLinksConfig" } }, "required": [ @@ -11132,103 +11190,6 @@ "updated_by" ] }, -<<<<<<< feat/sidebar-menu - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, -======= ->>>>>>> beta "CreateShareMetadataDto": { "type": "object", "properties": { From 3f21faaa66bb54910eca9974af9f910b7c6ee7f1 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Mar 2026 11:25:55 -0300 Subject: [PATCH 21/27] FEAT: using batch route to update pipeline --- src/modules/inputs/inputs.controller.ts | 1 + src/modules/pipelinesV2/pipelines.service.ts | 157 ++++++++----------- 2 files changed, 63 insertions(+), 95 deletions(-) diff --git a/src/modules/inputs/inputs.controller.ts b/src/modules/inputs/inputs.controller.ts index 82ddd36..a8cc143 100644 --- a/src/modules/inputs/inputs.controller.ts +++ b/src/modules/inputs/inputs.controller.ts @@ -99,6 +99,7 @@ export class InputsController { customer: info.customer, }); + this.logger.info(JSON.stringify(body)) const response = await this.inputService.create({ body, info }); return response; diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index e23ce23..c0a1ad6 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -356,125 +356,92 @@ export class PipelinesService implements OnModuleInit { info ) - const requests = []; - this.logger.info('Dynamo Response', updateInputResponse); + const jobsUpdated = []; for (const [index, table] of updateInputDTO.tables.entries()) { - const id = `${pipelineIdFormat}_${index}`; + const jobUpdate = { + job_id: `${pipelineIdFormat}_${index}`, + } + + if (table.memory) { + jobUpdate["memory"] = { + amount: table.memory * 1000 + } + } + this.logger.info('Updating input reference for table', table.name); - const body = {} + let hasUpdateSyncMode = false; + + const jobSyncMode = {} + if (table.columns) { - body['column_include_list'] = table.columns; + hasUpdateSyncMode = true; + jobSyncMode['column_include_list'] = table.columns; } if (table.reference_column) { - body['incremental_column_name'] = table.reference_column.name; - body['incremental_column_type'] = table.reference_column.type; + hasUpdateSyncMode = true; + jobSyncMode['incremental_column_name'] = table.reference_column.name; + jobSyncMode['incremental_column_type'] = table.reference_column.type; } if (table.identifier_columns) { - body['primary_keys'] = table.identifier_columns; - } - - this.logger.info('Request body', body); - const updateCollumns = this.platformAPI.proxy( - 'PATCH', - `/jobs/${id}/input`, - user, - body - ) - requests.push(updateCollumns); - - if (table.memory) { - this.logger.info('Updating memory allocation for table', table.name); - const updateMemory = this.platformAPI.proxy( - 'PUT', - `/jobs/${id}/memory`, - user, - { - amount: table.memory - } - ) - requests.push(updateMemory); + hasUpdateSyncMode = true; + jobSyncMode['primary_keys'] = table.identifier_columns; } if (table.type) { - const updateSyncMode = this.updatePipelineSyncMode(table, id, user); - requests.push(updateSyncMode); + hasUpdateSyncMode = true; + jobSyncMode['target_load_type'] = table.type; + } - } - this.logger.info('Create Platform Request for each JOB'); + if(hasUpdateSyncMode) { + jobUpdate["sync_mode"] = jobSyncMode; + } - if (updateInputDTO.cron) { - const crnUpdatedRequest = new Promise(async (resolve, reject) => { - const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user); - - if (response.error) { - this.logger.error('Error updating pipeline cron', response.error); - return reject(new ErrorBuilder(response.error)); + if (Object.keys(table.destinations).length > 1) { + let hasChanges = false + const jobRenameTables = { + raw: {}, + qualify: {} } - this.logger.error('Pipeline cron updated successfully', response); - return resolve(response); - }); - requests.push(crnUpdatedRequest); + + if (Object.keys(table.destinations.raw).length > 1) { + hasChanges = true; + jobRenameTables.raw = table.destinations.raw; + } + + if (Object.keys(table.destinations.qualify).length > 1) { + hasChanges = true; + jobRenameTables.qualify = table.destinations.qualify; + } + + if (hasChanges) { + jobUpdate['rename_tables'] = jobRenameTables; + } + } + + jobsUpdated.push(jobUpdate); } - this.logger.info('Executing all request for the platform api'); + this.logger.info('Request body:' + JSON.stringify({ + jobs_updated: jobsUpdated + })); - const results = await Promise.allSettled(requests); - this.logger.info('Platform api response', results); + const response = await this.platformAPI.proxy( + 'POST', + `/pipelines/${pipelineIdFormat}/batch-update`, + user, + { + job_updates: jobsUpdated + } + ) + this.logger.info('Platform api response: ' + JSON.stringify(response)); return updateInputResponse; } - private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) { - const body = { - target_load_type: table.type - } - - if (table.type === 'incremental_with_qualify') { - body['incremental_column_name'] = table.reference_column.name; - body['incremental_column_type'] = table.reference_column.type; - body['primary_keys'] = table.identifier_columns; - } - - if (table.type === 'incremental') { - body['incremental_column_name'] = table.reference_column.name; - body['incremental_column_type'] = table.reference_column.type; - } - - this.logger.info('Updating pipeline sync mode', { - pipelineId, - body - }); - - return this.platformAPI.proxy( - "POST", - `/jobs/jdbc/${pipelineId}/sync-mode`, - user, - body - ) - } - - private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) { - try { - const response = await this.platformAPI.proxy( - 'PATCH', - `/pipeline/${pipelineId}`, - user, - { - cron - } - ); - - return response - } catch (error) { - return { - error: error.message - } - } - } } From 74b3bd6b463a20c1ecfaf5abd47b0fc01f76937e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 19 Mar 2026 08:14:56 -0300 Subject: [PATCH 22/27] FEAT: update CustomerSidebarSection and related DTOs to use object type for title; update protospack-v2 version; update from id to customerId --- docsfera.json | 2 +- package-lock.json | 126 +++++++++++++-------- package.json | 2 +- src/modules/customers/customers.service.ts | 4 +- src/modules/customers/dtos/customers.ts | 12 +- 5 files changed, 88 insertions(+), 58 deletions(-) diff --git a/docsfera.json b/docsfera.json index b0bb337..bcdd005 100644 --- a/docsfera.json +++ b/docsfera.json @@ -10544,7 +10544,7 @@ "type": "object", "properties": { "title": { - "type": "string" + "type": "object" }, "items": { "type": "array", diff --git a/package-lock.json b/package-lock.json index 526a96d..20ee1e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,11 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", -<<<<<<< feat/sidebar-menu - "@dadosfera/protospack-v2": "^3.38.0-beta.29", -======= - "@dadosfera/protospack-v2": "3.38.0-beta.28", ->>>>>>> beta + "@dadosfera/protospack-v2": "^3.38.0-beta.30", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1748,16 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { -<<<<<<< feat/sidebar-menu - "version": "3.38.0-beta.29", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.29.tgz", - "integrity": "sha512-1TUHKoSi2h/Pt+YEno9KOj8Zv2uAjT4qPS3xK26Tk2Wxg/OTlbxevPc+iwy8EI6YsT24WsbZfkb5JfKRlvKFHA==", -======= - "version": "3.38.0-beta.28", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz", - "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==", - "license": "ISC", ->>>>>>> beta + "version": "3.38.0-beta.30", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz", + "integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -2938,6 +2927,20 @@ "node": ">= 0.6" } }, + "node_modules/@nestjs/platform-express/node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, "node_modules/@nestjs/platform-express/node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -3107,6 +3110,24 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/@nestjs/platform-express/node_modules/multer": { + "version": "1.4.4-lts.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", + "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==", + "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/@nestjs/platform-express/node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -3131,6 +3152,25 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/@nestjs/platform-express/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/@nestjs/platform-express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3205,6 +3245,19 @@ "node": ">= 0.8" } }, + "node_modules/@nestjs/platform-express/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/@nestjs/platform-express/node_modules/tslib": { "version": "2.5.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", @@ -6512,21 +6565,6 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -9138,6 +9176,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -10731,24 +10774,6 @@ "node": ">=18" } }, - "node_modules/multer": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", - "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", - "license": "MIT", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.6.0", - "concat-stream": "^2.0.0", - "mkdirp": "^0.5.6", - "object-assign": "^4.1.1", - "type-is": "^1.6.18", - "xtend": "^4.0.2" - }, - "engines": { - "node": ">= 10.16.0" - } - }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -11738,6 +11763,11 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "node_modules/process-warning": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", diff --git a/package.json b/package.json index 9a13d42..4b109cd 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.29", + "@dadosfera/protospack-v2": "^3.38.0-beta.30", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index f0a161c..8915ab9 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -70,7 +70,7 @@ export class CustomersService implements OnModuleInit { async getLinks(customerId: string): Promise { try { const result = await lastValueFrom( - this.customerService.CustomerGetLinks({ id: customerId }), + this.customerService.CustomerGetLinks({ customerId }), ); return (result.links as CustomerLinksConfig) || null; } catch (err) { @@ -88,7 +88,7 @@ export class CustomersService implements OnModuleInit { try { return await firstValueFrom( this.customerService.CustomerSetLinks({ - id: customerId, + customerId, links: links as CustomerSetLinksRequest['links'], }), ); diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index 479bace..d914f07 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -14,8 +14,8 @@ export class CustomerLinkItem { export class CustomerSidebarLinkItem { @ApiProperty() type: 'link'; - @ApiProperty() - title: string; + @ApiProperty({ type: Object }) + title: Record; @ApiProperty() link: string; @ApiPropertyOptional() @@ -25,8 +25,8 @@ export class CustomerSidebarLinkItem { export class CustomerSidebarMenuItem { @ApiProperty() type: 'menu'; - @ApiProperty() - title: string; + @ApiProperty({ type: Object }) + title: Record; @ApiPropertyOptional() icon?: string; @ApiProperty({ type: [CustomerSidebarLinkItem] }) @@ -34,8 +34,8 @@ export class CustomerSidebarMenuItem { } export class CustomerSidebarSection { - @ApiProperty() - title: string; + @ApiProperty({ type: Object }) + title: Record; @ApiProperty({ type: 'array', items: { From b7171556b87e45668910c81704c4881ec42da64e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 19 Mar 2026 13:58:02 -0300 Subject: [PATCH 23/27] =?UTF-8?q?FIX:=20add=20multer@2.0.2=20and=20update?= =?UTF-8?q?=20package-lock.json=20=E2=80=94=20missing=20transitive=20dep?= =?UTF-8?q?=20of=20protospack-v2@3.38.0-beta.30?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- package-lock.json | 32 ++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 33 insertions(+) diff --git a/package-lock.json b/package-lock.json index 20ee1e3..78cfaf2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", + "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", @@ -6565,6 +6566,20 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -10774,6 +10789,23 @@ "node": ">=18" } }, + "node_modules/multer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", + "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", + "object-assign": "^4.1.1", + "type-is": "^1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">= 10.16.0" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", diff --git a/package.json b/package.json index 4b109cd..ac6d540 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", + "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", From 2e3a13d42107b256302cb9c9d5c6eac06e3a995e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 19 Mar 2026 14:35:14 -0300 Subject: [PATCH 24/27] FIX: send correct name and email from user payload --- src/modules/auth/auth.controller.ts | 1 + src/modules/auth/auth.service.ts | 3 ++- src/modules/auth/dtos/login.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 385d60c..dbff860 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -488,6 +488,7 @@ export class AuthController { const userDto = { id: api_key.user_id, name: api_key.username, + email: api_key.username, customer: { id: api_key.customer_id, name: api_key.customer_name, diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 6d67b95..8539d0e 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit { const userDto: UserDTO = { id: user.id, - name: user.username, + name: user.name, + email: user.email, jobTitle: user?.jobTitle || null, department: user?.department || null, hierarchy: user?.hierarchy || null, diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 3a484c8..2e22363 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -144,6 +144,7 @@ export interface BulkEditResponse { export type UserDTO = { id: string, name: string, + email: string, jobTitle?: string, department?: string, hierarchy?: string, From f10953e949f45271b8775be252f6ad9199f6dec6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 19 Mar 2026 14:45:28 -0300 Subject: [PATCH 25/27] FIX: multer package --- package-lock.json | 106 ---------------------------------------------- package.json | 1 - 2 files changed, 107 deletions(-) diff --git a/package-lock.json b/package-lock.json index 78cfaf2..9bc18e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,7 +47,6 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", - "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", @@ -2928,20 +2927,6 @@ "node": ">= 0.6" } }, - "node_modules/@nestjs/platform-express/node_modules/concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "engines": [ - "node >= 0.8" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, "node_modules/@nestjs/platform-express/node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -3111,24 +3096,6 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, - "node_modules/@nestjs/platform-express/node_modules/multer": { - "version": "1.4.4-lts.1", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", - "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==", - "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.0.0", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.4", - "object-assign": "^4.1.1", - "type-is": "^1.6.4", - "xtend": "^4.0.0" - }, - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/@nestjs/platform-express/node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -3153,25 +3120,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/@nestjs/platform-express/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/@nestjs/platform-express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3246,19 +3194,6 @@ "node": ">= 0.8" } }, - "node_modules/@nestjs/platform-express/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/@nestjs/platform-express/node_modules/tslib": { "version": "2.5.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", @@ -6566,20 +6501,6 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -9191,11 +9112,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -10789,23 +10705,6 @@ "node": ">=18" } }, - "node_modules/multer": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", - "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.6.0", - "concat-stream": "^2.0.0", - "mkdirp": "^0.5.6", - "object-assign": "^4.1.1", - "type-is": "^1.6.18", - "xtend": "^4.0.2" - }, - "engines": { - "node": ">= 10.16.0" - } - }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -11795,11 +11694,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/process-warning": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", diff --git a/package.json b/package.json index ac6d540..4b109cd 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", - "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", From 959210e354505c47e24c36b3f87fcd8944f1eb9c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 19 Mar 2026 14:56:58 -0300 Subject: [PATCH 26/27] FIX: npm ci --- package-lock.json | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/package-lock.json b/package-lock.json index 9bc18e0..b9f5209 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6501,6 +6501,21 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -10705,6 +10720,24 @@ "node": ">=18" } }, + "node_modules/multer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", + "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", + "object-assign": "^4.1.1", + "type-is": "^1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">= 10.16.0" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", From 21c57e5620646e6f5cb75f9f0f0c9ce3d90f7e5c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 24 Mar 2026 12:28:20 -0300 Subject: [PATCH 27/27] FEAT: update nimbus after input --- docsfera.json | 18 +----- package-lock.json | 15 ++--- package.json | 2 +- src/modules/inputs/inputs.service.ts | 21 ++++--- .../pipelinesV2/pipelines.controller.ts | 3 +- src/modules/pipelinesV2/pipelines.module.ts | 7 ++- src/modules/pipelinesV2/pipelines.service.ts | 49 ++++++++++++---- .../platform-api/platform-api.service.ts | 6 +- src/services/nimbus/nimbus.module.ts | 9 +++ src/services/nimbus/nimbus.service.ts | 56 +++++++++++++++++++ 10 files changed, 140 insertions(+), 46 deletions(-) create mode 100644 src/services/nimbus/nimbus.module.ts create mode 100644 src/services/nimbus/nimbus.service.ts diff --git a/docsfera.json b/docsfera.json index bcdd005..4e2ed5f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3001,14 +3001,7 @@ "parameters": [], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -3583,14 +3576,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/package-lock.json b/package-lock.json index b9f5209..eba19a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.30", + "@dadosfera/protospack-v2": "3.38.0-beta.31", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.30", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz", - "integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==", + "version": "3.38.0-beta.31", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.31.tgz", + "integrity": "sha512-IxnVEaVvQNumN/PQsnF+9CsFvmn6/0YyfXauoPQ1bccvqU8Lo+hKZv4qTBrR+urd05OWyg+dJBv/QtW45b04XQ==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -5533,9 +5534,9 @@ } }, "node_modules/axios": { - "version": "0.30.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz", - "integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==", + "version": "0.30.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz", + "integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.4", diff --git a/package.json b/package.json index 4b109cd..7da4765 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.30", + "@dadosfera/protospack-v2": "3.38.0-beta.31", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index cb9d25d..44c7ddd 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -17,6 +17,7 @@ import { InputCreateGenericRequest, InputCreateS3Request, InputNewCreateRequest, + InputUpdateResponse, TestConnectionRequest, } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @@ -71,8 +72,8 @@ export class InputsService { objectCamelToSnake(createInputResponse); return createInputResponse; }, - update: async (updateInputDTO: UpdateInputRequest) => { - this.logger.info('InputClientService - Update'); + update: async (updateInputDTO: UpdateInputRequest): Promise => { + this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO)); const updateInputResponse = await lastValueFrom( this.inputWriteService.InputUpdate(updateInputDTO), ); @@ -207,16 +208,22 @@ export class InputsService { async update(id: string, data, info: Info) { // this.validateCron({ ...data, info }); try { - const updateInputResponse: any = await this.OLD_inputClient.update({ + const { + nimbusUpdates, + input + } = await this.OLD_inputClient.update({ id, - info, ...data, + info, }); - updateInputResponse.input = this.adjustInputPayload( - updateInputResponse?.input, + const updateInputResponse = this.adjustInputPayload( + input, ); - return updateInputResponse; + return { + input: updateInputResponse, + nimbusUpdates + }; } catch (err) { throw new HttpException(err.message, HttpStatus.NOT_FOUND); } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 11e7b10..1c00bcd 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -294,13 +294,14 @@ export class PipelinesController { ) { this.logger.info('PipelinesController - update', { user }); - const { customer_id, customer_name, user_id, username } = user; const info: Info = { user_id: user.user_id, customer: user.customer_name, customer_id: user.customer_id, + pipeline_id: pipelineId }; + const metadata = PackTheMetadata({ customer_id, customer_name, diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 9550aeb..d0007b8 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -12,6 +12,8 @@ import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; import { PlatformApiModule } from '../platform-api/platform-api.module'; +import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module'; +import { NimbusService } from 'src/services/nimbus/nimbus.service'; const client = new PipelinesClientConfiguration(); @@ -22,10 +24,11 @@ const client = new PipelinesClientConfiguration(); ConnectorModule, InputsModule, TransformationsModule, - PlatformApiModule + PlatformApiModule, + NimbusServicesModule ], controllers: [PipelinesController], - providers: [PipelinesService, DadosferaLogger], + providers: [PipelinesService, DadosferaLogger, NimbusService], exports: [PipelinesService], }) export class PipelinesV2Module {} diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index c0a1ad6..db04131 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -28,7 +28,10 @@ import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; import { PlatformApiService } from '../platform-api/platform-api.service'; -import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; +import axios from 'axios'; +import { NimbusService } from 'src/services/nimbus/nimbus.service'; export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; @@ -42,7 +45,8 @@ export class PipelinesService implements OnModuleInit { private readonly connectorService: ConnectorClientService, private readonly inputsService: InputsService, private readonly transformationsService: TransformationsService, - private readonly platformAPI: PlatformApiService + private readonly platformAPI: PlatformApiService, + private readonly nimbusService: NimbusService ) { this.logger = dadosferaLogger.logger; } @@ -350,18 +354,46 @@ export class PipelinesService implements OnModuleInit { this.logger.info('Update Dynamo Reference'); const pipelineIdFormat = pipelineId.split('-').join('_'); + console.log("updatePipelineInput", info, updateInputDTO) const updateInputResponse = await this.inputsService.update( inputId, updateInputDTO, info ) - this.logger.info('Dynamo Response', updateInputResponse); + this.logger.info(JSON.stringify(updateInputResponse)) + await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates); + + await this.updatePlatformJobs( + pipelineIdFormat, + updateInputDTO, + user + ); + + return updateInputResponse; + } + + private async updateNimbus(customer: string, changes: NimbusUpdate[]) { + this.logger.info('Nimbus Changes: ' + JSON.stringify(changes)); + if(!changes || changes.length === 0) return; + + const requests = changes.map(change => { + return this.nimbusService.renameTable(customer, change.database, change.old, change.new); + }) + + const values = await Promise.allSettled(requests); + + const success = values.map(request => request.status === "fulfilled") + + this.logger.info("Updates with succes: " + success.length); + } + + async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) { const jobsUpdated = []; for (const [index, table] of updateInputDTO.tables.entries()) { const jobUpdate = { - job_id: `${pipelineIdFormat}_${index}`, + job_id: `${pipelineId}_${index}`, } if (table.memory) { @@ -370,7 +402,7 @@ export class PipelinesService implements OnModuleInit { } } - this.logger.info('Updating input reference for table', table.name); + this.logger.info('Updating input reference for table: ' + table.name); let hasUpdateSyncMode = false; const jobSyncMode = {} @@ -431,17 +463,14 @@ export class PipelinesService implements OnModuleInit { })); const response = await this.platformAPI.proxy( - 'POST', - `/pipelines/${pipelineIdFormat}/batch-update`, + 'PUT', + `/pipeline/${pipelineId}/jobs`, user, { job_updates: jobsUpdated } ) this.logger.info('Platform api response: ' + JSON.stringify(response)); - - return updateInputResponse; - } } diff --git a/src/modules/platform-api/platform-api.service.ts b/src/modules/platform-api/platform-api.service.ts index f204678..7fe9928 100644 --- a/src/modules/platform-api/platform-api.service.ts +++ b/src/modules/platform-api/platform-api.service.ts @@ -89,12 +89,12 @@ export class PlatformApiService { // Propagate non-2xx responses as HttpExceptions if (response.status >= 400) { - this.logger.error('Platform API upstream error', { + this.logger.error('Platform API upstream error' + JSON.stringify({ status: response.status, data: response.data, path, method: method.toUpperCase(), - }); + })); throw new HttpException(response.data, response.status); } @@ -107,6 +107,8 @@ export class PlatformApiService { method: method.toUpperCase(), }); + this.logger.error(error) + if (error instanceof HttpException) { throw error; } diff --git a/src/services/nimbus/nimbus.module.ts b/src/services/nimbus/nimbus.module.ts new file mode 100644 index 0000000..de7b60f --- /dev/null +++ b/src/services/nimbus/nimbus.module.ts @@ -0,0 +1,9 @@ +import { Module } from "@nestjs/common"; +import { NimbusService } from "./nimbus.service"; +import DadosferaLogger from "@dadosfera/dadosfera-logs"; + +@Module({ + providers: [NimbusService, DadosferaLogger], + exports: [NimbusService], +}) +export class NimbusServicesModule {} diff --git a/src/services/nimbus/nimbus.service.ts b/src/services/nimbus/nimbus.service.ts new file mode 100644 index 0000000..031aae7 --- /dev/null +++ b/src/services/nimbus/nimbus.service.ts @@ -0,0 +1,56 @@ +import DadosferaLogger from "@dadosfera/dadosfera-logs"; +import { Inject, Injectable } from "@nestjs/common"; +import axios from "axios"; + +type TableUpdate = { + table_schema: string; + table_name: string; +} + +@Injectable() +export class NimbusService { + private logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + private buildUrl(customerName: string) { + if (process.env.ENV === 'prd') { + return `https://nimbus-${customerName}.dadosfera.ai`; + } + + return `https://nimbus-${customerName}.${process.env.ENV.replace( + 'local', + 'stg', + )}.dadosfera.ai`; + } + + async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) { + const nimbusUrl = this.buildUrl(customerName); + + const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`; + + try { + this.logger.info("Request for PATCH" + nimbusUrl + path); + this.logger.info("Payload: " + JSON.stringify(update)); + const { data } = await axios.patch(nimbusUrl + path, { + table_name: update.table_name, + table_schema: update.table_schema + }) + + return data; + } catch (error) { + this.logger.error(error); + return { + message: error.message, + database, + old, + update + } + } + + } +}