From b5d933eaf3438e6a9acdec055b04610194000de5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 4 Feb 2026 16:43:49 -0300 Subject: [PATCH 01/47] FEAT: qualify contract --- package-lock.json | 12 ++++-------- package.json | 2 +- src/modules/inputs/dtos/input.model.ts | 7 +++++++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4d90eb7..69f7636 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -95,6 +95,7 @@ "node": "18.17" } }, + "@dadosfera/protospack-v2@3.38.0-beta.27": {}, "node_modules/@angular-devkit/core": { "version": "16.0.1", "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-16.0.1.tgz", @@ -1744,13 +1745,8 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.26", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", - "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", - "dependencies": { - "@grpc/grpc-js": "^1.9.3", - "rxjs": "^7.5.5" - } + "resolved": "@dadosfera/protospack-v2@3.38.0-beta.27", + "link": true }, "node_modules/@elastic/ecs-helpers": { "version": "2.1.1", diff --git a/package.json b/package.json index 4fd10bd..cb4bbf4 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 7b40d1e..3b34a75 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -14,6 +14,13 @@ export class TableColumns { @ApiProperty() references: Column[]; @ApiProperty() + identifier_columns: string[]; + @ApiProperty() + destination: Record<'raw' | 'qualify', { + table_name: string; + table_schema: string; + }> | null; + @ApiProperty() type: string; } export class AvailableEntity { From 0e169a3cbc6aec4b86d6ee17128c16b14ed6d0c0 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 4 Feb 2026 16:50:20 -0300 Subject: [PATCH 02/47] FIX: protospack version --- docsfera.json | 41 +++++++++++------------------------------ package-lock.json | 13 +++++++++---- package.json | 2 +- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/docsfera.json b/docsfera.json index d13bcd6..6515e6e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8440,36 +8440,6 @@ ] } }, - "/storage-explorer/storage/link-to-table": { - "post": { - "operationId": "StorageExplorerController_linkFileToTable", - "summary": "Link an existing file in storage to a table", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/storage-explorer/storage/metadata": { "get": { "operationId": "StorageExplorerController_getFileMetadata", @@ -10054,6 +10024,15 @@ "type": "string" } }, + "identifier_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "destination": { + "type": "object" + }, "type": { "type": "string" } @@ -10062,6 +10041,8 @@ "name", "columns", "references", + "identifier_columns", + "destination", "type" ] }, diff --git a/package-lock.json b/package-lock.json index 69f7636..7d2079b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -95,7 +95,6 @@ "node": "18.17" } }, - "@dadosfera/protospack-v2@3.38.0-beta.27": {}, "node_modules/@angular-devkit/core": { "version": "16.0.1", "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-16.0.1.tgz", @@ -1745,8 +1744,14 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "resolved": "@dadosfera/protospack-v2@3.38.0-beta.27", - "link": true + "version": "3.38.0-beta.27", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz", + "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==", + "license": "ISC", + "dependencies": { + "@grpc/grpc-js": "^1.9.3", + "rxjs": "^7.5.5" + } }, "node_modules/@elastic/ecs-helpers": { "version": "2.1.1", diff --git a/package.json b/package.json index cb4bbf4..0fd3006 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "@dadosfera/protospack-v2@3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.27", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 66309c7bbe103fca4fbf42a5f03120b220f37eab Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 5 Feb 2026 10:29:26 -0300 Subject: [PATCH 03/47] FIX: platform routes --- docsfera.json | 78 +++++++++++++++++++ .../platform-api/platform-api.controller.ts | 37 +++++++++ src/modules/platform-api/platform-api.dto.ts | 9 +++ 3 files changed, 124 insertions(+) create mode 100644 src/modules/platform-api/platform-api.dto.ts diff --git a/docsfera.json b/docsfera.json index 6515e6e..6225efb 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7275,6 +7275,70 @@ ] } }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get available schemas", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate Table and Schema", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationTableDTO" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/pipeline/{pipelineId}/pipeline_run": { "get": { "operationId": "PlatformApiController_getPipelineRuns", @@ -11451,6 +11515,20 @@ "required": [ "providers" ] + }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] } } } diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index e0d59f2..e761e96 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -25,6 +25,7 @@ import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; +import { ValidationTableDTO } from './platform-api.dto'; @ApiTags('Platform API') @Controller('platform') @@ -720,6 +721,42 @@ export class PlatformApiController { ); } + // ==================== PIPELINE VALIDATION ==================== + + @Get('pipelines/catalog/schemas') + @ApiOperation({ summary: 'Get available schemas' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getAvailableSchemas( + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'GET', + `/catalog/schemas`, + user, + undefined, + query, + ); + } + + @Post('pipelines/catalog/tables/validate') + @ApiOperation({ summary: 'Validate Table and Schema' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async validateTableAndSchema( + @Body() payload: ValidationTableDTO, + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'POST', + `/catalog/tables/validate`, + user, + payload, + query, + ); + } + + // ==================== PIPELINE RUN ROUTES ==================== @Get('pipeline/:pipelineId/pipeline_run') diff --git a/src/modules/platform-api/platform-api.dto.ts b/src/modules/platform-api/platform-api.dto.ts new file mode 100644 index 0000000..3f6b89f --- /dev/null +++ b/src/modules/platform-api/platform-api.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class ValidationTableDTO { + @ApiProperty() + tables: Array<{ + table_name: string; + table_schema: string; + }> +} \ No newline at end of file From 30a41ba1445a4952228eb3788004f70546b138be Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Feb 2026 17:48:05 -0300 Subject: [PATCH 04/47] FEAT: qualify route update --- docsfera.json | 2583 +++++++++-------- src/modules/inputs/inputs.service.ts | 2 +- src/modules/pipelinesV2/interfaces.ts | 36 + .../pipelinesV2/pipelines.controller.ts | 45 + src/modules/pipelinesV2/pipelines.module.ts | 2 + src/modules/pipelinesV2/pipelines.service.ts | 140 +- 6 files changed, 1542 insertions(+), 1266 deletions(-) diff --git a/docsfera.json b/docsfera.json index 6225efb..0ae93e6 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3548,6 +3548,61 @@ ] } }, + "/pipelinesV2/{pipelineId}/inputs/{id}": { + "patch": { + "operationId": "PipelinesController_updatePipelineInput", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2/init-upload": { "post": { "operationId": "PipelinesController_initUploadFile", @@ -3960,6 +4015,1195 @@ ] } }, + "/platform/pipeline": { + "post": { + "operationId": "PlatformApiController_createPipeline", + "summary": "Create a new pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines": { + "get": { + "operationId": "PlatformApiController_getPipelines", + "summary": "List all pipelines for customer", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}": { + "get": { + "operationId": "PlatformApiController_getPipeline", + "summary": "Get pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_updatePipeline", + "summary": "Update pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "PlatformApiController_deletePipeline", + "summary": "Delete pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/execute": { + "post": { + "operationId": "PlatformApiController_executePipeline", + "summary": "Execute a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pause": { + "post": { + "operationId": "PlatformApiController_pausePipeline", + "summary": "Pause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/unpause": { + "post": { + "operationId": "PlatformApiController_unpausePipeline", + "summary": "Unpause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/memory": { + "put": { + "operationId": "PlatformApiController_updatePipelineMemory", + "summary": "Update pipeline memory configuration", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/metadata": { + "put": { + "operationId": "PlatformApiController_updatePipelineMetadata", + "summary": "Update pipeline metadata", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/metadata": { + "get": { + "operationId": "PlatformApiController_getPipelinesMetadata", + "summary": "Get all pipelines metadata", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get available schemas", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate Table and Schema", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationTableDTO" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run": { + "get": { + "operationId": "PlatformApiController_getPipelineRuns", + "summary": "Get pipeline runs for a pipeline", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { + "get": { + "operationId": "PlatformApiController_getPipelineRun", + "summary": "Get specific pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pipeline_run/{runId}/logs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunLogs", + "summary": "Get pipeline run logs", + "parameters": [ + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/input": { + "put": { + "operationId": "PlatformApiController_updateJobInput", + "summary": "Update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_patchJobInput", + "summary": "Partial update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/memory": { + "put": { + "operationId": "PlatformApiController_updateJobMemory", + "summary": "Update job memory configuration", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/reset-state": { + "post": { + "operationId": "PlatformApiController_resetJobState", + "summary": "Reset job state", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}": { + "get": { + "operationId": "PlatformApiController_getJdbcJob", + "summary": "Get JDBC job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateJdbcSyncMode", + "summary": "Update JDBC job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/configs/allowed_datatypes": { + "get": { + "operationId": "PlatformApiController_getJdbcAllowedDatatypes", + "summary": "Get allowed datatypes for JDBC", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}": { + "get": { + "operationId": "PlatformApiController_getSingerJob", + "summary": "Get Singer job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateSingerSyncMode", + "summary": "Update Singer job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/s3/{jobId}": { + "get": { + "operationId": "PlatformApiController_getS3Job", + "summary": "Get S3 job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/health": { + "get": { + "operationId": "PlatformApiController_healthCheck", + "summary": "Platform API health check", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/links": { + "get": { + "operationId": "CustomersController_getCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinksResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_setCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinkRequest" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/token": { + "get": { + "operationId": "CustomersController_getCustomerToken", + "parameters": [ + { + "name": "exp", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/monitoring-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMonitoringDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/logs-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/organization-info": { + "get": { + "operationId": "CustomersController_getOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_updateOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information updated" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog": { "get": { "operationId": "CatalogController_searchCatalog", @@ -5371,301 +6615,6 @@ ] } }, - "/customers/{id}/mfa": { - "post": { - "operationId": "CustomersController_enableMfaEnforce", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EnforceMfa" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/links": { - "get": { - "operationId": "CustomersController_getCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinksResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_setCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinkRequest" - } - } - } - }, - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/token": { - "get": { - "operationId": "CustomersController_getCustomerToken", - "parameters": [ - { - "name": "exp", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/monitoring-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMonitoringDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/logs-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/access-dashboard": { - "get": { - "operationId": "CustomersController_getAccessDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/organization-info": { - "get": { - "operationId": "CustomersController_getOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_updateOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information updated" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/catalog/data-asset/share/{id}": { "get": { "operationId": "ShareController_getShareDataAsset", @@ -6937,900 +7886,6 @@ } } }, - "/platform/pipeline": { - "post": { - "operationId": "PlatformApiController_createPipeline", - "summary": "Create a new pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines": { - "get": { - "operationId": "PlatformApiController_getPipelines", - "summary": "List all pipelines for customer", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}": { - "get": { - "operationId": "PlatformApiController_getPipeline", - "summary": "Get pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_updatePipeline", - "summary": "Update pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "delete": { - "operationId": "PlatformApiController_deletePipeline", - "summary": "Delete pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/execute": { - "post": { - "operationId": "PlatformApiController_executePipeline", - "summary": "Execute a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pause": { - "post": { - "operationId": "PlatformApiController_pausePipeline", - "summary": "Pause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/unpause": { - "post": { - "operationId": "PlatformApiController_unpausePipeline", - "summary": "Unpause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/memory": { - "put": { - "operationId": "PlatformApiController_updatePipelineMemory", - "summary": "Update pipeline memory configuration", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/metadata": { - "put": { - "operationId": "PlatformApiController_updatePipelineMetadata", - "summary": "Update pipeline metadata", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/metadata": { - "get": { - "operationId": "PlatformApiController_getPipelinesMetadata", - "summary": "Get all pipelines metadata", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/schemas": { - "get": { - "operationId": "PlatformApiController_getAvailableSchemas", - "summary": "Get available schemas", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables/validate": { - "post": { - "operationId": "PlatformApiController_validateTableAndSchema", - "summary": "Validate Table and Schema", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ValidationTableDTO" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run": { - "get": { - "operationId": "PlatformApiController_getPipelineRuns", - "summary": "Get pipeline runs for a pipeline", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { - "get": { - "operationId": "PlatformApiController_getPipelineRun", - "summary": "Get specific pipeline run", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pipeline_run/{runId}/logs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunLogs", - "summary": "Get pipeline run logs", - "parameters": [ - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/input": { - "put": { - "operationId": "PlatformApiController_updateJobInput", - "summary": "Update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_patchJobInput", - "summary": "Partial update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/memory": { - "put": { - "operationId": "PlatformApiController_updateJobMemory", - "summary": "Update job memory configuration", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/reset-state": { - "post": { - "operationId": "PlatformApiController_resetJobState", - "summary": "Reset job state", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}": { - "get": { - "operationId": "PlatformApiController_getJdbcJob", - "summary": "Get JDBC job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateJdbcSyncMode", - "summary": "Update JDBC job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/configs/allowed_datatypes": { - "get": { - "operationId": "PlatformApiController_getJdbcAllowedDatatypes", - "summary": "Get allowed datatypes for JDBC", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}": { - "get": { - "operationId": "PlatformApiController_getSingerJob", - "summary": "Get Singer job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateSingerSyncMode", - "summary": "Update Singer job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/s3/{jobId}": { - "get": { - "operationId": "PlatformApiController_getS3Job", - "summary": "Get S3 job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/health": { - "get": { - "operationId": "PlatformApiController_healthCheck", - "summary": "Platform API health check", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -10386,6 +10441,81 @@ "engine" ] }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] + }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLink": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, "IRole": { "type": "object", "properties": { @@ -10975,67 +11105,6 @@ "updated_by" ] }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLink": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, "CreateShareMetadataDto": { "type": "object", "properties": { @@ -11515,20 +11584,6 @@ "required": [ "providers" ] - }, - "ValidationTableDTO": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tables" - ] } } } diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 94dd27a..c71ecc7 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -200,7 +200,7 @@ export class InputsService { } async update(id: string, data, info: Info) { - this.validateCron({ ...data, info }); + // this.validateCron({ ...data, info }); try { const updateInputResponse: any = await this.OLD_inputClient.update({ id, diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index db9a201..252b94c 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -1,6 +1,15 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; +export class PipelineInputsDTO { + @ApiProperty() + tables: Array<{ + name: string, + type: string, + + }> +} + export class IPipelineV2 { @ApiProperty() id: string; @@ -123,3 +132,30 @@ export class PipelineFindAllReq { @ApiPropertyOptional() type?: string | undefined; } + +export interface UpdateTableDTO { + name: string; + type: string; + columns: string[]; + destinations: { + raw: { + table_schema: string; + table_name: string; + }; + qualify: { + table_schema: string; + table_name: string; + }; + }; + identifier_columns: string[]; + reference_column: { + name: string; + type: string; + }; + memory: number; +} + +export interface UpdatePlatformInputRequest { + cron: string; + tables: Array; +} diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 4f5ff1d..11e7b10 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -43,11 +43,15 @@ import { IPipelineV2, IInitUploadCSVFile, PipelineFindAllReq, + UpdatePlatformInputRequest, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; +import { TableColumns } from '../inputs/dtos/input.model'; +import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -223,6 +227,7 @@ export class PipelinesController { .then((res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} let tables = JSON.parse(res.pipeline.config.tables); + const input_id = tables?.input_id; if (tables?.tables) tables = tables.tables; Object.assign(res.pipeline, { transformations: res.pipeline.transformations @@ -231,6 +236,7 @@ export class PipelinesController { config: { cron: res.pipeline.config.cron, tables, + input_id }, properties: res.pipeline.properties ? JSON.parse(res.pipeline.properties) @@ -277,6 +283,45 @@ export class PipelinesController { return response; } + @Patch('/:pipelineId/inputs/:id') + @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async updatePipelineInput( + @Language() language: LanguageEnum, + @Body() pipelineInputDTO: UpdatePlatformInputRequest, + @Param('id') inputId: string, + @Param('pipelineId') pipelineId: string, + @User() user: RequestUser, + ) { + this.logger.info('PipelinesController - update', { user }); + + + const { customer_id, customer_name, user_id, username } = user; + const info: Info = { + user_id: user.user_id, + customer: user.customer_name, + customer_id: user.customer_id, + }; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + language, + }); + + const response = await this.pipelinesClientService.updatePipelineInput( + pipelineId, + inputId, + pipelineInputDTO, + info, + user, + metadata, + ); + + this.logger.info('PipelinesController - update: OK', { user }); + return response; + } + @ApiInternalOnlyEndpoint() @Put('/:id') @ApiOperation({ diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 7bc7cc2..9550aeb 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; +import { PlatformApiModule } from '../platform-api/platform-api.module'; const client = new PipelinesClientConfiguration(); @@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration(); ConnectorModule, InputsModule, TransformationsModule, + PlatformApiModule ], controllers: [PipelinesController], providers: [PipelinesService, DadosferaLogger], diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index f10b8ee..e23ce23 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-async-promise-executor */ import { BadRequestException, HttpException, @@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { ICreatePipelineV2Req } from './interfaces'; +import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { Metadata } from '@grpc/grpc-js'; import { ConnectorClientService } from '../connector/client.service'; @@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; +import { PlatformApiService } from '../platform-api/platform-api.service'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; @@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit { private readonly connectorService: ConnectorClientService, private readonly inputsService: InputsService, private readonly transformationsService: TransformationsService, + private readonly platformAPI: PlatformApiService ) { this.logger = dadosferaLogger.logger; } @@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit { const findOnePipelineResponse = await lastValueFrom( this.pipelineReadService.PipelineV2FindOne(data, metadata), ); + console.log('pipeline find one response', findOnePipelineResponse); this.logger.info('Done'); return findOnePipelineResponse; @@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit { return res; } + + async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) { + this.logger.info('InputClientService - Update'); + + this.logger.info('Update Dynamo Reference'); + const pipelineIdFormat = pipelineId.split('-').join('_'); + const updateInputResponse = await this.inputsService.update( + inputId, + updateInputDTO, + info + ) + + const requests = []; + + this.logger.info('Dynamo Response', updateInputResponse); + + for (const [index, table] of updateInputDTO.tables.entries()) { + const id = `${pipelineIdFormat}_${index}`; + this.logger.info('Updating input reference for table', table.name); + const body = {} + if (table.columns) { + body['column_include_list'] = table.columns; + } + + if (table.reference_column) { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + } + + if (table.identifier_columns) { + body['primary_keys'] = table.identifier_columns; + } + + this.logger.info('Request body', body); + const updateCollumns = this.platformAPI.proxy( + 'PATCH', + `/jobs/${id}/input`, + user, + body + ) + requests.push(updateCollumns); + + if (table.memory) { + this.logger.info('Updating memory allocation for table', table.name); + const updateMemory = this.platformAPI.proxy( + 'PUT', + `/jobs/${id}/memory`, + user, + { + amount: table.memory + } + ) + requests.push(updateMemory); + } + + if (table.type) { + const updateSyncMode = this.updatePipelineSyncMode(table, id, user); + requests.push(updateSyncMode); + } + } + + this.logger.info('Create Platform Request for each JOB'); + + if (updateInputDTO.cron) { + const crnUpdatedRequest = new Promise(async (resolve, reject) => { + const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user); + + if (response.error) { + this.logger.error('Error updating pipeline cron', response.error); + return reject(new ErrorBuilder(response.error)); + } + this.logger.error('Pipeline cron updated successfully', response); + return resolve(response); + }); + requests.push(crnUpdatedRequest); + } + + this.logger.info('Executing all request for the platform api'); + + const results = await Promise.allSettled(requests); + this.logger.info('Platform api response', results); + + return updateInputResponse; + + } + + private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) { + const body = { + target_load_type: table.type + } + + if (table.type === 'incremental_with_qualify') { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + body['primary_keys'] = table.identifier_columns; + } + + if (table.type === 'incremental') { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + } + + this.logger.info('Updating pipeline sync mode', { + pipelineId, + body + }); + + return this.platformAPI.proxy( + "POST", + `/jobs/jdbc/${pipelineId}/sync-mode`, + user, + body + ) + } + + private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) { + try { + const response = await this.platformAPI.proxy( + 'PATCH', + `/pipeline/${pipelineId}`, + user, + { + cron + } + ); + + return response + } catch (error) { + return { + error: error.message + } + } + } } From 30eec733b2592e75aa16466f1c8f407b4c834cf7 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 13 Feb 2026 10:01:46 -0300 Subject: [PATCH 05/47] FEAT: update proto --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d2079b..89fbba9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.28", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.27", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz", - "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==", + "version": "3.38.0-beta.28", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz", + "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 0fd3006..bbd9c19 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.27", + "@dadosfera/protospack-v2": "3.38.0-beta.28", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 72ed637640a2048c651a74f0718be6a28ab2d50b Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 16 Feb 2026 18:11:49 -0300 Subject: [PATCH 06/47] FIX: update storage port --- deploy/helm-chart/values-stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index a9aa543..2c1737d 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,7 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com - storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api" + storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" hostname: maestro.stg.dadosfera.ai From cf8eed35a36b05489526bd1f099cc9bcdeaa9339 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Feb 2026 15:32:05 -0300 Subject: [PATCH 07/47] FIX: storage url --- src/modules/storage-explorer/storage-explorer.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts index 02d919f..8f4e4b0 100644 --- a/src/modules/storage-explorer/storage-explorer.service.ts +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -113,7 +113,7 @@ export class StorageExplorerService { } // Get customer-specific storage-explorer URL - const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name); const url = new URL(`${baseUrl}${path}`); // Add query params From 38a9e21f5f737e2e93859ed7beff7dd4c4be7c90 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 5 Mar 2026 17:29:12 -0300 Subject: [PATCH 08/47] feat: add rename-tables endpoint with catalog sync and rollback Add POST /platform/jobs/:jobId/rename-tables that renames Snowflake tables via platform-api and syncs the rename to Elasticsearch and Nimbus (table-metadata, column-metadata, data-preview). If catalog sync fails, all completed catalog steps are rolled back in reverse order and the Snowflake rename is reverted. - Support any connector type (jdbc, singer, s3) via getJobByAnyConnectorType - Resolve old table names from output_config (raw/qualify) - Skip qualify sync when output_config.qualify has no table_name - Add findDataAssetByPipelineAndTable and updateDataAsset to ElasticsearchService - Add renameTableOnNimbus, renameColumnMetadataOnNimbus, renameDataPreviewOnNimbus to CatalogService - Add upstream error logging to PlatformApiService Co-Authored-By: Claude Opus 4.6 --- src/modules/catalog/catalog.service.ts | 58 +++++ .../platform-api/platform-api.controller.ts | 207 ++++++++++++++++++ .../platform-api/platform-api.module.ts | 3 +- .../platform-api/platform-api.service.ts | 6 + .../elasticsearch/elasticsearch.service.ts | 78 +++++++ 5 files changed, 351 insertions(+), 1 deletion(-) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 89dcf84..ebb888c 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit { } } + async renameTableOnNimbus( + nimbusUrl: string, + nimbusId: number, + changes: { table_name?: string; table_schema?: string; display_name?: string }, + ): Promise { + const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`; + this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes }); + await axios.patch(endpoint, changes); + } + + async renameColumnMetadataOnNimbus( + nimbusUrl: string, + databaseName: string, + oldTableName: string, + oldTableSchema: string, + newTableName: string, + newTableSchema: string, + ): Promise { + const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`; + this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint }); + const { data: columns } = await axios.get(listEndpoint); + + const filtered = Array.isArray(columns) ? columns : []; + + for (const column of filtered) { + const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`; + await axios.patch(patchEndpoint, { + table_name: newTableName, + table_schema: newTableSchema, + }); + } + this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`); + } + + async renameDataPreviewOnNimbus( + nimbusUrl: string, + databaseName: string, + oldTableName: string, + oldTableSchema: string, + newTableName: string, + newTableSchema: string, + ): Promise { + const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`; + this.logger.info(`Fetching data-preview records to rename`, { listEndpoint }); + const { data: previews } = await axios.get(listEndpoint); + + const filtered = Array.isArray(previews) ? previews : []; + + for (const preview of filtered) { + const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`; + await axios.patch(patchEndpoint, { + table_name: newTableName, + table_schema: newTableSchema, + }); + } + this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`); + } + async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 978f464..319dcd4 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -10,6 +10,7 @@ import { Query, Inject, BadRequestException, + HttpException, } from '@nestjs/common'; import { ApiTags, ApiOperation } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; @@ -25,6 +26,8 @@ import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; +import { CatalogService } from '../catalog/catalog.service'; +import { PackTheMetadata } from '../../utils/PackTheMetadata'; type ValidateTablesDTO = { @@ -34,6 +37,11 @@ type ValidateTablesDTO = { }> } +type RenameTablesBody = { + raw?: { table_name: string; table_schema: string }; + qualify?: { table_name: string; table_schema: string }; +} + @ApiTags('Platform API') @Controller('platform') export class PlatformApiController { @@ -44,6 +52,7 @@ export class PlatformApiController { private readonly elasticsearchService: ElasticsearchService, private readonly dynamoDBService: DynamoDBService, private readonly customersService: CustomersService, + private readonly catalogService: CatalogService, @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, ) { this.logger = dadosferaLogger.logger; @@ -75,6 +84,23 @@ export class PlatformApiController { return jobId?.replace(/-/g, '_') || ''; } + private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise { + const connectorTypes = ['jdbc', 'singer', 's3']; + for (const type of connectorTypes) { + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${type}/${normalizedJobId}`, + user, + ); + return job; + } catch (error) { + // Continue to next connector type + } + } + throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404); + } + /** * Extract the pipeline ID (base UUID) from a job ID. * Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1"). @@ -968,6 +994,187 @@ export class PlatformApiController { return result; } + @Post('jobs/:jobId/rename-tables') + @ApiOperation({ summary: 'Rename job output tables and sync to catalog' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async renameJobTables( + @Param('jobId') jobId: string, + @Body() body: RenameTablesBody, + @User() user: RequestUser, + ) { + const normalizedJobId = this.normalizeJobId(jobId); + + const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user); + + const result = await this.platformApiService.proxy( + 'POST', + `/jobs/${normalizedJobId}/rename-tables`, + user, + body, + ); + + try { + await this.syncTableRenameToCatalog(jobId, body, currentJob, user); + } catch (error) { + this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message }); + + const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {}); + if (reverseBody) { + try { + await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody); + this.logger.info('Snowflake rename rolled back', { jobId }); + } catch (rollbackError) { + this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message }); + } + } + + throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500); + } + + return result; + } + + private buildSnowflakeRollbackBody( + body: RenameTablesBody, + outputConfig: any, + ): RenameTablesBody | null { + const reverse: RenameTablesBody = {}; + + if (body.raw) { + const nested = outputConfig.raw; + const oldTableName = nested?.table_name || outputConfig.table_name; + const oldTableSchema = nested?.table_schema || 'PUBLIC'; + if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema }; + } + + if (body.qualify) { + const nested = outputConfig.qualify; + if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' }; + } + + return Object.keys(reverse).length > 0 ? reverse : null; + } + + /** + * Sync table rename to Elasticsearch and Nimbus. + * + * For each target (raw, qualify): + * 1. Resolve old table name from output_config + * 2. Find the ES data asset by pipeline + table + schema + * 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview + * 4. If any step fails, rollback all completed steps for that target + */ + private async syncTableRenameToCatalog( + jobId: string, + body: RenameTablesBody, + currentJob: any, + user: RequestUser, + ): Promise { + const pipelineId = this.extractPipelineIdFromJobId(jobId); + const outputConfig = currentJob.output_config || {}; + const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } }); + const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`; + + const targets = this.buildRenameTargets(body, outputConfig); + + for (const { key, oldTableName, oldTableSchema, newValues } of targets) { + const rollbackSteps: Array<() => Promise> = []; + + try { + const dataAsset = await this.elasticsearchService.findDataAssetByPipelineAndTable( + user.customer_name, pipelineId, oldTableName, oldTableSchema, + ); + + if (!dataAsset) { + this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema }); + continue; + } + + const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset; + const oldValues = { table_name: oldTableName, table_schema: oldTableSchema }; + + // ES update + const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name }; + await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields); + rollbackSteps.push(() => this.elasticsearchService.updateDataAsset( + user.customer_name, esAssetId, + { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName }, + )); + + // Nimbus table-metadata + if (nimbusId) { + await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, newValues); + rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, oldValues)); + } + + // Nimbus column-metadata + await this.catalogService.renameColumnMetadataOnNimbus( + nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + ); + rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus( + nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + )); + + // Nimbus data-preview + await this.catalogService.renameDataPreviewOnNimbus( + nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + ); + rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus( + nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + )); + + this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name }); + } catch (error) { + this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message }); + await this.executeRollback(rollbackSteps, key, jobId); + throw error; + } + } + } + + private buildRenameTargets( + body: RenameTablesBody, + outputConfig: any, + ): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> { + const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' }; + const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = []; + + for (const key of ['raw', 'qualify'] as const) { + if (!body[key]) continue; + + const nested = outputConfig[key]; + + // qualify: only sync if output_config.qualify already exists + if (key === 'qualify' && !nested?.table_name) continue; + + const oldTableName = nested?.table_name || outputConfig.table_name; + if (!oldTableName) continue; + + targets.push({ + key, + oldTableName, + oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key], + newValues: body[key], + }); + } + + return targets; + } + + private async executeRollback( + steps: Array<() => Promise>, + targetKey: string, + jobId: string, + ): Promise { + for (const rollback of steps.reverse()) { + try { + await rollback(); + } catch (error) { + this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message }); + } + } + } + @Get('jobs/jdbc/configs/allowed_datatypes') @ApiOperation({ summary: 'Get allowed datatypes for JDBC' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index c2b31e4..b78e680 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -7,9 +7,10 @@ import { PlatformApiService } from './platform-api.service'; import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; +import { CatalogModule } from '../catalog/catalog.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/modules/platform-api/platform-api.service.ts b/src/modules/platform-api/platform-api.service.ts index 97797a3..f204678 100644 --- a/src/modules/platform-api/platform-api.service.ts +++ b/src/modules/platform-api/platform-api.service.ts @@ -89,6 +89,12 @@ export class PlatformApiService { // Propagate non-2xx responses as HttpExceptions if (response.status >= 400) { + this.logger.error('Platform API upstream error', { + status: response.status, + data: response.data, + path, + method: method.toUpperCase(), + }); throw new HttpException(response.data, response.status); } diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index e2c1857..8fff83e 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -358,6 +358,84 @@ export class ElasticsearchService { } } + private getDataAssetIndex(customerName: string): string { + return `${customerName}_data_assets_catalog`; + } + + async findDataAssetByPipelineAndTable( + customerName: string, + pipelineId: string, + tableName: string, + tableSchema: string, + ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Searching data asset', { + index, + pipelineId, + tableName, + tableSchema, + }); + + try { + const response = await this.client.post(`/${index}/_search`, { + query: { + bool: { + must: [ + { term: { 'pipeline_id.keyword': pipelineId } }, + { term: { 'table_name.keyword': tableName } }, + { term: { 'table_schema.keyword': tableSchema } }, + ], + }, + }, + size: 1, + }); + + const hits = response.data.hits?.hits || []; + if (hits.length === 0) { + this.logger.warn('Elasticsearch: Data asset not found', { pipelineId, tableName, tableSchema, index }); + return null; + } + + return { ...hits[0]._source, _es_id: hits[0]._id }; + } catch (error) { + this.handleError('findDataAssetByPipelineAndTable', error, { pipelineId, tableName, tableSchema, index }); + throw error; + } + } + + async updateDataAsset( + customerName: string, + assetId: string, + updates: Record, + ): Promise { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Updating data asset', { + index, + assetId, + fields: Object.keys(updates), + }); + + try { + const response = await this.client.post( + `/${index}/_update/${assetId}`, + { doc: updates }, + { params: { refresh: 'wait_for' } }, + ); + + this.logger.info('Elasticsearch: Data asset updated', { + assetId, + result: response.data.result, + }); + + return response.data; + } catch (error) { + this.handleError('updateDataAsset', error, { assetId, index }); + throw error; + } + } + private handleError( operation: string, error: any, From 61a4f724ef7cadf2a3f20015b7bb172fb36d607f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Mar 2026 14:43:11 -0300 Subject: [PATCH 09/47] feat: list headers --- src/modules/auth/auth.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 7c3e073..5eb25f5 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -478,6 +478,7 @@ export class AuthController { @Get('me') async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') + this.logger.info(req.headers); // Check for API key header first const apiKey = req.get('X-Api-key'); From 93f452ae0515f7f23cf0e757258bd49531bb8d50 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Mar 2026 15:19:18 -0300 Subject: [PATCH 10/47] FIX: ghost commit --- src/main.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.ts b/src/main.ts index 1085fb3..85e460a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) { ); } bootstrap(); + From b8bdc5beeaef3107798b0e3ac9d0d9475424d35c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Mar 2026 16:00:39 -0300 Subject: [PATCH 11/47] FIX: stringify headers --- src/modules/auth/auth.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 5eb25f5..bcf1ff8 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -478,7 +478,7 @@ export class AuthController { @Get('me') async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') - this.logger.info(req.headers); + this.logger.info(JSON.stringify(req.headers)); // Check for API key header first const apiKey = req.get('X-Api-key'); From 99c2a9ecf5856901a2dfcbb71a819eca57c00501 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 12 Mar 2026 18:02:49 -0300 Subject: [PATCH 12/47] UPDATE: force deployment of maestro --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 028ca81..4432ce9 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ # Maestro + Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. ```mermaid From ff3999a6aa21f7557347acc9b1d4df7fbd08c27f Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 13 Mar 2026 17:35:56 -0300 Subject: [PATCH 13/47] FEAT: update customer links handling and DTOs for improved structure sidebar. update protospack-v2 --- docsfera.json | 118 +++++++++++++++++++-- package-lock.json | 8 +- package.json | 2 +- src/modules/customers/customers.service.ts | 18 ++-- src/modules/customers/dtos/customers.ts | 53 +++++++-- src/modules/inputs/dtos/input.model.ts | 6 +- src/modules/inputs/inputs.service.ts | 5 + 7 files changed, 175 insertions(+), 35 deletions(-) diff --git a/docsfera.json b/docsfera.json index 801658d..0acfbbe 100644 --- a/docsfera.json +++ b/docsfera.json @@ -10100,6 +10100,21 @@ "entities" ] }, + "Column": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "name", + "type" + ] + }, "TableColumns": { "type": "object", "properties": { @@ -10115,17 +10130,25 @@ "references": { "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/Column" } }, "type": { "type": "string" + }, + "identifier_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "reference_column": { + "$ref": "#/components/schemas/Column" } }, "required": [ "name", "columns", - "references", "type" ] }, @@ -11005,7 +11028,7 @@ "enabled" ] }, - "CustomerLink": { + "CustomerLinkItem": { "type": "object", "properties": { "href": { @@ -11027,28 +11050,101 @@ "description" ] }, - "CustomerLinksResponse": { + "CustomerSidebarLinkItem": { "type": "object", "properties": { - "links": { + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "link": { + "type": "string" + }, + "icon": { + "type": "string" + } + }, + "required": [ + "type", + "title", + "link" + ] + }, + "CustomerSidebarMenuItem": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "items": { "type": "array", "items": { - "$ref": "#/components/schemas/CustomerLink" + "$ref": "#/components/schemas/CustomerSidebarLinkItem" } } }, "required": [ - "links" + "type", + "title", + "items" ] }, + "CustomerSidebarSection": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + } + } + }, + "required": [ + "title", + "items" + ] + }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, "CustomerLinkRequest": { "type": "object", "properties": { "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } + "$ref": "#/components/schemas/CustomerLinksConfig" } }, "required": [ diff --git a/package-lock.json b/package-lock.json index 4d90eb7..615ef97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "^3.38.0-beta.29", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.26", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", - "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", + "version": "3.38.0-beta.29", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.29.tgz", + "integrity": "sha512-1TUHKoSi2h/Pt+YEno9KOj8Zv2uAjT4qPS3xK26Tk2Wxg/OTlbxevPc+iwy8EI6YsT24WsbZfkb5JfKRlvKFHA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 4fd10bd..9a13d42 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.26", + "@dadosfera/protospack-v2": "^3.38.0-beta.29", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 602943b..bcbd11d 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -9,12 +9,12 @@ import { } from '@nestjs/common'; import { firstValueFrom, lastValueFrom } from 'rxjs'; -import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { DucClient } from '../duc/client.config'; import { ClientGrpc } from '@nestjs/microservices'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; -import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; +import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { CustomerLinksConfig } from './dtos/customers'; import ErrorCodes from 'src/utils/errorCodes'; import jwt from 'jsonwebtoken'; import { @@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit { ) } - async getLinks(customerId: string) { + async getLinks(customerId: string): Promise { try { const result = await lastValueFrom( - this.customerService.CustomerFindOneById({ id: customerId }), + this.customerService.CustomerGetLinks({ id: customerId }), ); - return result.customer?.links || []; + return (result.links as unknown as CustomerLinksConfig) || null; } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) throw new HttpException(err.details, HttpStatus.NOT_FOUND); @@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit { } } - async setLinks(customerId: string, links: Link[]) { + async setLinks(customerId: string, links: CustomerLinksConfig) { if (!customerId || !links) { throw new HttpException(null, HttpStatus.BAD_REQUEST); } try { return await firstValueFrom( - this.customerService.CustomerUpdate({ + this.customerService.CustomerSetLinks({ id: customerId, - links, - } as CustomerUpdateRequest), + links: links as unknown as CustomerSetLinksRequest['links'], + }), ); } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index db56ee3..7299b23 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -1,7 +1,6 @@ -import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; -export class CustomerLink implements Link { +export class CustomerLinkItem { @ApiProperty() href: string; @ApiProperty() @@ -9,15 +8,51 @@ export class CustomerLink implements Link { @ApiProperty() description: string; @ApiPropertyOptional() - iconSrc: string; + iconSrc?: string; } + +export class CustomerSidebarLinkItem { + @ApiProperty() + type: 'link'; + @ApiProperty() + title: string; + @ApiProperty() + link: string; + @ApiPropertyOptional() + icon?: string; +} + +export class CustomerSidebarMenuItem { + @ApiProperty() + type: 'menu'; + @ApiProperty() + title: string; + @ApiPropertyOptional() + icon?: string; + @ApiProperty({ type: [CustomerSidebarLinkItem] }) + items: CustomerSidebarLinkItem[]; +} + +export class CustomerSidebarSection { + @ApiProperty() + title: string; + @ApiProperty({ type: [CustomerSidebarMenuItem] }) + items: CustomerSidebarMenuItem[]; +} + +export class CustomerLinksConfig { + @ApiPropertyOptional({ type: [CustomerLinkItem] }) + home?: CustomerLinkItem[]; + @ApiPropertyOptional({ type: [CustomerSidebarSection] }) + sidebar?: CustomerSidebarSection[]; +} + export class CustomerLinkRequest { - @ApiProperty({ type: [CustomerLink] }) - links: CustomerLink[]; + @ApiProperty({ type: CustomerLinksConfig }) + links: CustomerLinksConfig; } export class CustomerLinksResponse { - @ApiProperty({ type: [CustomerLink] }) - links: CustomerLink[]; -} - + @ApiPropertyOptional({ type: CustomerLinksConfig }) + links?: CustomerLinksConfig; +} \ No newline at end of file diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 7b40d1e..ef558be 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -11,10 +11,14 @@ export class TableColumns { name: string; @ApiProperty() columns: string[]; - @ApiProperty() + @ApiPropertyOptional({ type: [Column] }) references: Column[]; @ApiProperty() type: string; + @ApiPropertyOptional({ type: [String] }) + identifier_columns?: string[]; + @ApiPropertyOptional({ type: Column }) + reference_column?: Column; } export class AvailableEntity { @ApiProperty() diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 94dd27a..d258215 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -164,6 +164,11 @@ export class InputsService { const inputCreateGenericRequest: InputCreateGenericRequest = { input: { ...body, + tables: (body.tables || []).map((table) => ({ + ...table, + identifier_columns: table.identifier_columns || [], + reference_column: table.reference_column || table.references?.[0], + })), }, info, }; From 12c61d9b5dd7c0bbe033ba864d9d6ce8120273c8 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Mar 2026 18:08:39 -0300 Subject: [PATCH 14/47] FIX: rename-tables proxy path and ES lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix proxy path: /jobs/jdbc/:jobId/rename-tables → /jobs/:jobId/rename-tables - Remove pipeline_id from ES data asset lookup, search by table_name + table_schema only Co-Authored-By: Claude Opus 4.6 --- src/modules/platform-api/platform-api.controller.ts | 4 ++-- src/services/elasticsearch/elasticsearch.service.ts | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 97b0a08..31bf1c3 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -1066,8 +1066,8 @@ export class PlatformApiController { const rollbackSteps: Array<() => Promise> = []; try { - const dataAsset = await this.elasticsearchService.findDataAssetByPipelineAndTable( - user.customer_name, pipelineId, oldTableName, oldTableSchema, + const dataAsset = await this.elasticsearchService.findDataAssetByTable( + user.customer_name, oldTableName, oldTableSchema, ); if (!dataAsset) { diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index 8fff83e..8e2dfe7 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -362,9 +362,8 @@ export class ElasticsearchService { return `${customerName}_data_assets_catalog`; } - async findDataAssetByPipelineAndTable( + async findDataAssetByTable( customerName: string, - pipelineId: string, tableName: string, tableSchema: string, ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { @@ -372,7 +371,6 @@ export class ElasticsearchService { this.logger.info('Elasticsearch: Searching data asset', { index, - pipelineId, tableName, tableSchema, }); @@ -382,7 +380,6 @@ export class ElasticsearchService { query: { bool: { must: [ - { term: { 'pipeline_id.keyword': pipelineId } }, { term: { 'table_name.keyword': tableName } }, { term: { 'table_schema.keyword': tableSchema } }, ], @@ -393,13 +390,13 @@ export class ElasticsearchService { const hits = response.data.hits?.hits || []; if (hits.length === 0) { - this.logger.warn('Elasticsearch: Data asset not found', { pipelineId, tableName, tableSchema, index }); + this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index }); return null; } return { ...hits[0]._source, _es_id: hits[0]._id }; } catch (error) { - this.handleError('findDataAssetByPipelineAndTable', error, { pipelineId, tableName, tableSchema, index }); + this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index }); throw error; } } From e3099aa2b2ec2a57906764b7b1ea4bd572a417eb Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 16 Mar 2026 10:17:56 -0300 Subject: [PATCH 15/47] FIX: uppercase table_name and table_schema in ES lookup Co-Authored-By: Claude Opus 4.6 --- src/services/elasticsearch/elasticsearch.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index 8e2dfe7..ab2791a 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -380,8 +380,8 @@ export class ElasticsearchService { query: { bool: { must: [ - { term: { 'table_name.keyword': tableName } }, - { term: { 'table_schema.keyword': tableSchema } }, + { term: { 'table_name.keyword': tableName.toUpperCase() } }, + { term: { 'table_schema.keyword': tableSchema.toUpperCase() } }, ], }, }, From 051fb6e4dd7680cdb6716147409afabae3015250 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 16 Mar 2026 10:39:55 -0300 Subject: [PATCH 16/47] FIX: uppercase table_name and table_schema in Nimbus rename calls Co-Authored-By: Claude Opus 4.6 --- .../platform-api/platform-api.controller.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 31bf1c3..044f64b 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -1086,26 +1086,31 @@ export class PlatformApiController { { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName }, )); + const newTableNameUpper = newValues.table_name.toUpperCase(); + const newTableSchemaUpper = newValues.table_schema.toUpperCase(); + const oldTableNameUpper = oldTableName.toUpperCase(); + const oldTableSchemaUpper = oldTableSchema.toUpperCase(); + // Nimbus table-metadata if (nimbusId) { - await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, newValues); - rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, oldValues)); + await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper }); + rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper })); } // Nimbus column-metadata await this.catalogService.renameColumnMetadataOnNimbus( - nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, ); rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus( - nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, )); // Nimbus data-preview await this.catalogService.renameDataPreviewOnNimbus( - nimbusUrl, databaseName, oldTableName, oldTableSchema, newValues.table_name, newValues.table_schema, + nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, ); rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus( - nimbusUrl, databaseName, newValues.table_name, newValues.table_schema, oldTableName, oldTableSchema, + nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, )); this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name }); From 9ef4c51ba1878b4eeb16178b87b65aa0758e3dc3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:16:15 -0300 Subject: [PATCH 17/47] FEAT: update CustomerSidebarSection to support both menu and link items --- docsfera.json | 56 ++++------------------ src/modules/customers/customers.service.ts | 4 +- src/modules/customers/dtos/customers.ts | 12 ++++- 3 files changed, 20 insertions(+), 52 deletions(-) diff --git a/docsfera.json b/docsfera.json index ef933ea..5cdfb82 100644 --- a/docsfera.json +++ b/docsfera.json @@ -11166,53 +11166,6 @@ "description" ] }, - "CustomerSidebarLinkItem": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "title": { - "type": "string" - }, - "link": { - "type": "string" - }, - "icon": { - "type": "string" - } - }, - "required": [ - "type", - "title", - "link" - ] - }, - "CustomerSidebarMenuItem": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "title": { - "type": "string" - }, - "icon": { - "type": "string" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - } - }, - "required": [ - "type", - "title", - "items" - ] - }, "CustomerSidebarSection": { "type": "object", "properties": { @@ -11222,7 +11175,14 @@ "items": { "type": "array", "items": { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] } } }, diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index bcbd11d..f0a161c 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -72,7 +72,7 @@ export class CustomersService implements OnModuleInit { const result = await lastValueFrom( this.customerService.CustomerGetLinks({ id: customerId }), ); - return (result.links as unknown as CustomerLinksConfig) || null; + return (result.links as CustomerLinksConfig) || null; } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) throw new HttpException(err.details, HttpStatus.NOT_FOUND); @@ -89,7 +89,7 @@ export class CustomersService implements OnModuleInit { return await firstValueFrom( this.customerService.CustomerSetLinks({ id: customerId, - links: links as unknown as CustomerSetLinksRequest['links'], + links: links as CustomerSetLinksRequest['links'], }), ); } catch (err) { diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index 7299b23..479bace 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -36,8 +36,16 @@ export class CustomerSidebarMenuItem { export class CustomerSidebarSection { @ApiProperty() title: string; - @ApiProperty({ type: [CustomerSidebarMenuItem] }) - items: CustomerSidebarMenuItem[]; + @ApiProperty({ + type: 'array', + items: { + oneOf: [ + { $ref: '#/components/schemas/CustomerSidebarMenuItem' }, + { $ref: '#/components/schemas/CustomerSidebarLinkItem' }, + ], + }, + }) + items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[]; } export class CustomerLinksConfig { From 8b0bf84d3464aa659190b70c02c6053932d3cc11 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:34:12 -0300 Subject: [PATCH 18/47] FIX: resolve merge conflict in package.json for protospack-v2 version Co-Authored-By: Claude Sonnet 4.6 --- package.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package.json b/package.json index 5b727d6..9a13d42 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", -<<<<<<< feat/sidebar-menu "@dadosfera/protospack-v2": "^3.38.0-beta.29", -======= - "@dadosfera/protospack-v2": "3.38.0-beta.28", ->>>>>>> beta "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 6523f707e330037af72af3ebd02ad089ece48cac Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:37:08 -0300 Subject: [PATCH 19/47] FIX: remove duplicate identifier_columns declaration in TableColumns DTO Co-Authored-By: Claude Sonnet 4.6 --- src/modules/inputs/dtos/input.model.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 504e39a..c026f8d 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -14,8 +14,6 @@ export class TableColumns { @ApiPropertyOptional({ type: [Column] }) references: Column[]; @ApiProperty() - identifier_columns: string[]; - @ApiProperty() destination: Record<'raw' | 'qualify', { table_name: string; table_schema: string; From bf19d29a1dd00aa5827f12e333602f7143574ca3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 17 Mar 2026 15:38:35 -0300 Subject: [PATCH 20/47] FIX: docsfera.json --- docsfera.json | 195 ++++++++++++++++++++------------------------------ 1 file changed, 78 insertions(+), 117 deletions(-) diff --git a/docsfera.json b/docsfera.json index 5cdfb82..b0bb337 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4747,6 +4747,42 @@ ] } }, + "/platform/jobs/{jobId}/rename-tables": { + "post": { + "operationId": "PlatformApiController_renameJobTables", + "summary": "Rename job output tables and sync to catalog", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/jdbc/configs/allowed_datatypes": { "get": { "operationId": "PlatformApiController_getJdbcAllowedDatatypes", @@ -10158,12 +10194,6 @@ "$ref": "#/components/schemas/Column" } }, - "identifier_columns": { - "type": "array", - "items": { - "type": "string" - } - }, "destination": { "type": "object" }, @@ -10183,12 +10213,7 @@ "required": [ "name", "columns", -<<<<<<< feat/sidebar-menu -======= - "references", - "identifier_columns", "destination", ->>>>>>> beta "type" ] }, @@ -10493,7 +10518,7 @@ "enabled" ] }, - "CustomerLink": { + "CustomerLinkItem": { "type": "object", "properties": { "href": { @@ -10515,28 +10540,61 @@ "description" ] }, - "CustomerLinksResponse": { + "CustomerSidebarSection": { "type": "object", "properties": { - "links": { + "title": { + "type": "string" + }, + "items": { "type": "array", "items": { - "$ref": "#/components/schemas/CustomerLink" + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] } } }, "required": [ - "links" + "title", + "items" ] }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, "CustomerLinkRequest": { "type": "object", "properties": { "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } + "$ref": "#/components/schemas/CustomerLinksConfig" } }, "required": [ @@ -11132,103 +11190,6 @@ "updated_by" ] }, -<<<<<<< feat/sidebar-menu - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, -======= ->>>>>>> beta "CreateShareMetadataDto": { "type": "object", "properties": { From 3f21faaa66bb54910eca9974af9f910b7c6ee7f1 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Mar 2026 11:25:55 -0300 Subject: [PATCH 21/47] FEAT: using batch route to update pipeline --- src/modules/inputs/inputs.controller.ts | 1 + src/modules/pipelinesV2/pipelines.service.ts | 157 ++++++++----------- 2 files changed, 63 insertions(+), 95 deletions(-) diff --git a/src/modules/inputs/inputs.controller.ts b/src/modules/inputs/inputs.controller.ts index 82ddd36..a8cc143 100644 --- a/src/modules/inputs/inputs.controller.ts +++ b/src/modules/inputs/inputs.controller.ts @@ -99,6 +99,7 @@ export class InputsController { customer: info.customer, }); + this.logger.info(JSON.stringify(body)) const response = await this.inputService.create({ body, info }); return response; diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index e23ce23..c0a1ad6 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -356,125 +356,92 @@ export class PipelinesService implements OnModuleInit { info ) - const requests = []; - this.logger.info('Dynamo Response', updateInputResponse); + const jobsUpdated = []; for (const [index, table] of updateInputDTO.tables.entries()) { - const id = `${pipelineIdFormat}_${index}`; + const jobUpdate = { + job_id: `${pipelineIdFormat}_${index}`, + } + + if (table.memory) { + jobUpdate["memory"] = { + amount: table.memory * 1000 + } + } + this.logger.info('Updating input reference for table', table.name); - const body = {} + let hasUpdateSyncMode = false; + + const jobSyncMode = {} + if (table.columns) { - body['column_include_list'] = table.columns; + hasUpdateSyncMode = true; + jobSyncMode['column_include_list'] = table.columns; } if (table.reference_column) { - body['incremental_column_name'] = table.reference_column.name; - body['incremental_column_type'] = table.reference_column.type; + hasUpdateSyncMode = true; + jobSyncMode['incremental_column_name'] = table.reference_column.name; + jobSyncMode['incremental_column_type'] = table.reference_column.type; } if (table.identifier_columns) { - body['primary_keys'] = table.identifier_columns; - } - - this.logger.info('Request body', body); - const updateCollumns = this.platformAPI.proxy( - 'PATCH', - `/jobs/${id}/input`, - user, - body - ) - requests.push(updateCollumns); - - if (table.memory) { - this.logger.info('Updating memory allocation for table', table.name); - const updateMemory = this.platformAPI.proxy( - 'PUT', - `/jobs/${id}/memory`, - user, - { - amount: table.memory - } - ) - requests.push(updateMemory); + hasUpdateSyncMode = true; + jobSyncMode['primary_keys'] = table.identifier_columns; } if (table.type) { - const updateSyncMode = this.updatePipelineSyncMode(table, id, user); - requests.push(updateSyncMode); + hasUpdateSyncMode = true; + jobSyncMode['target_load_type'] = table.type; + } - } - this.logger.info('Create Platform Request for each JOB'); + if(hasUpdateSyncMode) { + jobUpdate["sync_mode"] = jobSyncMode; + } - if (updateInputDTO.cron) { - const crnUpdatedRequest = new Promise(async (resolve, reject) => { - const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user); - - if (response.error) { - this.logger.error('Error updating pipeline cron', response.error); - return reject(new ErrorBuilder(response.error)); + if (Object.keys(table.destinations).length > 1) { + let hasChanges = false + const jobRenameTables = { + raw: {}, + qualify: {} } - this.logger.error('Pipeline cron updated successfully', response); - return resolve(response); - }); - requests.push(crnUpdatedRequest); + + if (Object.keys(table.destinations.raw).length > 1) { + hasChanges = true; + jobRenameTables.raw = table.destinations.raw; + } + + if (Object.keys(table.destinations.qualify).length > 1) { + hasChanges = true; + jobRenameTables.qualify = table.destinations.qualify; + } + + if (hasChanges) { + jobUpdate['rename_tables'] = jobRenameTables; + } + } + + jobsUpdated.push(jobUpdate); } - this.logger.info('Executing all request for the platform api'); + this.logger.info('Request body:' + JSON.stringify({ + jobs_updated: jobsUpdated + })); - const results = await Promise.allSettled(requests); - this.logger.info('Platform api response', results); + const response = await this.platformAPI.proxy( + 'POST', + `/pipelines/${pipelineIdFormat}/batch-update`, + user, + { + job_updates: jobsUpdated + } + ) + this.logger.info('Platform api response: ' + JSON.stringify(response)); return updateInputResponse; } - private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) { - const body = { - target_load_type: table.type - } - - if (table.type === 'incremental_with_qualify') { - body['incremental_column_name'] = table.reference_column.name; - body['incremental_column_type'] = table.reference_column.type; - body['primary_keys'] = table.identifier_columns; - } - - if (table.type === 'incremental') { - body['incremental_column_name'] = table.reference_column.name; - body['incremental_column_type'] = table.reference_column.type; - } - - this.logger.info('Updating pipeline sync mode', { - pipelineId, - body - }); - - return this.platformAPI.proxy( - "POST", - `/jobs/jdbc/${pipelineId}/sync-mode`, - user, - body - ) - } - - private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) { - try { - const response = await this.platformAPI.proxy( - 'PATCH', - `/pipeline/${pipelineId}`, - user, - { - cron - } - ); - - return response - } catch (error) { - return { - error: error.message - } - } - } } From 74b3bd6b463a20c1ecfaf5abd47b0fc01f76937e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 19 Mar 2026 08:14:56 -0300 Subject: [PATCH 22/47] FEAT: update CustomerSidebarSection and related DTOs to use object type for title; update protospack-v2 version; update from id to customerId --- docsfera.json | 2 +- package-lock.json | 126 +++++++++++++-------- package.json | 2 +- src/modules/customers/customers.service.ts | 4 +- src/modules/customers/dtos/customers.ts | 12 +- 5 files changed, 88 insertions(+), 58 deletions(-) diff --git a/docsfera.json b/docsfera.json index b0bb337..bcdd005 100644 --- a/docsfera.json +++ b/docsfera.json @@ -10544,7 +10544,7 @@ "type": "object", "properties": { "title": { - "type": "string" + "type": "object" }, "items": { "type": "array", diff --git a/package-lock.json b/package-lock.json index 526a96d..20ee1e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,11 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", -<<<<<<< feat/sidebar-menu - "@dadosfera/protospack-v2": "^3.38.0-beta.29", -======= - "@dadosfera/protospack-v2": "3.38.0-beta.28", ->>>>>>> beta + "@dadosfera/protospack-v2": "^3.38.0-beta.30", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1748,16 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { -<<<<<<< feat/sidebar-menu - "version": "3.38.0-beta.29", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.29.tgz", - "integrity": "sha512-1TUHKoSi2h/Pt+YEno9KOj8Zv2uAjT4qPS3xK26Tk2Wxg/OTlbxevPc+iwy8EI6YsT24WsbZfkb5JfKRlvKFHA==", -======= - "version": "3.38.0-beta.28", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz", - "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==", - "license": "ISC", ->>>>>>> beta + "version": "3.38.0-beta.30", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz", + "integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -2938,6 +2927,20 @@ "node": ">= 0.6" } }, + "node_modules/@nestjs/platform-express/node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, "node_modules/@nestjs/platform-express/node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -3107,6 +3110,24 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/@nestjs/platform-express/node_modules/multer": { + "version": "1.4.4-lts.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", + "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==", + "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/@nestjs/platform-express/node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -3131,6 +3152,25 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/@nestjs/platform-express/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/@nestjs/platform-express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3205,6 +3245,19 @@ "node": ">= 0.8" } }, + "node_modules/@nestjs/platform-express/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/@nestjs/platform-express/node_modules/tslib": { "version": "2.5.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", @@ -6512,21 +6565,6 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -9138,6 +9176,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -10731,24 +10774,6 @@ "node": ">=18" } }, - "node_modules/multer": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", - "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", - "license": "MIT", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.6.0", - "concat-stream": "^2.0.0", - "mkdirp": "^0.5.6", - "object-assign": "^4.1.1", - "type-is": "^1.6.18", - "xtend": "^4.0.2" - }, - "engines": { - "node": ">= 10.16.0" - } - }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -11738,6 +11763,11 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "node_modules/process-warning": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", diff --git a/package.json b/package.json index 9a13d42..4b109cd 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.29", + "@dadosfera/protospack-v2": "^3.38.0-beta.30", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index f0a161c..8915ab9 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -70,7 +70,7 @@ export class CustomersService implements OnModuleInit { async getLinks(customerId: string): Promise { try { const result = await lastValueFrom( - this.customerService.CustomerGetLinks({ id: customerId }), + this.customerService.CustomerGetLinks({ customerId }), ); return (result.links as CustomerLinksConfig) || null; } catch (err) { @@ -88,7 +88,7 @@ export class CustomersService implements OnModuleInit { try { return await firstValueFrom( this.customerService.CustomerSetLinks({ - id: customerId, + customerId, links: links as CustomerSetLinksRequest['links'], }), ); diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index 479bace..d914f07 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -14,8 +14,8 @@ export class CustomerLinkItem { export class CustomerSidebarLinkItem { @ApiProperty() type: 'link'; - @ApiProperty() - title: string; + @ApiProperty({ type: Object }) + title: Record; @ApiProperty() link: string; @ApiPropertyOptional() @@ -25,8 +25,8 @@ export class CustomerSidebarLinkItem { export class CustomerSidebarMenuItem { @ApiProperty() type: 'menu'; - @ApiProperty() - title: string; + @ApiProperty({ type: Object }) + title: Record; @ApiPropertyOptional() icon?: string; @ApiProperty({ type: [CustomerSidebarLinkItem] }) @@ -34,8 +34,8 @@ export class CustomerSidebarMenuItem { } export class CustomerSidebarSection { - @ApiProperty() - title: string; + @ApiProperty({ type: Object }) + title: Record; @ApiProperty({ type: 'array', items: { From b7171556b87e45668910c81704c4881ec42da64e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 19 Mar 2026 13:58:02 -0300 Subject: [PATCH 23/47] =?UTF-8?q?FIX:=20add=20multer@2.0.2=20and=20update?= =?UTF-8?q?=20package-lock.json=20=E2=80=94=20missing=20transitive=20dep?= =?UTF-8?q?=20of=20protospack-v2@3.38.0-beta.30?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- package-lock.json | 32 ++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 33 insertions(+) diff --git a/package-lock.json b/package-lock.json index 20ee1e3..78cfaf2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", + "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", @@ -6565,6 +6566,20 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -10774,6 +10789,23 @@ "node": ">=18" } }, + "node_modules/multer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", + "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", + "object-assign": "^4.1.1", + "type-is": "^1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">= 10.16.0" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", diff --git a/package.json b/package.json index 4b109cd..ac6d540 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", + "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", From 2e3a13d42107b256302cb9c9d5c6eac06e3a995e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 19 Mar 2026 14:35:14 -0300 Subject: [PATCH 24/47] FIX: send correct name and email from user payload --- src/modules/auth/auth.controller.ts | 1 + src/modules/auth/auth.service.ts | 3 ++- src/modules/auth/dtos/login.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 385d60c..dbff860 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -488,6 +488,7 @@ export class AuthController { const userDto = { id: api_key.user_id, name: api_key.username, + email: api_key.username, customer: { id: api_key.customer_id, name: api_key.customer_name, diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 6d67b95..8539d0e 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit { const userDto: UserDTO = { id: user.id, - name: user.username, + name: user.name, + email: user.email, jobTitle: user?.jobTitle || null, department: user?.department || null, hierarchy: user?.hierarchy || null, diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 3a484c8..2e22363 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -144,6 +144,7 @@ export interface BulkEditResponse { export type UserDTO = { id: string, name: string, + email: string, jobTitle?: string, department?: string, hierarchy?: string, From f10953e949f45271b8775be252f6ad9199f6dec6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 19 Mar 2026 14:45:28 -0300 Subject: [PATCH 25/47] FIX: multer package --- package-lock.json | 106 ---------------------------------------------- package.json | 1 - 2 files changed, 107 deletions(-) diff --git a/package-lock.json b/package-lock.json index 78cfaf2..9bc18e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,7 +47,6 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", - "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", @@ -2928,20 +2927,6 @@ "node": ">= 0.6" } }, - "node_modules/@nestjs/platform-express/node_modules/concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "engines": [ - "node >= 0.8" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, "node_modules/@nestjs/platform-express/node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -3111,24 +3096,6 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, - "node_modules/@nestjs/platform-express/node_modules/multer": { - "version": "1.4.4-lts.1", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", - "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==", - "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.0.0", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.4", - "object-assign": "^4.1.1", - "type-is": "^1.6.4", - "xtend": "^4.0.0" - }, - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/@nestjs/platform-express/node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -3153,25 +3120,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/@nestjs/platform-express/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/@nestjs/platform-express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3246,19 +3194,6 @@ "node": ">= 0.8" } }, - "node_modules/@nestjs/platform-express/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/@nestjs/platform-express/node_modules/tslib": { "version": "2.5.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", @@ -6566,20 +6501,6 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -9191,11 +9112,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -10789,23 +10705,6 @@ "node": ">=18" } }, - "node_modules/multer": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", - "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.6.0", - "concat-stream": "^2.0.0", - "mkdirp": "^0.5.6", - "object-assign": "^4.1.1", - "type-is": "^1.6.18", - "xtend": "^4.0.2" - }, - "engines": { - "node": ">= 10.16.0" - } - }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -11795,11 +11694,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/process-warning": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", diff --git a/package.json b/package.json index ac6d540..4b109cd 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "jwk-to-pem": "^2.0.5", "mixpanel": "^0.17.0", "ms": "^3.0.0-canary.1", - "multer": "^2.0.2", "openid-client": "^5.7.1", "passport": "^0.6.0", "passport-facebook": "^3.0.0", From 959210e354505c47e24c36b3f87fcd8944f1eb9c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 19 Mar 2026 14:56:58 -0300 Subject: [PATCH 26/47] FIX: npm ci --- package-lock.json | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/package-lock.json b/package-lock.json index 9bc18e0..b9f5209 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6501,6 +6501,21 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/consola": { "version": "2.15.3", "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", @@ -10705,6 +10720,24 @@ "node": ">=18" } }, + "node_modules/multer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", + "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", + "object-assign": "^4.1.1", + "type-is": "^1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">= 10.16.0" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", From 21c57e5620646e6f5cb75f9f0f0c9ce3d90f7e5c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 24 Mar 2026 12:28:20 -0300 Subject: [PATCH 27/47] FEAT: update nimbus after input --- docsfera.json | 18 +----- package-lock.json | 15 ++--- package.json | 2 +- src/modules/inputs/inputs.service.ts | 21 ++++--- .../pipelinesV2/pipelines.controller.ts | 3 +- src/modules/pipelinesV2/pipelines.module.ts | 7 ++- src/modules/pipelinesV2/pipelines.service.ts | 49 ++++++++++++---- .../platform-api/platform-api.service.ts | 6 +- src/services/nimbus/nimbus.module.ts | 9 +++ src/services/nimbus/nimbus.service.ts | 56 +++++++++++++++++++ 10 files changed, 140 insertions(+), 46 deletions(-) create mode 100644 src/services/nimbus/nimbus.module.ts create mode 100644 src/services/nimbus/nimbus.service.ts diff --git a/docsfera.json b/docsfera.json index bcdd005..4e2ed5f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3001,14 +3001,7 @@ "parameters": [], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -3583,14 +3576,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/package-lock.json b/package-lock.json index b9f5209..eba19a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.30", + "@dadosfera/protospack-v2": "3.38.0-beta.31", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.30", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz", - "integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==", + "version": "3.38.0-beta.31", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.31.tgz", + "integrity": "sha512-IxnVEaVvQNumN/PQsnF+9CsFvmn6/0YyfXauoPQ1bccvqU8Lo+hKZv4qTBrR+urd05OWyg+dJBv/QtW45b04XQ==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -5533,9 +5534,9 @@ } }, "node_modules/axios": { - "version": "0.30.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz", - "integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==", + "version": "0.30.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz", + "integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.4", diff --git a/package.json b/package.json index 4b109cd..7da4765 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.30", + "@dadosfera/protospack-v2": "3.38.0-beta.31", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index cb9d25d..44c7ddd 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -17,6 +17,7 @@ import { InputCreateGenericRequest, InputCreateS3Request, InputNewCreateRequest, + InputUpdateResponse, TestConnectionRequest, } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @@ -71,8 +72,8 @@ export class InputsService { objectCamelToSnake(createInputResponse); return createInputResponse; }, - update: async (updateInputDTO: UpdateInputRequest) => { - this.logger.info('InputClientService - Update'); + update: async (updateInputDTO: UpdateInputRequest): Promise => { + this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO)); const updateInputResponse = await lastValueFrom( this.inputWriteService.InputUpdate(updateInputDTO), ); @@ -207,16 +208,22 @@ export class InputsService { async update(id: string, data, info: Info) { // this.validateCron({ ...data, info }); try { - const updateInputResponse: any = await this.OLD_inputClient.update({ + const { + nimbusUpdates, + input + } = await this.OLD_inputClient.update({ id, - info, ...data, + info, }); - updateInputResponse.input = this.adjustInputPayload( - updateInputResponse?.input, + const updateInputResponse = this.adjustInputPayload( + input, ); - return updateInputResponse; + return { + input: updateInputResponse, + nimbusUpdates + }; } catch (err) { throw new HttpException(err.message, HttpStatus.NOT_FOUND); } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 11e7b10..1c00bcd 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -294,13 +294,14 @@ export class PipelinesController { ) { this.logger.info('PipelinesController - update', { user }); - const { customer_id, customer_name, user_id, username } = user; const info: Info = { user_id: user.user_id, customer: user.customer_name, customer_id: user.customer_id, + pipeline_id: pipelineId }; + const metadata = PackTheMetadata({ customer_id, customer_name, diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 9550aeb..d0007b8 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -12,6 +12,8 @@ import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; import { PlatformApiModule } from '../platform-api/platform-api.module'; +import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module'; +import { NimbusService } from 'src/services/nimbus/nimbus.service'; const client = new PipelinesClientConfiguration(); @@ -22,10 +24,11 @@ const client = new PipelinesClientConfiguration(); ConnectorModule, InputsModule, TransformationsModule, - PlatformApiModule + PlatformApiModule, + NimbusServicesModule ], controllers: [PipelinesController], - providers: [PipelinesService, DadosferaLogger], + providers: [PipelinesService, DadosferaLogger, NimbusService], exports: [PipelinesService], }) export class PipelinesV2Module {} diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index c0a1ad6..db04131 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -28,7 +28,10 @@ import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; import { PlatformApiService } from '../platform-api/platform-api.service'; -import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; +import axios from 'axios'; +import { NimbusService } from 'src/services/nimbus/nimbus.service'; export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; @@ -42,7 +45,8 @@ export class PipelinesService implements OnModuleInit { private readonly connectorService: ConnectorClientService, private readonly inputsService: InputsService, private readonly transformationsService: TransformationsService, - private readonly platformAPI: PlatformApiService + private readonly platformAPI: PlatformApiService, + private readonly nimbusService: NimbusService ) { this.logger = dadosferaLogger.logger; } @@ -350,18 +354,46 @@ export class PipelinesService implements OnModuleInit { this.logger.info('Update Dynamo Reference'); const pipelineIdFormat = pipelineId.split('-').join('_'); + console.log("updatePipelineInput", info, updateInputDTO) const updateInputResponse = await this.inputsService.update( inputId, updateInputDTO, info ) - this.logger.info('Dynamo Response', updateInputResponse); + this.logger.info(JSON.stringify(updateInputResponse)) + await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates); + + await this.updatePlatformJobs( + pipelineIdFormat, + updateInputDTO, + user + ); + + return updateInputResponse; + } + + private async updateNimbus(customer: string, changes: NimbusUpdate[]) { + this.logger.info('Nimbus Changes: ' + JSON.stringify(changes)); + if(!changes || changes.length === 0) return; + + const requests = changes.map(change => { + return this.nimbusService.renameTable(customer, change.database, change.old, change.new); + }) + + const values = await Promise.allSettled(requests); + + const success = values.map(request => request.status === "fulfilled") + + this.logger.info("Updates with succes: " + success.length); + } + + async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) { const jobsUpdated = []; for (const [index, table] of updateInputDTO.tables.entries()) { const jobUpdate = { - job_id: `${pipelineIdFormat}_${index}`, + job_id: `${pipelineId}_${index}`, } if (table.memory) { @@ -370,7 +402,7 @@ export class PipelinesService implements OnModuleInit { } } - this.logger.info('Updating input reference for table', table.name); + this.logger.info('Updating input reference for table: ' + table.name); let hasUpdateSyncMode = false; const jobSyncMode = {} @@ -431,17 +463,14 @@ export class PipelinesService implements OnModuleInit { })); const response = await this.platformAPI.proxy( - 'POST', - `/pipelines/${pipelineIdFormat}/batch-update`, + 'PUT', + `/pipeline/${pipelineId}/jobs`, user, { job_updates: jobsUpdated } ) this.logger.info('Platform api response: ' + JSON.stringify(response)); - - return updateInputResponse; - } } diff --git a/src/modules/platform-api/platform-api.service.ts b/src/modules/platform-api/platform-api.service.ts index f204678..7fe9928 100644 --- a/src/modules/platform-api/platform-api.service.ts +++ b/src/modules/platform-api/platform-api.service.ts @@ -89,12 +89,12 @@ export class PlatformApiService { // Propagate non-2xx responses as HttpExceptions if (response.status >= 400) { - this.logger.error('Platform API upstream error', { + this.logger.error('Platform API upstream error' + JSON.stringify({ status: response.status, data: response.data, path, method: method.toUpperCase(), - }); + })); throw new HttpException(response.data, response.status); } @@ -107,6 +107,8 @@ export class PlatformApiService { method: method.toUpperCase(), }); + this.logger.error(error) + if (error instanceof HttpException) { throw error; } diff --git a/src/services/nimbus/nimbus.module.ts b/src/services/nimbus/nimbus.module.ts new file mode 100644 index 0000000..de7b60f --- /dev/null +++ b/src/services/nimbus/nimbus.module.ts @@ -0,0 +1,9 @@ +import { Module } from "@nestjs/common"; +import { NimbusService } from "./nimbus.service"; +import DadosferaLogger from "@dadosfera/dadosfera-logs"; + +@Module({ + providers: [NimbusService, DadosferaLogger], + exports: [NimbusService], +}) +export class NimbusServicesModule {} diff --git a/src/services/nimbus/nimbus.service.ts b/src/services/nimbus/nimbus.service.ts new file mode 100644 index 0000000..031aae7 --- /dev/null +++ b/src/services/nimbus/nimbus.service.ts @@ -0,0 +1,56 @@ +import DadosferaLogger from "@dadosfera/dadosfera-logs"; +import { Inject, Injectable } from "@nestjs/common"; +import axios from "axios"; + +type TableUpdate = { + table_schema: string; + table_name: string; +} + +@Injectable() +export class NimbusService { + private logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + private buildUrl(customerName: string) { + if (process.env.ENV === 'prd') { + return `https://nimbus-${customerName}.dadosfera.ai`; + } + + return `https://nimbus-${customerName}.${process.env.ENV.replace( + 'local', + 'stg', + )}.dadosfera.ai`; + } + + async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) { + const nimbusUrl = this.buildUrl(customerName); + + const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`; + + try { + this.logger.info("Request for PATCH" + nimbusUrl + path); + this.logger.info("Payload: " + JSON.stringify(update)); + const { data } = await axios.patch(nimbusUrl + path, { + table_name: update.table_name, + table_schema: update.table_schema + }) + + return data; + } catch (error) { + this.logger.error(error); + return { + message: error.message, + database, + old, + update + } + } + + } +} From ce619942a343d474c24c7920f69e6e0a533a237c Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 23 Mar 2026 17:02:48 -0300 Subject: [PATCH 28/47] FEAT: add endpoint to cancel running pipeline runs and update Elasticsearch status --- docsfera.json | 2641 +++++++++-------- .../platform-api/platform-api.controller.ts | 35 + .../elasticsearch/elasticsearch.service.ts | 116 + 3 files changed, 1479 insertions(+), 1313 deletions(-) diff --git a/docsfera.json b/docsfera.json index 425263d..bf0a836 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3548,6 +3548,61 @@ ] } }, + "/pipelinesV2/{pipelineId}/inputs/{id}": { + "patch": { + "operationId": "PipelinesController_updatePipelineInput", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2/init-upload": { "post": { "operationId": "PipelinesController_initUploadFile", @@ -3960,6 +4015,1275 @@ ] } }, + "/platform/pipeline": { + "post": { + "operationId": "PlatformApiController_createPipeline", + "summary": "Create a new pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines": { + "get": { + "operationId": "PlatformApiController_getPipelines", + "summary": "List all pipelines for customer", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}": { + "get": { + "operationId": "PlatformApiController_getPipeline", + "summary": "Get pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_updatePipeline", + "summary": "Update pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "PlatformApiController_deletePipeline", + "summary": "Delete pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/execute": { + "post": { + "operationId": "PlatformApiController_executePipeline", + "summary": "Execute a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pause": { + "post": { + "operationId": "PlatformApiController_pausePipeline", + "summary": "Pause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/unpause": { + "post": { + "operationId": "PlatformApiController_unpausePipeline", + "summary": "Unpause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/memory": { + "put": { + "operationId": "PlatformApiController_updatePipelineMemory", + "summary": "Update pipeline memory configuration", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/metadata": { + "put": { + "operationId": "PlatformApiController_updatePipelineMetadata", + "summary": "Update pipeline metadata", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/metadata": { + "get": { + "operationId": "PlatformApiController_getPipelinesMetadata", + "summary": "Get all pipelines metadata", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get available schemas", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate Table and Schema", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationTableDTO" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run": { + "get": { + "operationId": "PlatformApiController_getPipelineRuns", + "summary": "Get pipeline runs for a pipeline", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { + "get": { + "operationId": "PlatformApiController_getPipelineRun", + "summary": "Get specific pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pipeline_run/{runId}/logs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunLogs", + "summary": "Get pipeline run logs", + "parameters": [ + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}/cancel": { + "post": { + "operationId": "PlatformApiController_cancelPipelineRun", + "summary": "Cancel a running pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/input": { + "put": { + "operationId": "PlatformApiController_updateJobInput", + "summary": "Update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_patchJobInput", + "summary": "Partial update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/memory": { + "put": { + "operationId": "PlatformApiController_updateJobMemory", + "summary": "Update job memory configuration", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/reset-state": { + "post": { + "operationId": "PlatformApiController_resetJobState", + "summary": "Reset job state", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}": { + "get": { + "operationId": "PlatformApiController_getJdbcJob", + "summary": "Get JDBC job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateJdbcSyncMode", + "summary": "Update JDBC job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/rename-tables": { + "post": { + "operationId": "PlatformApiController_renameJobTables", + "summary": "Rename job output tables and sync to catalog", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/configs/allowed_datatypes": { + "get": { + "operationId": "PlatformApiController_getJdbcAllowedDatatypes", + "summary": "Get allowed datatypes for JDBC", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}": { + "get": { + "operationId": "PlatformApiController_getSingerJob", + "summary": "Get Singer job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateSingerSyncMode", + "summary": "Update Singer job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/s3/{jobId}": { + "get": { + "operationId": "PlatformApiController_getS3Job", + "summary": "Get S3 job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/health": { + "get": { + "operationId": "PlatformApiController_healthCheck", + "summary": "Platform API health check", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/links": { + "get": { + "operationId": "CustomersController_getCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinksResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_setCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinkRequest" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/token": { + "get": { + "operationId": "CustomersController_getCustomerToken", + "parameters": [ + { + "name": "exp", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/monitoring-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMonitoringDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/logs-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/organization-info": { + "get": { + "operationId": "CustomersController_getOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_updateOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information updated" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog": { "get": { "operationId": "CatalogController_searchCatalog", @@ -5371,301 +6695,6 @@ ] } }, - "/customers/{id}/mfa": { - "post": { - "operationId": "CustomersController_enableMfaEnforce", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EnforceMfa" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/links": { - "get": { - "operationId": "CustomersController_getCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinksResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_setCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinkRequest" - } - } - } - }, - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/token": { - "get": { - "operationId": "CustomersController_getCustomerToken", - "parameters": [ - { - "name": "exp", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/monitoring-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMonitoringDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/logs-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/access-dashboard": { - "get": { - "operationId": "CustomersController_getAccessDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/organization-info": { - "get": { - "operationId": "CustomersController_getOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_updateOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information updated" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/catalog/data-asset/share/{id}": { "get": { "operationId": "ShareController_getShareDataAsset", @@ -6937,930 +7966,6 @@ } } }, - "/platform/pipeline": { - "post": { - "operationId": "PlatformApiController_createPipeline", - "summary": "Create a new pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines": { - "get": { - "operationId": "PlatformApiController_getPipelines", - "summary": "List all pipelines for customer", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}": { - "get": { - "operationId": "PlatformApiController_getPipeline", - "summary": "Get pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_updatePipeline", - "summary": "Update pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "delete": { - "operationId": "PlatformApiController_deletePipeline", - "summary": "Delete pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/execute": { - "post": { - "operationId": "PlatformApiController_executePipeline", - "summary": "Execute a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pause": { - "post": { - "operationId": "PlatformApiController_pausePipeline", - "summary": "Pause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/unpause": { - "post": { - "operationId": "PlatformApiController_unpausePipeline", - "summary": "Unpause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/memory": { - "put": { - "operationId": "PlatformApiController_updatePipelineMemory", - "summary": "Update pipeline memory configuration", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/metadata": { - "put": { - "operationId": "PlatformApiController_updatePipelineMetadata", - "summary": "Update pipeline metadata", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/metadata": { - "get": { - "operationId": "PlatformApiController_getPipelinesMetadata", - "summary": "Get all pipelines metadata", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables": { - "get": { - "operationId": "PlatformApiController_getAvailableTables", - "summary": "Get all tables available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/schemas": { - "get": { - "operationId": "PlatformApiController_getAvailableSchemas", - "summary": "Get all schemas available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables/validate": { - "post": { - "operationId": "PlatformApiController_validateTableAndSchema", - "summary": "Validate tables and schemas", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run": { - "get": { - "operationId": "PlatformApiController_getPipelineRuns", - "summary": "Get pipeline runs for a pipeline", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { - "get": { - "operationId": "PlatformApiController_getPipelineRun", - "summary": "Get specific pipeline run", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pipeline_run/{runId}/logs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunLogs", - "summary": "Get pipeline run logs", - "parameters": [ - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/input": { - "put": { - "operationId": "PlatformApiController_updateJobInput", - "summary": "Update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_patchJobInput", - "summary": "Partial update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/memory": { - "put": { - "operationId": "PlatformApiController_updateJobMemory", - "summary": "Update job memory configuration", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/reset-state": { - "post": { - "operationId": "PlatformApiController_resetJobState", - "summary": "Reset job state", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}": { - "get": { - "operationId": "PlatformApiController_getJdbcJob", - "summary": "Get JDBC job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateJdbcSyncMode", - "summary": "Update JDBC job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/configs/allowed_datatypes": { - "get": { - "operationId": "PlatformApiController_getJdbcAllowedDatatypes", - "summary": "Get allowed datatypes for JDBC", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}": { - "get": { - "operationId": "PlatformApiController_getSingerJob", - "summary": "Get Singer job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateSingerSyncMode", - "summary": "Update Singer job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/s3/{jobId}": { - "get": { - "operationId": "PlatformApiController_getS3Job", - "summary": "Get S3 job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/health": { - "get": { - "operationId": "PlatformApiController_healthCheck", - "summary": "Platform API health check", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -10133,6 +10238,9 @@ "$ref": "#/components/schemas/Column" } }, + "destination": { + "type": "object" + }, "type": { "type": "string" }, @@ -10149,6 +10257,7 @@ "required": [ "name", "columns", + "destination", "type" ] }, @@ -11125,100 +11234,6 @@ "updated_by" ] }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, "CreateShareMetadataDto": { "type": "object", "properties": { diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 978f464..0d7fc71 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -834,6 +834,41 @@ export class PlatformApiController { ); } + @Post('pipeline/:pipelineId/pipeline_run/:runId/cancel') + @ApiOperation({ summary: 'Cancel a running pipeline run' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async cancelPipelineRun( + @Param('pipelineId') pipelineId: string, + @Param('runId') runId: string, + @User() user: RequestUser, + ) { + const normalizedPipelineId = this.normalizePipelineId(pipelineId); + const normalizedRunId = this.normalizePipelineId(runId); + + const result = await this.platformApiService.proxy( + 'POST', + `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`, + user, + ); + + // Update Elasticsearch to reflect the canceled run status + try { + await this.elasticsearchService.updateLastRunStatus( + user.customer_name, + pipelineId, + 'CANCELED', + ); + } catch (esError) { + this.logger.warn('Failed to update Elasticsearch after pipeline run cancel', { + pipelineId, + runId, + error: esError.message, + }); + } + + return result; + } + // ==================== JOBS - COLUMN EDITING ROUTES ==================== @Put('jobs/:jobId/input') diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index e2c1857..0309d36 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -50,6 +50,9 @@ interface PipelineDocument { type: string; in_use: number; keywords: MultiLangArray; + // Pipeline run tracking + last_run_status?: string; + last_run_canceled_at?: string; } @Injectable() @@ -358,6 +361,119 @@ export class ElasticsearchService { } } + async updateLastRunStatus( + customerName: string, + pipelineId: string, + lastRunStatus: string, + ): Promise { + const index = this.getIndex(customerName); + const now = new Date().toISOString(); + + this.logger.info('Elasticsearch: Updating last run status', { + index, + pipelineId, + lastRunStatus, + }); + + try { + const response = await this.client.post( + `/${index}/_update/${pipelineId}`, + { + doc: { + last_run_status: lastRunStatus, + ...(lastRunStatus === 'CANCELED' ? { last_run_canceled_at: now } : {}), + }, + }, + { params: { refresh: 'wait_for' } }, + ); + + this.logger.info('Elasticsearch: Last run status updated', { + pipelineId, + result: response.data.result, + }); + + return response.data; + } catch (error) { + this.handleError('updateLastRunStatus', error, { pipelineId, index }); + throw error; + } + } + + private getDataAssetIndex(customerName: string): string { + return `${customerName}_data_assets_catalog`; + } + + async findDataAssetByTable( + customerName: string, + tableName: string, + tableSchema: string, + ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Searching data asset', { + index, + tableName, + tableSchema, + }); + + try { + const response = await this.client.post(`/${index}/_search`, { + query: { + bool: { + must: [ + { term: { 'table_name.keyword': tableName.toUpperCase() } }, + { term: { 'table_schema.keyword': tableSchema.toUpperCase() } }, + ], + }, + }, + size: 1, + }); + + const hits = response.data.hits?.hits || []; + if (hits.length === 0) { + this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index }); + return null; + } + + return { ...hits[0]._source, _es_id: hits[0]._id }; + } catch (error) { + this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index }); + throw error; + } + } + + async updateDataAsset( + customerName: string, + assetId: string, + updates: Record, + ): Promise { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Updating data asset', { + index, + assetId, + fields: Object.keys(updates), + }); + + try { + const response = await this.client.post( + `/${index}/_update/${assetId}`, + { doc: updates }, + { params: { refresh: 'wait_for' } }, + ); + + this.logger.info('Elasticsearch: Data asset updated', { + assetId, + result: response.data.result, + }); + + return response.data; + } catch (error) { + this.handleError('updateDataAsset', error, { assetId, index }); + throw error; + } + } + private handleError( operation: string, error: any, From d3aeca12cd01ae9e2d3489141d267f22c66c6f67 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 24 Mar 2026 08:33:57 -0300 Subject: [PATCH 29/47] FIX: remove last_run_canceled_at from PipelineDocument and update last_run_status handling --- src/services/elasticsearch/elasticsearch.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index 0309d36..7945acf 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -52,7 +52,6 @@ interface PipelineDocument { keywords: MultiLangArray; // Pipeline run tracking last_run_status?: string; - last_run_canceled_at?: string; } @Injectable() @@ -381,7 +380,7 @@ export class ElasticsearchService { { doc: { last_run_status: lastRunStatus, - ...(lastRunStatus === 'CANCELED' ? { last_run_canceled_at: now } : {}), + updated_at: now, }, }, { params: { refresh: 'wait_for' } }, From 89834308891f3fc5ea7cea61ffbd0d68f23bb54e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 24 Mar 2026 09:34:38 -0300 Subject: [PATCH 30/47] REF: remove last_run_status tracking and related Elasticsearch update logic from pipeline cancellation --- .../platform-api/platform-api.controller.ts | 19 +-------- .../elasticsearch/elasticsearch.service.ts | 40 ------------------- 2 files changed, 1 insertion(+), 58 deletions(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 0d7fc71..7c1c025 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -845,28 +845,11 @@ export class PlatformApiController { const normalizedPipelineId = this.normalizePipelineId(pipelineId); const normalizedRunId = this.normalizePipelineId(runId); - const result = await this.platformApiService.proxy( + return this.platformApiService.proxy( 'POST', `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`, user, ); - - // Update Elasticsearch to reflect the canceled run status - try { - await this.elasticsearchService.updateLastRunStatus( - user.customer_name, - pipelineId, - 'CANCELED', - ); - } catch (esError) { - this.logger.warn('Failed to update Elasticsearch after pipeline run cancel', { - pipelineId, - runId, - error: esError.message, - }); - } - - return result; } // ==================== JOBS - COLUMN EDITING ROUTES ==================== diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index 7945acf..ab2791a 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -50,8 +50,6 @@ interface PipelineDocument { type: string; in_use: number; keywords: MultiLangArray; - // Pipeline run tracking - last_run_status?: string; } @Injectable() @@ -360,44 +358,6 @@ export class ElasticsearchService { } } - async updateLastRunStatus( - customerName: string, - pipelineId: string, - lastRunStatus: string, - ): Promise { - const index = this.getIndex(customerName); - const now = new Date().toISOString(); - - this.logger.info('Elasticsearch: Updating last run status', { - index, - pipelineId, - lastRunStatus, - }); - - try { - const response = await this.client.post( - `/${index}/_update/${pipelineId}`, - { - doc: { - last_run_status: lastRunStatus, - updated_at: now, - }, - }, - { params: { refresh: 'wait_for' } }, - ); - - this.logger.info('Elasticsearch: Last run status updated', { - pipelineId, - result: response.data.result, - }); - - return response.data; - } catch (error) { - this.handleError('updateLastRunStatus', error, { pipelineId, index }); - throw error; - } - } - private getDataAssetIndex(customerName: string): string { return `${customerName}_data_assets_catalog`; } From 98282471a995860339095248faac58035197b936 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 26 Mar 2026 11:08:40 -0300 Subject: [PATCH 31/47] FIX: remove unused data asset methods from ElasticsearchService --- .../elasticsearch/elasticsearch.service.ts | 75 ------------------- 1 file changed, 75 deletions(-) diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index ab2791a..e2c1857 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -358,81 +358,6 @@ export class ElasticsearchService { } } - private getDataAssetIndex(customerName: string): string { - return `${customerName}_data_assets_catalog`; - } - - async findDataAssetByTable( - customerName: string, - tableName: string, - tableSchema: string, - ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { - const index = this.getDataAssetIndex(customerName); - - this.logger.info('Elasticsearch: Searching data asset', { - index, - tableName, - tableSchema, - }); - - try { - const response = await this.client.post(`/${index}/_search`, { - query: { - bool: { - must: [ - { term: { 'table_name.keyword': tableName.toUpperCase() } }, - { term: { 'table_schema.keyword': tableSchema.toUpperCase() } }, - ], - }, - }, - size: 1, - }); - - const hits = response.data.hits?.hits || []; - if (hits.length === 0) { - this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index }); - return null; - } - - return { ...hits[0]._source, _es_id: hits[0]._id }; - } catch (error) { - this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index }); - throw error; - } - } - - async updateDataAsset( - customerName: string, - assetId: string, - updates: Record, - ): Promise { - const index = this.getDataAssetIndex(customerName); - - this.logger.info('Elasticsearch: Updating data asset', { - index, - assetId, - fields: Object.keys(updates), - }); - - try { - const response = await this.client.post( - `/${index}/_update/${assetId}`, - { doc: updates }, - { params: { refresh: 'wait_for' } }, - ); - - this.logger.info('Elasticsearch: Data asset updated', { - assetId, - result: response.data.result, - }); - - return response.data; - } catch (error) { - this.handleError('updateDataAsset', error, { assetId, index }); - throw error; - } - } - private handleError( operation: string, error: any, From 3bcd78bb329fbdeea4db30f4610fc5bbe8782796 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 26 Mar 2026 14:53:33 -0300 Subject: [PATCH 32/47] FEAT: update protospack-v2 to version 3.40.0-beta.1 and adjust related scripts --- docsfera.json | 18 ++---------------- package-lock.json | 8 ++++---- package.json | 4 ++-- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/docsfera.json b/docsfera.json index bf0a836..a99d60e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3001,14 +3001,7 @@ "parameters": [], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -3583,14 +3576,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/package-lock.json b/package-lock.json index 592117e..9b97c19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.39.0", + "@dadosfera/protospack-v2": "3.40.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.39.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.39.0.tgz", - "integrity": "sha512-z6q+ZRpbDT6apfmly1+X818DgAFUCilFO0ATP7xR5+SH+GAUAJLY1dJRQDEkq5vFIfXnC2Nmm9aODZAs+9/5vw==", + "version": "3.40.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.1.tgz", + "integrity": "sha512-Oa0AgYDxo9JT/yQ+sGsgCBe1dJZX5ER0kmtjqlzqSg4pGnx1achQOlWOzv23/u54yUBh0TkvKvwffgI4HkgOew==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 6afe185..ffd7a4c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ }, "scripts": { "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1", - "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact", + "proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact", "prebuild": "rimraf dist", "build": "nest build", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.39.0", + "@dadosfera/protospack-v2": "3.40.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 7fbce8b5be9edb0b60e15b75ff23a1c8b6446d52 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 30 Mar 2026 12:02:27 -0300 Subject: [PATCH 33/47] FEAT: update input --- package-lock.json | 8 +- package.json | 2 +- src/modules/inputs/inputs.service.ts | 22 ++- src/modules/pipelinesV2/pipelines.service.ts | 135 ++++++++++++++++--- src/services/nimbus/nimbus.service.ts | 2 +- 5 files changed, 145 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index eba19a9..fc5d425 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.31", + "@dadosfera/protospack-v2": "3.40.0-beta.2", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.31", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.31.tgz", - "integrity": "sha512-IxnVEaVvQNumN/PQsnF+9CsFvmn6/0YyfXauoPQ1bccvqU8Lo+hKZv4qTBrR+urd05OWyg+dJBv/QtW45b04XQ==", + "version": "3.40.0-beta.2", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz", + "integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 7da4765..059f151 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.31", + "@dadosfera/protospack-v2": "3.40.0-beta.2", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 44c7ddd..46ba2a4 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -18,6 +18,7 @@ import { InputCreateS3Request, InputNewCreateRequest, InputUpdateResponse, + RollbackInputRequest, TestConnectionRequest, } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @@ -209,7 +210,8 @@ export class InputsService { // this.validateCron({ ...data, info }); try { const { - nimbusUpdates, + tablesUpdate, + dataAssetUpdate, input } = await this.OLD_inputClient.update({ id, @@ -222,13 +224,29 @@ export class InputsService { ); return { input: updateInputResponse, - nimbusUpdates + tablesUpdate, + dataAssetUpdate }; } catch (err) { throw new HttpException(err.message, HttpStatus.NOT_FOUND); } } + async rollbackUpdate( + data: RollbackInputRequest + ) { + this.logger.info('PipelinesClientService - rollbackUpdate'); + this.logger.info('Rolling back input update with data: ' + JSON.stringify(data)); + const updatePipelineResponse = await lastValueFrom( + this.inputWriteService.RollbackInputUpdate( + data + ), + ); + this.logger.info('Done'); + + return updatePipelineResponse; + } + async remove(idRequest: IIdRequest) { return lastValueFrom(this.inputWriteService.InputRemove(idRequest)); } diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index db04131..17eb41b 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -28,11 +28,13 @@ import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; import { PlatformApiService } from '../platform-api/platform-api.service'; -import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; -import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; -import axios from 'axios'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; +import { AxiosError } from 'axios'; import { NimbusService } from 'src/services/nimbus/nimbus.service'; +type RollbackPromise = () => Promise; + export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; pipelineWriteService: WriteService.PipelineV2WriteService; @@ -352,33 +354,116 @@ export class PipelinesService implements OnModuleInit { async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) { this.logger.info('InputClientService - Update'); - this.logger.info('Update Dynamo Reference'); + const { + input: oldInput + } = await this.inputsService.findOne({ + id: inputId, + info: info + }); + + this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput)); const pipelineIdFormat = pipelineId.split('-').join('_'); - console.log("updatePipelineInput", info, updateInputDTO) + const rollback: RollbackPromise[] = []; + const updateInputResponse = await this.inputsService.update( inputId, updateInputDTO, info - ) - - this.logger.info(JSON.stringify(updateInputResponse)) - await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates); - - await this.updatePlatformJobs( - pipelineIdFormat, - updateInputDTO, - user ); + const inputRollback = () => { + this.logger.info("exec rollback to input: " + JSON.stringify(oldInput)); + return this.inputsService.rollbackUpdate( + { + id: inputId, + dataAssetUpdate: updateInputResponse.dataAssetUpdate, + tables: oldInput.tables, + info + } + ) as Promise; + } + + rollback.push(inputRollback); + + this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse)) + + const nimbusUpdates = updateInputResponse?.tablesUpdate || []; + + nimbusUpdates.forEach(update => { + const nimbusRollback = () => { + return this.nimbusService.renameTable( + info.customer, + update.database, + { + table_name: update.table_name, + table_schema: update.table_schema + }, + { + table_name: update.old_table_name, + table_schema: update.old_table_schema + } + ); + } + rollback.push(nimbusRollback); + }); + + try { + await this.updateNimbus(info.customer, nimbusUpdates); + } catch (error) { + this.logger.error(error); + if (error instanceof AxiosError) { + this.logger.error(JSON.stringify(error.response.data)); + } + await this.executeRenameRollback(rollback); + + throw new Error("Error Nimbus updating tables"); + } + + try { + await this.updatePlatformJobs( + pipelineIdFormat, + updateInputDTO, + user + ); + } catch (error) { + this.logger.error(error); + await this.executeRenameRollback(rollback) + throw new Error("Error Platform API updating jobs"); + } + return updateInputResponse; } - private async updateNimbus(customer: string, changes: NimbusUpdate[]) { + private async executeRenameRollback(request: RollbackPromise[]) { + this.logger.info('rollback steps: ' + request.length) + const result = await Promise.allSettled(request.map(func => func())); + result.forEach(promise => { + this.logger.info("Promise finish with status: " + promise.status) + + if (promise.status === "rejected") { + this.logger.error("reject with: " + JSON.stringify(promise.reason || {})) + } + + if (promise.status === "fulfilled") { + this.logger.info("success with: " + JSON.stringify(promise.value || {})) + } + }); + + } + + private async updateNimbus(customer: string, changes: TableUpdate[]) { + // throw new Error("teste error nimbus"); this.logger.info('Nimbus Changes: ' + JSON.stringify(changes)); if(!changes || changes.length === 0) return; const requests = changes.map(change => { - return this.nimbusService.renameTable(customer, change.database, change.old, change.new); + return this.nimbusService.renameTable(customer, change.database, { + table_name: change.old_table_name, + table_schema: change.old_table_schema + }, { + table_name: change.table_name, + table_schema: change.table_schema + }); }) const values = await Promise.allSettled(requests); @@ -386,6 +471,20 @@ export class PipelinesService implements OnModuleInit { const success = values.map(request => request.status === "fulfilled") this.logger.info("Updates with succes: " + success.length); + + values.forEach(promise => { + this.logger.info("Promise finish with status: " + promise.status) + + if (promise.status === "rejected") { + this.logger.error("reject with: " + JSON.stringify(promise.reason || {})); + throw new Error(promise.reason ); + } + + if (promise.status === "fulfilled") { + this.logger.info("success with: " + JSON.stringify(promise.value || {})); + } + }); + } async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) { @@ -396,6 +495,10 @@ export class PipelinesService implements OnModuleInit { job_id: `${pipelineId}_${index}`, } + if (table.type !== "incremental_with_qualify") { + delete table.destinations?.qualify; + } + if (table.memory) { jobUpdate["memory"] = { amount: table.memory * 1000 diff --git a/src/services/nimbus/nimbus.service.ts b/src/services/nimbus/nimbus.service.ts index 031aae7..03789ed 100644 --- a/src/services/nimbus/nimbus.service.ts +++ b/src/services/nimbus/nimbus.service.ts @@ -34,7 +34,7 @@ export class NimbusService { const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`; try { - this.logger.info("Request for PATCH" + nimbusUrl + path); + this.logger.info("Request for PATCH " + nimbusUrl + path); this.logger.info("Payload: " + JSON.stringify(update)); const { data } = await axios.patch(nimbusUrl + path, { table_name: update.table_name, From 2d46ac32133a0702fec8d675f25182677479c3fd Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 30 Mar 2026 22:24:10 -0300 Subject: [PATCH 34/47] FEAT: add delete job endpoint and update related services; update protospack-v2 version to 3.40.0-beta.3 --- docsfera.json | 1162 +++++++++++++++-- package-lock.json | 9 +- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 9 +- .../platform-api/platform-api.controller.ts | 74 ++ .../platform-api/platform-api.module.ts | 3 +- src/services/dynamodb/dynamodb.service.ts | 62 +- 7 files changed, 1199 insertions(+), 122 deletions(-) diff --git a/docsfera.json b/docsfera.json index a99d60e..deb8c8f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7952,6 +7952,966 @@ } } }, + "/platform/pipeline": { + "post": { + "operationId": "PlatformApiController_createPipeline", + "summary": "Create a new pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines": { + "get": { + "operationId": "PlatformApiController_getPipelines", + "summary": "List all pipelines for customer", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}": { + "get": { + "operationId": "PlatformApiController_getPipeline", + "summary": "Get pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_updatePipeline", + "summary": "Update pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "PlatformApiController_deletePipeline", + "summary": "Delete pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/execute": { + "post": { + "operationId": "PlatformApiController_executePipeline", + "summary": "Execute a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pause": { + "post": { + "operationId": "PlatformApiController_pausePipeline", + "summary": "Pause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/unpause": { + "post": { + "operationId": "PlatformApiController_unpausePipeline", + "summary": "Unpause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/memory": { + "put": { + "operationId": "PlatformApiController_updatePipelineMemory", + "summary": "Update pipeline memory configuration", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/metadata": { + "put": { + "operationId": "PlatformApiController_updatePipelineMetadata", + "summary": "Update pipeline metadata", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/metadata": { + "get": { + "operationId": "PlatformApiController_getPipelinesMetadata", + "summary": "Get all pipelines metadata", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables": { + "get": { + "operationId": "PlatformApiController_getAvailableTables", + "summary": "Get all tables available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get all schemas available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate tables and schemas", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run": { + "get": { + "operationId": "PlatformApiController_getPipelineRuns", + "summary": "Get pipeline runs for a pipeline", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { + "get": { + "operationId": "PlatformApiController_getPipelineRun", + "summary": "Get specific pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pipeline_run/{runId}/logs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunLogs", + "summary": "Get pipeline run logs", + "parameters": [ + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/input": { + "put": { + "operationId": "PlatformApiController_updateJobInput", + "summary": "Update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_patchJobInput", + "summary": "Partial update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/memory": { + "put": { + "operationId": "PlatformApiController_updateJobMemory", + "summary": "Update job memory configuration", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/reset-state": { + "post": { + "operationId": "PlatformApiController_resetJobState", + "summary": "Reset job state", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}": { + "delete": { + "operationId": "PlatformApiController_deleteJob", + "summary": "Delete a job and sync pipeline data", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}": { + "get": { + "operationId": "PlatformApiController_getJdbcJob", + "summary": "Get JDBC job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateJdbcSyncMode", + "summary": "Update JDBC job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/configs/allowed_datatypes": { + "get": { + "operationId": "PlatformApiController_getJdbcAllowedDatatypes", + "summary": "Get allowed datatypes for JDBC", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}": { + "get": { + "operationId": "PlatformApiController_getSingerJob", + "summary": "Get Singer job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateSingerSyncMode", + "summary": "Update Singer job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/s3/{jobId}": { + "get": { + "operationId": "PlatformApiController_getS3Job", + "summary": "Get S3 job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/health": { + "get": { + "operationId": "PlatformApiController_healthCheck", + "summary": "Platform API health check", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -10523,114 +11483,6 @@ "engine" ] }, - "ValidationTableDTO": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tables" - ] - }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "object" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, "IRole": { "type": "object", "properties": { @@ -11220,6 +12072,100 @@ "updated_by" ] }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLinkItem": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerSidebarSection": { + "type": "object", + "properties": { + "title": { + "type": "object" + }, + "items": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] + } + } + }, + "required": [ + "title", + "items" + ] + }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + }, + "required": [ + "links" + ] + }, "CreateShareMetadataDto": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index adacb76..de2fca5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.2", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,10 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz", - "integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==", - "license": "ISC", + "version": "3.40.0-beta.3", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.3.tgz", + "integrity": "sha512-m77RSqAO+hZkjUgbdrMqC+6yyyo7indH6fWP3WuygCHGYwdN6FsWTA+EGTAhXQRSgNlI8h358ZpRoILnnGzLMg==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index fd480de..586d74c 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.2", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 1c00bcd..ab39d38 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -224,11 +224,12 @@ export class PipelinesController { const result = await this.pipelinesClientService .findOne({ id }, metadata) - .then((res) => { + .then(async (res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} - let tables = JSON.parse(res.pipeline.config.tables); - const input_id = tables?.input_id; - if (tables?.tables) tables = tables.tables; + const parsed = JSON.parse(res.pipeline.config.tables); + const input_id = parsed?.input_id; + const tables = parsed?.tables ?? parsed; + Object.assign(res.pipeline, { transformations: res.pipeline.transformations ? JSON.parse(res.pipeline.transformations) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 8c42102..74caaf9 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -962,6 +962,80 @@ export class PlatformApiController { ); } + @Delete('jobs/:jobId') + @ApiOperation({ summary: 'Delete a job and sync pipeline data' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) + async deleteJob( + @Param('jobId') jobId: string, + @User() user: RequestUser, + ) { + // Get job details before deletion to extract table_name and connector type + const normalizedJobId = this.normalizeJobId(jobId); + let tableName: string | undefined; + + this.logger.info('[deleteJob] Start', { jobId }); + + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${normalizedJobId}`, + user, + ); + tableName = job?.source_config?.table_name; + this.logger.info('[deleteJob] Job fetched', { jobId, tableName }); + } catch (err) { + this.logger.warn('[deleteJob] Could not fetch job before deletion', { jobId, error: err.message }); + } + + const result = await this.platformApiService.proxy( + 'DELETE', + `/jobs/${normalizedJobId}`, + user, + ); + this.logger.info('[deleteJob] Job deleted from platform-api', { jobId }); + + // Mark the table as deleted via in-factory (which has the correct DynamoDB credentials). + // config.tables in ES stores the DynamoDB input ID (legacy naming). + if (tableName) { + try { + const esPipelineId = this.extractPipelineIdFromJobId(jobId); + const esPipeline = await this.elasticsearchService.getPipeline( + user.customer_name, + esPipelineId, + ); + // config.tables holds the DynamoDB input ID (see syncJobInputToDynamoDB for context) + const dynamoInputId = esPipeline?.config?.tables; + + if (dynamoInputId) { + const info = { customer_id: user.customer_id, user_id: user.user_id, customer: user.customer_name }; + + // Read current input to get full tables array (adjustInputPayload flattens the response) + const { input } = await this.inputsService.findOne({ id: dynamoInputId, info }); + const currentTables: any[] = input?.tables ?? []; + + const updatedTables = currentTables.map((t: any) => + t.name === tableName ? { ...t, status: 'deleted' } : t, + ); + + await this.inputsService.update(dynamoInputId, { tables: updatedTables }, info); + this.logger.info('[deleteJob] Marked table as deleted via in-factory', { jobId, tableName }); + } else { + this.logger.warn('[deleteJob] No input ID in ES pipeline, skipping DynamoDB update', { jobId, tableName }); + } + } catch (error) { + this.logger.error('[deleteJob] Failed to mark table as deleted via in-factory', { + jobId, + tableName, + error: error.message, + }); + } + } else { + this.logger.warn('[deleteJob] tableName not resolved, skipping DynamoDB update', { jobId }); + } + + return result; + } + // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== @Get('jobs/jdbc/:jobId') diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index b78e680..d202700 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; +import { PipelinesV2Module } from '../pipelinesV2/pipelines.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, PipelinesV2Module], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index 15dde41..a0d3537 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -30,6 +30,7 @@ export interface InputDocument { type: string; columns?: string[]; reference_column?: ReferenceColumn; + status?: string; }>; credentials?: Record; } @@ -125,9 +126,13 @@ export class DynamoDBService { }, }); - const { Item } = await this.documentClient.send(getCommand); - - return Item as InputDocument | null; + try { + const { Item } = await this.documentClient.send(getCommand); + return Item as InputDocument | null; + } catch (error) { + this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message }); + throw error; + } } async deleteInput(clientId: string, inputId: string): Promise { @@ -163,6 +168,7 @@ export class DynamoDBService { type?: string; columns?: string[]; reference_column?: ReferenceColumn | null; + status?: string; }, ): Promise { const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); @@ -203,6 +209,7 @@ export class DynamoDBService { updatedTable.reference_column = changes.reference_column; } } + if ('status' in changes) updatedTable.status = changes.status; tables[tableIndex] = updatedTable; @@ -231,4 +238,53 @@ export class DynamoDBService { throw error; } } + + /** + * Remove a specific table entry from the input document. + * Fetches the current document, removes the matching table, and saves. + */ + async removeInputTable( + clientId: string, + inputId: string, + tableName: string, + ): Promise { + const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); + + this.logger.info('DynamoDB: Removing input table', { + inputId, + tableName, + }); + + const current = await this.findInput(clientId, inputId); + if (!current) { + this.logger.warn('DynamoDB: Input not found for table removal', { inputId }); + return; + } + + const tables = (current.tables || []).filter((t) => t.name !== tableName); + + const putCommand = new PutCommand({ + TableName: dynamoTableName, + Item: { + ...current, + tables, + updated_at: new Date().toISOString(), + }, + }); + + try { + await this.documentClient.send(putCommand); + this.logger.info('DynamoDB: Input table removed successfully', { + inputId, + tableName, + }); + } catch (error) { + this.logger.error('DynamoDB: Failed to remove input table', { + inputId, + tableName, + error: error.message, + }); + throw error; + } + } } From efc1f49d92bc5d1473d4f7ad8a7da9ae75b35d47 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 30 Mar 2026 22:39:40 -0300 Subject: [PATCH 35/47] FEAT: update delete job endpoint summary and remove unused InputsModule from platform-api module --- docsfera.json | 2 +- .../platform-api/platform-api.controller.ts | 89 ++++++++----------- 2 files changed, 36 insertions(+), 55 deletions(-) diff --git a/docsfera.json b/docsfera.json index deb8c8f..ec823b4 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8645,7 +8645,7 @@ "/platform/jobs/{jobId}": { "delete": { "operationId": "PlatformApiController_deleteJob", - "summary": "Delete a job and sync pipeline data", + "summary": "Delete a job and mark its table as deleted in DynamoDB", "parameters": [ { "name": "jobId", diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 74caaf9..a83b7bd 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -963,74 +963,55 @@ export class PlatformApiController { } @Delete('jobs/:jobId') - @ApiOperation({ summary: 'Delete a job and sync pipeline data' }) + @ApiOperation({ summary: 'Delete a job and mark its table as deleted in DynamoDB' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteJob( @Param('jobId') jobId: string, @User() user: RequestUser, ) { - // Get job details before deletion to extract table_name and connector type const normalizedJobId = this.normalizeJobId(jobId); - let tableName: string | undefined; + const esPipelineId = this.extractPipelineIdFromJobId(jobId); - this.logger.info('[deleteJob] Start', { jobId }); + // Fetch job details and ES pipeline in parallel — both are reads with no mutual dependency + const [jobResult, esPipelineResult] = await Promise.allSettled([ + this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user), + this.elasticsearchService.getPipeline(user.customer_name, esPipelineId), + ]); - try { - const job = await this.platformApiService.proxy( - 'GET', - `/jobs/${normalizedJobId}`, - user, - ); - tableName = job?.source_config?.table_name; - this.logger.info('[deleteJob] Job fetched', { jobId, tableName }); - } catch (err) { - this.logger.warn('[deleteJob] Could not fetch job before deletion', { jobId, error: err.message }); + const tableName: string | undefined = jobResult.status === 'fulfilled' + ? jobResult.value?.source_config?.table_name + : undefined; + + if (jobResult.status === 'rejected') { + this.logger.warn('deleteJob: could not fetch job before deletion', { jobId, error: jobResult.reason?.message }); } - const result = await this.platformApiService.proxy( - 'DELETE', - `/jobs/${normalizedJobId}`, - user, - ); - this.logger.info('[deleteJob] Job deleted from platform-api', { jobId }); + // Main operation + const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); - // Mark the table as deleted via in-factory (which has the correct DynamoDB credentials). - // config.tables in ES stores the DynamoDB input ID (legacy naming). - if (tableName) { - try { - const esPipelineId = this.extractPipelineIdFromJobId(jobId); - const esPipeline = await this.elasticsearchService.getPipeline( - user.customer_name, - esPipelineId, - ); - // config.tables holds the DynamoDB input ID (see syncJobInputToDynamoDB for context) - const dynamoInputId = esPipeline?.config?.tables; + // Sync deleted status to DynamoDB — follows syncJobInputToDynamoDB pattern + // config.tables in ES stores the DynamoDB input ID (legacy field naming) + const dynamoInputId: string | undefined = esPipelineResult.status === 'fulfilled' + ? esPipelineResult.value?.config?.tables + : undefined; - if (dynamoInputId) { - const info = { customer_id: user.customer_id, user_id: user.user_id, customer: user.customer_name }; + if (esPipelineResult.status === 'rejected') { + this.logger.warn('deleteJob: could not fetch ES pipeline for DynamoDB sync', { jobId, error: esPipelineResult.reason?.message }); + } - // Read current input to get full tables array (adjustInputPayload flattens the response) - const { input } = await this.inputsService.findOne({ id: dynamoInputId, info }); - const currentTables: any[] = input?.tables ?? []; - - const updatedTables = currentTables.map((t: any) => - t.name === tableName ? { ...t, status: 'deleted' } : t, - ); - - await this.inputsService.update(dynamoInputId, { tables: updatedTables }, info); - this.logger.info('[deleteJob] Marked table as deleted via in-factory', { jobId, tableName }); - } else { - this.logger.warn('[deleteJob] No input ID in ES pipeline, skipping DynamoDB update', { jobId, tableName }); - } - } catch (error) { - this.logger.error('[deleteJob] Failed to mark table as deleted via in-factory', { - jobId, - tableName, - error: error.message, - }); - } + if (tableName && dynamoInputId) { + await this.dynamoDBService.updateInputTable( + user.customer_id, + dynamoInputId, + tableName, + { status: 'deleted' }, + ).catch((error) => { + this.logger.error('Failed to sync job deletion to DynamoDB', { jobId, tableName, error: error.message }); + }); + } else if (!tableName) { + this.logger.warn('deleteJob: table name not resolved, skipping DynamoDB sync', { jobId }); } else { - this.logger.warn('[deleteJob] tableName not resolved, skipping DynamoDB update', { jobId }); + this.logger.warn('deleteJob: no input ID in ES pipeline, skipping DynamoDB sync', { jobId }); } return result; From ecca9106f0d9ebd73613185310085ad13e1e4207 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 31 Mar 2026 11:17:32 -0300 Subject: [PATCH 36/47] FEAT: remove PipelinesV2Module from PlatformApiModule imports --- src/modules/platform-api/platform-api.module.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index d202700..b78e680 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,10 +8,9 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; -import { PipelinesV2Module } from '../pipelinesV2/pipelines.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, PipelinesV2Module], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], From 0369f10b5c75e3bcefe0dfbf15060994777f0453 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 31 Mar 2026 14:52:52 -0300 Subject: [PATCH 37/47] FIX: preventing the axios vulnerability --- Dockerfile | 4 ++-- Dockerfile.local | 2 +- package-lock.json | 8 ++++---- package.json | 6 +++++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7a2c3e6..607f833 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ # run aws cli without mounting secret, because CI already has AWS credentials FROM build_base AS ci_image RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 -RUN npm ci +RUN npm ci --ignore-scripts COPY . . @@ -37,7 +37,7 @@ FROM build_base AS dev RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \ aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 # flag --build-from-source is required to force-build sqlite3 -RUN npm ci +RUN npm ci --ignore-scripts COPY . . ENTRYPOINT npm run start:dev diff --git a/Dockerfile.local b/Dockerfile.local index fc8afaf..83d2af0 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -22,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ FROM build_base AS build RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \ aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 -RUN npm ci +RUN npm ci --ignore-scripts COPY . . RUN npm run build diff --git a/package-lock.json b/package-lock.json index 592117e..0fa30be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ "@nestjs/schematics": "^9.2.0", "@nestjs/swagger": "^6.3.0", "@nestjs/testing": "^9.4.3", - "axios": "^0.30.2", + "axios": "0.30.3", "cache-manager": "^5.1.4", "cache-manager-ioredis-yet": "^1.1.0", "class-transformer": "^0.5.1", @@ -5598,9 +5598,9 @@ } }, "node_modules/axios": { - "version": "0.30.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz", - "integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==", + "version": "0.30.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz", + "integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.4", diff --git a/package.json b/package.json index 6afe185..ec7ac26 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "@nestjs/schematics": "^9.2.0", "@nestjs/swagger": "^6.3.0", "@nestjs/testing": "^9.4.3", - "axios": "^0.30.2", + "axios": "0.30.3", "cache-manager": "^5.1.4", "cache-manager-ioredis-yet": "^1.1.0", "class-transformer": "^0.5.1", @@ -81,6 +81,7 @@ "swagger-ui-express": "^4.6.3" }, "overrides": { + "axios": "0.30.3", "form-data": "^4.0.4", "body-parser": "^1.20.3", "cross-spawn": "^7.0.5", @@ -117,5 +118,8 @@ "ts-node": "^10.9.1", "tsconfig-paths": "^3.14.2", "typescript": "^4.9.5" + }, + "resolutions": { + "axios": "0.30.3" } } From 59efdb6272cf4072342946e9f4935bacbb8ec471 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 31 Mar 2026 16:36:59 -0300 Subject: [PATCH 38/47] FEAT: update job deletion and mark associated table as deleted in DynamoDB. Update protospack-v2 --- docsfera.json | 1198 ++--------------- package-lock.json | 8 +- package.json | 2 +- src/modules/inputs/inputs.service.ts | 4 + .../pipelinesV2/pipelines.controller.ts | 55 +- .../platform-api/platform-api.controller.ts | 64 +- .../platform-api/platform-api.module.ts | 3 +- 7 files changed, 216 insertions(+), 1118 deletions(-) diff --git a/docsfera.json b/docsfera.json index ec823b4..c78fbec 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,6 +4705,42 @@ ] } }, + "/platform/jobs/{jobId}": { + "delete": { + "operationId": "PlatformApiController_deleteJob", + "summary": "Delete a job and mark its table as deleted via in-factory", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/jdbc/{jobId}": { "get": { "operationId": "PlatformApiController_getJdbcJob", @@ -7952,966 +7988,6 @@ } } }, - "/platform/pipeline": { - "post": { - "operationId": "PlatformApiController_createPipeline", - "summary": "Create a new pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines": { - "get": { - "operationId": "PlatformApiController_getPipelines", - "summary": "List all pipelines for customer", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}": { - "get": { - "operationId": "PlatformApiController_getPipeline", - "summary": "Get pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_updatePipeline", - "summary": "Update pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "delete": { - "operationId": "PlatformApiController_deletePipeline", - "summary": "Delete pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/execute": { - "post": { - "operationId": "PlatformApiController_executePipeline", - "summary": "Execute a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pause": { - "post": { - "operationId": "PlatformApiController_pausePipeline", - "summary": "Pause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/unpause": { - "post": { - "operationId": "PlatformApiController_unpausePipeline", - "summary": "Unpause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/memory": { - "put": { - "operationId": "PlatformApiController_updatePipelineMemory", - "summary": "Update pipeline memory configuration", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/metadata": { - "put": { - "operationId": "PlatformApiController_updatePipelineMetadata", - "summary": "Update pipeline metadata", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/metadata": { - "get": { - "operationId": "PlatformApiController_getPipelinesMetadata", - "summary": "Get all pipelines metadata", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables": { - "get": { - "operationId": "PlatformApiController_getAvailableTables", - "summary": "Get all tables available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/schemas": { - "get": { - "operationId": "PlatformApiController_getAvailableSchemas", - "summary": "Get all schemas available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables/validate": { - "post": { - "operationId": "PlatformApiController_validateTableAndSchema", - "summary": "Validate tables and schemas", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run": { - "get": { - "operationId": "PlatformApiController_getPipelineRuns", - "summary": "Get pipeline runs for a pipeline", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { - "get": { - "operationId": "PlatformApiController_getPipelineRun", - "summary": "Get specific pipeline run", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pipeline_run/{runId}/logs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunLogs", - "summary": "Get pipeline run logs", - "parameters": [ - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/input": { - "put": { - "operationId": "PlatformApiController_updateJobInput", - "summary": "Update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_patchJobInput", - "summary": "Partial update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/memory": { - "put": { - "operationId": "PlatformApiController_updateJobMemory", - "summary": "Update job memory configuration", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/reset-state": { - "post": { - "operationId": "PlatformApiController_resetJobState", - "summary": "Reset job state", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}": { - "delete": { - "operationId": "PlatformApiController_deleteJob", - "summary": "Delete a job and mark its table as deleted in DynamoDB", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}": { - "get": { - "operationId": "PlatformApiController_getJdbcJob", - "summary": "Get JDBC job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateJdbcSyncMode", - "summary": "Update JDBC job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/configs/allowed_datatypes": { - "get": { - "operationId": "PlatformApiController_getJdbcAllowedDatatypes", - "summary": "Get allowed datatypes for JDBC", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}": { - "get": { - "operationId": "PlatformApiController_getSingerJob", - "summary": "Get Singer job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateSingerSyncMode", - "summary": "Update Singer job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/s3/{jobId}": { - "get": { - "operationId": "PlatformApiController_getS3Job", - "summary": "Get S3 job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/health": { - "get": { - "operationId": "PlatformApiController_healthCheck", - "summary": "Platform API health check", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -11483,6 +10559,114 @@ "engine" ] }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] + }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLinkItem": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerSidebarSection": { + "type": "object", + "properties": { + "title": { + "type": "object" + }, + "items": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] + } + } + }, + "required": [ + "title", + "items" + ] + }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + }, + "required": [ + "links" + ] + }, "IRole": { "type": "object", "properties": { @@ -12072,100 +11256,6 @@ "updated_by" ] }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "object" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, "CreateShareMetadataDto": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index de2fca5..098aea0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.3", + "@dadosfera/protospack-v2": "^3.40.0-beta.4", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.3", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.3.tgz", - "integrity": "sha512-m77RSqAO+hZkjUgbdrMqC+6yyyo7indH6fWP3WuygCHGYwdN6FsWTA+EGTAhXQRSgNlI8h358ZpRoILnnGzLMg==", + "version": "3.40.0-beta.4", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.4.tgz", + "integrity": "sha512-ftK4edb3fgGw3qLx+cT79cHYL+1gBoWXd8+SBFQXWHBrnxoRKbanBDp5Yn9PbaYZdRMSqTxSKsBwiTyJzLBunQ==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 586d74c..e4815e6 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.3", + "@dadosfera/protospack-v2": "^3.40.0-beta.4", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 46ba2a4..5ead744 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -288,4 +288,8 @@ export class InputsService { }; return formatedPayload; } + + async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) { + return lastValueFrom(this.inputWriteService.MarkTableDeleted(data)); + } } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index ab39d38..5e6456e 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -48,6 +48,7 @@ import { import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; +import { PlatformApiService } from '../platform-api/platform-api.service'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { TableColumns } from '../inputs/dtos/input.model'; import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; @@ -64,6 +65,7 @@ export class PipelinesController { dadosferaLogger: DadosferaLogger, private pipelinesClientService: PipelinesService, private oldPipelinesService: OldPipelineService, + private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } @@ -222,31 +224,40 @@ export class PipelinesController { language, }); - const result = await this.pipelinesClientService - .findOne({ id }, metadata) - .then(async (res) => { - //{pipeline:{tables: {tables: [], input_id: ''}}} - const parsed = JSON.parse(res.pipeline.config.tables); - const input_id = parsed?.input_id; - const tables = parsed?.tables ?? parsed; + const [pipelineRes, platformPipeline] = await Promise.all([ + this.pipelinesClientService.findOne({ id }, metadata), + this.platformApiService.proxy('GET', `/pipeline/${id.replace(/-/g, '_')}`, user).catch(() => null), + ]); - Object.assign(res.pipeline, { - transformations: res.pipeline.transformations - ? JSON.parse(res.pipeline.transformations) - : [], - config: { - cron: res.pipeline.config.cron, - tables, - input_id - }, - properties: res.pipeline.properties - ? JSON.parse(res.pipeline.properties) - : {}, - }); - return res; + //{pipeline:{tables: {tables: [], input_id: ''}}} + type PipelineTablesConfig = { input_id?: string; tables?: Array<{ name: string }> } | Array<{ name: string }>; + const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); + const input_id = (parsed as { input_id?: string })?.input_id; + let tables: Array = (parsed as { tables?: Array })?.tables ?? (parsed as Array); + + if (platformPipeline?.jobs?.length) { + tables = tables.map((table) => { + const job = (platformPipeline.jobs as Array<{ job_id: string; input?: { table_name: string } }>) + .find((j) => j.input?.table_name === table.name); + return job ? { ...table, job_id: job.job_id } : table; }); + } - return result; + Object.assign(pipelineRes.pipeline, { + transformations: pipelineRes.pipeline.transformations + ? JSON.parse(pipelineRes.pipeline.transformations) + : [], + config: { + cron: pipelineRes.pipeline.config.cron, + tables, + input_id, + }, + properties: pipelineRes.pipeline.properties + ? JSON.parse(pipelineRes.pipeline.properties) + : {}, + }); + + return pipelineRes; } @Patch('/:id') diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index a83b7bd..efe6592 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -11,6 +11,7 @@ import { Inject, BadRequestException, HttpException, + NotFoundException, } from '@nestjs/common'; import { ApiTags, ApiOperation } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; @@ -29,6 +30,7 @@ import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; import { CatalogService } from '../catalog/catalog.service'; import { PackTheMetadata } from '../../utils/PackTheMetadata'; import { ValidationTableDTO } from './platform-api.dto'; +import { InputsService } from '../inputs/inputs.service'; type ValidateTablesDTO = { @@ -54,6 +56,7 @@ export class PlatformApiController { private readonly dynamoDBService: DynamoDBService, private readonly customersService: CustomersService, private readonly catalogService: CatalogService, + private readonly inputsService: InputsService, @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, ) { this.logger = dadosferaLogger.logger; @@ -963,7 +966,7 @@ export class PlatformApiController { } @Delete('jobs/:jobId') - @ApiOperation({ summary: 'Delete a job and mark its table as deleted in DynamoDB' }) + @ApiOperation({ summary: 'Delete a job and mark its table as deleted via in-factory' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteJob( @Param('jobId') jobId: string, @@ -972,47 +975,36 @@ export class PlatformApiController { const normalizedJobId = this.normalizeJobId(jobId); const esPipelineId = this.extractPipelineIdFromJobId(jobId); - // Fetch job details and ES pipeline in parallel — both are reads with no mutual dependency - const [jobResult, esPipelineResult] = await Promise.allSettled([ - this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user), - this.elasticsearchService.getPipeline(user.customer_name, esPipelineId), - ]); + // Fetch job details first — fail fast if job doesn't exist or jobId is invalid + const jobDetails = await this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user); + const tableName: string | undefined = jobDetails?.source_config?.table_name; - const tableName: string | undefined = jobResult.status === 'fulfilled' - ? jobResult.value?.source_config?.table_name - : undefined; - - if (jobResult.status === 'rejected') { - this.logger.warn('deleteJob: could not fetch job before deletion', { jobId, error: jobResult.reason?.message }); + if (!tableName) { + throw new NotFoundException(`Job ${jobId} not found or has no associated table`); } - // Main operation + // Fetch ES pipeline to resolve the DynamoDB input ID + const esPipeline = await this.elasticsearchService.getPipeline(user.customer_name, esPipelineId); + // config.tables in ES stores the DynamoDB input ID (legacy field naming) + const inputId: string | undefined = esPipeline?.config?.tables; + + if (!inputId) { + throw new NotFoundException(`Pipeline data not found for job ${jobId}`); + } + + // Delete the job on platform-api const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); - // Sync deleted status to DynamoDB — follows syncJobInputToDynamoDB pattern - // config.tables in ES stores the DynamoDB input ID (legacy field naming) - const dynamoInputId: string | undefined = esPipelineResult.status === 'fulfilled' - ? esPipelineResult.value?.config?.tables - : undefined; + // Mark table as deleted in DynamoDB via in-factory gRPC + const info = { + customer_id: user.customer_id, + customer: user.customer_name, + user_id: user.user_id, + }; - if (esPipelineResult.status === 'rejected') { - this.logger.warn('deleteJob: could not fetch ES pipeline for DynamoDB sync', { jobId, error: esPipelineResult.reason?.message }); - } - - if (tableName && dynamoInputId) { - await this.dynamoDBService.updateInputTable( - user.customer_id, - dynamoInputId, - tableName, - { status: 'deleted' }, - ).catch((error) => { - this.logger.error('Failed to sync job deletion to DynamoDB', { jobId, tableName, error: error.message }); - }); - } else if (!tableName) { - this.logger.warn('deleteJob: table name not resolved, skipping DynamoDB sync', { jobId }); - } else { - this.logger.warn('deleteJob: no input ID in ES pipeline, skipping DynamoDB sync', { jobId }); - } + await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }).catch((error) => { + this.logger.error('deleteJob: failed to mark table as deleted via in-factory', { jobId, tableName, error: error.message }); + }); return result; } diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index b78e680..b2b0726 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; +import { InputsModule } from '../inputs/inputs.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], From 877cb9d2813eb00a10302e1d02a0e000424b8c7d Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 31 Mar 2026 18:38:25 -0300 Subject: [PATCH 39/47] FEAT: refine PipelineTablesConfig type definition and update parsing logic in PipelinesController --- src/modules/pipelinesV2/pipelines.controller.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 5e6456e..37b3a47 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -54,6 +54,9 @@ import { TableColumns } from '../inputs/dtos/input.model'; import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; +type PipelineTablesConfig = { input_id?: string; tables?: PipelineTable[] } | PipelineTable[]; + @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @UseFilters(new GrpcToHttpExceptionFilter()) @@ -63,6 +66,7 @@ export class PipelinesController { constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + private pipelinesClientService: PipelinesService, private oldPipelinesService: OldPipelineService, private platformApiService: PlatformApiService, @@ -229,11 +233,9 @@ export class PipelinesController { this.platformApiService.proxy('GET', `/pipeline/${id.replace(/-/g, '_')}`, user).catch(() => null), ]); - //{pipeline:{tables: {tables: [], input_id: ''}}} - type PipelineTablesConfig = { input_id?: string; tables?: Array<{ name: string }> } | Array<{ name: string }>; const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); - const input_id = (parsed as { input_id?: string })?.input_id; - let tables: Array = (parsed as { tables?: Array })?.tables ?? (parsed as Array); + const input_id = (parsed as { input_id?: string; tables?: PipelineTable[] })?.input_id; + let tables: PipelineTable[] = (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]); if (platformPipeline?.jobs?.length) { tables = tables.map((table) => { From 968f75b688ccd612a8ea66409b9847cb1af88b75 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 1 Apr 2026 14:26:55 -0300 Subject: [PATCH 40/47] FEAT: update job deletion endpoint to include inputId in path and implement rollback for table deletion --- docsfera.json | 10 ++++- src/modules/inputs/inputs.service.ts | 4 ++ .../pipelinesV2/pipelines.controller.ts | 20 +++------- .../platform-api/platform-api.controller.ts | 37 +++++++++---------- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/docsfera.json b/docsfera.json index c78fbec..7fb3e1a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,11 +4705,19 @@ ] } }, - "/platform/jobs/{jobId}": { + "/platform/inputs/{inputId}/jobs/{jobId}": { "delete": { "operationId": "PlatformApiController_deleteJob", "summary": "Delete a job and mark its table as deleted via in-factory", "parameters": [ + { + "name": "inputId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, { "name": "jobId", "required": true, diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 5ead744..96df891 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -292,4 +292,8 @@ export class InputsService { async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) { return lastValueFrom(this.inputWriteService.MarkTableDeleted(data)); } + + async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) { + return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data)); + } } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 37b3a47..71091ad 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -48,7 +48,6 @@ import { import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; -import { PlatformApiService } from '../platform-api/platform-api.service'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { TableColumns } from '../inputs/dtos/input.model'; import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; @@ -69,7 +68,6 @@ export class PipelinesController { private pipelinesClientService: PipelinesService, private oldPipelinesService: OldPipelineService, - private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } @@ -228,22 +226,14 @@ export class PipelinesController { language, }); - const [pipelineRes, platformPipeline] = await Promise.all([ - this.pipelinesClientService.findOne({ id }, metadata), - this.platformApiService.proxy('GET', `/pipeline/${id.replace(/-/g, '_')}`, user).catch(() => null), - ]); + const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); const input_id = (parsed as { input_id?: string; tables?: PipelineTable[] })?.input_id; - let tables: PipelineTable[] = (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]); - - if (platformPipeline?.jobs?.length) { - tables = tables.map((table) => { - const job = (platformPipeline.jobs as Array<{ job_id: string; input?: { table_name: string } }>) - .find((j) => j.input?.table_name === table.name); - return job ? { ...table, job_id: job.job_id } : table; - }); - } + const normalizedId = id.replace(/-/g, '_'); + const tables: PipelineTable[] = ( + (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]) + ).map((table, index) => ({ ...table, job_id: `${normalizedId}_${index}` })); Object.assign(pipelineRes.pipeline, { transformations: pipelineRes.pipeline.transformations diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index efe6592..b880a9b 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -965,17 +965,16 @@ export class PlatformApiController { ); } - @Delete('jobs/:jobId') + @Delete('inputs/:inputId/jobs/:jobId') @ApiOperation({ summary: 'Delete a job and mark its table as deleted via in-factory' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteJob( + @Param('inputId') inputId: string, @Param('jobId') jobId: string, @User() user: RequestUser, ) { const normalizedJobId = this.normalizeJobId(jobId); - const esPipelineId = this.extractPipelineIdFromJobId(jobId); - // Fetch job details first — fail fast if job doesn't exist or jobId is invalid const jobDetails = await this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user); const tableName: string | undefined = jobDetails?.source_config?.table_name; @@ -983,30 +982,28 @@ export class PlatformApiController { throw new NotFoundException(`Job ${jobId} not found or has no associated table`); } - // Fetch ES pipeline to resolve the DynamoDB input ID - const esPipeline = await this.elasticsearchService.getPipeline(user.customer_name, esPipelineId); - // config.tables in ES stores the DynamoDB input ID (legacy field naming) - const inputId: string | undefined = esPipeline?.config?.tables; - - if (!inputId) { - throw new NotFoundException(`Pipeline data not found for job ${jobId}`); - } - - // Delete the job on platform-api - const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); - - // Mark table as deleted in DynamoDB via in-factory gRPC const info = { customer_id: user.customer_id, customer: user.customer_name, user_id: user.user_id, }; - await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }).catch((error) => { - this.logger.error('deleteJob: failed to mark table as deleted via in-factory', { jobId, tableName, error: error.message }); - }); + const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }); - return result; + try { + await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); + + const updatedTable = updatedInput.tables?.find((t: any) => t.name === tableName); + return updatedTable ?? { name: tableName, is_deleted: true }; + } catch (error) { + this.logger.error('deleteJob: platform-api delete failed, attempting rollback', { jobId, tableName, error: error.message }); + try { + await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info }); + } catch (rollbackError) { + this.logger.error('deleteJob: rollback failed', { jobId, tableName, error: rollbackError.message }); + } + throw error; + } } // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== From d0122a9c20d5ae4e175ec979ecd67e54be6d3575 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 1 Apr 2026 14:43:51 -0300 Subject: [PATCH 41/47] FEAT: remove unused status field from InputDocument and updateInputTable method --- src/services/dynamodb/dynamodb.service.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index a0d3537..90e5efa 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -30,7 +30,6 @@ export interface InputDocument { type: string; columns?: string[]; reference_column?: ReferenceColumn; - status?: string; }>; credentials?: Record; } @@ -168,7 +167,6 @@ export class DynamoDBService { type?: string; columns?: string[]; reference_column?: ReferenceColumn | null; - status?: string; }, ): Promise { const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); @@ -209,8 +207,6 @@ export class DynamoDBService { updatedTable.reference_column = changes.reference_column; } } - if ('status' in changes) updatedTable.status = changes.status; - tables[tableIndex] = updatedTable; // Save updated document From 837a9d7265be8f6af0c51f70f18335cf68ce4a84 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 2 Apr 2026 15:40:07 -0300 Subject: [PATCH 42/47] FEAT: update job deletion logic to mark tables as deleted and adjust API endpoints accordingly --- docsfera.json | 25 +++++----- package-lock.json | 8 ++-- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 9 ++-- .../platform-api/platform-api.controller.ts | 39 ++++++++------- src/services/dynamodb/dynamodb.service.ts | 48 ------------------- 6 files changed, 42 insertions(+), 89 deletions(-) diff --git a/docsfera.json b/docsfera.json index 7fb3e1a..29f2ff3 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,10 +4705,10 @@ ] } }, - "/platform/inputs/{inputId}/jobs/{jobId}": { + "/platform/inputs/{inputId}/tables/{tableName}": { "delete": { - "operationId": "PlatformApiController_deleteJob", - "summary": "Delete a job and mark its table as deleted via in-factory", + "operationId": "PlatformApiController_deleteTable", + "summary": "Mark a table as deleted and delete its associated job via platform-api", "parameters": [ { "name": "inputId", @@ -4719,24 +4719,25 @@ } }, { - "name": "jobId", + "name": "tableName", "required": true, "in": "path", "schema": { "type": "string" } + }, + { + "name": "pipelineId", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/package-lock.json b/package-lock.json index 098aea0..a7f9147 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.4", + "@dadosfera/protospack-v2": "^3.40.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.4", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.4.tgz", - "integrity": "sha512-ftK4edb3fgGw3qLx+cT79cHYL+1gBoWXd8+SBFQXWHBrnxoRKbanBDp5Yn9PbaYZdRMSqTxSKsBwiTyJzLBunQ==", + "version": "3.40.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.5.tgz", + "integrity": "sha512-iocKv/XXp2jKAasO5ONgm31cKfLgNsU4pEKZMr6YnR7nQaH11WcW7rnuagNxWoik++wLUqbYyf0bZWRDzMlCPA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index e4815e6..18bb538 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.4", + "@dadosfera/protospack-v2": "^3.40.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 71091ad..e91d217 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -54,7 +54,7 @@ import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; -type PipelineTablesConfig = { input_id?: string; tables?: PipelineTable[] } | PipelineTable[]; +type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -229,11 +229,8 @@ export class PipelinesController { const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); - const input_id = (parsed as { input_id?: string; tables?: PipelineTable[] })?.input_id; - const normalizedId = id.replace(/-/g, '_'); - const tables: PipelineTable[] = ( - (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]) - ).map((table, index) => ({ ...table, job_id: `${normalizedId}_${index}` })); + const input_id = parsed.input_id; + const tables: PipelineTable[] = parsed.tables ?? []; Object.assign(pipelineRes.pipeline, { transformations: pipelineRes.pipeline.transformations diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index b880a9b..e850f3e 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -965,42 +965,45 @@ export class PlatformApiController { ); } - @Delete('inputs/:inputId/jobs/:jobId') - @ApiOperation({ summary: 'Delete a job and mark its table as deleted via in-factory' }) + @Delete('inputs/:inputId/tables/:tableName') + @ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) - async deleteJob( + async deleteTable( @Param('inputId') inputId: string, - @Param('jobId') jobId: string, + @Param('tableName') tableName: string, + @Query('pipelineId') pipelineId: string, @User() user: RequestUser, ) { - const normalizedJobId = this.normalizeJobId(jobId); - - const jobDetails = await this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user); - const tableName: string | undefined = jobDetails?.source_config?.table_name; - - if (!tableName) { - throw new NotFoundException(`Job ${jobId} not found or has no associated table`); - } - const info = { customer_id: user.customer_id, customer: user.customer_name, user_id: user.user_id, }; + this.logger.info('deleteTable: marking table as deleted', { inputId, tableName }); const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }); + this.logger.info('deleteTable: table marked as deleted', { inputId, tableName }); try { - await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); + const normalizedPipelineId = this.normalizeJobId(pipelineId); + this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId }); + const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user); + this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length }); - const updatedTable = updatedInput.tables?.find((t: any) => t.name === tableName); - return updatedTable ?? { name: tableName, is_deleted: true }; + const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName); + if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`); + + this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id }); + await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user); + this.logger.info('deleteTable: job deleted', { jobId: job.job_id }); + + return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at }; } catch (error) { - this.logger.error('deleteJob: platform-api delete failed, attempting rollback', { jobId, tableName, error: error.message }); + this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message }); try { await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info }); } catch (rollbackError) { - this.logger.error('deleteJob: rollback failed', { jobId, tableName, error: rollbackError.message }); + this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message }); } throw error; } diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index 90e5efa..0022b35 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -235,52 +235,4 @@ export class DynamoDBService { } } - /** - * Remove a specific table entry from the input document. - * Fetches the current document, removes the matching table, and saves. - */ - async removeInputTable( - clientId: string, - inputId: string, - tableName: string, - ): Promise { - const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); - - this.logger.info('DynamoDB: Removing input table', { - inputId, - tableName, - }); - - const current = await this.findInput(clientId, inputId); - if (!current) { - this.logger.warn('DynamoDB: Input not found for table removal', { inputId }); - return; - } - - const tables = (current.tables || []).filter((t) => t.name !== tableName); - - const putCommand = new PutCommand({ - TableName: dynamoTableName, - Item: { - ...current, - tables, - updated_at: new Date().toISOString(), - }, - }); - - try { - await this.documentClient.send(putCommand); - this.logger.info('DynamoDB: Input table removed successfully', { - inputId, - tableName, - }); - } catch (error) { - this.logger.error('DynamoDB: Failed to remove input table', { - inputId, - tableName, - error: error.message, - }); - throw error; - } - } } From 8a9b58b6121ef758f30c5c5028b87ec39465b77e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 2 Apr 2026 16:40:35 -0300 Subject: [PATCH 43/47] FIX: rename method normalizeJobId to normalizePipelineId for clarity --- src/modules/platform-api/platform-api.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index e850f3e..9e5419c 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -985,7 +985,7 @@ export class PlatformApiController { this.logger.info('deleteTable: table marked as deleted', { inputId, tableName }); try { - const normalizedPipelineId = this.normalizeJobId(pipelineId); + const normalizedPipelineId = this.normalizePipelineId(pipelineId); this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId }); const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user); this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length }); From efb0b5864806aa4d9bc8fed0b61aeb2de690cee1 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 2 Apr 2026 17:28:42 -0300 Subject: [PATCH 44/47] FEAT: update deleteTable endpoint to use pipelineId in path and refactor parameters --- docsfera.json | 26 +++++++------------ .../platform-api/platform-api.controller.ts | 7 ++--- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/docsfera.json b/docsfera.json index 29f2ff3..fc5e27b 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,11 +4705,19 @@ ] } }, - "/platform/inputs/{inputId}/tables/{tableName}": { + "/platform/pipelines/{pipelineId}/inputs/{inputId}": { "delete": { "operationId": "PlatformApiController_deleteTable", "summary": "Mark a table as deleted and delete its associated job via platform-api", "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, { "name": "inputId", "required": true, @@ -4717,22 +4725,6 @@ "schema": { "type": "string" } - }, - { - "name": "tableName", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "pipelineId", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 9e5419c..5fd0b94 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -965,15 +965,16 @@ export class PlatformApiController { ); } - @Delete('inputs/:inputId/tables/:tableName') + @Delete('pipelines/:pipelineId/inputs/:inputId') @ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteTable( + @Param('pipelineId') pipelineId: string, @Param('inputId') inputId: string, - @Param('tableName') tableName: string, - @Query('pipelineId') pipelineId: string, + @Body() body: { table_name: string }, @User() user: RequestUser, ) { + const tableName = body.table_name; const info = { customer_id: user.customer_id, customer: user.customer_name, From 0086528d51ad937d24e85a1bc25f51e8aab92a52 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 7 Apr 2026 18:05:30 -0300 Subject: [PATCH 45/47] FEAT: update connectors --- src/modules/pipelinesV2/pipelines.service.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 17eb41b..7475a1d 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -422,6 +422,7 @@ export class PipelinesService implements OnModuleInit { try { await this.updatePlatformJobs( pipelineIdFormat, + updateInputResponse.input.type, updateInputDTO, user ); @@ -487,7 +488,7 @@ export class PipelinesService implements OnModuleInit { } - async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) { + async updatePlatformJobs(pipelineId: string, pipelineType: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) { const jobsUpdated = []; for (const [index, table] of updateInputDTO.tables.entries()) { @@ -528,8 +529,14 @@ export class PipelinesService implements OnModuleInit { if (table.type) { hasUpdateSyncMode = true; + jobSyncMode['target_load_type'] = table.type; + } + if (table.type === "full_load" && pipelineType === "application") { + hasUpdateSyncMode = true; + + jobSyncMode['target_load_type'] = "FULL_TABLE"; } if(hasUpdateSyncMode) { From 4ab81471493d1de5b58ae7e41a40c624a854b54a Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 9 Apr 2026 17:23:58 -0300 Subject: [PATCH 46/47] FEAT: remove endpoints --- .../platform-api/platform-api.controller.ts | 265 ------------------ 1 file changed, 265 deletions(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 5fd0b94..ff5539a 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -1012,225 +1012,6 @@ export class PlatformApiController { // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== - @Get('jobs/jdbc/:jobId') - @ApiOperation({ summary: 'Get JDBC job details' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getJdbcJob(@Param('jobId') jobId: string, @User() user: RequestUser) { - // Normalize job ID for Platform API (replace - with _) - const normalizedJobId = this.normalizeJobId(jobId); - return this.platformApiService.proxy('GET', `/jobs/jdbc/${normalizedJobId}`, user); - } - - @Post('jobs/jdbc/:jobId/sync-mode') - @ApiOperation({ summary: 'Update JDBC job sync mode' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) - async updateJdbcSyncMode( - @Param('jobId') jobId: string, - @Body() body: any, - @User() user: RequestUser, - ) { - // Normalize job ID for Platform API (replace - with _) - const normalizedJobId = this.normalizeJobId(jobId); - - const result = await this.platformApiService.proxy( - 'POST', - `/jobs/jdbc/${normalizedJobId}/sync-mode`, - user, - body, - ); - - // Sync to DynamoDB (pass raw jobId for pipeline extraction) - await this.syncJdbcSyncModeToDynamoDB(jobId, body, user); - - return result; - } - - @Post('jobs/:jobId/rename-tables') - @ApiOperation({ summary: 'Rename job output tables and sync to catalog' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) - async renameJobTables( - @Param('jobId') jobId: string, - @Body() body: RenameTablesBody, - @User() user: RequestUser, - ) { - const normalizedJobId = this.normalizeJobId(jobId); - - const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user); - - const result = await this.platformApiService.proxy( - 'POST', - `/jobs/${normalizedJobId}/rename-tables`, - user, - body, - ); - - try { - await this.syncTableRenameToCatalog(jobId, body, currentJob, user); - } catch (error) { - this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message }); - - const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {}); - if (reverseBody) { - try { - await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody); - this.logger.info('Snowflake rename rolled back', { jobId }); - } catch (rollbackError) { - this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message }); - } - } - - throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500); - } - - return result; - } - - private buildSnowflakeRollbackBody( - body: RenameTablesBody, - outputConfig: any, - ): RenameTablesBody | null { - const reverse: RenameTablesBody = {}; - - if (body.raw) { - const nested = outputConfig.raw; - const oldTableName = nested?.table_name || outputConfig.table_name; - const oldTableSchema = nested?.table_schema || 'PUBLIC'; - if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema }; - } - - if (body.qualify) { - const nested = outputConfig.qualify; - if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' }; - } - - return Object.keys(reverse).length > 0 ? reverse : null; - } - - /** - * Sync table rename to Elasticsearch and Nimbus. - * - * For each target (raw, qualify): - * 1. Resolve old table name from output_config - * 2. Find the ES data asset by pipeline + table + schema - * 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview - * 4. If any step fails, rollback all completed steps for that target - */ - private async syncTableRenameToCatalog( - jobId: string, - body: RenameTablesBody, - currentJob: any, - user: RequestUser, - ): Promise { - const pipelineId = this.extractPipelineIdFromJobId(jobId); - const outputConfig = currentJob.output_config || {}; - const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } }); - const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`; - - const targets = this.buildRenameTargets(body, outputConfig); - - for (const { key, oldTableName, oldTableSchema, newValues } of targets) { - const rollbackSteps: Array<() => Promise> = []; - - try { - const dataAsset = await this.elasticsearchService.findDataAssetByTable( - user.customer_name, oldTableName, oldTableSchema, - ); - - if (!dataAsset) { - this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema }); - continue; - } - - const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset; - const oldValues = { table_name: oldTableName, table_schema: oldTableSchema }; - - // ES update - const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name }; - await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields); - rollbackSteps.push(() => this.elasticsearchService.updateDataAsset( - user.customer_name, esAssetId, - { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName }, - )); - - const newTableNameUpper = newValues.table_name.toUpperCase(); - const newTableSchemaUpper = newValues.table_schema.toUpperCase(); - const oldTableNameUpper = oldTableName.toUpperCase(); - const oldTableSchemaUpper = oldTableSchema.toUpperCase(); - - // Nimbus table-metadata - if (nimbusId) { - await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper }); - rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper })); - } - - // Nimbus column-metadata - await this.catalogService.renameColumnMetadataOnNimbus( - nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, - ); - rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus( - nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, - )); - - // Nimbus data-preview - await this.catalogService.renameDataPreviewOnNimbus( - nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, - ); - rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus( - nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, - )); - - this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name }); - } catch (error) { - this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message }); - await this.executeRollback(rollbackSteps, key, jobId); - throw error; - } - } - } - - private buildRenameTargets( - body: RenameTablesBody, - outputConfig: any, - ): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> { - const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' }; - const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = []; - - for (const key of ['raw', 'qualify'] as const) { - if (!body[key]) continue; - - const nested = outputConfig[key]; - - // qualify: only sync if output_config.qualify already exists - if (key === 'qualify' && !nested?.table_name) continue; - - const oldTableName = nested?.table_name || outputConfig.table_name; - if (!oldTableName) continue; - - targets.push({ - key, - oldTableName, - oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key], - newValues: body[key], - }); - } - - return targets; - } - - private async executeRollback( - steps: Array<() => Promise>, - targetKey: string, - jobId: string, - ): Promise { - for (const rollback of steps.reverse()) { - try { - await rollback(); - } catch (error) { - this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message }); - } - } - } - @Get('jobs/jdbc/configs/allowed_datatypes') @ApiOperation({ summary: 'Get allowed datatypes for JDBC' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) @@ -1242,52 +1023,6 @@ export class PlatformApiController { ); } - // ==================== JOBS - SINGER REPLICATION ROUTES ==================== - - @Get('jobs/singer/:jobId') - @ApiOperation({ summary: 'Get Singer job details' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getSingerJob(@Param('jobId') jobId: string, @User() user: RequestUser) { - // Normalize job ID for Platform API (replace - with _) - const normalizedJobId = this.normalizeJobId(jobId); - return this.platformApiService.proxy('GET', `/jobs/singer/${normalizedJobId}`, user); - } - - @Post('jobs/singer/:jobId/sync-mode') - @ApiOperation({ summary: 'Update Singer job sync mode' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) - async updateSingerSyncMode( - @Param('jobId') jobId: string, - @Body() body: any, - @User() user: RequestUser, - ) { - // Normalize job ID for Platform API (replace - with _) - const normalizedJobId = this.normalizeJobId(jobId); - - const result = await this.platformApiService.proxy( - 'POST', - `/jobs/singer/${normalizedJobId}/sync-mode`, - user, - body, - ); - - // Sync to DynamoDB (pass raw jobId for pipeline extraction) - await this.syncSingerSyncModeToDynamoDB(jobId, body, user); - - return result; - } - - // ==================== JOBS - S3 ROUTES ==================== - - @Get('jobs/s3/:jobId') - @ApiOperation({ summary: 'Get S3 job details' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getS3Job(@Param('jobId') jobId: string, @User() user: RequestUser) { - // Normalize job ID for Platform API (replace - with _) - const normalizedJobId = this.normalizeJobId(jobId); - return this.platformApiService.proxy('GET', `/jobs/s3/${normalizedJobId}`, user); - } - // ==================== HEALTH ROUTE ==================== @Get('health') From 901518d26c2fa430fa85089af9ebca6c48a8ca1e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 9 Apr 2026 17:38:28 -0300 Subject: [PATCH 47/47] FIX: remove full table --- src/modules/pipelinesV2/pipelines.service.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 7475a1d..650bb1c 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -533,12 +533,6 @@ export class PipelinesService implements OnModuleInit { jobSyncMode['target_load_type'] = table.type; } - if (table.type === "full_load" && pipelineType === "application") { - hasUpdateSyncMode = true; - - jobSyncMode['target_load_type'] = "FULL_TABLE"; - } - if(hasUpdateSyncMode) { jobUpdate["sync_mode"] = jobSyncMode; }