From 2d46ac32133a0702fec8d675f25182677479c3fd Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 30 Mar 2026 22:24:10 -0300 Subject: [PATCH 01/10] FEAT: add delete job endpoint and update related services; update protospack-v2 version to 3.40.0-beta.3 --- docsfera.json | 1162 +++++++++++++++-- package-lock.json | 9 +- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 9 +- .../platform-api/platform-api.controller.ts | 74 ++ .../platform-api/platform-api.module.ts | 3 +- src/services/dynamodb/dynamodb.service.ts | 62 +- 7 files changed, 1199 insertions(+), 122 deletions(-) diff --git a/docsfera.json b/docsfera.json index a99d60e..deb8c8f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7952,6 +7952,966 @@ } } }, + "/platform/pipeline": { + "post": { + "operationId": "PlatformApiController_createPipeline", + "summary": "Create a new pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines": { + "get": { + "operationId": "PlatformApiController_getPipelines", + "summary": "List all pipelines for customer", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}": { + "get": { + "operationId": "PlatformApiController_getPipeline", + "summary": "Get pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_updatePipeline", + "summary": "Update pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "PlatformApiController_deletePipeline", + "summary": "Delete pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/execute": { + "post": { + "operationId": "PlatformApiController_executePipeline", + "summary": "Execute a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pause": { + "post": { + "operationId": "PlatformApiController_pausePipeline", + "summary": "Pause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/unpause": { + "post": { + "operationId": "PlatformApiController_unpausePipeline", + "summary": "Unpause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/memory": { + "put": { + "operationId": "PlatformApiController_updatePipelineMemory", + "summary": "Update pipeline memory configuration", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/metadata": { + "put": { + "operationId": "PlatformApiController_updatePipelineMetadata", + "summary": "Update pipeline metadata", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/metadata": { + "get": { + "operationId": "PlatformApiController_getPipelinesMetadata", + "summary": "Get all pipelines metadata", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables": { + "get": { + "operationId": "PlatformApiController_getAvailableTables", + "summary": "Get all tables available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get all schemas available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate tables and schemas", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run": { + "get": { + "operationId": "PlatformApiController_getPipelineRuns", + "summary": "Get pipeline runs for a pipeline", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { + "get": { + "operationId": "PlatformApiController_getPipelineRun", + "summary": "Get specific pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pipeline_run/{runId}/logs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunLogs", + "summary": "Get pipeline run logs", + "parameters": [ + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/input": { + "put": { + "operationId": "PlatformApiController_updateJobInput", + "summary": "Update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_patchJobInput", + "summary": "Partial update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/memory": { + "put": { + "operationId": "PlatformApiController_updateJobMemory", + "summary": "Update job memory configuration", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/reset-state": { + "post": { + "operationId": "PlatformApiController_resetJobState", + "summary": "Reset job state", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}": { + "delete": { + "operationId": "PlatformApiController_deleteJob", + "summary": "Delete a job and sync pipeline data", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}": { + "get": { + "operationId": "PlatformApiController_getJdbcJob", + "summary": "Get JDBC job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateJdbcSyncMode", + "summary": "Update JDBC job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/configs/allowed_datatypes": { + "get": { + "operationId": "PlatformApiController_getJdbcAllowedDatatypes", + "summary": "Get allowed datatypes for JDBC", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}": { + "get": { + "operationId": "PlatformApiController_getSingerJob", + "summary": "Get Singer job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateSingerSyncMode", + "summary": "Update Singer job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/s3/{jobId}": { + "get": { + "operationId": "PlatformApiController_getS3Job", + "summary": "Get S3 job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/health": { + "get": { + "operationId": "PlatformApiController_healthCheck", + "summary": "Platform API health check", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -10523,114 +11483,6 @@ "engine" ] }, - "ValidationTableDTO": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tables" - ] - }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "object" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, "IRole": { "type": "object", "properties": { @@ -11220,6 +12072,100 @@ "updated_by" ] }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLinkItem": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerSidebarSection": { + "type": "object", + "properties": { + "title": { + "type": "object" + }, + "items": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] + } + } + }, + "required": [ + "title", + "items" + ] + }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + }, + "required": [ + "links" + ] + }, "CreateShareMetadataDto": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index adacb76..de2fca5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.2", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,10 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz", - "integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==", - "license": "ISC", + "version": "3.40.0-beta.3", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.3.tgz", + "integrity": "sha512-m77RSqAO+hZkjUgbdrMqC+6yyyo7indH6fWP3WuygCHGYwdN6FsWTA+EGTAhXQRSgNlI8h358ZpRoILnnGzLMg==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index fd480de..586d74c 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.2", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 1c00bcd..ab39d38 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -224,11 +224,12 @@ export class PipelinesController { const result = await this.pipelinesClientService .findOne({ id }, metadata) - .then((res) => { + .then(async (res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} - let tables = JSON.parse(res.pipeline.config.tables); - const input_id = tables?.input_id; - if (tables?.tables) tables = tables.tables; + const parsed = JSON.parse(res.pipeline.config.tables); + const input_id = parsed?.input_id; + const tables = parsed?.tables ?? parsed; + Object.assign(res.pipeline, { transformations: res.pipeline.transformations ? JSON.parse(res.pipeline.transformations) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 8c42102..74caaf9 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -962,6 +962,80 @@ export class PlatformApiController { ); } + @Delete('jobs/:jobId') + @ApiOperation({ summary: 'Delete a job and sync pipeline data' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) + async deleteJob( + @Param('jobId') jobId: string, + @User() user: RequestUser, + ) { + // Get job details before deletion to extract table_name and connector type + const normalizedJobId = this.normalizeJobId(jobId); + let tableName: string | undefined; + + this.logger.info('[deleteJob] Start', { jobId }); + + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${normalizedJobId}`, + user, + ); + tableName = job?.source_config?.table_name; + this.logger.info('[deleteJob] Job fetched', { jobId, tableName }); + } catch (err) { + this.logger.warn('[deleteJob] Could not fetch job before deletion', { jobId, error: err.message }); + } + + const result = await this.platformApiService.proxy( + 'DELETE', + `/jobs/${normalizedJobId}`, + user, + ); + this.logger.info('[deleteJob] Job deleted from platform-api', { jobId }); + + // Mark the table as deleted via in-factory (which has the correct DynamoDB credentials). + // config.tables in ES stores the DynamoDB input ID (legacy naming). + if (tableName) { + try { + const esPipelineId = this.extractPipelineIdFromJobId(jobId); + const esPipeline = await this.elasticsearchService.getPipeline( + user.customer_name, + esPipelineId, + ); + // config.tables holds the DynamoDB input ID (see syncJobInputToDynamoDB for context) + const dynamoInputId = esPipeline?.config?.tables; + + if (dynamoInputId) { + const info = { customer_id: user.customer_id, user_id: user.user_id, customer: user.customer_name }; + + // Read current input to get full tables array (adjustInputPayload flattens the response) + const { input } = await this.inputsService.findOne({ id: dynamoInputId, info }); + const currentTables: any[] = input?.tables ?? []; + + const updatedTables = currentTables.map((t: any) => + t.name === tableName ? { ...t, status: 'deleted' } : t, + ); + + await this.inputsService.update(dynamoInputId, { tables: updatedTables }, info); + this.logger.info('[deleteJob] Marked table as deleted via in-factory', { jobId, tableName }); + } else { + this.logger.warn('[deleteJob] No input ID in ES pipeline, skipping DynamoDB update', { jobId, tableName }); + } + } catch (error) { + this.logger.error('[deleteJob] Failed to mark table as deleted via in-factory', { + jobId, + tableName, + error: error.message, + }); + } + } else { + this.logger.warn('[deleteJob] tableName not resolved, skipping DynamoDB update', { jobId }); + } + + return result; + } + // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== @Get('jobs/jdbc/:jobId') diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index b78e680..d202700 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; +import { PipelinesV2Module } from '../pipelinesV2/pipelines.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, PipelinesV2Module], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index 15dde41..a0d3537 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -30,6 +30,7 @@ export interface InputDocument { type: string; columns?: string[]; reference_column?: ReferenceColumn; + status?: string; }>; credentials?: Record; } @@ -125,9 +126,13 @@ export class DynamoDBService { }, }); - const { Item } = await this.documentClient.send(getCommand); - - return Item as InputDocument | null; + try { + const { Item } = await this.documentClient.send(getCommand); + return Item as InputDocument | null; + } catch (error) { + this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message }); + throw error; + } } async deleteInput(clientId: string, inputId: string): Promise { @@ -163,6 +168,7 @@ export class DynamoDBService { type?: string; columns?: string[]; reference_column?: ReferenceColumn | null; + status?: string; }, ): Promise { const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); @@ -203,6 +209,7 @@ export class DynamoDBService { updatedTable.reference_column = changes.reference_column; } } + if ('status' in changes) updatedTable.status = changes.status; tables[tableIndex] = updatedTable; @@ -231,4 +238,53 @@ export class DynamoDBService { throw error; } } + + /** + * Remove a specific table entry from the input document. + * Fetches the current document, removes the matching table, and saves. + */ + async removeInputTable( + clientId: string, + inputId: string, + tableName: string, + ): Promise { + const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); + + this.logger.info('DynamoDB: Removing input table', { + inputId, + tableName, + }); + + const current = await this.findInput(clientId, inputId); + if (!current) { + this.logger.warn('DynamoDB: Input not found for table removal', { inputId }); + return; + } + + const tables = (current.tables || []).filter((t) => t.name !== tableName); + + const putCommand = new PutCommand({ + TableName: dynamoTableName, + Item: { + ...current, + tables, + updated_at: new Date().toISOString(), + }, + }); + + try { + await this.documentClient.send(putCommand); + this.logger.info('DynamoDB: Input table removed successfully', { + inputId, + tableName, + }); + } catch (error) { + this.logger.error('DynamoDB: Failed to remove input table', { + inputId, + tableName, + error: error.message, + }); + throw error; + } + } } From efc1f49d92bc5d1473d4f7ad8a7da9ae75b35d47 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 30 Mar 2026 22:39:40 -0300 Subject: [PATCH 02/10] FEAT: update delete job endpoint summary and remove unused InputsModule from platform-api module --- docsfera.json | 2 +- .../platform-api/platform-api.controller.ts | 89 ++++++++----------- 2 files changed, 36 insertions(+), 55 deletions(-) diff --git a/docsfera.json b/docsfera.json index deb8c8f..ec823b4 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8645,7 +8645,7 @@ "/platform/jobs/{jobId}": { "delete": { "operationId": "PlatformApiController_deleteJob", - "summary": "Delete a job and sync pipeline data", + "summary": "Delete a job and mark its table as deleted in DynamoDB", "parameters": [ { "name": "jobId", diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 74caaf9..a83b7bd 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -963,74 +963,55 @@ export class PlatformApiController { } @Delete('jobs/:jobId') - @ApiOperation({ summary: 'Delete a job and sync pipeline data' }) + @ApiOperation({ summary: 'Delete a job and mark its table as deleted in DynamoDB' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteJob( @Param('jobId') jobId: string, @User() user: RequestUser, ) { - // Get job details before deletion to extract table_name and connector type const normalizedJobId = this.normalizeJobId(jobId); - let tableName: string | undefined; + const esPipelineId = this.extractPipelineIdFromJobId(jobId); - this.logger.info('[deleteJob] Start', { jobId }); + // Fetch job details and ES pipeline in parallel — both are reads with no mutual dependency + const [jobResult, esPipelineResult] = await Promise.allSettled([ + this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user), + this.elasticsearchService.getPipeline(user.customer_name, esPipelineId), + ]); - try { - const job = await this.platformApiService.proxy( - 'GET', - `/jobs/${normalizedJobId}`, - user, - ); - tableName = job?.source_config?.table_name; - this.logger.info('[deleteJob] Job fetched', { jobId, tableName }); - } catch (err) { - this.logger.warn('[deleteJob] Could not fetch job before deletion', { jobId, error: err.message }); + const tableName: string | undefined = jobResult.status === 'fulfilled' + ? jobResult.value?.source_config?.table_name + : undefined; + + if (jobResult.status === 'rejected') { + this.logger.warn('deleteJob: could not fetch job before deletion', { jobId, error: jobResult.reason?.message }); } - const result = await this.platformApiService.proxy( - 'DELETE', - `/jobs/${normalizedJobId}`, - user, - ); - this.logger.info('[deleteJob] Job deleted from platform-api', { jobId }); + // Main operation + const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); - // Mark the table as deleted via in-factory (which has the correct DynamoDB credentials). - // config.tables in ES stores the DynamoDB input ID (legacy naming). - if (tableName) { - try { - const esPipelineId = this.extractPipelineIdFromJobId(jobId); - const esPipeline = await this.elasticsearchService.getPipeline( - user.customer_name, - esPipelineId, - ); - // config.tables holds the DynamoDB input ID (see syncJobInputToDynamoDB for context) - const dynamoInputId = esPipeline?.config?.tables; + // Sync deleted status to DynamoDB — follows syncJobInputToDynamoDB pattern + // config.tables in ES stores the DynamoDB input ID (legacy field naming) + const dynamoInputId: string | undefined = esPipelineResult.status === 'fulfilled' + ? esPipelineResult.value?.config?.tables + : undefined; - if (dynamoInputId) { - const info = { customer_id: user.customer_id, user_id: user.user_id, customer: user.customer_name }; + if (esPipelineResult.status === 'rejected') { + this.logger.warn('deleteJob: could not fetch ES pipeline for DynamoDB sync', { jobId, error: esPipelineResult.reason?.message }); + } - // Read current input to get full tables array (adjustInputPayload flattens the response) - const { input } = await this.inputsService.findOne({ id: dynamoInputId, info }); - const currentTables: any[] = input?.tables ?? []; - - const updatedTables = currentTables.map((t: any) => - t.name === tableName ? { ...t, status: 'deleted' } : t, - ); - - await this.inputsService.update(dynamoInputId, { tables: updatedTables }, info); - this.logger.info('[deleteJob] Marked table as deleted via in-factory', { jobId, tableName }); - } else { - this.logger.warn('[deleteJob] No input ID in ES pipeline, skipping DynamoDB update', { jobId, tableName }); - } - } catch (error) { - this.logger.error('[deleteJob] Failed to mark table as deleted via in-factory', { - jobId, - tableName, - error: error.message, - }); - } + if (tableName && dynamoInputId) { + await this.dynamoDBService.updateInputTable( + user.customer_id, + dynamoInputId, + tableName, + { status: 'deleted' }, + ).catch((error) => { + this.logger.error('Failed to sync job deletion to DynamoDB', { jobId, tableName, error: error.message }); + }); + } else if (!tableName) { + this.logger.warn('deleteJob: table name not resolved, skipping DynamoDB sync', { jobId }); } else { - this.logger.warn('[deleteJob] tableName not resolved, skipping DynamoDB update', { jobId }); + this.logger.warn('deleteJob: no input ID in ES pipeline, skipping DynamoDB sync', { jobId }); } return result; From ecca9106f0d9ebd73613185310085ad13e1e4207 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 31 Mar 2026 11:17:32 -0300 Subject: [PATCH 03/10] FEAT: remove PipelinesV2Module from PlatformApiModule imports --- src/modules/platform-api/platform-api.module.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index d202700..b78e680 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,10 +8,9 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; -import { PipelinesV2Module } from '../pipelinesV2/pipelines.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, PipelinesV2Module], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], From 59efdb6272cf4072342946e9f4935bacbb8ec471 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 31 Mar 2026 16:36:59 -0300 Subject: [PATCH 04/10] FEAT: update job deletion and mark associated table as deleted in DynamoDB. Update protospack-v2 --- docsfera.json | 1198 ++--------------- package-lock.json | 8 +- package.json | 2 +- src/modules/inputs/inputs.service.ts | 4 + .../pipelinesV2/pipelines.controller.ts | 55 +- .../platform-api/platform-api.controller.ts | 64 +- .../platform-api/platform-api.module.ts | 3 +- 7 files changed, 216 insertions(+), 1118 deletions(-) diff --git a/docsfera.json b/docsfera.json index ec823b4..c78fbec 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,6 +4705,42 @@ ] } }, + "/platform/jobs/{jobId}": { + "delete": { + "operationId": "PlatformApiController_deleteJob", + "summary": "Delete a job and mark its table as deleted via in-factory", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/jdbc/{jobId}": { "get": { "operationId": "PlatformApiController_getJdbcJob", @@ -7952,966 +7988,6 @@ } } }, - "/platform/pipeline": { - "post": { - "operationId": "PlatformApiController_createPipeline", - "summary": "Create a new pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines": { - "get": { - "operationId": "PlatformApiController_getPipelines", - "summary": "List all pipelines for customer", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}": { - "get": { - "operationId": "PlatformApiController_getPipeline", - "summary": "Get pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_updatePipeline", - "summary": "Update pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "delete": { - "operationId": "PlatformApiController_deletePipeline", - "summary": "Delete pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/execute": { - "post": { - "operationId": "PlatformApiController_executePipeline", - "summary": "Execute a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pause": { - "post": { - "operationId": "PlatformApiController_pausePipeline", - "summary": "Pause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/unpause": { - "post": { - "operationId": "PlatformApiController_unpausePipeline", - "summary": "Unpause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/memory": { - "put": { - "operationId": "PlatformApiController_updatePipelineMemory", - "summary": "Update pipeline memory configuration", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/metadata": { - "put": { - "operationId": "PlatformApiController_updatePipelineMetadata", - "summary": "Update pipeline metadata", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/metadata": { - "get": { - "operationId": "PlatformApiController_getPipelinesMetadata", - "summary": "Get all pipelines metadata", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables": { - "get": { - "operationId": "PlatformApiController_getAvailableTables", - "summary": "Get all tables available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/schemas": { - "get": { - "operationId": "PlatformApiController_getAvailableSchemas", - "summary": "Get all schemas available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables/validate": { - "post": { - "operationId": "PlatformApiController_validateTableAndSchema", - "summary": "Validate tables and schemas", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run": { - "get": { - "operationId": "PlatformApiController_getPipelineRuns", - "summary": "Get pipeline runs for a pipeline", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { - "get": { - "operationId": "PlatformApiController_getPipelineRun", - "summary": "Get specific pipeline run", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pipeline_run/{runId}/logs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunLogs", - "summary": "Get pipeline run logs", - "parameters": [ - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/input": { - "put": { - "operationId": "PlatformApiController_updateJobInput", - "summary": "Update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_patchJobInput", - "summary": "Partial update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/memory": { - "put": { - "operationId": "PlatformApiController_updateJobMemory", - "summary": "Update job memory configuration", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/reset-state": { - "post": { - "operationId": "PlatformApiController_resetJobState", - "summary": "Reset job state", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}": { - "delete": { - "operationId": "PlatformApiController_deleteJob", - "summary": "Delete a job and mark its table as deleted in DynamoDB", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}": { - "get": { - "operationId": "PlatformApiController_getJdbcJob", - "summary": "Get JDBC job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateJdbcSyncMode", - "summary": "Update JDBC job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/configs/allowed_datatypes": { - "get": { - "operationId": "PlatformApiController_getJdbcAllowedDatatypes", - "summary": "Get allowed datatypes for JDBC", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}": { - "get": { - "operationId": "PlatformApiController_getSingerJob", - "summary": "Get Singer job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateSingerSyncMode", - "summary": "Update Singer job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/s3/{jobId}": { - "get": { - "operationId": "PlatformApiController_getS3Job", - "summary": "Get S3 job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/health": { - "get": { - "operationId": "PlatformApiController_healthCheck", - "summary": "Platform API health check", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -11483,6 +10559,114 @@ "engine" ] }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] + }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLinkItem": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerSidebarSection": { + "type": "object", + "properties": { + "title": { + "type": "object" + }, + "items": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/CustomerSidebarMenuItem" + }, + { + "$ref": "#/components/schemas/CustomerSidebarLinkItem" + } + ] + } + } + }, + "required": [ + "title", + "items" + ] + }, + "CustomerLinksConfig": { + "type": "object", + "properties": { + "home": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLinkItem" + } + }, + "sidebar": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerSidebarSection" + } + } + } + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + } + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "$ref": "#/components/schemas/CustomerLinksConfig" + } + }, + "required": [ + "links" + ] + }, "IRole": { "type": "object", "properties": { @@ -12072,100 +11256,6 @@ "updated_by" ] }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "object" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, "CreateShareMetadataDto": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index de2fca5..098aea0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.3", + "@dadosfera/protospack-v2": "^3.40.0-beta.4", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.3", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.3.tgz", - "integrity": "sha512-m77RSqAO+hZkjUgbdrMqC+6yyyo7indH6fWP3WuygCHGYwdN6FsWTA+EGTAhXQRSgNlI8h358ZpRoILnnGzLMg==", + "version": "3.40.0-beta.4", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.4.tgz", + "integrity": "sha512-ftK4edb3fgGw3qLx+cT79cHYL+1gBoWXd8+SBFQXWHBrnxoRKbanBDp5Yn9PbaYZdRMSqTxSKsBwiTyJzLBunQ==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 586d74c..e4815e6 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.3", + "@dadosfera/protospack-v2": "^3.40.0-beta.4", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 46ba2a4..5ead744 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -288,4 +288,8 @@ export class InputsService { }; return formatedPayload; } + + async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) { + return lastValueFrom(this.inputWriteService.MarkTableDeleted(data)); + } } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index ab39d38..5e6456e 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -48,6 +48,7 @@ import { import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; +import { PlatformApiService } from '../platform-api/platform-api.service'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { TableColumns } from '../inputs/dtos/input.model'; import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; @@ -64,6 +65,7 @@ export class PipelinesController { dadosferaLogger: DadosferaLogger, private pipelinesClientService: PipelinesService, private oldPipelinesService: OldPipelineService, + private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } @@ -222,31 +224,40 @@ export class PipelinesController { language, }); - const result = await this.pipelinesClientService - .findOne({ id }, metadata) - .then(async (res) => { - //{pipeline:{tables: {tables: [], input_id: ''}}} - const parsed = JSON.parse(res.pipeline.config.tables); - const input_id = parsed?.input_id; - const tables = parsed?.tables ?? parsed; + const [pipelineRes, platformPipeline] = await Promise.all([ + this.pipelinesClientService.findOne({ id }, metadata), + this.platformApiService.proxy('GET', `/pipeline/${id.replace(/-/g, '_')}`, user).catch(() => null), + ]); - Object.assign(res.pipeline, { - transformations: res.pipeline.transformations - ? JSON.parse(res.pipeline.transformations) - : [], - config: { - cron: res.pipeline.config.cron, - tables, - input_id - }, - properties: res.pipeline.properties - ? JSON.parse(res.pipeline.properties) - : {}, - }); - return res; + //{pipeline:{tables: {tables: [], input_id: ''}}} + type PipelineTablesConfig = { input_id?: string; tables?: Array<{ name: string }> } | Array<{ name: string }>; + const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); + const input_id = (parsed as { input_id?: string })?.input_id; + let tables: Array = (parsed as { tables?: Array })?.tables ?? (parsed as Array); + + if (platformPipeline?.jobs?.length) { + tables = tables.map((table) => { + const job = (platformPipeline.jobs as Array<{ job_id: string; input?: { table_name: string } }>) + .find((j) => j.input?.table_name === table.name); + return job ? { ...table, job_id: job.job_id } : table; }); + } - return result; + Object.assign(pipelineRes.pipeline, { + transformations: pipelineRes.pipeline.transformations + ? JSON.parse(pipelineRes.pipeline.transformations) + : [], + config: { + cron: pipelineRes.pipeline.config.cron, + tables, + input_id, + }, + properties: pipelineRes.pipeline.properties + ? JSON.parse(pipelineRes.pipeline.properties) + : {}, + }); + + return pipelineRes; } @Patch('/:id') diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index a83b7bd..efe6592 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -11,6 +11,7 @@ import { Inject, BadRequestException, HttpException, + NotFoundException, } from '@nestjs/common'; import { ApiTags, ApiOperation } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; @@ -29,6 +30,7 @@ import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; import { CatalogService } from '../catalog/catalog.service'; import { PackTheMetadata } from '../../utils/PackTheMetadata'; import { ValidationTableDTO } from './platform-api.dto'; +import { InputsService } from '../inputs/inputs.service'; type ValidateTablesDTO = { @@ -54,6 +56,7 @@ export class PlatformApiController { private readonly dynamoDBService: DynamoDBService, private readonly customersService: CustomersService, private readonly catalogService: CatalogService, + private readonly inputsService: InputsService, @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, ) { this.logger = dadosferaLogger.logger; @@ -963,7 +966,7 @@ export class PlatformApiController { } @Delete('jobs/:jobId') - @ApiOperation({ summary: 'Delete a job and mark its table as deleted in DynamoDB' }) + @ApiOperation({ summary: 'Delete a job and mark its table as deleted via in-factory' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteJob( @Param('jobId') jobId: string, @@ -972,47 +975,36 @@ export class PlatformApiController { const normalizedJobId = this.normalizeJobId(jobId); const esPipelineId = this.extractPipelineIdFromJobId(jobId); - // Fetch job details and ES pipeline in parallel — both are reads with no mutual dependency - const [jobResult, esPipelineResult] = await Promise.allSettled([ - this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user), - this.elasticsearchService.getPipeline(user.customer_name, esPipelineId), - ]); + // Fetch job details first — fail fast if job doesn't exist or jobId is invalid + const jobDetails = await this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user); + const tableName: string | undefined = jobDetails?.source_config?.table_name; - const tableName: string | undefined = jobResult.status === 'fulfilled' - ? jobResult.value?.source_config?.table_name - : undefined; - - if (jobResult.status === 'rejected') { - this.logger.warn('deleteJob: could not fetch job before deletion', { jobId, error: jobResult.reason?.message }); + if (!tableName) { + throw new NotFoundException(`Job ${jobId} not found or has no associated table`); } - // Main operation + // Fetch ES pipeline to resolve the DynamoDB input ID + const esPipeline = await this.elasticsearchService.getPipeline(user.customer_name, esPipelineId); + // config.tables in ES stores the DynamoDB input ID (legacy field naming) + const inputId: string | undefined = esPipeline?.config?.tables; + + if (!inputId) { + throw new NotFoundException(`Pipeline data not found for job ${jobId}`); + } + + // Delete the job on platform-api const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); - // Sync deleted status to DynamoDB — follows syncJobInputToDynamoDB pattern - // config.tables in ES stores the DynamoDB input ID (legacy field naming) - const dynamoInputId: string | undefined = esPipelineResult.status === 'fulfilled' - ? esPipelineResult.value?.config?.tables - : undefined; + // Mark table as deleted in DynamoDB via in-factory gRPC + const info = { + customer_id: user.customer_id, + customer: user.customer_name, + user_id: user.user_id, + }; - if (esPipelineResult.status === 'rejected') { - this.logger.warn('deleteJob: could not fetch ES pipeline for DynamoDB sync', { jobId, error: esPipelineResult.reason?.message }); - } - - if (tableName && dynamoInputId) { - await this.dynamoDBService.updateInputTable( - user.customer_id, - dynamoInputId, - tableName, - { status: 'deleted' }, - ).catch((error) => { - this.logger.error('Failed to sync job deletion to DynamoDB', { jobId, tableName, error: error.message }); - }); - } else if (!tableName) { - this.logger.warn('deleteJob: table name not resolved, skipping DynamoDB sync', { jobId }); - } else { - this.logger.warn('deleteJob: no input ID in ES pipeline, skipping DynamoDB sync', { jobId }); - } + await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }).catch((error) => { + this.logger.error('deleteJob: failed to mark table as deleted via in-factory', { jobId, tableName, error: error.message }); + }); return result; } diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index b78e680..b2b0726 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; +import { InputsModule } from '../inputs/inputs.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], From 877cb9d2813eb00a10302e1d02a0e000424b8c7d Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 31 Mar 2026 18:38:25 -0300 Subject: [PATCH 05/10] FEAT: refine PipelineTablesConfig type definition and update parsing logic in PipelinesController --- src/modules/pipelinesV2/pipelines.controller.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 5e6456e..37b3a47 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -54,6 +54,9 @@ import { TableColumns } from '../inputs/dtos/input.model'; import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; +type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; +type PipelineTablesConfig = { input_id?: string; tables?: PipelineTable[] } | PipelineTable[]; + @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @UseFilters(new GrpcToHttpExceptionFilter()) @@ -63,6 +66,7 @@ export class PipelinesController { constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + private pipelinesClientService: PipelinesService, private oldPipelinesService: OldPipelineService, private platformApiService: PlatformApiService, @@ -229,11 +233,9 @@ export class PipelinesController { this.platformApiService.proxy('GET', `/pipeline/${id.replace(/-/g, '_')}`, user).catch(() => null), ]); - //{pipeline:{tables: {tables: [], input_id: ''}}} - type PipelineTablesConfig = { input_id?: string; tables?: Array<{ name: string }> } | Array<{ name: string }>; const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); - const input_id = (parsed as { input_id?: string })?.input_id; - let tables: Array = (parsed as { tables?: Array })?.tables ?? (parsed as Array); + const input_id = (parsed as { input_id?: string; tables?: PipelineTable[] })?.input_id; + let tables: PipelineTable[] = (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]); if (platformPipeline?.jobs?.length) { tables = tables.map((table) => { From 968f75b688ccd612a8ea66409b9847cb1af88b75 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 1 Apr 2026 14:26:55 -0300 Subject: [PATCH 06/10] FEAT: update job deletion endpoint to include inputId in path and implement rollback for table deletion --- docsfera.json | 10 ++++- src/modules/inputs/inputs.service.ts | 4 ++ .../pipelinesV2/pipelines.controller.ts | 20 +++------- .../platform-api/platform-api.controller.ts | 37 +++++++++---------- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/docsfera.json b/docsfera.json index c78fbec..7fb3e1a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,11 +4705,19 @@ ] } }, - "/platform/jobs/{jobId}": { + "/platform/inputs/{inputId}/jobs/{jobId}": { "delete": { "operationId": "PlatformApiController_deleteJob", "summary": "Delete a job and mark its table as deleted via in-factory", "parameters": [ + { + "name": "inputId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, { "name": "jobId", "required": true, diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 5ead744..96df891 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -292,4 +292,8 @@ export class InputsService { async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) { return lastValueFrom(this.inputWriteService.MarkTableDeleted(data)); } + + async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) { + return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data)); + } } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 37b3a47..71091ad 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -48,7 +48,6 @@ import { import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; -import { PlatformApiService } from '../platform-api/platform-api.service'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { TableColumns } from '../inputs/dtos/input.model'; import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; @@ -69,7 +68,6 @@ export class PipelinesController { private pipelinesClientService: PipelinesService, private oldPipelinesService: OldPipelineService, - private platformApiService: PlatformApiService, ) { this.logger = dadosferaLogger.logger; } @@ -228,22 +226,14 @@ export class PipelinesController { language, }); - const [pipelineRes, platformPipeline] = await Promise.all([ - this.pipelinesClientService.findOne({ id }, metadata), - this.platformApiService.proxy('GET', `/pipeline/${id.replace(/-/g, '_')}`, user).catch(() => null), - ]); + const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); const input_id = (parsed as { input_id?: string; tables?: PipelineTable[] })?.input_id; - let tables: PipelineTable[] = (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]); - - if (platformPipeline?.jobs?.length) { - tables = tables.map((table) => { - const job = (platformPipeline.jobs as Array<{ job_id: string; input?: { table_name: string } }>) - .find((j) => j.input?.table_name === table.name); - return job ? { ...table, job_id: job.job_id } : table; - }); - } + const normalizedId = id.replace(/-/g, '_'); + const tables: PipelineTable[] = ( + (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]) + ).map((table, index) => ({ ...table, job_id: `${normalizedId}_${index}` })); Object.assign(pipelineRes.pipeline, { transformations: pipelineRes.pipeline.transformations diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index efe6592..b880a9b 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -965,17 +965,16 @@ export class PlatformApiController { ); } - @Delete('jobs/:jobId') + @Delete('inputs/:inputId/jobs/:jobId') @ApiOperation({ summary: 'Delete a job and mark its table as deleted via in-factory' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteJob( + @Param('inputId') inputId: string, @Param('jobId') jobId: string, @User() user: RequestUser, ) { const normalizedJobId = this.normalizeJobId(jobId); - const esPipelineId = this.extractPipelineIdFromJobId(jobId); - // Fetch job details first — fail fast if job doesn't exist or jobId is invalid const jobDetails = await this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user); const tableName: string | undefined = jobDetails?.source_config?.table_name; @@ -983,30 +982,28 @@ export class PlatformApiController { throw new NotFoundException(`Job ${jobId} not found or has no associated table`); } - // Fetch ES pipeline to resolve the DynamoDB input ID - const esPipeline = await this.elasticsearchService.getPipeline(user.customer_name, esPipelineId); - // config.tables in ES stores the DynamoDB input ID (legacy field naming) - const inputId: string | undefined = esPipeline?.config?.tables; - - if (!inputId) { - throw new NotFoundException(`Pipeline data not found for job ${jobId}`); - } - - // Delete the job on platform-api - const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); - - // Mark table as deleted in DynamoDB via in-factory gRPC const info = { customer_id: user.customer_id, customer: user.customer_name, user_id: user.user_id, }; - await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }).catch((error) => { - this.logger.error('deleteJob: failed to mark table as deleted via in-factory', { jobId, tableName, error: error.message }); - }); + const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }); - return result; + try { + await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); + + const updatedTable = updatedInput.tables?.find((t: any) => t.name === tableName); + return updatedTable ?? { name: tableName, is_deleted: true }; + } catch (error) { + this.logger.error('deleteJob: platform-api delete failed, attempting rollback', { jobId, tableName, error: error.message }); + try { + await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info }); + } catch (rollbackError) { + this.logger.error('deleteJob: rollback failed', { jobId, tableName, error: rollbackError.message }); + } + throw error; + } } // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== From d0122a9c20d5ae4e175ec979ecd67e54be6d3575 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 1 Apr 2026 14:43:51 -0300 Subject: [PATCH 07/10] FEAT: remove unused status field from InputDocument and updateInputTable method --- src/services/dynamodb/dynamodb.service.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index a0d3537..90e5efa 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -30,7 +30,6 @@ export interface InputDocument { type: string; columns?: string[]; reference_column?: ReferenceColumn; - status?: string; }>; credentials?: Record; } @@ -168,7 +167,6 @@ export class DynamoDBService { type?: string; columns?: string[]; reference_column?: ReferenceColumn | null; - status?: string; }, ): Promise { const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); @@ -209,8 +207,6 @@ export class DynamoDBService { updatedTable.reference_column = changes.reference_column; } } - if ('status' in changes) updatedTable.status = changes.status; - tables[tableIndex] = updatedTable; // Save updated document From 837a9d7265be8f6af0c51f70f18335cf68ce4a84 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 2 Apr 2026 15:40:07 -0300 Subject: [PATCH 08/10] FEAT: update job deletion logic to mark tables as deleted and adjust API endpoints accordingly --- docsfera.json | 25 +++++----- package-lock.json | 8 ++-- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 9 ++-- .../platform-api/platform-api.controller.ts | 39 ++++++++------- src/services/dynamodb/dynamodb.service.ts | 48 ------------------- 6 files changed, 42 insertions(+), 89 deletions(-) diff --git a/docsfera.json b/docsfera.json index 7fb3e1a..29f2ff3 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,10 +4705,10 @@ ] } }, - "/platform/inputs/{inputId}/jobs/{jobId}": { + "/platform/inputs/{inputId}/tables/{tableName}": { "delete": { - "operationId": "PlatformApiController_deleteJob", - "summary": "Delete a job and mark its table as deleted via in-factory", + "operationId": "PlatformApiController_deleteTable", + "summary": "Mark a table as deleted and delete its associated job via platform-api", "parameters": [ { "name": "inputId", @@ -4719,24 +4719,25 @@ } }, { - "name": "jobId", + "name": "tableName", "required": true, "in": "path", "schema": { "type": "string" } + }, + { + "name": "pipelineId", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ diff --git a/package-lock.json b/package-lock.json index 098aea0..a7f9147 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.4", + "@dadosfera/protospack-v2": "^3.40.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.4", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.4.tgz", - "integrity": "sha512-ftK4edb3fgGw3qLx+cT79cHYL+1gBoWXd8+SBFQXWHBrnxoRKbanBDp5Yn9PbaYZdRMSqTxSKsBwiTyJzLBunQ==", + "version": "3.40.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.5.tgz", + "integrity": "sha512-iocKv/XXp2jKAasO5ONgm31cKfLgNsU4pEKZMr6YnR7nQaH11WcW7rnuagNxWoik++wLUqbYyf0bZWRDzMlCPA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index e4815e6..18bb538 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.40.0-beta.4", + "@dadosfera/protospack-v2": "^3.40.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 71091ad..e91d217 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -54,7 +54,7 @@ import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any }; -type PipelineTablesConfig = { input_id?: string; tables?: PipelineTable[] } | PipelineTable[]; +type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] }; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -229,11 +229,8 @@ export class PipelinesController { const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata); const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables); - const input_id = (parsed as { input_id?: string; tables?: PipelineTable[] })?.input_id; - const normalizedId = id.replace(/-/g, '_'); - const tables: PipelineTable[] = ( - (parsed as { tables?: PipelineTable[] })?.tables ?? (parsed as PipelineTable[]) - ).map((table, index) => ({ ...table, job_id: `${normalizedId}_${index}` })); + const input_id = parsed.input_id; + const tables: PipelineTable[] = parsed.tables ?? []; Object.assign(pipelineRes.pipeline, { transformations: pipelineRes.pipeline.transformations diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index b880a9b..e850f3e 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -965,42 +965,45 @@ export class PlatformApiController { ); } - @Delete('inputs/:inputId/jobs/:jobId') - @ApiOperation({ summary: 'Delete a job and mark its table as deleted via in-factory' }) + @Delete('inputs/:inputId/tables/:tableName') + @ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) - async deleteJob( + async deleteTable( @Param('inputId') inputId: string, - @Param('jobId') jobId: string, + @Param('tableName') tableName: string, + @Query('pipelineId') pipelineId: string, @User() user: RequestUser, ) { - const normalizedJobId = this.normalizeJobId(jobId); - - const jobDetails = await this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user); - const tableName: string | undefined = jobDetails?.source_config?.table_name; - - if (!tableName) { - throw new NotFoundException(`Job ${jobId} not found or has no associated table`); - } - const info = { customer_id: user.customer_id, customer: user.customer_name, user_id: user.user_id, }; + this.logger.info('deleteTable: marking table as deleted', { inputId, tableName }); const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info }); + this.logger.info('deleteTable: table marked as deleted', { inputId, tableName }); try { - await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user); + const normalizedPipelineId = this.normalizeJobId(pipelineId); + this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId }); + const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user); + this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length }); - const updatedTable = updatedInput.tables?.find((t: any) => t.name === tableName); - return updatedTable ?? { name: tableName, is_deleted: true }; + const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName); + if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`); + + this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id }); + await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user); + this.logger.info('deleteTable: job deleted', { jobId: job.job_id }); + + return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at }; } catch (error) { - this.logger.error('deleteJob: platform-api delete failed, attempting rollback', { jobId, tableName, error: error.message }); + this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message }); try { await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info }); } catch (rollbackError) { - this.logger.error('deleteJob: rollback failed', { jobId, tableName, error: rollbackError.message }); + this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message }); } throw error; } diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index 90e5efa..0022b35 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -235,52 +235,4 @@ export class DynamoDBService { } } - /** - * Remove a specific table entry from the input document. - * Fetches the current document, removes the matching table, and saves. - */ - async removeInputTable( - clientId: string, - inputId: string, - tableName: string, - ): Promise { - const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); - - this.logger.info('DynamoDB: Removing input table', { - inputId, - tableName, - }); - - const current = await this.findInput(clientId, inputId); - if (!current) { - this.logger.warn('DynamoDB: Input not found for table removal', { inputId }); - return; - } - - const tables = (current.tables || []).filter((t) => t.name !== tableName); - - const putCommand = new PutCommand({ - TableName: dynamoTableName, - Item: { - ...current, - tables, - updated_at: new Date().toISOString(), - }, - }); - - try { - await this.documentClient.send(putCommand); - this.logger.info('DynamoDB: Input table removed successfully', { - inputId, - tableName, - }); - } catch (error) { - this.logger.error('DynamoDB: Failed to remove input table', { - inputId, - tableName, - error: error.message, - }); - throw error; - } - } } From 8a9b58b6121ef758f30c5c5028b87ec39465b77e Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 2 Apr 2026 16:40:35 -0300 Subject: [PATCH 09/10] FIX: rename method normalizeJobId to normalizePipelineId for clarity --- src/modules/platform-api/platform-api.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index e850f3e..9e5419c 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -985,7 +985,7 @@ export class PlatformApiController { this.logger.info('deleteTable: table marked as deleted', { inputId, tableName }); try { - const normalizedPipelineId = this.normalizeJobId(pipelineId); + const normalizedPipelineId = this.normalizePipelineId(pipelineId); this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId }); const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user); this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length }); From efb0b5864806aa4d9bc8fed0b61aeb2de690cee1 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Thu, 2 Apr 2026 17:28:42 -0300 Subject: [PATCH 10/10] FEAT: update deleteTable endpoint to use pipelineId in path and refactor parameters --- docsfera.json | 26 +++++++------------ .../platform-api/platform-api.controller.ts | 7 ++--- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/docsfera.json b/docsfera.json index 29f2ff3..fc5e27b 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,11 +4705,19 @@ ] } }, - "/platform/inputs/{inputId}/tables/{tableName}": { + "/platform/pipelines/{pipelineId}/inputs/{inputId}": { "delete": { "operationId": "PlatformApiController_deleteTable", "summary": "Mark a table as deleted and delete its associated job via platform-api", "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, { "name": "inputId", "required": true, @@ -4717,22 +4725,6 @@ "schema": { "type": "string" } - }, - { - "name": "tableName", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "pipelineId", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 9e5419c..5fd0b94 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -965,15 +965,16 @@ export class PlatformApiController { ); } - @Delete('inputs/:inputId/tables/:tableName') + @Delete('pipelines/:pipelineId/inputs/:inputId') @ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) async deleteTable( + @Param('pipelineId') pipelineId: string, @Param('inputId') inputId: string, - @Param('tableName') tableName: string, - @Query('pipelineId') pipelineId: string, + @Body() body: { table_name: string }, @User() user: RequestUser, ) { + const tableName = body.table_name; const info = { customer_id: user.customer_id, customer: user.customer_name,