From da8cf2d590a6a16f8f1c88e7dac377d522202d59 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 30 Mar 2026 22:24:10 -0300 Subject: [PATCH] FEAT: add delete job endpoint and update related services; update protospack-v2 version to 3.40.0-beta.3 --- docsfera.json | 150 +++++------------- package-lock.json | 8 +- package.json | 2 +- .../pipelinesV2/pipelines.controller.ts | 7 +- .../platform-api/platform-api.controller.ts | 76 +++++++++ .../platform-api/platform-api.module.ts | 4 +- src/services/dynamodb/dynamodb.service.ts | 62 +++++++- 7 files changed, 188 insertions(+), 121 deletions(-) diff --git a/docsfera.json b/docsfera.json index 425263d..137ea2f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7627,6 +7627,42 @@ ] } }, + "/platform/jobs/{jobId}": { + "delete": { + "operationId": "PlatformApiController_deleteJob", + "summary": "Delete a job and sync pipeline data", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/jdbc/{jobId}": { "get": { "operationId": "PlatformApiController_getJdbcJob", @@ -10133,6 +10169,9 @@ "$ref": "#/components/schemas/Column" } }, + "destination": { + "type": "object" + }, "type": { "type": "string" }, @@ -10149,6 +10188,7 @@ "required": [ "name", "columns", + "destination", "type" ] }, @@ -10428,114 +10468,6 @@ "engine" ] }, - "ValidationTableDTO": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tables" - ] - }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLinkItem": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerSidebarSection": { - "type": "object", - "properties": { - "title": { - "type": "object" - }, - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/CustomerSidebarMenuItem" - }, - { - "$ref": "#/components/schemas/CustomerSidebarLinkItem" - } - ] - } - } - }, - "required": [ - "title", - "items" - ] - }, - "CustomerLinksConfig": { - "type": "object", - "properties": { - "home": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLinkItem" - } - }, - "sidebar": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerSidebarSection" - } - } - } - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - } - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "$ref": "#/components/schemas/CustomerLinksConfig" - } - }, - "required": [ - "links" - ] - }, "IRole": { "type": "object", "properties": { @@ -11162,7 +11094,7 @@ "type": "object", "properties": { "title": { - "type": "string" + "type": "object" }, "items": { "type": "array", diff --git a/package-lock.json b/package-lock.json index 592117e..de2fca5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.39.0", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1745,9 +1745,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.39.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.39.0.tgz", - "integrity": "sha512-z6q+ZRpbDT6apfmly1+X818DgAFUCilFO0ATP7xR5+SH+GAUAJLY1dJRQDEkq5vFIfXnC2Nmm9aODZAs+9/5vw==", + "version": "3.40.0-beta.3", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.3.tgz", + "integrity": "sha512-m77RSqAO+hZkjUgbdrMqC+6yyyo7indH6fWP3WuygCHGYwdN6FsWTA+EGTAhXQRSgNlI8h358ZpRoILnnGzLMg==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 6afe185..235e02d 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.39.0", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 4f5ff1d..31433c3 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -220,10 +220,11 @@ export class PipelinesController { const result = await this.pipelinesClientService .findOne({ id }, metadata) - .then((res) => { + .then(async (res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} - let tables = JSON.parse(res.pipeline.config.tables); - if (tables?.tables) tables = tables.tables; + const parsed = JSON.parse(res.pipeline.config.tables); + const tables = parsed?.tables ?? parsed; + Object.assign(res.pipeline, { transformations: res.pipeline.transformations ? JSON.parse(res.pipeline.transformations) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 978f464..0d06347 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -25,6 +25,7 @@ import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; +import { InputsService } from '../inputs/inputs.service'; type ValidateTablesDTO = { @@ -44,6 +45,7 @@ export class PlatformApiController { private readonly elasticsearchService: ElasticsearchService, private readonly dynamoDBService: DynamoDBService, private readonly customersService: CustomersService, + private readonly inputsService: InputsService, @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, ) { this.logger = dadosferaLogger.logger; @@ -933,6 +935,80 @@ export class PlatformApiController { ); } + @Delete('jobs/:jobId') + @ApiOperation({ summary: 'Delete a job and sync pipeline data' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) + async deleteJob( + @Param('jobId') jobId: string, + @User() user: RequestUser, + ) { + // Get job details before deletion to extract table_name and connector type + const normalizedJobId = this.normalizeJobId(jobId); + let tableName: string | undefined; + + this.logger.info('[deleteJob] Start', { jobId }); + + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${normalizedJobId}`, + user, + ); + tableName = job?.source_config?.table_name; + this.logger.info('[deleteJob] Job fetched', { jobId, tableName }); + } catch (err) { + this.logger.warn('[deleteJob] Could not fetch job before deletion', { jobId, error: err.message }); + } + + const result = await this.platformApiService.proxy( + 'DELETE', + `/jobs/${normalizedJobId}`, + user, + ); + this.logger.info('[deleteJob] Job deleted from platform-api', { jobId }); + + // Mark the table as deleted via in-factory (which has the correct DynamoDB credentials). + // config.tables in ES stores the DynamoDB input ID (legacy naming). + if (tableName) { + try { + const esPipelineId = this.extractPipelineIdFromJobId(jobId); + const esPipeline = await this.elasticsearchService.getPipeline( + user.customer_name, + esPipelineId, + ); + // config.tables holds the DynamoDB input ID (see syncJobInputToDynamoDB for context) + const dynamoInputId = esPipeline?.config?.tables; + + if (dynamoInputId) { + const info = { customer_id: user.customer_id, user_id: user.user_id, customer: user.customer_name }; + + // Read current input to get full tables array (adjustInputPayload flattens the response) + const { input } = await this.inputsService.findOne({ id: dynamoInputId, info }); + const currentTables: any[] = input?.tables ?? []; + + const updatedTables = currentTables.map((t: any) => + t.name === tableName ? { ...t, status: 'deleted' } : t, + ); + + await this.inputsService.update(dynamoInputId, { tables: updatedTables }, info); + this.logger.info('[deleteJob] Marked table as deleted via in-factory', { jobId, tableName }); + } else { + this.logger.warn('[deleteJob] No input ID in ES pipeline, skipping DynamoDB update', { jobId, tableName }); + } + } catch (error) { + this.logger.error('[deleteJob] Failed to mark table as deleted via in-factory', { + jobId, + tableName, + error: error.message, + }); + } + } else { + this.logger.warn('[deleteJob] tableName not resolved, skipping DynamoDB update', { jobId }); + } + + return result; + } + // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== @Get('jobs/jdbc/:jobId') diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index c2b31e4..3c9ff86 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -7,9 +7,11 @@ import { PlatformApiService } from './platform-api.service'; import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; +import { InputsModule } from '../inputs/inputs.module'; +import { PipelinesV2Module } from '../pipelinesV2/pipelines.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, InputsModule, PipelinesV2Module], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index 15dde41..a0d3537 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -30,6 +30,7 @@ export interface InputDocument { type: string; columns?: string[]; reference_column?: ReferenceColumn; + status?: string; }>; credentials?: Record; } @@ -125,9 +126,13 @@ export class DynamoDBService { }, }); - const { Item } = await this.documentClient.send(getCommand); - - return Item as InputDocument | null; + try { + const { Item } = await this.documentClient.send(getCommand); + return Item as InputDocument | null; + } catch (error) { + this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message }); + throw error; + } } async deleteInput(clientId: string, inputId: string): Promise { @@ -163,6 +168,7 @@ export class DynamoDBService { type?: string; columns?: string[]; reference_column?: ReferenceColumn | null; + status?: string; }, ): Promise { const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); @@ -203,6 +209,7 @@ export class DynamoDBService { updatedTable.reference_column = changes.reference_column; } } + if ('status' in changes) updatedTable.status = changes.status; tables[tableIndex] = updatedTable; @@ -231,4 +238,53 @@ export class DynamoDBService { throw error; } } + + /** + * Remove a specific table entry from the input document. + * Fetches the current document, removes the matching table, and saves. + */ + async removeInputTable( + clientId: string, + inputId: string, + tableName: string, + ): Promise { + const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); + + this.logger.info('DynamoDB: Removing input table', { + inputId, + tableName, + }); + + const current = await this.findInput(clientId, inputId); + if (!current) { + this.logger.warn('DynamoDB: Input not found for table removal', { inputId }); + return; + } + + const tables = (current.tables || []).filter((t) => t.name !== tableName); + + const putCommand = new PutCommand({ + TableName: dynamoTableName, + Item: { + ...current, + tables, + updated_at: new Date().toISOString(), + }, + }); + + try { + await this.documentClient.send(putCommand); + this.logger.info('DynamoDB: Input table removed successfully', { + inputId, + tableName, + }); + } catch (error) { + this.logger.error('DynamoDB: Failed to remove input table', { + inputId, + tableName, + error: error.message, + }); + throw error; + } + } }