MERGE: resolve package conflicts

FEAT: update protospack-v2 to version 3.40.0-beta.1 and adjust relate…

docsfera.json

@@ -3001,14 +3001,7 @@
           "parameters": [],
           "responses": {
             "200": {
-              "description": "",
+              "description": ""
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
             }
           },
           "tags": [
@@ -3583,14 +3576,7 @@
           ],
           "responses": {
             "200": {
-              "description": "",
+              "description": ""
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
             }
           },
           "tags": [
@@ -4533,6 +4519,50 @@
           ]
         }
       },
+      "/platform/pipeline/{pipelineId}/pipeline_run/{runId}/cancel": {
+        "post": {
+          "operationId": "PlatformApiController_cancelPipelineRun",
+          "summary": "Cancel a running pipeline run",
+          "parameters": [
+            {
+              "name": "pipelineId",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "runId",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Platform API"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/platform/jobs/{jobId}/input": {
         "put": {
           "operationId": "PlatformApiController_updateJobInput",

package-lock.json

@@ -17,7 +17,7 @@
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "^3.38.0-beta.30",
+        "@dadosfera/protospack-v2": "3.40.0-beta.2",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1745,9 +1745,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.38.0-beta.30",
+      "version": "3.40.0-beta.2",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz",
-      "integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==",
+      "integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -10,7 +10,7 @@
   },
   "scripts": {
     "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
-    "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
+    "proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
     "prebuild": "rimraf dist",
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
@@ -35,7 +35,7 @@
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "^3.38.0-beta.30",
+    "@dadosfera/protospack-v2": "3.40.0-beta.2",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",
@@ -81,7 +81,6 @@
     "swagger-ui-express": "^4.6.3"
   },
   "overrides": {
-    "multer": "2.0.2",
     "form-data": "^4.0.4",
     "body-parser": "^1.20.3",
     "cross-spawn": "^7.0.5",

src/modules/auth/auth.controller.ts

@@ -489,6 +489,7 @@ export class AuthController {
       const userDto = {
         id: api_key.user_id,
         name: api_key.username,
+        email: api_key.username,
         customer: {
           id: api_key.customer_id,
           name: api_key.customer_name,

src/modules/auth/auth.service.ts

@@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit {
 
     const userDto: UserDTO = {
       id: user.id,
-      name: user.username,
+      name: user.name,
+      email: user.email,
       jobTitle: user?.jobTitle || null,
       department: user?.department || null,
       hierarchy: user?.hierarchy || null,

src/modules/auth/dtos/login.ts

@@ -144,6 +144,7 @@ export interface BulkEditResponse {
 export type UserDTO = {
   id: string,
   name: string,
+  email: string,
   jobTitle?: string,
   department?: string,
   hierarchy?: string,

src/modules/inputs/inputs.controller.ts

@@ -99,6 +99,7 @@ export class InputsController {
       customer: info.customer,
     });
 
+    this.logger.info(JSON.stringify(body))
     const response = await this.inputService.create({ body, info });
 
     return response;

src/modules/inputs/inputs.service.ts

@@ -17,6 +17,8 @@ import {
   InputCreateGenericRequest,
   InputCreateS3Request,
   InputNewCreateRequest,
+  InputUpdateResponse,
+  RollbackInputRequest,
   TestConnectionRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -71,8 +73,8 @@ export class InputsService {
       objectCamelToSnake(createInputResponse);
       return createInputResponse;
     },
-    update: async (updateInputDTO: UpdateInputRequest) => {
+    update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
-      this.logger.info('InputClientService - Update');
+      this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
       const updateInputResponse = await lastValueFrom(
         this.inputWriteService.InputUpdate(updateInputDTO),
       );
@@ -207,21 +209,44 @@ export class InputsService {
   async update(id: string, data, info: Info) {
     // this.validateCron({ ...data, info });
     try {
-      const updateInputResponse: any = await this.OLD_inputClient.update({
+      const {
+        tablesUpdate,
+        dataAssetUpdate,
+        input
+      } = await this.OLD_inputClient.update({
         id,
-        info,
         ...data,
+        info,
       });
 
-      updateInputResponse.input = this.adjustInputPayload(
+      const updateInputResponse = this.adjustInputPayload(
-        updateInputResponse?.input,
+        input,
       );
-      return updateInputResponse;
+      return {
+        input: updateInputResponse,
+        tablesUpdate,
+        dataAssetUpdate
+      };
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
 
+  async rollbackUpdate(
+    data: RollbackInputRequest
+  ) {
+    this.logger.info('PipelinesClientService - rollbackUpdate');
+    this.logger.info('Rolling back input update with data: ' + JSON.stringify(data));
+    const updatePipelineResponse = await lastValueFrom(
+      this.inputWriteService.RollbackInputUpdate(
+        data
+      ),
+    );
+    this.logger.info('Done');
+
+    return updatePipelineResponse;
+  }
+
   async remove(idRequest: IIdRequest) {
     return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
   }

src/modules/pipelinesV2/pipelines.controller.ts

@@ -294,13 +294,14 @@ export class PipelinesController {
   ) {
     this.logger.info('PipelinesController - update', { user });
 
-
     const { customer_id, customer_name, user_id, username } = user;
     const info: Info = {
       user_id: user.user_id,
       customer: user.customer_name,
       customer_id: user.customer_id,
+      pipeline_id: pipelineId
     };
+
     const metadata = PackTheMetadata({
       customer_id,
       customer_name,

src/modules/pipelinesV2/pipelines.module.ts

@@ -12,6 +12,8 @@ import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
 import { PlatformApiModule } from '../platform-api/platform-api.module';
+import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
+import { NimbusService } from 'src/services/nimbus/nimbus.service';
 
 const client = new PipelinesClientConfiguration();
 
@@ -22,10 +24,11 @@ const client = new PipelinesClientConfiguration();
     ConnectorModule,
     InputsModule,
     TransformationsModule,
-    PlatformApiModule
+    PlatformApiModule,
+    NimbusServicesModule
   ],
   controllers: [PipelinesController],
-  providers: [PipelinesService, DadosferaLogger],
+  providers: [PipelinesService, DadosferaLogger, NimbusService],
   exports: [PipelinesService],
 })
 export class PipelinesV2Module {}

src/modules/pipelinesV2/pipelines.service.ts

@@ -29,6 +29,11 @@ import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
 import { PlatformApiService } from '../platform-api/platform-api.service';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
+import { AxiosError } from 'axios';
+import { NimbusService } from 'src/services/nimbus/nimbus.service';
+
+type RollbackPromise = () => Promise<any>;
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -42,7 +47,8 @@ export class PipelinesService implements OnModuleInit {
     private readonly connectorService: ConnectorClientService,
     private readonly inputsService: InputsService,
     private readonly transformationsService: TransformationsService,
-    private readonly platformAPI: PlatformApiService
+    private readonly platformAPI: PlatformApiService,
+    private readonly nimbusService: NimbusService
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -348,133 +354,226 @@ export class PipelinesService implements OnModuleInit {
   async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
     this.logger.info('InputClientService - Update');
 
-    this.logger.info('Update Dynamo Reference');
+    const {
+      input: oldInput
+    } = await this.inputsService.findOne({
+      id: inputId,
+      info: info
+    });
+
+    this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
     const pipelineIdFormat = pipelineId.split('-').join('_');
+    const rollback: RollbackPromise[] = [];
+
     const updateInputResponse = await this.inputsService.update(
       inputId,
       updateInputDTO,
       info
-    )
+    );
 
-    const requests = [];
+    const inputRollback = () => {
+      this.logger.info("exec rollback to input: " + JSON.stringify(oldInput));
+      return this.inputsService.rollbackUpdate(
+        {
+          id: inputId,
+          dataAssetUpdate: updateInputResponse.dataAssetUpdate,
+          tables: oldInput.tables,
+          info
+        }
+      ) as Promise<any>;
+    }
 
-    this.logger.info('Dynamo Response', updateInputResponse);
+    rollback.push(inputRollback);
+
+    this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse))
+
+    const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
+
+    nimbusUpdates.forEach(update => {
+      const nimbusRollback = () => {
+        return this.nimbusService.renameTable(
+          info.customer,
+          update.database,
+          {
+            table_name: update.table_name,
+            table_schema: update.table_schema
+          },
+          {
+            table_name: update.old_table_name,
+            table_schema: update.old_table_schema
+          }
+        );
+      }
+      rollback.push(nimbusRollback);
+    });
+
+    try {
+      await this.updateNimbus(info.customer, nimbusUpdates);
+    } catch (error) {
+      this.logger.error(error);
+      if (error instanceof AxiosError) {
+        this.logger.error(JSON.stringify(error.response.data));
+      }
+      await this.executeRenameRollback(rollback);
+
+      throw new Error("Error Nimbus updating tables");
+    }
+
+    try {
+      await this.updatePlatformJobs(
+        pipelineIdFormat,
+        updateInputDTO,
+        user
+      );
+    } catch (error) {
+      this.logger.error(error);
+      await this.executeRenameRollback(rollback)
+      throw new Error("Error Platform API updating jobs");
+    }
+
+    return updateInputResponse;
+  }
+
+  private async executeRenameRollback(request: RollbackPromise[]) {
+    this.logger.info('rollback steps: ' + request.length)
+    const result = await Promise.allSettled(request.map(func => func()));
+    result.forEach(promise => {
+      this.logger.info("Promise finish with status: " + promise.status)
+
+      if (promise.status === "rejected") {
+        this.logger.error("reject with: " + JSON.stringify(promise.reason || {}))
+      }
+
+      if (promise.status === "fulfilled") {
+        this.logger.info("success with: " + JSON.stringify(promise.value || {}))
+      }
+    });
+
+  }
+
+  private async updateNimbus(customer: string, changes: TableUpdate[]) {
+    // throw new Error("teste error nimbus");
+    this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
+    if(!changes || changes.length === 0) return;
+
+    const requests = changes.map(change => {
+      return this.nimbusService.renameTable(customer, change.database, {
+        table_name: change.old_table_name,
+        table_schema: change.old_table_schema
+      }, {
+        table_name: change.table_name,
+        table_schema: change.table_schema
+      });
+    })
+
+    const values = await Promise.allSettled(requests);
+
+    const success = values.map(request => request.status === "fulfilled")
+
+    this.logger.info("Updates with succes: " + success.length);
+
+    values.forEach(promise => {
+      this.logger.info("Promise finish with status: " + promise.status)
+
+      if (promise.status === "rejected") {
+        this.logger.error("reject with: " + JSON.stringify(promise.reason || {}));
+        throw new Error(promise.reason );
+      }
+
+      if (promise.status === "fulfilled") {
+        this.logger.info("success with: " + JSON.stringify(promise.value || {}));
+      }
+    });
+
+  }
+
+  async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
+    const jobsUpdated = [];
 
     for (const [index, table] of updateInputDTO.tables.entries()) {
-      const id = `${pipelineIdFormat}_${index}`;
+      const jobUpdate = {
-      this.logger.info('Updating input reference for table', table.name);
+        job_id: `${pipelineId}_${index}`,
-      const body = {}
+      }
+
+      if (table.type !== "incremental_with_qualify") {
+        delete table.destinations?.qualify;
+      }
+
+      if (table.memory) {
+        jobUpdate["memory"] = {
+          amount: table.memory * 1000
+        }
+      }
+      
+      this.logger.info('Updating input reference for table: ' + table.name);
+      let hasUpdateSyncMode = false;
+
+      const jobSyncMode = {}
+
       if (table.columns) {
-        body['column_include_list'] = table.columns;
+        hasUpdateSyncMode = true;
+        jobSyncMode['column_include_list'] = table.columns;
       }
 
       if (table.reference_column) {
-        body['incremental_column_name'] = table.reference_column.name;
+        hasUpdateSyncMode = true;
-        body['incremental_column_type'] = table.reference_column.type;
+        jobSyncMode['incremental_column_name'] = table.reference_column.name;
+        jobSyncMode['incremental_column_type'] = table.reference_column.type;
       }
 
       if (table.identifier_columns) {
-        body['primary_keys'] = table.identifier_columns;
+        hasUpdateSyncMode = true;
-      }
+        jobSyncMode['primary_keys'] = table.identifier_columns;
-
-      this.logger.info('Request body', body);
-      const updateCollumns = this.platformAPI.proxy(
-        'PATCH',
-        `/jobs/${id}/input`,
-        user,
-        body
-      )
-      requests.push(updateCollumns);
-
-      if (table.memory) {
-        this.logger.info('Updating memory allocation for table', table.name);
-        const updateMemory = this.platformAPI.proxy(
-          'PUT',
-          `/jobs/${id}/memory`,
-          user,
-          {
-            amount: table.memory
-          }
-        )
-        requests.push(updateMemory);
       }
 
       if (table.type) {
-          const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
+        hasUpdateSyncMode = true;
-          requests.push(updateSyncMode);
+        jobSyncMode['target_load_type'] = table.type;
+
       }
-    }
 
-    this.logger.info('Create Platform Request for each JOB');
+      if(hasUpdateSyncMode) {
+        jobUpdate["sync_mode"] = jobSyncMode;
+      }
 
-    if (updateInputDTO.cron) {
+      if (Object.keys(table.destinations).length > 1) {
-      const crnUpdatedRequest = new Promise(async (resolve, reject) => {
+        let hasChanges = false
-        const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
+        const jobRenameTables = {
-
+            raw: {},
-        if (response.error) {
+            qualify: {}
-          this.logger.error('Error updating pipeline cron', response.error);
-          return reject(new ErrorBuilder(response.error));
         }
-        this.logger.error('Pipeline cron updated successfully', response);
+
-        return resolve(response);
+        if (Object.keys(table.destinations.raw).length > 1) {
-      });
+          hasChanges = true;
-      requests.push(crnUpdatedRequest);
+          jobRenameTables.raw = table.destinations.raw;
+        }
+
+        if (Object.keys(table.destinations.qualify).length > 1) {
+          hasChanges = true;
+          jobRenameTables.qualify = table.destinations.qualify;
+        }
+
+        if (hasChanges) {
+          jobUpdate['rename_tables'] = jobRenameTables;
+        }
+      }
+
+      jobsUpdated.push(jobUpdate);
     }
 
-    this.logger.info('Executing all request for the platform api');
+    this.logger.info('Request body:' + JSON.stringify({
+      jobs_updated: jobsUpdated
+    }));
 
-    const results = await Promise.allSettled(requests);
+    const response = await this.platformAPI.proxy(
-    this.logger.info('Platform api response', results);
+      'PUT',
-
+        `/pipeline/${pipelineId}/jobs`,
-    return updateInputResponse;
-    
-  }
-
-  private async updatePipelineSyncMode(table: UpdateTableDTO,  pipelineId: string, user: RequestUser) {
-    const body = {
-      target_load_type: table.type
-    }
-
-    if (table.type === 'incremental_with_qualify') {
-      body['incremental_column_name'] = table.reference_column.name;
-      body['incremental_column_type'] = table.reference_column.type;
-      body['primary_keys'] = table.identifier_columns;
-    }
-
-    if (table.type === 'incremental') {
-      body['incremental_column_name'] = table.reference_column.name;
-      body['incremental_column_type'] = table.reference_column.type;
-    }
-
-    this.logger.info('Updating pipeline sync mode', {
-      pipelineId,
-      body
-    });
-
-    return this.platformAPI.proxy(
-      "POST",
-      `/jobs/jdbc/${pipelineId}/sync-mode`,
       user,
-      body
+      {
+        job_updates: jobsUpdated
+      }
     )
+    this.logger.info('Platform api response: ' + JSON.stringify(response));
   }
 
-  private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
-    try {
-      const response = await this.platformAPI.proxy(
-        'PATCH',
-        `/pipeline/${pipelineId}`,
-        user,
-        {
-          cron
-        }
-      );
-
-      return response
-    } catch (error) {
-        return {
-          error: error.message
-        }
-    }
-  }
 }

src/modules/platform-api/platform-api.controller.ts

@@ -845,6 +845,24 @@ export class PlatformApiController {
     );
   }
 
+  @Post('pipeline/:pipelineId/pipeline_run/:runId/cancel')
+  @ApiOperation({ summary: 'Cancel a running pipeline run' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  async cancelPipelineRun(
+    @Param('pipelineId') pipelineId: string,
+    @Param('runId') runId: string,
+    @User() user: RequestUser,
+  ) {
+    const normalizedPipelineId = this.normalizePipelineId(pipelineId);
+    const normalizedRunId = this.normalizePipelineId(runId);
+
+    return this.platformApiService.proxy(
+      'POST',
+      `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
+      user,
+    );
+  }
+
   // ==================== JOBS - COLUMN EDITING ROUTES ====================
 
   @Put('jobs/:jobId/input')

src/modules/platform-api/platform-api.service.ts

@@ -89,12 +89,12 @@ export class PlatformApiService {
 
       // Propagate non-2xx responses as HttpExceptions
       if (response.status >= 400) {
-        this.logger.error('Platform API upstream error', {
+        this.logger.error('Platform API upstream error' + JSON.stringify({
           status: response.status,
           data: response.data,
           path,
           method: method.toUpperCase(),
-        });
+        }));
         throw new HttpException(response.data, response.status);
       }
 
@@ -107,6 +107,8 @@ export class PlatformApiService {
         method: method.toUpperCase(),
       });
 
+      this.logger.error(error)
+
       if (error instanceof HttpException) {
         throw error;
       }

src/services/nimbus/nimbus.module.ts

@@ -0,0 +1,9 @@
+import { Module } from "@nestjs/common";
+import { NimbusService } from "./nimbus.service";
+import DadosferaLogger from "@dadosfera/dadosfera-logs";
+
+@Module({
+  providers: [NimbusService, DadosferaLogger],
+  exports: [NimbusService],
+})
+export class NimbusServicesModule {}

src/services/nimbus/nimbus.service.ts

@@ -0,0 +1,56 @@
+import DadosferaLogger from "@dadosfera/dadosfera-logs";
+import { Inject, Injectable } from "@nestjs/common";
+import axios from "axios";
+
+type TableUpdate = {
+    table_schema: string;
+    table_name: string;
+}
+
+@Injectable()
+export class NimbusService {
+    private logger: DadosferaLogger;
+  
+    constructor(
+      @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
+    ) {
+      this.logger = dadosferaLogger.logger;
+    }
+
+    private buildUrl(customerName: string) {
+        if (process.env.ENV === 'prd') {
+            return `https://nimbus-${customerName}.dadosfera.ai`;
+          }
+      
+          return `https://nimbus-${customerName}.${process.env.ENV.replace(
+            'local',
+            'stg',
+          )}.dadosfera.ai`;
+    }
+
+    async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) {
+        const nimbusUrl = this.buildUrl(customerName);
+
+        const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
+
+        try {
+            this.logger.info("Request for PATCH " + nimbusUrl + path);
+            this.logger.info("Payload: " + JSON.stringify(update));
+            const { data } = await axios.patch(nimbusUrl + path, {
+                table_name: update.table_name,
+                table_schema: update.table_schema
+            })
+
+            return data;
+        } catch (error) {
+            this.logger.error(error);
+            return {
+                message: error.message,
+                database,
+                old, 
+                update
+            }
+        }
+
+    }
+}