FIX: npm ci

FIX: send correct name and email from user payload

package-lock.json

@@ -47,7 +47,6 @@
         "jwk-to-pem": "^2.0.5",
         "mixpanel": "^0.17.0",
         "ms": "^3.0.0-canary.1",
-        "multer": "^2.0.2",
         "openid-client": "^5.7.1",
         "passport": "^0.6.0",
         "passport-facebook": "^3.0.0",
@@ -2928,20 +2927,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/@nestjs/platform-express/node_modules/concat-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
-      "engines": [
-        "node >= 0.8"
-      ],
-      "dependencies": {
-        "buffer-from": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^2.2.2",
-        "typedarray": "^0.0.6"
-      }
-    },
     "node_modules/@nestjs/platform-express/node_modules/content-disposition": {
       "version": "0.5.4",
       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3111,24 +3096,6 @@
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "license": "MIT"
     },
-    "node_modules/@nestjs/platform-express/node_modules/multer": {
-      "version": "1.4.4-lts.1",
-      "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
-      "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
-      "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
-      "dependencies": {
-        "append-field": "^1.0.0",
-        "busboy": "^1.0.0",
-        "concat-stream": "^1.5.2",
-        "mkdirp": "^0.5.4",
-        "object-assign": "^4.1.1",
-        "type-is": "^1.6.4",
-        "xtend": "^4.0.0"
-      },
-      "engines": {
-        "node": ">= 6.0.0"
-      }
-    },
     "node_modules/@nestjs/platform-express/node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3153,25 +3120,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@nestjs/platform-express/node_modules/readable-stream": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
-      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
-      "dependencies": {
-        "core-util-is": "~1.0.0",
-        "inherits": "~2.0.3",
-        "isarray": "~1.0.0",
-        "process-nextick-args": "~2.0.0",
-        "safe-buffer": "~5.1.1",
-        "string_decoder": "~1.1.1",
-        "util-deprecate": "~1.0.1"
-      }
-    },
-    "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
-    },
     "node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3246,19 +3194,6 @@
         "node": ">= 0.8"
       }
     },
-    "node_modules/@nestjs/platform-express/node_modules/string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "dependencies": {
-        "safe-buffer": "~5.1.0"
-      }
-    },
-    "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
-    },
     "node_modules/@nestjs/platform-express/node_modules/tslib": {
       "version": "2.5.3",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -6573,6 +6508,7 @@
       "engines": [
         "node >= 6.0"
       ],
+      "license": "MIT",
       "dependencies": {
         "buffer-from": "^1.0.0",
         "inherits": "^2.0.3",
@@ -9191,11 +9127,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
-    },
     "node_modules/isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10793,6 +10724,7 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
       "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
+      "license": "MIT",
       "dependencies": {
         "append-field": "^1.0.0",
         "busboy": "^1.6.0",
@@ -11795,11 +11727,6 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/process-nextick-args": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
-    },
     "node_modules/process-warning": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",

package.json

@@ -65,7 +65,6 @@
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
     "ms": "^3.0.0-canary.1",
-    "multer": "^2.0.2",
     "openid-client": "^5.7.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",

src/modules/auth/auth.controller.ts

@@ -489,6 +489,7 @@ export class AuthController {
       const userDto = {
         id: api_key.user_id,
         name: api_key.username,
+        email: api_key.username,
         customer: {
           id: api_key.customer_id,
           name: api_key.customer_name,

src/modules/auth/auth.service.ts

@@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit {
 
     const userDto: UserDTO = {
       id: user.id,
-      name: user.username,
+      name: user.name,
+      email: user.email,
       jobTitle: user?.jobTitle || null,
       department: user?.department || null,
       hierarchy: user?.hierarchy || null,

src/modules/auth/dtos/login.ts

@@ -144,6 +144,7 @@ export interface BulkEditResponse {
 export type UserDTO = {
   id: string,
   name: string,
+  email: string,
   jobTitle?: string,
   department?: string,
   hierarchy?: string,

src/modules/inputs/inputs.controller.ts

@@ -99,6 +99,7 @@ export class InputsController {
       customer: info.customer,
     });
 
+    this.logger.info(JSON.stringify(body))
     const response = await this.inputService.create({ body, info });
 
     return response;

src/modules/pipelinesV2/pipelines.service.ts

@@ -356,125 +356,92 @@ export class PipelinesService implements OnModuleInit {
       info
     )
 
-    const requests = [];
-
     this.logger.info('Dynamo Response', updateInputResponse);
+    const jobsUpdated = [];
 
     for (const [index, table] of updateInputDTO.tables.entries()) {
-      const id = `${pipelineIdFormat}_${index}`;
+      const jobUpdate = {
+        job_id: `${pipelineIdFormat}_${index}`,
+      }
+
+      if (table.memory) {
+        jobUpdate["memory"] = {
+          amount: table.memory * 1000
+        }
+      }
+      
       this.logger.info('Updating input reference for table', table.name);
-      const body = {}
+      let hasUpdateSyncMode = false;
+
+      const jobSyncMode = {}
+
       if (table.columns) {
-        body['column_include_list'] = table.columns;
+        hasUpdateSyncMode = true;
+        jobSyncMode['column_include_list'] = table.columns;
       }
 
       if (table.reference_column) {
-        body['incremental_column_name'] = table.reference_column.name;
-        body['incremental_column_type'] = table.reference_column.type;
+        hasUpdateSyncMode = true;
+        jobSyncMode['incremental_column_name'] = table.reference_column.name;
+        jobSyncMode['incremental_column_type'] = table.reference_column.type;
       }
 
       if (table.identifier_columns) {
-        body['primary_keys'] = table.identifier_columns;
-      }
-
-      this.logger.info('Request body', body);
-      const updateCollumns = this.platformAPI.proxy(
-        'PATCH',
-        `/jobs/${id}/input`,
-        user,
-        body
-      )
-      requests.push(updateCollumns);
-
-      if (table.memory) {
-        this.logger.info('Updating memory allocation for table', table.name);
-        const updateMemory = this.platformAPI.proxy(
-          'PUT',
-          `/jobs/${id}/memory`,
-          user,
-          {
-            amount: table.memory
-          }
-        )
-        requests.push(updateMemory);
+        hasUpdateSyncMode = true;
+        jobSyncMode['primary_keys'] = table.identifier_columns;
       }
 
       if (table.type) {
-          const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
-          requests.push(updateSyncMode);
+        hasUpdateSyncMode = true;
+        jobSyncMode['target_load_type'] = table.type;
+
       }
-    }
 
-    this.logger.info('Create Platform Request for each JOB');
+      if(hasUpdateSyncMode) {
+        jobUpdate["sync_mode"] = jobSyncMode;
+      }
 
-    if (updateInputDTO.cron) {
-      const crnUpdatedRequest = new Promise(async (resolve, reject) => {
-        const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
-
-        if (response.error) {
-          this.logger.error('Error updating pipeline cron', response.error);
-          return reject(new ErrorBuilder(response.error));
+      if (Object.keys(table.destinations).length > 1) {
+        let hasChanges = false
+        const jobRenameTables = {
+            raw: {},
+            qualify: {}
         }
-        this.logger.error('Pipeline cron updated successfully', response);
-        return resolve(response);
-      });
-      requests.push(crnUpdatedRequest);
+
+        if (Object.keys(table.destinations.raw).length > 1) {
+          hasChanges = true;
+          jobRenameTables.raw = table.destinations.raw;
+        }
+
+        if (Object.keys(table.destinations.qualify).length > 1) {
+          hasChanges = true;
+          jobRenameTables.qualify = table.destinations.qualify;
+        }
+
+        if (hasChanges) {
+          jobUpdate['rename_tables'] = jobRenameTables;
+        }
+      }
+
+      jobsUpdated.push(jobUpdate);
     }
 
-    this.logger.info('Executing all request for the platform api');
+    this.logger.info('Request body:' + JSON.stringify({
+      jobs_updated: jobsUpdated
+    }));
 
-    const results = await Promise.allSettled(requests);
-    this.logger.info('Platform api response', results);
+    const response = await this.platformAPI.proxy(
+      'POST',
+        `/pipelines/${pipelineIdFormat}/batch-update`,
+      user,
+      {
+        job_updates: jobsUpdated
+      }
+    )
+    this.logger.info('Platform api response: ' + JSON.stringify(response));
 
     return updateInputResponse;
     
   }
 
-  private async updatePipelineSyncMode(table: UpdateTableDTO,  pipelineId: string, user: RequestUser) {
-    const body = {
-      target_load_type: table.type
-    }
-
-    if (table.type === 'incremental_with_qualify') {
-      body['incremental_column_name'] = table.reference_column.name;
-      body['incremental_column_type'] = table.reference_column.type;
-      body['primary_keys'] = table.identifier_columns;
-    }
-
-    if (table.type === 'incremental') {
-      body['incremental_column_name'] = table.reference_column.name;
-      body['incremental_column_type'] = table.reference_column.type;
-    }
-
-    this.logger.info('Updating pipeline sync mode', {
-      pipelineId,
-      body
-    });
-
-    return this.platformAPI.proxy(
-      "POST",
-      `/jobs/jdbc/${pipelineId}/sync-mode`,
-      user,
-      body
-    )
-  }
-
-  private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
-    try {
-      const response = await this.platformAPI.proxy(
-        'PATCH',
-        `/pipeline/${pipelineId}`,
-        user,
-        {
-          cron
-        }
-      );
-
-      return response
-    } catch (error) {
-        return {
-          error: error.message
-        }
-    }
-  }
 }