Merge pull request #98 from dadosfera/beta

Beta -> main

.gitignore

@@ -29,7 +29,7 @@ lerna-debug.log*
 
 # IDE - VSCode
 .vscode/*
-!.vscode/settings.json
+#!.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json

README.md

@@ -1,4 +1,4 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-  <h1 align="center">Maestro</h1> 
+  <h1 align="center">Maestro</h1>  
 </p>  

package-lock.json

@@ -19,7 +19,7 @@
         "@nestjs/platform-express": "^8.4.3",
         "@nestjs/schedule": "^1.1.0",
         "@nestjs/swagger": "^5.2.1",
-        "@victorradael/protospack": "^1.8.0",
+        "@victorradael/protospack": "2.3.0",
         "axios": "^0.25.0",
         "dotenv": "^14.2.0",
         "helmet": "^5.0.2",
@@ -2359,9 +2359,9 @@
       }
     },
     "node_modules/@victorradael/protospack": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.8.0.tgz",
-      "integrity": "sha512-vixXFliu7hPtJshGyxyekXBMsY1VCDo6WKu+aDKGHxAvd1cSG46UjAOYvYX5yPL4/J6fD//C4mdEYNW9l5TLvQ==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.3.0.tgz",
+      "integrity": "sha512-D2iy5Ff8G2FhH6Grs4MHM0qyqc6EU7iGuD3Bit6Tm6GbHmhDbLER9uBowClPjbPSAQ6jP7WTYxwn/TgDK4lNaA==",
       "dependencies": {
         "rxjs": "^7.5.5"
       }
@@ -4738,20 +4738,6 @@
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
       "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
     },
-    "node_modules/fsevents": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
-      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
-      "dev": true,
-      "hasInstallScript": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
-      }
-    },
     "node_modules/function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -10955,9 +10941,9 @@
       }
     },
     "@victorradael/protospack": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.8.0.tgz",
-      "integrity": "sha512-vixXFliu7hPtJshGyxyekXBMsY1VCDo6WKu+aDKGHxAvd1cSG46UjAOYvYX5yPL4/J6fD//C4mdEYNW9l5TLvQ==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.3.0.tgz",
+      "integrity": "sha512-D2iy5Ff8G2FhH6Grs4MHM0qyqc6EU7iGuD3Bit6Tm6GbHmhDbLER9uBowClPjbPSAQ6jP7WTYxwn/TgDK4lNaA==",
       "requires": {
         "rxjs": "^7.5.5"
       }
@@ -12828,13 +12814,6 @@
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
       "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
     },
-    "fsevents": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
-      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
-      "dev": true,
-      "optional": true
-    },
     "function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",

package.json

@@ -31,7 +31,7 @@
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "2.2.0",
+    "@victorradael/protospack": "2.3.0",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "helmet": "^5.0.2",

src/clients/auth/client.config.ts

@@ -10,7 +10,9 @@ export class AuthClient {
       options: {
         url: process.env.DUC_URL,
         package: DucPackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: DucProtoFilePath,
         loader: {
           enums: String,

src/clients/inputs/client.config.ts

@@ -10,7 +10,9 @@ export class InputsClientConfiguration {
       options: {
         url: process.env.INFACTORY_URL,
         package: InputPackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: InputProtoFilePath,
         loader: {
           enums: String,

src/clients/inputs/client.service.ts

@@ -1,6 +1,7 @@
 import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
+  InputCreateResponse,
   InputCreateS3Request,
   InputNewCreateRequest,
   InputService,
@@ -42,11 +43,9 @@ export class InputsClientService implements OnModuleInit {
             console.log('done');
           },
         });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
+    }).catch((err) => {
+      throw new Error(err);
+    });
 
     return createInputResponse;
   }

src/clients/outputs/client.config.ts

@@ -9,7 +9,9 @@ export class OutputsClientConfiguration {
       options: {
         url: process.env.OTFACTORY_URL,
         package: OutputPackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: OutputProtoFilePath,
         loader: {
           enums: String,

src/clients/pipelines/client.config.ts

@@ -12,7 +12,9 @@ export class PipelinesClientConfiguration {
       options: {
         url: process.env.PIFACTORY_URL,
         package: PipelinePackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: PipelineProtoFilePath,
         loader: {
           enums: String,

src/clients/pipelines/client.service.ts

@@ -61,7 +61,7 @@ export class PipelinesClientService implements OnModuleInit {
     const findAllPipelineResponse = await new Promise((resolve, reject) => {
       this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
         next(x) {
-          resolve(x);
+          resolve(x.pipelines);
         },
         error(err) {
           console.log('Observable Error');

src/modules/inputs/inputs.service.ts

@@ -14,10 +14,18 @@ export class InputsService {
         data.plugin == 'json' ||
         data.plugin == 'parquet'
       ) {
-        const createInputResponse = await this.inputClient.createS3Inputs(data);
+        const { info, ...input } = data;
+        const inputPayload = this.generateInputS3Payload(input);
+        const createInputResponse = await this.inputClient.createS3Inputs({
+          input: inputPayload,
+          info,
+        });
         return createInputResponse;
       } else {
-        const createInputResponse = await this.inputClient.create(data);
+        const createInputResponse: any = await this.inputClient.create(data);
+        if (createInputResponse?.input?.input_jdbc) {
+          return { input: createInputResponse?.input?.input_jdbc };
+        }
         return createInputResponse;
       }
     } catch (err) {
@@ -91,4 +99,29 @@ export class InputsService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  generateInputS3Payload(payload) {
+    const { credentials, plugin } = payload;
+    if (!credentials) return payload;
+
+    const {
+      client_aws_access_key_id,
+      client_aws_secret_access_key,
+      file_format_params,
+      format_file_params,
+      client_bucket,
+      file_to_extract,
+    } = credentials;
+    const formatedPayload = {
+      plugin,
+      source_bucket: client_bucket,
+      source_prefix: file_to_extract,
+      auth_parameters: {
+        aws_access_key_id: client_aws_access_key_id,
+        aws_secret_access_key: client_aws_secret_access_key,
+      },
+      file_format_params: file_format_params || format_file_params,
+    };
+    return formatedPayload;
+  }
 }

src/modules/pipelines/pipelines.service.ts

@@ -7,13 +7,20 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
 export class PipelinesService {
   constructor(private pipelineClient: PipelinesClientService) {}
 
+  adjustPayload(payload) {
+    payload.input = payload.input?.input_s3 || payload.input?.input_jdbc;
+  }
+
   async create(createPipelineDto) {
     try {
       const createPipelineResponse = await this.pipelineClient.create(
         createPipelineDto,
       );
 
-      return objectCamelToSnake(createPipelineResponse);
+      const pipeline = objectCamelToSnake(createPipelineResponse);
+      this.adjustPayload(pipeline.pipeline);
+
+      return pipeline;
     } catch (err) {
       console.log(err);
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -23,7 +30,11 @@ export class PipelinesService {
   async findOne(data: IIdRequest) {
     try {
       const findOnePipelineResponse = await this.pipelineClient.findOne(data);
-      return objectCamelToSnake(findOnePipelineResponse);
+
+      const pipeline = objectCamelToSnake(findOnePipelineResponse);
+      this.adjustPayload(pipeline.pipeline);
+
+      return pipeline;
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
@@ -33,7 +44,11 @@ export class PipelinesService {
     try {
       const findAllPipelineResponse = await this.pipelineClient.findAll(data);
 
-      return objectCamelToSnake(findAllPipelineResponse);
+      const pipelines = objectCamelToSnake(findAllPipelineResponse);
+      pipelines.forEach((pipeline) => {
+        this.adjustPayload(pipeline);
+      });
+      return { pipelines };
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
@@ -47,7 +62,10 @@ export class PipelinesService {
         ...data,
       });
 
-      return objectCamelToSnake(updatePipelineResponse);
+      const pipeline = objectCamelToSnake(updatePipelineResponse);
+      this.adjustPayload(pipeline);
+
+      return pipeline;
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }