CI: .

Beta -> main

.ebextensions/imdsv2.config

@@ -0,0 +1,7 @@
+Resources:
+  AWSEBAutoScalingLaunchConfiguration:
+    Properties:
+      MetadataOptions:
+        HttpEndpoint: enabled
+        HttpPutResponseHopLimit: 2
+        HttpTokens: required

.github/workflows/deploy-manually.yml

@@ -2,119 +2,111 @@ name: Deploy
 on:
   push:
     branches:
-      - alpha
-      - beta
       - main
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Deployment environment'
+        required: true
+        type: choice
+        options:
+          - dev
+          - stg
+          - prd
 
 jobs:
   extract_environment:
     runs-on: ubuntu-latest
     outputs:
       environment: ${{ steps.extract_environment.outputs.environment }}
+    env:
+      EVENT: ${{ github.event_name }}
+      DEPLOY_ENV: ${{ github.event.inputs.environment }}
     steps:
       - name: Extract Environment
         run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
+          if [ ${EVENT} == "workflow_dispatch" ]; then
+            echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
+          elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
             echo "##[set-output name=environment;]$(echo "prd")"
-          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
-            echo "##[set-output name=environment;]$(echo "dev")"
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "##[set-output name=environment;]$(echo "stg")"
           fi
         id: extract_environment
 
-  deploy:
-    needs: extract_environment
-    runs-on:
-      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+  semantic_release:
+    runs-on: ubuntu-latest
+    outputs:
+      #new_release_published: ${{ steps.semantic.outputs.last_release_version != steps.semantic.outputs.new_release_version }}
+      new_release_published: ${{ steps.semantic.outputs.new_release_published }}
+      new_release_version: ${{ steps.semantic.outputs.new_release_version }}
+      test: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
+      testt: ${{ (needs.semantic_release.outputs.new_release_published == 'true' && needs.semantic_release.outputs.new_release_version) || '1.0.0' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      # Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
-      - name: BKP .npmrc Step 1
-        run: |
-          cp .npmrc .npmrcbkp
-
-      - name: Semantic Release
+      - if: github.event_name != 'workflow_dispatch'
+        name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2
         id: semantic
         with:
           semantic_version: 16
           extra_plugins: |
-            @saithodev/semantic-release-backmerge
             conventional-changelog-eslint
           branches: |
             [
-              'main',
-              {
-                name: 'alpha',
-                prerelease: true
-              },
-              {
-                name: 'beta',
-                prerelease: true
-              }
+              'main'
             ]
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      # Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
-      - name: BKP .npmrc Step 2
-        run: |
-          cp .npmrcbkp .npmrc
+  deploy:
+    if: ${{ (github.event_name != 'workflow_dispatch' && needs.semantic_release.outputs.new_release_published == 'true') || 'true' }}
+    needs: [extract_environment, semantic_release]
+    runs-on:
+      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+
+    steps:
+      - name: print
+        env:
+          EVENT: ${{ github.event_name }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.test }}
+          IMAGE_TAGG: ${{ needs.semantic_release.outputs.testT }}
+        run: echo ${IMAGE_TAG} ${IMAGE_TAGG} ${EVENT}
+
+      - name: Checkout
+        uses: actions/checkout@v2
 
       - name: Update Pip
-        if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install --upgrade pip
 
       - name: Install Docker Compose
-        if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install docker-compose --upgrade
 
       - name: Install AWS CLI
-        if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awscli --upgrade
 
       - name: Install AWS Elastic Beanstalk CLI
-        if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        if: steps.semantic.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
         id: aws
         with:
           aws-region: us-east-1
 
-      - name: Extract Environment
-        if: steps.semantic.outputs.new_release_published == 'true'
-        shell: bash
-        run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=result;]$(echo "prd")"
-          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
-            echo "##[set-output name=result;]$(echo "dev")"
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "##[set-output name=result;]$(echo "stg")"
-          fi
-        id: extract_environment
-
       - name: Login to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
         id: login_ecr
         uses: aws-actions/amazon-ecr-login@v1
 
       - name: Build, Tag, and Push Image to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
         env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
@@ -122,10 +114,9 @@ jobs:
           docker-compose -f build.docker-compose.yml push
 
       - name: Create ZIP file to Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
         env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
@@ -134,13 +125,13 @@ jobs:
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
         env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
           AWS_REGION: us-east-1
-          APP_NAME: maestro
+          APP_NAME: ${{ github.event.repository.name }}
         run: |
           eb use $APP_NAME-$ENV
+          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
           eb deploy
 
       - name: Remove Docker's Trash

.gitignore

@@ -29,7 +29,7 @@ lerna-debug.log*
 
 # IDE - VSCode
 .vscode/*
-!.vscode/settings.json
+#!.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json

.releaserc.json

@@ -26,18 +26,6 @@
         "preset": "eslint"
       }
     ],
-    [
-      "@saithodev/semantic-release-backmerge",
-      {
-        "branches": [
-          { "from": "main", "to": "alpha" },
-          { "from": "main", "to": "beta" }
-        ],
-        "backmergeStrategy": "merge",
-        "clearWorkspace": true,
-        "restoreWorkspace": true
-      }
-    ],
     "@semantic-release/npm",
     "@semantic-release/github"
   ]

README.md

@@ -1,4 +1,4 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-  <h1 align="center">Maestro</h1> 
-</p>
+  <h1 align="center">Maestro</h1>  
+</p>  

package.json

@@ -29,11 +29,12 @@
     "@nestjs/mapped-types": "*",
     "@nestjs/microservices": "^8.4.3",
     "@nestjs/platform-express": "^8.4.3",
+    "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.4.2",
+    "@victorradael/protospack": "2.3.0",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
-    "grpc": "^1.24.11",
+    "helmet": "^5.0.2",
     "jsonwebtoken": "^8.5.1",
     "jwk-to-pem": "^2.0.5",
     "reflect-metadata": "^0.1.13",

src/app.module.ts

@@ -29,6 +29,8 @@ import { TransformationsClientConfiguration } from './clients/transformations/cl
 import { AuthClient } from './clients/auth/client.config';
 import { InputsClientConfiguration } from './clients/inputs/client.config';
 import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
+import { CatalogController } from './modules/catalog/catalog.controller';
+import { CatalogService } from './modules/catalog/catalog.service';
 
 const authClient = new AuthClient();
 const inputClient = new InputsClientConfiguration();
@@ -44,6 +46,7 @@ const transformationClient = new TransformationsClientConfiguration();
     PipelinesController,
     AuthController,
     HealthController,
+    CatalogController,
   ],
   providers: [
     InputsService,
@@ -57,6 +60,7 @@ const transformationClient = new TransformationsClientConfiguration();
     OutputsClientService,
     PipelinesClientService,
     AuthClientService,
+    CatalogService,
   ],
   imports: [
     ConfigModule.forRoot({
@@ -97,6 +101,7 @@ export class AppModule implements NestModule {
         TransformationsController,
         OutputsController,
         PipelinesController,
+        CatalogController,
       );
   }
 }

src/clients/auth/client.config.ts

@@ -1,7 +1,7 @@
 import { credentials } from '@grpc/grpc-js';
 import { ClientOptions, Transport } from '@nestjs/microservices';
 
-import { DucProtofile } from '@victorradael/protospack';
+import { DucProtoFilePath, DucPackages } from '@victorradael/protospack';
 
 export class AuthClient {
   config(): ClientOptions {
@@ -9,9 +9,11 @@ export class AuthClient {
       transport: Transport.GRPC,
       options: {
         url: process.env.DUC_URL,
-        package: 'duc',
-        credentials: credentials.createSsl(),
-        protoPath: DucProtofile,
+        package: DucPackages,
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
+        protoPath: DucProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/auth/client.service.ts

@@ -1,27 +1,124 @@
 import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 
-import { CognitoServiceInterface } from '@victorradael/protospack';
-
-import { ILogin } from './interfaces';
+import {
+  DucServicesNames,
+  AuthServiceInterface,
+  AuthSignInRequest,
+  AuthRefreshAccessTokenRequest,
+  AuthConfirmationRequest,
+  AuthResendConfirmationCodeRequest,
+  AuthEnableTotpMfaRequest,
+  AuthDisableTotpMfaRequest,
+  AuthDismissTotpMfaRequest,
+  AuthVerifyTotpMfaRequest,
+} from '@victorradael/protospack';
 
 export class AuthClientService implements OnModuleInit {
-  private cognitoService: CognitoServiceInterface;
+  private authService: AuthServiceInterface;
   constructor(
     @Inject('AUTH_PACKAGE') private readonly grpcClient: ClientGrpc,
   ) {}
 
   onModuleInit() {
-    this.cognitoService = this.grpcClient.getService<CognitoServiceInterface>(
-      'CognitoProtoService',
+    this.authService = this.grpcClient.getService<AuthServiceInterface>(
+      DucServicesNames.AuthProtoService,
     );
   }
 
-  async signIn({ username, password }: ILogin): Promise<any> {
-    console.log('CognitoClientService', 'SignIn');
+  async signIn({ username, password, totp }: AuthSignInRequest): Promise<any> {
+    console.log('AuthClientService', 'SignIn');
+
+    return new Promise((resolve, reject) => {
+      this.authService.signIn({ username, password, totp }).subscribe({
+        next: resolve,
+        //error: (err) => reject(err.details),
+        error: (err) => {
+          console.log(err);
+          reject(err.details);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    });
+  }
+
+  async enableTotpMFA({
+    accessToken,
+    password,
+  }: AuthEnableTotpMfaRequest): Promise<any> {
+    console.log('AuthClientService', 'enableTotpMFA');
+
+    return new Promise((resolve, reject) => {
+      this.authService.enableTotpMfa({ accessToken, password }).subscribe({
+        next: resolve,
+        error: (err) => reject(err.details),
+        complete() {
+          console.log('done');
+        },
+      });
+    });
+  }
+
+  async disableTotpMFA({
+    accessToken,
+    password,
+  }: AuthDisableTotpMfaRequest): Promise<any> {
+    console.log('AuthClientService', 'disableTotpMFA');
+
+    return new Promise((resolve, reject) => {
+      this.authService.disableTotpMfa({ accessToken, password }).subscribe({
+        next: resolve,
+        error: (err) => reject(err.details),
+        complete() {
+          console.log('done');
+        },
+      });
+    });
+  }
+
+  async dismissTotpMFA({
+    accessToken,
+  }: AuthDismissTotpMfaRequest): Promise<any> {
+    console.log('AuthClientService', 'dismissTotpMFA');
+
+    return new Promise((resolve, reject) => {
+      this.authService.dismissTotpMfa({ accessToken }).subscribe({
+        next: resolve,
+        error: (err) => reject(err.details),
+        complete() {
+          console.log('done');
+        },
+      });
+    });
+  }
+
+  async verifyTotp({
+    accessToken,
+    totp,
+  }: AuthVerifyTotpMfaRequest): Promise<any> {
+    console.log('AuthClientService', 'disableTotpMFA');
+
+    return new Promise((resolve, reject) => {
+      this.authService.verifyTotp({ accessToken, totp }).subscribe({
+        next: resolve,
+        error: (err) => reject(err.details),
+        complete() {
+          console.log('done');
+        },
+      });
+    });
+  }
+
+  async confirmRegister({
+    username,
+    code,
+  }: AuthConfirmationRequest): Promise<any> {
+    console.log('AuthClientService', 'confirmRegister');
 
     const tokens = await new Promise((resolve, reject) => {
-      this.cognitoService.signIn({ username, password }).subscribe({
+      this.authService.confirmRegister({ username, code }).subscribe({
         next(x) {
           resolve(x);
         },
@@ -40,4 +137,51 @@ export class AuthClientService implements OnModuleInit {
       });
     return tokens;
   }
+
+  async resendeConfirmationCode({
+    username,
+  }: AuthResendConfirmationCodeRequest): Promise<any> {
+    console.log('AuthClientService', 'resendConfirmationCode');
+
+    const authResendConfirmationCodeRequestReturn = await new Promise(
+      (resolve, reject) => {
+        this.authService.resendConfirmationCode({ username }).subscribe({
+          next(x) {
+            resolve(x);
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
+      },
+    )
+      .then((res) => res)
+      .catch((err) => {
+        throw new Error(err);
+      });
+    return authResendConfirmationCodeRequestReturn;
+  }
+
+  async refreshAccessToken({
+    username,
+    refreshToken,
+  }: AuthRefreshAccessTokenRequest): Promise<any> {
+    console.log('AuthClientService', 'RefreshAccessToken');
+
+    return new Promise((resolve, reject) => {
+      this.authService
+        .refreshAccessToken({ username, refreshToken })
+        .subscribe({
+          next: resolve,
+          error: (err) => reject(err.details),
+          complete() {
+            console.log('done');
+          },
+        });
+    });
+  }
 }

src/clients/auth/interfaces.d.ts

@@ -1,4 +0,0 @@
-export interface ILogin {
-  username: string;
-  password: string;
-}

src/clients/inputs/client.config.ts

@@ -1,7 +1,7 @@
 import { credentials } from '@grpc/grpc-js';
 import { ClientOptions, Transport } from '@nestjs/microservices';
 
-import { InputProtofile } from '@victorradael/protospack';
+import { InputPackages, InputProtoFilePath } from '@victorradael/protospack';
 
 export class InputsClientConfiguration {
   config(): ClientOptions {
@@ -9,9 +9,11 @@ export class InputsClientConfiguration {
       transport: Transport.GRPC,
       options: {
         url: process.env.INFACTORY_URL,
-        package: 'input',
-        credentials: credentials.createSsl(),
-        protoPath: InputProtofile,
+        package: InputPackages,
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
+        protoPath: InputProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/inputs/client.service.ts

@@ -1,16 +1,18 @@
 import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
-import { InputService } from '@victorradael/protospack';
+import {
+  InputCreateResponse,
+  InputCreateS3Request,
+  InputNewCreateRequest,
+  InputService,
+  TestConnectionGetColumnsRequest,
+  TestConnectionRequest,
+} from '@victorradael/protospack';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
-import {
-  ICreateInputRequest,
-  IIdRequest,
-  ITestConnectionRequest,
-  UpdateInputRequest,
-} from './interfaces';
+import { IIdRequest, UpdateInputRequest } from './interfaces';
 
 export class InputsClientService implements OnModuleInit {
   private inputService: InputService;
@@ -23,11 +25,36 @@ export class InputsClientService implements OnModuleInit {
       this.grpcClient.getService<InputService>('InputService');
   }
 
-  async create(createInputDto: ICreateInputRequest) {
+  async create(createInputDto: InputNewCreateRequest) {
     console.log('InputClientService', 'Create');
 
     const createInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Create(objectSnakeToCamel(createInputDto)).subscribe({
+      this.inputService
+        .NewCreate(objectSnakeToCamel(createInputDto))
+        .subscribe({
+          next(x) {
+            resolve(objectCamelToSnake(x));
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
+    }).catch((err) => {
+      throw new Error(err);
+    });
+
+    return createInputResponse;
+  }
+
+  async createS3Inputs(createInputDto: InputCreateS3Request) {
+    console.log('InputClientService', 'Create');
+
+    const createInputResponse = await new Promise((resolve, reject) => {
+      this.inputService.CreateS3(objectSnakeToCamel(createInputDto)).subscribe({
         next(x) {
           resolve(objectCamelToSnake(x));
         },
@@ -147,10 +174,10 @@ export class InputsClientService implements OnModuleInit {
     return removeInputResponse;
   }
 
-  async testConnection(data: ITestConnectionRequest) {
+  async testConnection(data: TestConnectionRequest) {
     console.log('InputClientService', 'TestConnection');
     const testConnectionResponse = await new Promise((resolve, reject) => {
-      this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({
+      this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
         next(x) {
           resolve(objectCamelToSnake(x));
         },
@@ -170,4 +197,28 @@ export class InputsClientService implements OnModuleInit {
 
     return testConnectionResponse;
   }
+
+  async getColumns(data: TestConnectionGetColumnsRequest) {
+    console.log('InputClientService', 'TestConnection/Get-Columns');
+    const getColumnsResponse = await new Promise((resolve, reject) => {
+      this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
+        next(x) {
+          resolve(objectCamelToSnake(x));
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    })
+      .then((res) => res)
+      .catch((err) => {
+        throw new Error(err);
+      });
+
+    return getColumnsResponse;
+  }
 }

src/clients/outputs/client.config.ts

@@ -1,4 +1,4 @@
-import { OutputProtofile } from '@victorradael/protospack';
+import { OutputPackages, OutputProtoFilePath } from '@victorradael/protospack';
 import { ClientOptions, Transport } from '@nestjs/microservices';
 import { credentials } from '@grpc/grpc-js';
 
@@ -8,9 +8,11 @@ export class OutputsClientConfiguration {
       transport: Transport.GRPC,
       options: {
         url: process.env.OTFACTORY_URL,
-        package: 'output',
-        credentials: credentials.createSsl(),
-        protoPath: OutputProtofile,
+        package: OutputPackages,
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
+        protoPath: OutputProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/outputs/client.service.ts

@@ -1,6 +1,9 @@
 import { Controller, Inject, OnModuleInit } from '@nestjs/common';
 import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { OutputsServiceInterface } from '@victorradael/protospack';
+import {
+  OutputServicesNames,
+  OutputsServiceInterface,
+} from '@victorradael/protospack';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
@@ -19,8 +22,9 @@ export class OutputsClientService implements OnModuleInit {
   ) {}
 
   onModuleInit() {
-    this.outputService =
-      this.grpcClient.getService<OutputsServiceInterface>('OutputService');
+    this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
+      OutputServicesNames.OutputService,
+    );
   }
 
   async create(@Payload() createOutputDto: ICreateOutputRequest) {

src/clients/pipelines/client.config.ts

@@ -1,5 +1,8 @@
 import { ClientOptions, Transport } from '@nestjs/microservices';
-import { PipelineProtofile } from '@victorradael/protospack';
+import {
+  PipelinePackages,
+  PipelineProtoFilePath,
+} from '@victorradael/protospack';
 import { credentials } from '@grpc/grpc-js';
 
 export class PipelinesClientConfiguration {
@@ -8,9 +11,11 @@ export class PipelinesClientConfiguration {
       transport: Transport.GRPC,
       options: {
         url: process.env.PIFACTORY_URL,
-        package: 'pipeline',
-        credentials: credentials.createSsl(),
-        protoPath: PipelineProtofile,
+        package: PipelinePackages,
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
+        protoPath: PipelineProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/pipelines/client.service.ts

@@ -1,13 +1,15 @@
 import { Inject, OnModuleInit } from '@nestjs/common';
 import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { PipelinesServiceInterface } from '@victorradael/protospack';
+import {
+  PipelineServicesNames,
+  PipelinesServiceInterface,
+} from '@victorradael/protospack';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
 import {
   ICreatePipelineDto,
-  IdRequest,
   IGetPipelineLogsRequest,
   IIdRequest,
   IUpdatePipelineRequest,
@@ -21,7 +23,9 @@ export class PipelinesClientService implements OnModuleInit {
 
   onModuleInit() {
     this.pipelineService =
-      this.grpcClient.getService<PipelinesServiceInterface>('PipelineService');
+      this.grpcClient.getService<PipelinesServiceInterface>(
+        PipelineServicesNames.PipelineService,
+      );
   }
 
   async create(@Payload() createPipelineDto: ICreatePipelineDto) {
@@ -29,7 +33,7 @@ export class PipelinesClientService implements OnModuleInit {
 
     const createPipelineResponse = await new Promise((resolve, reject) => {
       this.pipelineService
-        .create(objectSnakeToCamel(createPipelineDto))
+        .Create(objectSnakeToCamel(createPipelineDto))
         .subscribe({
           next(x) {
             resolve(x);
@@ -55,9 +59,9 @@ export class PipelinesClientService implements OnModuleInit {
     console.log('PipelinesClientService', 'FindAll');
 
     const findAllPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.findAll(objectSnakeToCamel(data)).subscribe({
+      this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
         next(x) {
-          resolve(x);
+          resolve(x.pipelines);
         },
         error(err) {
           console.log('Observable Error');
@@ -80,7 +84,7 @@ export class PipelinesClientService implements OnModuleInit {
     console.log('PipelinesClientService', 'FindOne');
 
     const findOnePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.findOne(objectSnakeToCamel(data)).subscribe({
+      this.pipelineService.FindOne(objectSnakeToCamel(data)).subscribe({
         next(x) {
           resolve(x);
         },
@@ -106,7 +110,7 @@ export class PipelinesClientService implements OnModuleInit {
 
     const updatePipelineResponse = await new Promise((resolve, reject) => {
       this.pipelineService
-        .update(objectSnakeToCamel(UpdatePipelineRequest))
+        .Update(objectSnakeToCamel(UpdatePipelineRequest))
         .subscribe({
           next(x) {
             resolve(x);
@@ -178,22 +182,31 @@ export class PipelinesClientService implements OnModuleInit {
     return logsPipelineResponse;
   }
 
-  async getPipelineStatus({ id }: IdRequest) {
+  async getPipelineStatus(data) {
     console.log('PipelinesClientService', 'GetPipelineStatus');
 
     const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.getPipelineStatus({ id }).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
+      this.pipelineService
+        .getPipelineStatus(objectSnakeToCamel(data))
+        .subscribe({
+          next(x) {
+            const statusArray = x.status.sort((a, b) => {
+              if (a.id < b.id) {
+                return 1;
+              } else {
+                return -1;
+              }
+            });
+            resolve({ status: statusArray });
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
     })
       .then((res) => res)
       .catch((err) => {
@@ -210,6 +223,11 @@ export class PipelinesClientService implements OnModuleInit {
         .triggerPipeline(objectSnakeToCamel({ id, info }))
         .subscribe({
           next(x) {
+            if (x.status == false) {
+              reject(
+                'This pipeline is not ready yet to execute, Try again later!',
+              );
+            }
             resolve(x);
           },
           error(err) {

src/clients/transformations/client.config.ts

@@ -1,5 +1,8 @@
 import { ClientOptions, Transport } from '@nestjs/microservices';
-import { TransformationProtofile } from '@victorradael/protospack';
+import {
+  TransformationPackages,
+  TransformationProtoFilePath,
+} from '@victorradael/protospack';
 import { credentials } from '@grpc/grpc-js';
 
 export class TransformationsClientConfiguration {
@@ -7,10 +10,10 @@ export class TransformationsClientConfiguration {
     return {
       transport: Transport.GRPC,
       options: {
-        url: process.env.TRFACTORY_URL,
-        package: 'transformation',
+        url: process.env.INFACTORY_URL,
+        package: TransformationPackages,
         credentials: credentials.createSsl(),
-        protoPath: TransformationProtofile,
+        protoPath: TransformationProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/transformations/client.service.ts

@@ -28,7 +28,7 @@ export class TransformationsClientService implements OnModuleInit {
     const createTransformationResponse = await new Promise(
       (resolve, reject) => {
         this.transformationService
-          .create(objectSnakeToCamel(createTransformationsDto))
+          .Create(objectSnakeToCamel(createTransformationsDto))
           .subscribe({
             next(x) {
               resolve(x);
@@ -62,7 +62,7 @@ export class TransformationsClientService implements OnModuleInit {
 
     const findAllTransformationResponse = await new Promise(
       (resolve, reject) => {
-        this.transformationService.findAll(objectSnakeToCamel(data)).subscribe({
+        this.transformationService.FindAll(objectSnakeToCamel(data)).subscribe({
           next(x) {
             resolve(x);
           },
@@ -93,7 +93,7 @@ export class TransformationsClientService implements OnModuleInit {
 
     const findOneTransformationResponse = await new Promise(
       (resolve, reject) => {
-        this.transformationService.findOne(objectSnakeToCamel(data)).subscribe({
+        this.transformationService.FindOne(objectSnakeToCamel(data)).subscribe({
           next(x) {
             resolve(x);
           },
@@ -121,7 +121,7 @@ export class TransformationsClientService implements OnModuleInit {
     const updateTransformationResponse = await new Promise(
       (resolve, reject) => {
         this.transformationService
-          .update(objectSnakeToCamel(updateTransformationDTO))
+          .Update(objectSnakeToCamel(updateTransformationDTO))
           .subscribe({
             next(x) {
               resolve(x);
@@ -149,7 +149,7 @@ export class TransformationsClientService implements OnModuleInit {
 
     const removeTransformationResponse = await new Promise(
       (resolve, reject) => {
-        this.transformationService.remove(objectSnakeToCamel(data)).subscribe({
+        this.transformationService.Remove(objectSnakeToCamel(data)).subscribe({
           next(x) {
             resolve(x);
           },

src/main.ts

@@ -1,6 +1,7 @@
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { writeFileSync } from 'fs';
+import helmet from 'helmet';
 
 import { AppModule } from './app.module';
 
@@ -14,6 +15,8 @@ async function bootstrap() {
     },
   });
 
+  app.use(helmet());
+
   const config = new DocumentBuilder()
     .setTitle('Maestro Grpc Documentation')
     .setDescription('Documentation for Maestro gateway')

src/middlewares/authentication.ts

@@ -17,9 +17,11 @@ export class LoggerMiddleware implements NestMiddleware {
     const idToken = request.get('Dadosfera-User');
     const accessToken = request.get('Authorization');
     const privateKey = process.env.JWT_PRIVATE_KEY;
-
+    let requiredRoute = '';
     const requiredMethod = request.method.trim();
-    const requiredRoute = request.route.path.split('/')[1].trim();
+    if (request.route.path.split('/').length >= 2) {
+      requiredRoute = request.route.path.split('/')[1].trim();
+    }
 
     verify(idToken, privateKey, (err) => {
       if (err) {
@@ -29,26 +31,33 @@ export class LoggerMiddleware implements NestMiddleware {
     const jwtDecoded: any = decode(idToken);
 
     const permissions = jwtDecoded.user.permissions;
+
     const clienId = jwtDecoded.user.customerId;
+    const customer = jwtDecoded.user.customer;
     const userId = jwtDecoded.user.id;
 
     await verifyToken(accessToken);
 
     let havePermission = false;
-    permissions.forEach((permission) => {
+    const permission = permissions.find((permission) => {
       permission = permission.split('/');
       const method = permission[0].trim();
       const route = permission[1].trim();
 
       if (method === requiredMethod && route === requiredRoute) {
-        havePermission = true;
+        return permission;
       }
     });
 
+    if (permission) {
+      havePermission = true;
+    }
+
     if (havePermission) {
       request.body.info = {
         customer_id: clienId,
         user_id: userId,
+        customer,
       };
 
       next();

src/modules/auth/auth.controller.ts

@@ -1,27 +1,148 @@
-import { Body, Controller, Post, UnauthorizedException } from '@nestjs/common';
+import {
+  Body,
+  Controller,
+  Headers,
+  Post,
+  HttpCode,
+  HttpStatus,
+  UnauthorizedException,
+} from '@nestjs/common';
+import {
+  AuthSignInRequest,
+  AuthRefreshAccessTokenRequest,
+  AuthEnableTotpMfaRequest,
+  AuthDisableTotpMfaRequest,
+  AuthVerifyTotpMfaRequest,
+} from '@victorradael/protospack';
 
 import { AuthClientService } from 'src/clients/auth/client.service';
 
-interface ISignIn {
-  username: string;
-  password: string;
-}
+import ErrorBuilder from '../../utils/ErrorBuilder';
 
 @Controller('auth')
 export class AuthController {
   constructor(private authClient: AuthClientService) {}
 
+  // DEPRECADO!
   @Post()
-  async signIn(@Body() { username, password }: ISignIn) {
+  @HttpCode(HttpStatus.OK)
+  async signIn_old(@Body() body: AuthSignInRequest) {
+    console.log(`/auth`, 'SignIn_old');
+
+    return this.signIn(body);
+  }
+
+  @Post('login')
+  @HttpCode(HttpStatus.OK)
+  async signIn_login(@Body() body: AuthSignInRequest) {
+    console.log(`/auth`, 'SignIn_login');
+
+    return this.signIn(body);
+  }
+
+  @Post('sign-in')
+  @HttpCode(HttpStatus.OK)
+  async signIn(@Body() { username, password, totp }: AuthSignInRequest) {
     console.log(`/auth`, 'SignIn');
 
     const tokens = await this.authClient
-      .signIn({ username, password })
-      .then((result) => result)
+      .signIn({ username, password, totp })
       .catch((err) => {
-        throw new UnauthorizedException(err.message);
+        throw ErrorBuilder(err);
       });
 
     return tokens;
   }
+
+  @Post('refresh-access-token')
+  @HttpCode(HttpStatus.OK)
+  async refreshAccessToken(
+    @Body() { username, refreshToken }: AuthRefreshAccessTokenRequest,
+  ) {
+    console.log(`/auth`, 'RefreshAccessToken');
+
+    const accessToken = await this.authClient
+      .refreshAccessToken({ username, refreshToken })
+      .catch((err) => {
+        throw ErrorBuilder(err);
+      });
+
+    return accessToken;
+  }
+
+  @Post('enable-totp')
+  @HttpCode(HttpStatus.OK)
+  async enableTotpMFA(
+    @Body() body: AuthEnableTotpMfaRequest,
+    @Headers() headers,
+  ) {
+    console.log(`/auth`, 'enable-totp');
+
+    const { password } = body;
+    const { authorization: accessToken } = headers;
+
+    const response = await this.authClient
+      .enableTotpMFA({ accessToken, password })
+      .catch((err) => {
+        throw ErrorBuilder(err);
+      });
+
+    return response;
+  }
+
+  @Post('disable-totp')
+  @HttpCode(HttpStatus.OK)
+  async disableTotpMFA(
+    @Body() body: AuthDisableTotpMfaRequest,
+    @Headers() headers,
+  ) {
+    console.log(`/auth`, 'disable-totp');
+
+    const { password } = body;
+    const { authorization: accessToken } = headers;
+
+    const response = await this.authClient
+      .disableTotpMFA({ accessToken, password })
+      .catch((err) => {
+        throw ErrorBuilder(err);
+      });
+
+    return response;
+  }
+
+  @Post('dismiss-totp')
+  @HttpCode(HttpStatus.OK)
+  async dismissTotpMFA(@Headers() headers) {
+    console.log(`/auth`, 'disable-totp');
+
+    const { authorization: accessToken } = headers;
+
+    const response = await this.authClient
+      .dismissTotpMFA({ accessToken })
+      .catch((err) => {
+        throw ErrorBuilder(err);
+      });
+
+    return response;
+  }
+
+  @Post('verify-totp')
+  @HttpCode(HttpStatus.OK)
+  async verifyTotp(
+    @Body() body: AuthVerifyTotpMfaRequest,
+    @Headers() headers,
+  ): Promise<any> {
+    console.log(`/auth`, 'enable-totp');
+
+    const { totp } = body;
+    const { authorization: accessToken } = headers;
+
+    const response = await this.authClient
+      .verifyTotp({ accessToken, totp })
+      .catch((err) => {
+        throw ErrorBuilder(err);
+      });
+
+    return response;
+  }
 }

src/modules/catalog/catalog.controller.ts

@@ -0,0 +1,302 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  Param,
+  Post,
+  Query,
+} from '@nestjs/common';
+import { CatalogService } from './catalog.service';
+
+@Controller('catalog')
+export class CatalogController {
+  constructor(private catalogService: CatalogService) {}
+
+  @Get('all')
+  async catalogAll(@Body() body) {
+    console.log(`/catalog`, 'ON CATALOG ALL ROUTE');
+
+    const res = await this.catalogService.catalogAll(body);
+
+    return res;
+  }
+
+  @Get('data_apps')
+  async dataAppsAll(@Body() body) {
+    console.log(`/catalog`, 'ON FIND ALL DATA APPS ROUTE');
+
+    const res = await this.catalogService.dataAppsAll(body);
+
+    return res;
+  }
+
+  @Delete('data_apps/:id')
+  async dataAppsOne(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON FIND ONE DATA APP ROUTE');
+    const { id } = params;
+
+    const res = await this.catalogService.dataAppsOne(id, body);
+
+    return res;
+  }
+
+  @Get('dashboard-metabase')
+  async getAllDashboardMetabase(@Body() body) {
+    console.log(`/catalog`, 'ON GET ALL DASHBOARDS METABASE ROUTE');
+
+    const res = await this.catalogService.getAllDashboardMetabase(body);
+
+    return res;
+  }
+
+  @Get('dashboard-metabase/:id')
+  async getOneDashboardMetabase(@Body() body, @Param() params) {
+    const { id } = params;
+    console.log(`/catalog`, 'ON GET ONE DASHBOARD METABASE ROUTE');
+
+    const res = await this.catalogService.getOneDashboardMetabase(id, body);
+
+    return res;
+  }
+
+  @Get('table-metadata')
+  async getAllTableMetadata(@Body() body, @Query() query) {
+    console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE');
+
+    if (!query) {
+      const res = await this.catalogService.getAllTableMetadata(body);
+      return res;
+    } else {
+      const res = await this.catalogService.getOneTableMetadata(body, query);
+      return res;
+    }
+  }
+
+  @Delete('table-metadata/:id')
+  async deleteOneTableMetadata(@Param() params, @Body() body) {
+    const { id } = params;
+    console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE');
+
+    const res = await this.catalogService.deleteOneTableMetadata(id, body);
+
+    return res;
+  }
+
+  @Get('column-metadata')
+  async getOneColumnMetadata(@Body() body, @Query() params) {
+    console.log(`/catalog`, 'ON GET ONE COLUMN METADATA ROUTE');
+
+    const res = await this.catalogService.getOneColumnMetadata(body, params);
+
+    return res;
+  }
+
+  @Delete('column-metadata/:id')
+  async deleteOneColumnMetadata(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE');
+    const { id } = params;
+
+    const res = await this.catalogService.deleteOneColumnMetadata(id, body);
+
+    return res;
+  }
+
+  @Get('data-preview')
+  async getOneDataPreview(@Body() body, @Query() params) {
+    console.log(`/catalog`, 'ON GET ONE DATAPREVIEW ROUTE');
+
+    const res = await this.catalogService.getOneDataPreview(body, params);
+
+    return res;
+  }
+
+  @Get('data-status')
+  async getDataStatus(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA STATUS ROUTE');
+
+    const res = await this.catalogService.getDataStatus(body);
+
+    return res;
+  }
+
+  @Get('data-status')
+  async createDataStatus(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE DATA STATUS ROUTE');
+
+    const res = await this.catalogService.createDataStatus(body);
+
+    return res;
+  }
+
+  @Get('data-description')
+  async getDataDescription(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA DESCRIPTION ROUTE');
+
+    const res = await this.catalogService.getDataDescription(body);
+
+    return res;
+  }
+
+  @Post('data-description')
+  async createDataDescription(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE DATA DESCRIPTION ROUTE');
+
+    const res = await this.catalogService.createDataDescription(body);
+
+    return res;
+  }
+
+  @Get('data-docs')
+  async getDataDocs(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA DOCS ROUTE');
+
+    const res = await this.catalogService.getDataDocs(body);
+
+    return res;
+  }
+
+  @Post('data-docs')
+  async createDataDocs(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE DATA DOCS ROUTE');
+
+    const res = await this.catalogService.createDataDocs(body);
+
+    return res;
+  }
+
+  @Get('data-rating')
+  async getDataRating(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
+
+    const res = await this.catalogService.getDataRating(body);
+
+    return res;
+  }
+
+  @Post('data-rating')
+  async createDataRating(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
+
+    const res = await this.catalogService.createDataRating(body);
+
+    return res;
+  }
+
+  @Get('summary-rating/:id')
+  async getSummaryRating() {
+    console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE');
+    //const { id } = params;
+
+    //
+
+    //const res = await this.catalogService.getSummaryRating(id, body);
+
+    return {
+      avg_rating: 0,
+      rating_count: 0,
+    };
+  }
+
+  @Get('data-comment')
+  async getDataComment(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
+
+    const res = await this.catalogService.getDataComment(body);
+
+    return res;
+  }
+
+  @Post('data-comment')
+  async createDataComment(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
+
+    const res = await this.catalogService.createDataComment(body);
+
+    return res;
+  }
+
+  @Get('data-review')
+  async getDataReview(@Body() body, @Query() params) {
+    console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE');
+
+    const res = await this.catalogService.getDataReview(body, params);
+
+    return res;
+  }
+
+  @Post('tags')
+  async createTags(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE TAG ROUTE');
+
+    const res = await this.catalogService.createTags(body);
+
+    return res;
+  }
+
+  @Get('tags')
+  async findAllTags(@Body() body) {
+    console.log(`/catalog`, 'ON FIND ALL TAGS ROUTE');
+
+    const res = await this.catalogService.findAllTags(body);
+
+    return res;
+  }
+
+  @Delete('tags/:id')
+  async deleteTags(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON DELETE TAG ROUTE');
+    const { id } = params;
+
+    const res = await this.catalogService.deleteTags(id, body);
+
+    return res;
+  }
+
+  @Post('table-tags')
+  async createTableTags(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE TABLE TAG ROUTE');
+
+    const res = await this.catalogService.createTableTags(body);
+
+    return res;
+  }
+
+  @Get('table-tags')
+  async getAllTableTags(@Body() body) {
+    console.log(`/catalog`, 'ON GET ALL TABLE TAGS ROUTE');
+
+    const res = await this.catalogService.findAllTableTags(body);
+
+    return res;
+  }
+
+  @Delete('table-tags/:id')
+  async deleteTableTags(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');
+    const { id } = params;
+
+    const res = await this.catalogService.deleteTableTags(id, body);
+
+    return res;
+  }
+
+  @Get('table-rules')
+  async getTableRules(@Body() body) {
+    console.log(`/catalog`, 'ON GET TABLE RULES ROUTE');
+
+    const res = await this.catalogService.getTableRules(body);
+
+    return res;
+  }
+
+  @Delete('table-rules/:id')
+  async deleteTableRules(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE');
+    const { id } = params;
+
+    const res = await this.catalogService.deleteTableRules(id, body);
+
+    return res;
+  }
+}

src/modules/catalog/catalog.service.ts

@@ -0,0 +1,263 @@
+import axios from 'axios';
+
+class CatalogService {
+  _getNimbusUrl(body) {
+    const customer = body.info.customer.toLowerCase();
+
+    if (process.env.ENV === 'prd') {
+      return `https://nimbus-${customer}.dadosfera.ai`;
+    }
+
+    return `https://nimbus-${customer}.${process.env.ENV}.dadosfera.ai`;
+  }
+
+  async catalogAll(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/all/`);
+
+    return data;
+  }
+
+  async dataAppsAll(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data_apps/`);
+    return data;
+  }
+
+  async dataAppsOne(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/data_apps/${id}/`,
+    );
+
+    return data;
+  }
+
+  async getAllDashboardMetabase(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/dashboard-metabase/`,
+    );
+    return data;
+  }
+
+  async getOneDashboardMetabase(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/dashboard-metabase/${id}`,
+    );
+    console.log(data);
+
+    // const [, path] = data.iframe_url.split('.dadosfera');
+    // const host = `httpss://metabase-${body.info.customer.toLowerCase()}.dadosfera.ai`;
+    // data.iframe_url = host + path;
+
+    return data;
+  }
+
+  async getAllTableMetadata(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/table-metadata/`,
+    );
+    return data;
+  }
+
+  async getOneTableMetadata(body, params) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/table-metadata/`,
+      { params: params },
+    );
+    return data;
+  }
+
+  async deleteOneTableMetadata(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.delete(
+      `${nimbusUrl}/api/catalog/table-metadata/${id}`,
+    );
+    return data;
+  }
+
+  async getOneColumnMetadata(body, params) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/column-metadata/`,
+      { params: params },
+    );
+    return data;
+  }
+
+  async deleteOneColumnMetadata(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.delete(
+      `${nimbusUrl}/api/catalog/column-metadata/${id}/`,
+    );
+    return data;
+  }
+
+  async getOneDataPreview(body, params) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-preview/`, {
+      params: params,
+    });
+    return data;
+  }
+
+  async getDataStatus(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-status/`);
+    return data;
+  }
+
+  async createDataStatus(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(
+      `${nimbusUrl}/api/catalog/data-status/`,
+      body,
+    );
+    return data;
+  }
+
+  async getDataDescription(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/data-description/`,
+    );
+    return data;
+  }
+
+  async createDataDescription(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(
+      `${nimbusUrl}/api/catalog/data-description/`,
+      body,
+    );
+    return data;
+  }
+
+  async getDataDocs(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-docs/`);
+    return data;
+  }
+
+  async createDataDocs(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(
+      `${nimbusUrl}/api/catalog/data-docs/`,
+      body,
+    );
+    return data;
+  }
+
+  async getDataRating(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-rating/`);
+    return data;
+  }
+
+  async createDataRating(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(
+      `${nimbusUrl}/api/catalog/data-rating/`,
+      body,
+    );
+    return data;
+  }
+
+  async getSummaryRating(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/summary-rating/${id}`,
+    );
+    return data;
+  }
+
+  async getDataComment(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-comment/`);
+    return data;
+  }
+
+  async createDataComment(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(
+      `${nimbusUrl}/api/catalog/data-comment/`,
+      body,
+    );
+    return data;
+  }
+
+  async getDataReview(body, params) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-review/`, {
+      params: params,
+    });
+    return data;
+  }
+
+  async createTags(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(`${nimbusUrl}/api/catalog/tags/`, body);
+    return data;
+  }
+
+  async findAllTags(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(`${nimbusUrl}/api/catalog/tags/`, body);
+    return data;
+  }
+
+  async findAllTableTags(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/table-tags/`,
+      body,
+    );
+    return data;
+  }
+
+  async deleteTags(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.delete(`${nimbusUrl}/api/catalog/tags/${id}`);
+    return data;
+  }
+
+  async createTableTags(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.post(
+      `${nimbusUrl}/api/catalog/table-tags/`,
+      body,
+    );
+    return data;
+  }
+
+  async deleteTableTags(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.delete(
+      `${nimbusUrl}/api/catalog/table-tags/${id}`,
+    );
+    return data;
+  }
+
+  async getTableRules(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.get(
+      `${nimbusUrl}/api/catalog/table-rules/`,
+      body,
+    );
+    return data;
+  }
+
+  async deleteTableRules(id, body) {
+    const nimbusUrl = this._getNimbusUrl(body);
+    const { data } = await axios.delete(
+      `${nimbusUrl}/api/catalog/table-rules/${id}`,
+    );
+    return data;
+  }
+}
+
+export { CatalogService };

src/modules/inputs/inputs.controller.ts

@@ -9,10 +9,8 @@ import {
 } from '@nestjs/common';
 import { InputsService } from './inputs.service';
 import { InputsClientService } from 'src/clients/inputs/client.service';
-import {
-  ICreateInputRequest,
-  UpdateInputRequest,
-} from 'src/clients/inputs/interfaces';
+import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
+import { InputNewCreateRequest } from '@victorradael/protospack';
 
 @Controller('inputs')
 export class InputsController {
@@ -28,8 +26,21 @@ export class InputsController {
     return response;
   }
 
+  @Post('/test-connection/get-columns')
+  async getColumns(@Body() data) {
+    console.log(
+      `/test-connection/get-columns`,
+      'ON TEST CONNECTION GET COLUMNS ROUTE',
+    );
+
+    const inputService = new InputsService(this.inputsClientService);
+    const response = await inputService.getColumns(data);
+
+    return response;
+  }
+
   @Post()
-  async create(@Body() createInputDto: ICreateInputRequest) {
+  async create(@Body() createInputDto: InputNewCreateRequest) {
     console.log(`/input`, 'ON CREATE ROUTE');
 
     const inputService = new InputsService(this.inputsClientService);

src/modules/inputs/inputs.service.ts

@@ -1,20 +1,33 @@
 import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import { Timeout } from '@nestjs/schedule';
 import { InputsClientService } from 'src/clients/inputs/client.service';
-import {
-  ICreateInputRequest,
-  IIdRequest,
-  Info,
-} from 'src/clients/inputs/interfaces';
+import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
 
 @Injectable()
 export class InputsService {
   constructor(private inputClient: InputsClientService) {}
 
-  async create(@Body() data: ICreateInputRequest) {
+  async create(@Body() data) {
     try {
-      const createInputResponse = await this.inputClient.create(data);
-
-      return createInputResponse;
+      if (
+        data.plugin == 'csv' ||
+        data.plugin == 'json' ||
+        data.plugin == 'parquet'
+      ) {
+        const { info, ...input } = data;
+        const inputPayload = this.generateInputS3Payload(input);
+        const createInputResponse = await this.inputClient.createS3Inputs({
+          input: inputPayload,
+          info,
+        });
+        return createInputResponse;
+      } else {
+        const createInputResponse: any = await this.inputClient.create(data);
+        if (createInputResponse?.input?.input_jdbc) {
+          return { input: createInputResponse?.input?.input_jdbc };
+        }
+        return createInputResponse;
+      }
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
@@ -63,6 +76,7 @@ export class InputsService {
     }
   }
 
+  @Timeout(60000 * 10) // Timeout set for 10 minutes
   async testConnection(data) {
     try {
       const testConnectionInputResponse = await this.inputClient.testConnection(
@@ -74,4 +88,40 @@ export class InputsService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  async getColumns(data) {
+    try {
+      const testConnectionGetColumnsResponse =
+        await this.inputClient.getColumns(data);
+
+      return testConnectionGetColumnsResponse;
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
+
+  generateInputS3Payload(payload) {
+    const { credentials, plugin } = payload;
+    if (!credentials) return payload;
+
+    const {
+      client_aws_access_key_id,
+      client_aws_secret_access_key,
+      file_format_params,
+      format_file_params,
+      client_bucket,
+      file_to_extract,
+    } = credentials;
+    const formatedPayload = {
+      plugin,
+      source_bucket: client_bucket,
+      source_prefix: file_to_extract,
+      auth_parameters: {
+        aws_access_key_id: client_aws_access_key_id,
+        aws_secret_access_key: client_aws_secret_access_key,
+      },
+      file_format_params: file_format_params || format_file_params,
+    };
+    return formatedPayload;
+  }
 }

src/modules/pipelines/pipelines.controller.ts

@@ -14,13 +14,13 @@ import { PipelinesService } from './pipelines.service';
 export class PipelinesController {
   constructor(private pipelinesClientService: PipelinesClientService) {}
 
-  @Get('start/:id')
+  @Post('start/:id')
   async activate(@Param() params, @Body() body) {
     const { id } = params;
     const { info } = body;
 
     console.log(
-      process.env.DEV_URL + `/transformation/start/${id}`,
+      process.env.DEV_URL + `/pipeline/start/${id}`,
       'ON START PIPELINE ROUTE',
     );
 
@@ -31,15 +31,17 @@ export class PipelinesController {
   }
 
   @Get(':id/status')
-  async getPipelineStatus(@Param() params) {
+  async getPipelineStatus(@Body() body, @Param() params) {
     const { id } = params;
+    body.id = id;
+
     console.log(
       process.env.DEV_URL + `/pipeline/${id}`,
       'ON GET PIPELINE STATUS ROUTE',
     );
 
     const pipelineService = new PipelinesService(this.pipelinesClientService);
-    const response = await pipelineService.getPipelineStatus(id);
+    const response = await pipelineService.getPipelineStatus(body);
 
     return response;
   }

src/modules/pipelines/pipelines.service.ts

@@ -7,13 +7,20 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
 export class PipelinesService {
   constructor(private pipelineClient: PipelinesClientService) {}
 
+  adjustPayload(payload) {
+    payload.input = payload.input?.input_s3 || payload.input?.input_jdbc;
+  }
+
   async create(createPipelineDto) {
     try {
       const createPipelineResponse = await this.pipelineClient.create(
         createPipelineDto,
       );
 
-      return objectCamelToSnake(createPipelineResponse);
+      const pipeline = objectCamelToSnake(createPipelineResponse);
+      this.adjustPayload(pipeline.pipeline);
+
+      return pipeline;
     } catch (err) {
       console.log(err);
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -23,7 +30,11 @@ export class PipelinesService {
   async findOne(data: IIdRequest) {
     try {
       const findOnePipelineResponse = await this.pipelineClient.findOne(data);
-      return objectCamelToSnake(findOnePipelineResponse);
+
+      const pipeline = objectCamelToSnake(findOnePipelineResponse);
+      this.adjustPayload(pipeline.pipeline);
+
+      return pipeline;
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
@@ -33,7 +44,11 @@ export class PipelinesService {
     try {
       const findAllPipelineResponse = await this.pipelineClient.findAll(data);
 
-      return objectCamelToSnake(findAllPipelineResponse);
+      const pipelines = objectCamelToSnake(findAllPipelineResponse);
+      pipelines.forEach((pipeline) => {
+        this.adjustPayload(pipeline);
+      });
+      return { pipelines };
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
@@ -47,7 +62,10 @@ export class PipelinesService {
         ...data,
       });
 
-      return objectCamelToSnake(updatePipelineResponse);
+      const pipeline = objectCamelToSnake(updatePipelineResponse);
+      this.adjustPayload(pipeline);
+
+      return pipeline;
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
@@ -74,10 +92,10 @@ export class PipelinesService {
     }
   }
 
-  async getPipelineStatus(id: string) {
+  async getPipelineStatus(data: IIdRequest) {
     try {
       const pipelineStatusResponse =
-        await this.pipelineClient.getPipelineStatus({ id });
+        await this.pipelineClient.getPipelineStatus(data);
 
       return objectCamelToSnake(pipelineStatusResponse);
     } catch (err) {

src/utils/ErrorBuilder.ts

@@ -0,0 +1,106 @@
+import { HttpStatus, HttpException } from '@nestjs/common';
+
+import ErrorCodes from './errorCodes';
+
+function Builder({ statusCode, message, error, code }) {
+  return new HttpException({ statusCode, message, error, code }, statusCode);
+}
+
+export default function ErrorBuilder(code: string) {
+  console.log(code);
+
+  switch (code) {
+    case ErrorCodes.AUTH.WRONG_CREDENTIALS:
+      return Builder({
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Não autenticado',
+        message: 'Usuário ou senha incorretos',
+        code,
+      });
+
+    case ErrorCodes.AUTH.WRONG_PASSWORD_CONFIRMATION:
+      return Builder({
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Senha incorreta',
+        message: 'Confirmação de senha incorreta',
+        code,
+      });
+
+    case ErrorCodes.AUTH.TOTP_NOT_ENABLED:
+      return Builder({
+        statusCode: HttpStatus.PRECONDITION_FAILED,
+        error: 'Não permitido',
+        message: 'A autenticação multifator não está habilitada',
+        code,
+      });
+
+    case ErrorCodes.AUTH.TOTP_ALREADY_ENABLED:
+      return Builder({
+        statusCode: HttpStatus.PRECONDITION_FAILED,
+        error: 'Não permitido',
+        message: 'A autenticação multifator já está habilitada',
+        code,
+      });
+
+    case ErrorCodes.AUTH.TOTP_ALREADY_DISABLED:
+      return Builder({
+        statusCode: HttpStatus.PRECONDITION_FAILED,
+        error: 'Não permitido',
+        message: 'A autenticação multifator já está desabilitada',
+        code,
+      });
+
+    case ErrorCodes.AUTH.TOTP_REQUIRED:
+      return Builder({
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Não autenticado',
+        message: 'Informe o token de autenticação multifator',
+        code,
+      });
+
+    case ErrorCodes.AUTH.CODE_MISMATCH:
+    case ErrorCodes.AUTH.CODE_ALREADY_USED:
+      return Builder({
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Não autenticado',
+        message: 'Token de autenticação multifator inválido',
+        code,
+      });
+
+    case ErrorCodes.RATE_LIMIT:
+      return Builder({
+        statusCode: HttpStatus.TOO_MANY_REQUESTS,
+        error: 'Limite excedido',
+        message:
+          'Você tentou realizar essa operação muitas vezes. Tente novamente mais tarde',
+        code,
+      });
+
+    case ErrorCodes.AUTH.UNAUTHORIZED:
+      return Builder({
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Não autenticado',
+        message: 'É necessário estar logado para realizar essa operação',
+        code,
+      });
+
+    case ErrorCodes.AUTH.FORBIDDEN:
+      return Builder({
+        statusCode: HttpStatus.FORBIDDEN,
+        error: 'Não autorizado',
+        message:
+          'Você não tem permissões suficientes para realizar essa operação',
+        code,
+      });
+
+    case ErrorCodes.UNKNOWN:
+    default:
+      return Builder({
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Desconhecido',
+        message:
+          'Erro desconhecido. Tente novamente ou entre em contato com o suporte',
+        code: ErrorCodes.UNKNOWN,
+      });
+  }
+}

src/utils/errorCodes.ts

@@ -0,0 +1,20 @@
+export const Auth = {
+  UNAUTHORIZED: 'AUTH.UNAUTHORIZED',
+  FORBIDDEN: 'AUTH.FORBIDDEN',
+  WRONG_CREDENTIALS: 'AUTH.WRONG_CREDENTIALS',
+  WRONG_PASSWORD_CONFIRMATION: 'AUTH.WRONG_PASSWORD_CONFIRMATION',
+  TOTP_NOT_ENABLED: 'AUTH.TOTP_NOT_ENABLED',
+  TOTP_ALREADY_ENABLED: 'AUTH.TOTP_ALREADY_ENABLED',
+  TOTP_ALREADY_DISABLED: 'AUTH.TOTP_ALREADY_DISABLED',
+  TOTP_REQUIRED: 'AUTH.TOTP_REQUIRED',
+  CODE_MISMATCH: 'AUTH.CODE_MISMATCH',
+  CODE_ALREADY_USED: 'AUTH.CODE_ALREADY_USED',
+};
+
+const ErrorCodes = {
+  UNKNOWN: 'UNKNOWN',
+  RATE_LIMIT: 'RATE_LIMIT',
+  AUTH: Auth,
+};
+
+export default ErrorCodes;