FIX: Fix runPipeline fail case return

Beta

.ebextensions/imdsv2.config

@@ -0,0 +1,7 @@
+Resources:
+  AWSEBAutoScalingLaunchConfiguration:
+    Properties:
+      MetadataOptions:
+        HttpEndpoint: enabled
+        HttpPutResponseHopLimit: 2
+        HttpTokens: required

.github/workflows/deploy.yml

@@ -23,19 +23,15 @@ jobs:
           fi
         id: extract_environment
 
-  deploy:
-    needs: extract_environment
-    runs-on:
-      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+  semantic_release:
+    runs-on: ubuntu-latest
+    outputs:
+      new_release_published: ${{ steps.semantic.outputs.new_release_published }}
+      new_release_version: ${{ steps.semantic.outputs.new_release_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      # Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
-      - name: BKP .npmrc Step 1
-        run: |
-          cp .npmrc .npmrcbkp
-
       - name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2
         id: semantic
@@ -60,40 +56,43 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      # Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
-      - name: BKP .npmrc Step 2
-        run: |
-          cp .npmrcbkp .npmrc
+  deploy:
+    needs: [extract_environment, semantic_release]
+    runs-on: [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
 
       - name: Update Pip
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install --upgrade pip
 
       - name: Install Docker Compose
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install docker-compose --upgrade
 
       - name: Install AWS CLI
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awscli --upgrade
 
       - name: Install AWS Elastic Beanstalk CLI
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
         id: aws
         with:
           aws-region: us-east-1
 
       - name: Extract Environment
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         shell: bash
         run: |
           if [ ${GITHUB_REF} == "refs/heads/main" ]; then
@@ -106,15 +105,15 @@ jobs:
         id: extract_environment
 
       - name: Login to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         id: login_ecr
         uses: aws-actions/amazon-ecr-login@v1
 
       - name: Build, Tag, and Push Image to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
@@ -122,10 +121,10 @@ jobs:
           docker-compose -f build.docker-compose.yml push
 
       - name: Create ZIP file to Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
@@ -134,13 +133,14 @@ jobs:
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
           AWS_REGION: us-east-1
           APP_NAME: maestro
         run: |
           eb use $APP_NAME-$ENV
+          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
           eb deploy
 
       - name: Remove Docker's Trash

package.json

@@ -30,10 +30,10 @@
     "@nestjs/microservices": "^8.4.3",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.5.5",
+    "@victorradael/protospack": "^1.5.9",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
-    "grpc": "^1.24.11",
+    "helmet": "^5.0.2",
     "jsonwebtoken": "^8.5.1",
     "jwk-to-pem": "^2.0.5",
     "reflect-metadata": "^0.1.13",

src/clients/inputs/client.service.ts

@@ -1,16 +1,16 @@
 import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
-import { InputService } from '@victorradael/protospack';
+import {
+  InputNewCreateRequest,
+  InputService,
+  TestConnectionGetColumnsRequest,
+  TestConnectionRequest,
+} from '@victorradael/protospack';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
-import {
-  ICreateInputRequest,
-  IIdRequest,
-  ITestConnectionRequest,
-  UpdateInputRequest,
-} from './interfaces';
+import { IIdRequest, UpdateInputRequest } from './interfaces';
 
 export class InputsClientService implements OnModuleInit {
   private inputService: InputService;
@@ -23,22 +23,24 @@ export class InputsClientService implements OnModuleInit {
       this.grpcClient.getService<InputService>('InputService');
   }
 
-  async create(createInputDto: ICreateInputRequest) {
+  async create(createInputDto: InputNewCreateRequest) {
     console.log('InputClientService', 'Create');
 
     const createInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Create(objectSnakeToCamel(createInputDto)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
+      this.inputService
+        .NewCreate(objectSnakeToCamel(createInputDto))
+        .subscribe({
+          next(x) {
+            resolve(objectCamelToSnake(x));
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
     })
       .then((res) => res)
       .catch((err) => {
@@ -147,10 +149,10 @@ export class InputsClientService implements OnModuleInit {
     return removeInputResponse;
   }
 
-  async testConnection(data: ITestConnectionRequest) {
+  async testConnection(data: TestConnectionRequest) {
     console.log('InputClientService', 'TestConnection');
     const testConnectionResponse = await new Promise((resolve, reject) => {
-      this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({
+      this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
         next(x) {
           resolve(objectCamelToSnake(x));
         },
@@ -170,4 +172,28 @@ export class InputsClientService implements OnModuleInit {
 
     return testConnectionResponse;
   }
+
+  async getColumns(data: TestConnectionGetColumnsRequest) {
+    console.log('InputClientService', 'TestConnection/Get-Columns');
+    const getColumnsResponse = await new Promise((resolve, reject) => {
+      this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
+        next(x) {
+          resolve(objectCamelToSnake(x));
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    })
+      .then((res) => res)
+      .catch((err) => {
+        throw new Error(err);
+      });
+
+    return getColumnsResponse;
+  }
 }

src/clients/pipelines/client.service.ts

@@ -182,22 +182,31 @@ export class PipelinesClientService implements OnModuleInit {
     return logsPipelineResponse;
   }
 
-  async getPipelineStatus(data: IIdRequest) {
+  async getPipelineStatus(data) {
     console.log('PipelinesClientService', 'GetPipelineStatus');
 
     const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.getPipelineStatus(data).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
+      this.pipelineService
+        .getPipelineStatus(objectSnakeToCamel(data))
+        .subscribe({
+          next(x) {
+            const statusArray = x.status.sort((a, b) => {
+              if (Date.parse(a.updated_at) < Date.parse(b.updated_at)) {
+                return 1;
+              } else {
+                return -1;
+              }
+            });
+            resolve(statusArray);
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
     })
       .then((res) => res)
       .catch((err) => {
@@ -214,6 +223,11 @@ export class PipelinesClientService implements OnModuleInit {
         .triggerPipeline(objectSnakeToCamel({ id, info }))
         .subscribe({
           next(x) {
+            if (x.status == false) {
+              reject(
+                'This pipeline is not ready yet to execute, Try again later!',
+              );
+            }
             resolve(x);
           },
           error(err) {

src/main.ts

@@ -1,6 +1,7 @@
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { writeFileSync } from 'fs';
+import helmet from 'helmet';
 
 import { AppModule } from './app.module';
 
@@ -14,6 +15,8 @@ async function bootstrap() {
     },
   });
 
+  app.use(helmet());
+
   const config = new DocumentBuilder()
     .setTitle('Maestro Grpc Documentation')
     .setDescription('Documentation for Maestro gateway')

src/modules/catalog/catalog.controller.ts

@@ -1,4 +1,12 @@
-import { Body, Controller, Delete, Get, Param, Post } from '@nestjs/common';
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  Param,
+  Post,
+  Query,
+} from '@nestjs/common';
 import { CatalogService } from './catalog.service';
 
 @Controller('catalog')
@@ -33,7 +41,7 @@ export class CatalogController {
     const { id } = params;
     const catalogService = new CatalogService();
 
-    const res = await catalogService.dataAppsOne(body, id);
+    const res = await catalogService.dataAppsOne(id, body);
 
     return res;
   }
@@ -56,64 +64,67 @@ export class CatalogController {
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.getOneDashboardMetabase(body, id);
+    const res = await catalogService.getOneDashboardMetabase(id, body);
 
     return res;
   }
 
   @Get('table-metadata')
-  async getAllTableMetadata(@Body() body) {
+  async getAllTableMetadata(@Body() body, @Query() query) {
     console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE');
 
     const catalogService = new CatalogService();
-
-    const res = await catalogService.getAllTableMetadata(body);
-
-    return res;
+    if (!query) {
+      const res = await catalogService.getAllTableMetadata(body);
+      return res;
+    } else {
+      const res = await catalogService.getOneTableMetadata(body, query);
+      return res;
+    }
   }
 
-  @Delete('table-metadata')
+  @Delete('table-metadata/:id')
   async deleteOneTableMetadata(@Param() params, @Body() body) {
     const { id } = params;
     console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE');
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.deleteOneTableMetadata(body, id);
+    const res = await catalogService.deleteOneTableMetadata(id, body);
 
     return res;
   }
 
   @Get('column-metadata')
-  async getOneColumnMetadata(@Body() body) {
+  async getOneColumnMetadata(@Body() body, @Query() params) {
     console.log(`/catalog`, 'ON GET ONE COLUMN METADATA ROUTE');
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.getOneColumnMetadata(body);
+    const res = await catalogService.getOneColumnMetadata(body, params);
 
     return res;
   }
 
-  @Delete('column-metadata')
+  @Delete('column-metadata/:id')
   async deleteOneColumnMetadata(@Param() params, @Body() body) {
     console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE');
     const { id } = params;
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.deleteOneColumnMetadata(body, id);
+    const res = await catalogService.deleteOneColumnMetadata(id, body);
 
     return res;
   }
 
   @Get('data-preview')
-  async getOneDataPreview(@Body() body) {
+  async getOneDataPreview(@Body() body, @Query() params) {
     console.log(`/catalog`, 'ON GET ONE DATAPREVIEW ROUTE');
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.getOneDataPreview(body);
+    const res = await catalogService.getOneDataPreview(body, params);
 
     return res;
   }
@@ -162,6 +173,17 @@ export class CatalogController {
     return res;
   }
 
+  @Post('data-rating')
+  async createDataRating(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.createDataRating(body);
+
+    return res;
+  }
+
   @Get('summary-rating/:id')
   async getSummaryRating(@Param() params, @Body() body) {
     console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE');
@@ -185,13 +207,24 @@ export class CatalogController {
     return res;
   }
 
+  @Post('data-comment')
+  async createDataComment(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.createDataComment(body);
+
+    return res;
+  }
+
   @Get('data-review')
-  async getDataReview(@Body() body) {
+  async getDataReview(@Body() body, @Query() params) {
     console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE');
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.getDataReview(body);
+    const res = await catalogService.getDataReview(body, params);
 
     return res;
   }
@@ -225,7 +258,7 @@ export class CatalogController {
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.deleteTags(body, id);
+    const res = await catalogService.deleteTags(id, body);
 
     return res;
   }
@@ -252,14 +285,14 @@ export class CatalogController {
     return res;
   }
 
-  @Delete('table-tags')
+  @Delete('table-tags/:id')
   async deleteTableTags(@Param() params, @Body() body) {
     console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');
     const { id } = params;
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.deleteTableTags(body, id);
+    const res = await catalogService.deleteTableTags(id, body);
 
     return res;
   }
@@ -275,14 +308,14 @@ export class CatalogController {
     return res;
   }
 
-  @Delete('table-rules')
+  @Delete('table-rules/:id')
   async deleteTableRules(@Param() params, @Body() body) {
     console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE');
     const { id } = params;
 
     const catalogService = new CatalogService();
 
-    const res = await catalogService.deleteTableRules(body, id);
+    const res = await catalogService.deleteTableRules(id, body);
 
     return res;
   }

src/modules/catalog/catalog.service.ts

@@ -51,7 +51,7 @@ class CatalogService {
     return data;
   }
 
-  async getOneDashboardMetabase(body, id) {
+  async getOneDashboardMetabase(id, body) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'
         ? ``
@@ -60,6 +60,11 @@ class CatalogService {
     const { data } = await axios.get(
       `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/dashboard-metabase/${id}`,
     );
+
+    const [, path] = data.iframe_url.split('.dadosfera');
+    const host = `https://metabase-${body.info.customer.toLowerCase()}.dadosfera.ai`;
+    data.iframe_url = host + path;
+
     return data;
   }
 
@@ -75,6 +80,19 @@ class CatalogService {
     return data;
   }
 
+  async getOneTableMetadata(body, params) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/`,
+      { params: params },
+    );
+    return data;
+  }
+
   async deleteOneTableMetadata(id, body) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'
@@ -87,7 +105,7 @@ class CatalogService {
     return data;
   }
 
-  async getOneColumnMetadata(body) {
+  async getOneColumnMetadata(body, params) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'
         ? ``
@@ -95,6 +113,7 @@ class CatalogService {
 
     const { data } = await axios.get(
       `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/column-metadata/`,
+      { params: params },
     );
     return data;
   }
@@ -111,7 +130,7 @@ class CatalogService {
     return data;
   }
 
-  async getOneDataPreview(body) {
+  async getOneDataPreview(body, params) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'
         ? ``
@@ -119,6 +138,7 @@ class CatalogService {
 
     const { data } = await axios.get(
       `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-preview/`,
+      { params: params },
     );
     return data;
   }
@@ -171,6 +191,19 @@ class CatalogService {
     return data;
   }
 
+  async createDataRating(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.post(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-rating/`,
+      body,
+    );
+    return data;
+  }
+
   async getSummaryRating(id, body) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'
@@ -194,7 +227,20 @@ class CatalogService {
     return data;
   }
 
-  async getDataReview(body) {
+  async createDataComment(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.post(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-comment/`,
+      body,
+    );
+    return data;
+  }
+
+  async getDataReview(body, params) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'
         ? ``
@@ -202,6 +248,7 @@ class CatalogService {
 
     const { data } = await axios.get(
       `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-review/`,
+      { params: params },
     );
     return data;
   }
@@ -245,7 +292,6 @@ class CatalogService {
     return data;
   }
 
-  
   async deleteTags(id, body) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'

src/modules/inputs/inputs.controller.ts

@@ -9,10 +9,8 @@ import {
 } from '@nestjs/common';
 import { InputsService } from './inputs.service';
 import { InputsClientService } from 'src/clients/inputs/client.service';
-import {
-  ICreateInputRequest,
-  UpdateInputRequest,
-} from 'src/clients/inputs/interfaces';
+import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
+import { InputNewCreateRequest } from '@victorradael/protospack';
 
 @Controller('inputs')
 export class InputsController {
@@ -28,8 +26,21 @@ export class InputsController {
     return response;
   }
 
+  @Post('/test-connection/get-columns')
+  async getColumns(@Body() data) {
+    console.log(
+      `/test-connection/get-columns`,
+      'ON TEST CONNECTION GET COLUMNS ROUTE',
+    );
+
+    const inputService = new InputsService(this.inputsClientService);
+    const response = await inputService.getColumns(data);
+
+    return response;
+  }
+
   @Post()
-  async create(@Body() createInputDto: ICreateInputRequest) {
+  async create(@Body() createInputDto: InputNewCreateRequest) {
     console.log(`/input`, 'ON CREATE ROUTE');
 
     const inputService = new InputsService(this.inputsClientService);

src/modules/inputs/inputs.service.ts

@@ -1,16 +1,13 @@
 import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import { InputNewCreateRequest } from '@victorradael/protospack';
 import { InputsClientService } from 'src/clients/inputs/client.service';
-import {
-  ICreateInputRequest,
-  IIdRequest,
-  Info,
-} from 'src/clients/inputs/interfaces';
+import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
 
 @Injectable()
 export class InputsService {
   constructor(private inputClient: InputsClientService) {}
 
-  async create(@Body() data: ICreateInputRequest) {
+  async create(@Body() data: InputNewCreateRequest) {
     try {
       const createInputResponse = await this.inputClient.create(data);
 
@@ -74,4 +71,15 @@ export class InputsService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  async getColumns(data) {
+    try {
+      const testConnectionGetColumnsResponse =
+        await this.inputClient.getColumns(data);
+
+      return testConnectionGetColumnsResponse;
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
 }

src/modules/pipelines/pipelines.controller.ts

@@ -14,13 +14,13 @@ import { PipelinesService } from './pipelines.service';
 export class PipelinesController {
   constructor(private pipelinesClientService: PipelinesClientService) {}
 
-  @Get('start/:id')
+  @Post('start/:id')
   async activate(@Param() params, @Body() body) {
     const { id } = params;
     const { info } = body;
 
     console.log(
-      process.env.DEV_URL + `/transformation/start/${id}`,
+      process.env.DEV_URL + `/pipeline/start/${id}`,
       'ON START PIPELINE ROUTE',
     );