Merge pull request #70 from dadosfera/beta

Beta

.github/workflows/deploy.yml

@@ -23,19 +23,15 @@ jobs:
           fi
         id: extract_environment
 
-  deploy:
-    needs: extract_environment
-    runs-on:
-      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+  semantic_release:
+    runs-on: ubuntu-latest
+    outputs:
+      new_release_published: ${{ steps.semantic.outputs.new_release_published }}
+      new_release_version: ${{ steps.semantic.outputs.new_release_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      # Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
-      - name: BKP .npmrc Step 1
-        run: |
-          cp .npmrc .npmrcbkp
-
       - name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2
         id: semantic
@@ -60,40 +56,43 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      # Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
-      - name: BKP .npmrc Step 2
-        run: |
-          cp .npmrcbkp .npmrc
+  deploy:
+    needs: [extract_environment, semantic_release]
+    runs-on: [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
 
       - name: Update Pip
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install --upgrade pip
 
       - name: Install Docker Compose
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install docker-compose --upgrade
 
       - name: Install AWS CLI
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awscli --upgrade
 
       - name: Install AWS Elastic Beanstalk CLI
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
         id: aws
         with:
           aws-region: us-east-1
 
       - name: Extract Environment
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         shell: bash
         run: |
           if [ ${GITHUB_REF} == "refs/heads/main" ]; then
@@ -106,15 +105,15 @@ jobs:
         id: extract_environment
 
       - name: Login to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         id: login_ecr
         uses: aws-actions/amazon-ecr-login@v1
 
       - name: Build, Tag, and Push Image to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
@@ -122,10 +121,10 @@ jobs:
           docker-compose -f build.docker-compose.yml push
 
       - name: Create ZIP file to Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
@@ -134,13 +133,14 @@ jobs:
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
+        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
           AWS_REGION: us-east-1
           APP_NAME: maestro
         run: |
           eb use $APP_NAME-$ENV
+          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
           eb deploy
 
       - name: Remove Docker's Trash

README.md

@@ -1,4 +1,4 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
   <h1 align="center">Maestro</h1> 
-</p> 
+</p>

package-lock.json

@@ -18,10 +18,11 @@
         "@nestjs/microservices": "^8.4.3",
         "@nestjs/platform-express": "^8.4.3",
         "@nestjs/swagger": "^5.2.1",
-        "@victorradael/protospack": "^1.4.5",
+        "@victorradael/protospack": "^1.5.5",
         "axios": "^0.25.0",
         "dotenv": "^14.2.0",
         "grpc": "^1.24.11",
+        "helmet": "^5.0.2",
         "jsonwebtoken": "^8.5.1",
         "jwk-to-pem": "^2.0.5",
         "reflect-metadata": "^0.1.13",
@@ -2372,9 +2373,9 @@
       }
     },
     "node_modules/@victorradael/protospack": {
-      "version": "1.4.5",
-      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.4.5.tgz",
-      "integrity": "sha512-xYJyN+C5GRwCfSbZOZyUcswBQZO99y1koMNnIIZHEY1TgHnMjehMTQv3F8lKmUelvsfUBhTebrrptjRoxhh4Vg==",
+      "version": "1.5.5",
+      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.5.5.tgz",
+      "integrity": "sha512-5yGW/EuZDY9N56CVMXJW1IrBkkv6KpalBepu0LLQvM5Y+Pr8lwY2i1p/1rlsjHo6Gk1RODKuWSXAL870jQEAdg==",
       "dependencies": {
         "rxjs": "^7.5.5"
       }
@@ -5232,6 +5233,14 @@
         "minimalistic-assert": "^1.0.1"
       }
     },
+    "node_modules/helmet": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/helmet/-/helmet-5.0.2.tgz",
+      "integrity": "sha512-QWlwUZZ8BtlvwYVTSDTBChGf8EOcQ2LkGMnQJxSzD1mUu8CCjXJZq/BXP8eWw4kikRnzlhtYo3lCk0ucmYA3Vg==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/hexoid": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",
@@ -11386,9 +11395,9 @@
       }
     },
     "@victorradael/protospack": {
-      "version": "1.4.5",
-      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.4.5.tgz",
-      "integrity": "sha512-xYJyN+C5GRwCfSbZOZyUcswBQZO99y1koMNnIIZHEY1TgHnMjehMTQv3F8lKmUelvsfUBhTebrrptjRoxhh4Vg==",
+      "version": "1.5.5",
+      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.5.5.tgz",
+      "integrity": "sha512-5yGW/EuZDY9N56CVMXJW1IrBkkv6KpalBepu0LLQvM5Y+Pr8lwY2i1p/1rlsjHo6Gk1RODKuWSXAL870jQEAdg==",
       "requires": {
         "rxjs": "^7.5.5"
       }
@@ -13620,6 +13629,11 @@
         "minimalistic-assert": "^1.0.1"
       }
     },
+    "helmet": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/helmet/-/helmet-5.0.2.tgz",
+      "integrity": "sha512-QWlwUZZ8BtlvwYVTSDTBChGf8EOcQ2LkGMnQJxSzD1mUu8CCjXJZq/BXP8eWw4kikRnzlhtYo3lCk0ucmYA3Vg=="
+    },
     "hexoid": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",

package.json

@@ -30,10 +30,11 @@
     "@nestjs/microservices": "^8.4.3",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.4.5",
+    "@victorradael/protospack": "^1.5.5",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "grpc": "^1.24.11",
+    "helmet": "^5.0.2",
     "jsonwebtoken": "^8.5.1",
     "jwk-to-pem": "^2.0.5",
     "reflect-metadata": "^0.1.13",

src/clients/auth/client.config.ts

@@ -1,7 +1,7 @@
 import { credentials } from '@grpc/grpc-js';
 import { ClientOptions, Transport } from '@nestjs/microservices';
 
-import { DucProtoFilePath } from '@victorradael/protospack';
+import { DucProtoFilePath, DucPackages } from '@victorradael/protospack';
 
 export class AuthClient {
   config(): ClientOptions {
@@ -9,7 +9,7 @@ export class AuthClient {
       transport: Transport.GRPC,
       options: {
         url: process.env.DUC_URL,
-        package: 'duc',
+        package: DucPackages,
         credentials: credentials.createSsl(),
         protoPath: DucProtoFilePath,
         loader: {

src/clients/auth/client.service.ts

@@ -1,27 +1,30 @@
 import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 
-import { CognitoServiceInterface } from '@victorradael/protospack';
+import {
+  AuthServiceInterface,
+  DucServicesNames,
+} from '@victorradael/protospack';
 
 import { ILogin } from './interfaces';
 
 export class AuthClientService implements OnModuleInit {
-  private cognitoService: CognitoServiceInterface;
+  private authService: AuthServiceInterface;
   constructor(
     @Inject('AUTH_PACKAGE') private readonly grpcClient: ClientGrpc,
   ) {}
 
   onModuleInit() {
-    this.cognitoService = this.grpcClient.getService<CognitoServiceInterface>(
-      'CognitoProtoService',
+    this.authService = this.grpcClient.getService<AuthServiceInterface>(
+      DucServicesNames.AuthProtoService,
     );
   }
 
   async signIn({ username, password }: ILogin): Promise<any> {
-    console.log('CognitoClientService', 'SignIn');
+    console.log('AuthClientService', 'SignIn');
 
     const tokens = await new Promise((resolve, reject) => {
-      this.cognitoService.signIn({ username, password }).subscribe({
+      this.authService.signIn({ username, password }).subscribe({
         next(x) {
           resolve(x);
         },

src/clients/outputs/client.config.ts

@@ -1,4 +1,4 @@
-import { OutputProtofile } from '@victorradael/protospack';
+import { OutputPackages, OutputProtoFilePath } from '@victorradael/protospack';
 import { ClientOptions, Transport } from '@nestjs/microservices';
 import { credentials } from '@grpc/grpc-js';
 
@@ -8,9 +8,9 @@ export class OutputsClientConfiguration {
       transport: Transport.GRPC,
       options: {
         url: process.env.OTFACTORY_URL,
-        package: 'output',
+        package: OutputPackages,
         credentials: credentials.createSsl(),
-        protoPath: OutputProtofile,
+        protoPath: OutputProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/outputs/client.service.ts

@@ -1,6 +1,9 @@
 import { Controller, Inject, OnModuleInit } from '@nestjs/common';
 import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { OutputsServiceInterface } from '@victorradael/protospack';
+import {
+  OutputServicesNames,
+  OutputsServiceInterface,
+} from '@victorradael/protospack';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
@@ -19,8 +22,9 @@ export class OutputsClientService implements OnModuleInit {
   ) {}
 
   onModuleInit() {
-    this.outputService =
-      this.grpcClient.getService<OutputsServiceInterface>('OutputService');
+    this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
+      OutputServicesNames.OutputService,
+    );
   }
 
   async create(@Payload() createOutputDto: ICreateOutputRequest) {

src/clients/pipelines/client.config.ts

@@ -1,5 +1,8 @@
 import { ClientOptions, Transport } from '@nestjs/microservices';
-import { PipelineProtofile } from '@victorradael/protospack';
+import {
+  PipelinePackages,
+  PipelineProtoFilePath,
+} from '@victorradael/protospack';
 import { credentials } from '@grpc/grpc-js';
 
 export class PipelinesClientConfiguration {
@@ -8,9 +11,9 @@ export class PipelinesClientConfiguration {
       transport: Transport.GRPC,
       options: {
         url: process.env.PIFACTORY_URL,
-        package: 'pipeline',
+        package: PipelinePackages,
         credentials: credentials.createSsl(),
-        protoPath: PipelineProtofile,
+        protoPath: PipelineProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/pipelines/client.service.ts

@@ -1,13 +1,15 @@
 import { Inject, OnModuleInit } from '@nestjs/common';
 import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { PipelinesServiceInterface } from '@victorradael/protospack';
+import {
+  PipelineServicesNames,
+  PipelinesServiceInterface,
+} from '@victorradael/protospack';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
 import {
   ICreatePipelineDto,
-  IdRequest,
   IGetPipelineLogsRequest,
   IIdRequest,
   IUpdatePipelineRequest,
@@ -21,7 +23,9 @@ export class PipelinesClientService implements OnModuleInit {
 
   onModuleInit() {
     this.pipelineService =
-      this.grpcClient.getService<PipelinesServiceInterface>('PipelineService');
+      this.grpcClient.getService<PipelinesServiceInterface>(
+        PipelineServicesNames.PipelineService,
+      );
   }
 
   async create(@Payload() createPipelineDto: ICreatePipelineDto) {
@@ -29,7 +33,7 @@ export class PipelinesClientService implements OnModuleInit {
 
     const createPipelineResponse = await new Promise((resolve, reject) => {
       this.pipelineService
-        .create(objectSnakeToCamel(createPipelineDto))
+        .Create(objectSnakeToCamel(createPipelineDto))
         .subscribe({
           next(x) {
             resolve(x);
@@ -55,7 +59,7 @@ export class PipelinesClientService implements OnModuleInit {
     console.log('PipelinesClientService', 'FindAll');
 
     const findAllPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.findAll(objectSnakeToCamel(data)).subscribe({
+      this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
         next(x) {
           resolve(x);
         },
@@ -80,7 +84,7 @@ export class PipelinesClientService implements OnModuleInit {
     console.log('PipelinesClientService', 'FindOne');
 
     const findOnePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.findOne(objectSnakeToCamel(data)).subscribe({
+      this.pipelineService.FindOne(objectSnakeToCamel(data)).subscribe({
         next(x) {
           resolve(x);
         },
@@ -106,7 +110,7 @@ export class PipelinesClientService implements OnModuleInit {
 
     const updatePipelineResponse = await new Promise((resolve, reject) => {
       this.pipelineService
-        .update(objectSnakeToCamel(UpdatePipelineRequest))
+        .Update(objectSnakeToCamel(UpdatePipelineRequest))
         .subscribe({
           next(x) {
             resolve(x);
@@ -178,11 +182,11 @@ export class PipelinesClientService implements OnModuleInit {
     return logsPipelineResponse;
   }
 
-  async getPipelineStatus({ id }: IdRequest) {
+  async getPipelineStatus(data: IIdRequest) {
     console.log('PipelinesClientService', 'GetPipelineStatus');
 
     const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.getPipelineStatus({ id }).subscribe({
+      this.pipelineService.getPipelineStatus(data).subscribe({
         next(x) {
           resolve(x);
         },

src/clients/transformations/client.config.ts

@@ -1,5 +1,8 @@
 import { ClientOptions, Transport } from '@nestjs/microservices';
-import { TransformationProtofile } from '@victorradael/protospack';
+import {
+  TransformationPackages,
+  TransformationProtoFilePath,
+} from '@victorradael/protospack';
 import { credentials } from '@grpc/grpc-js';
 
 export class TransformationsClientConfiguration {
@@ -8,9 +11,9 @@ export class TransformationsClientConfiguration {
       transport: Transport.GRPC,
       options: {
         url: process.env.TRFACTORY_URL,
-        package: 'transformation',
+        package: TransformationPackages,
         credentials: credentials.createSsl(),
-        protoPath: TransformationProtofile,
+        protoPath: TransformationProtoFilePath,
         loader: {
           enums: String,
           objects: true,

src/clients/transformations/client.service.ts

@@ -28,7 +28,7 @@ export class TransformationsClientService implements OnModuleInit {
     const createTransformationResponse = await new Promise(
       (resolve, reject) => {
         this.transformationService
-          .create(objectSnakeToCamel(createTransformationsDto))
+          .Create(objectSnakeToCamel(createTransformationsDto))
           .subscribe({
             next(x) {
               resolve(x);
@@ -62,7 +62,7 @@ export class TransformationsClientService implements OnModuleInit {
 
     const findAllTransformationResponse = await new Promise(
       (resolve, reject) => {
-        this.transformationService.findAll(objectSnakeToCamel(data)).subscribe({
+        this.transformationService.FindAll(objectSnakeToCamel(data)).subscribe({
           next(x) {
             resolve(x);
           },
@@ -93,7 +93,7 @@ export class TransformationsClientService implements OnModuleInit {
 
     const findOneTransformationResponse = await new Promise(
       (resolve, reject) => {
-        this.transformationService.findOne(objectSnakeToCamel(data)).subscribe({
+        this.transformationService.FindOne(objectSnakeToCamel(data)).subscribe({
           next(x) {
             resolve(x);
           },
@@ -121,7 +121,7 @@ export class TransformationsClientService implements OnModuleInit {
     const updateTransformationResponse = await new Promise(
       (resolve, reject) => {
         this.transformationService
-          .update(objectSnakeToCamel(updateTransformationDTO))
+          .Update(objectSnakeToCamel(updateTransformationDTO))
           .subscribe({
             next(x) {
               resolve(x);
@@ -149,7 +149,7 @@ export class TransformationsClientService implements OnModuleInit {
 
     const removeTransformationResponse = await new Promise(
       (resolve, reject) => {
-        this.transformationService.remove(objectSnakeToCamel(data)).subscribe({
+        this.transformationService.Remove(objectSnakeToCamel(data)).subscribe({
           next(x) {
             resolve(x);
           },

src/main.ts

@@ -1,6 +1,7 @@
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { writeFileSync } from 'fs';
+import helmet from 'helmet';
 
 import { AppModule } from './app.module';
 
@@ -14,6 +15,8 @@ async function bootstrap() {
     },
   });
 
+  app.use(helmet());
+
   const config = new DocumentBuilder()
     .setTitle('Maestro Grpc Documentation')
     .setDescription('Documentation for Maestro gateway')

src/modules/catalog/catalog.controller.ts

@@ -241,6 +241,17 @@ export class CatalogController {
     return res;
   }
 
+  @Get('table-tags')
+  async getAllTableTags(@Body() body) {
+    console.log(`/catalog`, 'ON GET ALL TABLE TAGS ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.findAllTableTags(body);
+
+    return res;
+  }
+
   @Delete('table-tags')
   async deleteTableTags(@Param() params, @Body() body) {
     console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');

src/modules/catalog/catalog.service.ts

@@ -232,6 +232,19 @@ class CatalogService {
     return data;
   }
 
+  async findAllTableTags(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-tags/`,
+      body,
+    );
+    return data;
+  }
+
   async deleteTags(id, body) {
     const customer =
       body.info.customer.toLowerCase() === 'dadosfera'

src/modules/pipelines/pipelines.controller.ts

@@ -31,15 +31,17 @@ export class PipelinesController {
   }
 
   @Get(':id/status')
-  async getPipelineStatus(@Param() params) {
+  async getPipelineStatus(@Body() body, @Param() params) {
     const { id } = params;
+    body.id = id;
+
     console.log(
       process.env.DEV_URL + `/pipeline/${id}`,
       'ON GET PIPELINE STATUS ROUTE',
     );
 
     const pipelineService = new PipelinesService(this.pipelinesClientService);
-    const response = await pipelineService.getPipelineStatus(id);
+    const response = await pipelineService.getPipelineStatus(body);
 
     return response;
   }

src/modules/pipelines/pipelines.service.ts

@@ -74,10 +74,10 @@ export class PipelinesService {
     }
   }
 
-  async getPipelineStatus(id: string) {
+  async getPipelineStatus(data: IIdRequest) {
     try {
       const pipelineStatusResponse =
-        await this.pipelineClient.getPipelineStatus({ id });
+        await this.pipelineClient.getPipelineStatus(data);
 
       return objectCamelToSnake(pipelineStatusResponse);
     } catch (err) {