Merge pull request #54 from dadosfera/beta

Beta

.github/workflows/deploy.yml

@@ -7,8 +7,26 @@ on:
       - main
 
 jobs:
+  extract_environment:
+    runs-on: ubuntu-latest
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
+            echo "##[set-output name=environment;]$(echo "prd")"
+          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
+            echo "##[set-output name=environment;]$(echo "dev")"
+          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
+            echo "##[set-output name=environment;]$(echo "stg")"
+          fi
+        id: extract_environment
+
   deploy:
-    runs-on: self-hosted
+    needs: extract_environment
+    runs-on:
+      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -70,6 +88,7 @@ jobs:
       - name: Configure AWS Region
         if: steps.semantic.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
+        id: aws
         with:
           aws-region: us-east-1
 
@@ -78,9 +97,11 @@ jobs:
         shell: bash
         run: |
           if [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=result;]$(echo "production")"
-          else
-            echo "##[set-output name=result;]$(echo ${GITHUB_REF#refs/heads/})"
+            echo "##[set-output name=result;]$(echo "prd")"
+          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
+            echo "##[set-output name=result;]$(echo "dev")"
+          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
+            echo "##[set-output name=result;]$(echo "stg")"
           fi
         id: extract_environment
 
@@ -94,6 +115,7 @@ jobs:
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
           IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
@@ -104,9 +126,11 @@ jobs:
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
           IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
           sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+          sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk

.github/workflows/test.yml

@@ -16,6 +16,7 @@ jobs:
       - name: Install Docker Compose
         run: |
           python3 -m pip install docker-compose --upgrade
+
       - name: Build
         env:
           ENV: test
@@ -24,13 +25,22 @@ jobs:
         run: |
           export ENV=test
           export IMAGE_TAG=test
+          export ACCOUNT_ID=test
           docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
+
       - name: Run Test
         env:
           ENV: test
           IMAGE_TAG: test
+          ACCOUNT_ID: test
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           export ENV=test
           export IMAGE_TAG=test
-          docker-compose -f build.docker-compose.yml run -e NPM_TOKEN=${NPM_TOKEN} --rm --entrypoint="npm run test" maestro
+          docker-compose -f build.docker-compose.yml run -e NPM_TOKEN=${NPM_TOKEN} --rm --entrypoint="npm run test" maestro
+
+      - name: Remove Docker's Trash
+        continue-on-error: true
+        run: |
+          docker system prune
+          docker rmi -f $(docker images -aq)

.releaserc.json

@@ -33,7 +33,9 @@
           { "from": "main", "to": "alpha" },
           { "from": "main", "to": "beta" }
         ],
-        "backmergeStrategy": "merge"
+        "backmergeStrategy": "merge",
+        "clearWorkspace": true,
+        "restoreWorkspace": true
       }
     ],
     "@semantic-release/npm",

README.md

@@ -1,6 +1,4 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-  <h1 align="center">Maestro</h1>
-</p>
-
-x
+  <h1 align="center">Maestro</h1> 
+</p>

build.docker-compose.yml

@@ -2,4 +2,4 @@ version: "3.8"
 services:
   maestro:
     build: .
-    image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.8"
 services:
   maestro:
-    image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
     container_name: maestro
     restart: always
     ports:

package-lock.json

@@ -18,7 +18,7 @@
         "@nestjs/microservices": "^8.4.3",
         "@nestjs/platform-express": "^8.4.3",
         "@nestjs/swagger": "^5.2.1",
-        "@victorradael/protospack": "^1.3.3",
+        "@victorradael/protospack": "^1.4.2",
         "axios": "^0.25.0",
         "dotenv": "^14.2.0",
         "grpc": "^1.24.11",
@@ -2405,9 +2405,9 @@
       }
     },
     "node_modules/@victorradael/protospack": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.3.3.tgz",
-      "integrity": "sha512-20f0Zu62WEOMFFiKRw+pm6mRjStrx/msPL4J3wQ6U3gBZP/MOJE65ZUontAiw/dx3iwiprR2PVjX5ymsqIPTtA==",
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.4.2.tgz",
+      "integrity": "sha512-UWEjCVtFag0uHoMLxb55JXG7Qfz0fn8Pk7v1H5dgZcBMLDhs9VARBVNq1JT4P6ypMyrnTbOyq08CoMukF8GHbQ==",
       "dependencies": {
         "rxjs": "^7.5.5"
       }
@@ -11532,9 +11532,9 @@
       }
     },
     "@victorradael/protospack": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.3.3.tgz",
-      "integrity": "sha512-20f0Zu62WEOMFFiKRw+pm6mRjStrx/msPL4J3wQ6U3gBZP/MOJE65ZUontAiw/dx3iwiprR2PVjX5ymsqIPTtA==",
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-1.4.2.tgz",
+      "integrity": "sha512-UWEjCVtFag0uHoMLxb55JXG7Qfz0fn8Pk7v1H5dgZcBMLDhs9VARBVNq1JT4P6ypMyrnTbOyq08CoMukF8GHbQ==",
       "requires": {
         "rxjs": "^7.5.5"
       }

package.json

@@ -30,7 +30,7 @@
     "@nestjs/microservices": "^8.4.3",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.3.3",
+    "@victorradael/protospack": "^1.4.2",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "grpc": "^1.24.11",

src/clients/inputs/client.service.ts

@@ -148,12 +148,11 @@ export class InputsClientService implements OnModuleInit {
   }
 
   async testConnection(data: ITestConnectionRequest) {
-    console.log('InputClientService', 'TestCOnnection');
-
+    console.log('InputClientService', 'TestConnection');
     const testConnectionResponse = await new Promise((resolve, reject) => {
-      this.inputService.TestConnection(data).subscribe({
+      this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({
         next(x) {
-          resolve(x);
+          resolve(objectCamelToSnake(x));
         },
         error(err) {
           console.log('Observable Error');

src/clients/pipelines/client.service.ts

@@ -202,4 +202,30 @@ export class PipelinesClientService implements OnModuleInit {
 
     return statusPipelineResponse;
   }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    console.log('PipelinesClientService', 'RunPipeline');
+    const statusPipelineResponse = await new Promise((resolve, reject) => {
+      this.pipelineService
+        .triggerPipeline(objectSnakeToCamel({ id, info }))
+        .subscribe({
+          next(x) {
+            resolve(x);
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
+    })
+      .then((res) => res)
+      .catch((err) => {
+        throw new Error(err);
+      });
+
+    return statusPipelineResponse;
+  }
 }

src/middlewares/authentication.ts

@@ -27,7 +27,7 @@ export class LoggerMiddleware implements NestMiddleware {
       }
     });
     const jwtDecoded: any = decode(idToken);
-    console.log(jwtDecoded);
+
     const permissions = jwtDecoded.user.permissions;
     const clienId = jwtDecoded.user.customerId;
     const userId = jwtDecoded.user.id;
@@ -94,29 +94,31 @@ const verifyToken = async (accessToken: string) => {
 
   try {
     await setUp(awsRegion, awsPoolId);
-    if (accessToken) {
-      const user: any = decode(accessToken, { complete: true });
-
-      if (user === null) {
-        throw new UnauthorizedException();
-      }
-
-      const { kid } = user.header;
-      const pem = pems.filter((item: any) => item[kid]);
-      const pemValue: any = pem[0][kid];
-
-      if (!pem) {
-        throw new UnauthorizedException();
-      }
-
-      verify(accessToken, pemValue, (err: any) => {
-        if (err) {
-          throw new UnauthorizedException();
-        }
-
-        return;
-      });
+    if (!accessToken) {
+      throw new UnauthorizedException();
     }
+
+    const user: any = decode(accessToken, { complete: true });
+
+    if (user === null) {
+      throw new UnauthorizedException();
+    }
+
+    const { kid } = user.header;
+    const pem = pems.filter((item: any) => item[kid]);
+    const pemValue: any = pem[0][kid];
+
+    if (!pem) {
+      throw new UnauthorizedException();
+    }
+
+    verify(accessToken, pemValue, (err: any) => {
+      if (err) {
+        throw new UnauthorizedException();
+      }
+
+      return;
+    });
   } catch (error) {
     throw new UnauthorizedException();
   }

src/modules/auth/auth.controller.ts

@@ -1,4 +1,4 @@
-import { Body, Controller, Post } from '@nestjs/common';
+import { Body, Controller, Post, UnauthorizedException } from '@nestjs/common';
 
 import { AuthClientService } from 'src/clients/auth/client.service';
 
@@ -18,7 +18,9 @@ export class AuthController {
     const tokens = await this.authClient
       .signIn({ username, password })
       .then((result) => result)
-      .catch((err) => console.log(err));
+      .catch((err) => {
+        throw new UnauthorizedException(err.message);
+      });
 
     return tokens;
   }

src/modules/inputs/inputs.controller.ts

@@ -18,12 +18,10 @@ import {
 export class InputsController {
   constructor(private inputsClientService: InputsClientService) {}
 
-  @Get('/test-connection')
+  @Post('/test-connection')
   async testConnection(@Body() data) {
     console.log(`/test-connection`, 'ON TEST CONNECTION ROUTE');
 
-    console.log(data);
-
     const inputService = new InputsService(this.inputsClientService);
     const response = await inputService.testConnection(data);
 

src/modules/pipelines/pipelines.controller.ts

@@ -14,6 +14,22 @@ import { PipelinesService } from './pipelines.service';
 export class PipelinesController {
   constructor(private pipelinesClientService: PipelinesClientService) {}
 
+  @Get('start/:id')
+  async activate(@Param() params, @Body() body) {
+    const { id } = params;
+    const { info } = body;
+
+    console.log(
+      process.env.DEV_URL + `/transformation/start/${id}`,
+      'ON START PIPELINE ROUTE',
+    );
+
+    const pipelineService = new PipelinesService(this.pipelinesClientService);
+    const response = await pipelineService.runPipeline({ id, info });
+
+    return response;
+  }
+
   @Get(':id/status')
   async getPipelineStatus(@Param() params) {
     const { id } = params;

src/modules/pipelines/pipelines.service.ts

@@ -84,4 +84,17 @@ export class PipelinesService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    try {
+      const triggerPipelineResponse = await this.pipelineClient.runPipeline({
+        id,
+        info,
+      });
+
+      return objectCamelToSnake(triggerPipelineResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
 }

src/modules/transformations/transformations.service.ts

@@ -11,15 +11,20 @@ export class TransformationsService {
   constructor(private transformationClient: TransformationsClientService) {}
 
   async create(createTransformationDto: ICreateTransformationsRequest) {
+    const { transformations, info } = createTransformationDto;
+
+    if (!transformations || transformations.length == 0) {
+      throw new HttpException('Missing transformations field.', 400);
+    }
+
     try {
-      const convertedDTO = createTransformationDto.transformations.map((tr) => {
+      const convertedDTO = transformations.map((tr) => {
         return trSnakeToCamel(tr);
       });
-
       const createTransformationResponse =
         await this.transformationClient.create({
           transformations: convertedDTO,
-          info: createTransformationDto.info,
+          info: info,
         });
 
       return createTransformationResponse;

src/utils/CaseConverter.ts

@@ -41,8 +41,8 @@ export const objectSnakeToCamel = (object) => {
       delete object[key];
     }
   });
-
   objectValues.forEach((value) => {
+    if (value == null) return;
     if (typeof value === 'object') {
       objectSnakeToCamel(value);
     }

swagger.json

@@ -1 +1 @@
-{"openapi":"3.0.0","paths":{"/inputs/test-connection":{"get":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"200":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"201":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}}
+{"openapi":"3.0.0","paths":{"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"201":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/start/{id}":{"get":{"operationId":"PipelinesController_activate","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"201":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}}