Merge pull request #54 from dadosfera/beta

Beta

.github/workflows/deploy.yml

@@ -24,7 +24,9 @@ jobs:
         id: extract_environment
 
   deploy:
-    runs-on: self-hosted
+    needs: extract_environment
+    runs-on:
+      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -86,6 +88,7 @@ jobs:
       - name: Configure AWS Region
         if: steps.semantic.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
+        id: aws
         with:
           aws-region: us-east-1
 
@@ -112,6 +115,7 @@ jobs:
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
           IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
@@ -122,9 +126,11 @@ jobs:
         env:
           ENV: ${{ steps.extract_environment.outputs.result }}
           IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
           sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+          sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk

.github/workflows/test.yml

@@ -25,12 +25,14 @@ jobs:
         run: |
           export ENV=test
           export IMAGE_TAG=test
+          export ACCOUNT_ID=test
           docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
 
       - name: Run Test
         env:
           ENV: test
           IMAGE_TAG: test
+          ACCOUNT_ID: test
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           export ENV=test

README.md

@@ -1,4 +1,4 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-  <h1 align="center">Maestro</h1>
+  <h1 align="center">Maestro</h1> 
 </p>

build.docker-compose.yml

@@ -2,4 +2,4 @@ version: "3.8"
 services:
   maestro:
     build: .
-    image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.8"
 services:
   maestro:
-    image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
     container_name: maestro
     restart: always
     ports:

package.json

@@ -30,7 +30,7 @@
     "@nestjs/microservices": "^8.4.3",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.3.9",
+    "@victorradael/protospack": "^1.4.2",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "grpc": "^1.24.11",

src/clients/pipelines/client.service.ts

@@ -202,4 +202,30 @@ export class PipelinesClientService implements OnModuleInit {
 
     return statusPipelineResponse;
   }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    console.log('PipelinesClientService', 'RunPipeline');
+    const statusPipelineResponse = await new Promise((resolve, reject) => {
+      this.pipelineService
+        .triggerPipeline(objectSnakeToCamel({ id, info }))
+        .subscribe({
+          next(x) {
+            resolve(x);
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
+    })
+      .then((res) => res)
+      .catch((err) => {
+        throw new Error(err);
+      });
+
+    return statusPipelineResponse;
+  }
 }

src/middlewares/authentication.ts

@@ -14,7 +14,6 @@ import { HttpExceptionFilter } from 'src/error/http-exception.filter';
 @UseFilters(new HttpExceptionFilter())
 export class LoggerMiddleware implements NestMiddleware {
   use = async (request: Request, response: Response, next: NextFunction) => {
-    console.log('Middle', new Date());
     const idToken = request.get('Dadosfera-User');
     const accessToken = request.get('Authorization');
     const privateKey = process.env.JWT_PRIVATE_KEY;

src/modules/auth/auth.controller.ts

@@ -1,4 +1,4 @@
-import { Body, Controller, Post } from '@nestjs/common';
+import { Body, Controller, Post, UnauthorizedException } from '@nestjs/common';
 
 import { AuthClientService } from 'src/clients/auth/client.service';
 
@@ -18,7 +18,9 @@ export class AuthController {
     const tokens = await this.authClient
       .signIn({ username, password })
       .then((result) => result)
-      .catch((err) => console.log(err));
+      .catch((err) => {
+        throw new UnauthorizedException(err.message);
+      });
 
     return tokens;
   }

src/modules/pipelines/pipelines.controller.ts

@@ -14,6 +14,22 @@ import { PipelinesService } from './pipelines.service';
 export class PipelinesController {
   constructor(private pipelinesClientService: PipelinesClientService) {}
 
+  @Get('start/:id')
+  async activate(@Param() params, @Body() body) {
+    const { id } = params;
+    const { info } = body;
+
+    console.log(
+      process.env.DEV_URL + `/transformation/start/${id}`,
+      'ON START PIPELINE ROUTE',
+    );
+
+    const pipelineService = new PipelinesService(this.pipelinesClientService);
+    const response = await pipelineService.runPipeline({ id, info });
+
+    return response;
+  }
+
   @Get(':id/status')
   async getPipelineStatus(@Param() params) {
     const { id } = params;

src/modules/pipelines/pipelines.service.ts

@@ -84,4 +84,17 @@ export class PipelinesService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    try {
+      const triggerPipelineResponse = await this.pipelineClient.runPipeline({
+        id,
+        info,
+      });
+
+      return objectCamelToSnake(triggerPipelineResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
 }

src/modules/transformations/transformations.service.ts

@@ -11,15 +11,20 @@ export class TransformationsService {
   constructor(private transformationClient: TransformationsClientService) {}
 
   async create(createTransformationDto: ICreateTransformationsRequest) {
+    const { transformations, info } = createTransformationDto;
+
+    if (!transformations || transformations.length == 0) {
+      throw new HttpException('Missing transformations field.', 400);
+    }
+
     try {
-      const convertedDTO = createTransformationDto.transformations.map((tr) => {
+      const convertedDTO = transformations.map((tr) => {
         return trSnakeToCamel(tr);
       });
-
       const createTransformationResponse =
         await this.transformationClient.create({
           transformations: convertedDTO,
-          info: createTransformationDto.info,
+          info: info,
         });
 
       return createTransformationResponse;

swagger.json

@@ -1 +1 @@
-{"openapi":"3.0.0","paths":{"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"201":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"201":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}}
+{"openapi":"3.0.0","paths":{"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"201":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/start/{id}":{"get":{"operationId":"PipelinesController_activate","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"201":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}}