CI: Fix

REFACTOR: Att Functions + Alpha Clients connection

.elasticbeanstalk/config.yml

@@ -0,0 +1,19 @@
+branch-defaults:
+  alpha:
+    environment: null
+global:
+  application_name: maestro
+  branch: null
+  default_ec2_keyname: null
+  default_platform: Docker running on 64bit Amazon Linux 2
+  default_region: us-east-1
+  include_git_submodules: true
+  instance_profile: null
+  platform_name: null
+  platform_version: null
+  profile: null
+  repository: null
+  sc: git
+  workspace_type: Application
+deploy:
+  artifact: deploy.zip

.github/workflows/deploy.yml

@@ -9,7 +9,6 @@ on:
 jobs:
   deploy:
     runs-on: self-hosted
-
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -18,6 +17,7 @@ jobs:
       - name: BKP .npmrc Step 1
         run: |
           cp .npmrc .npmrcbkp
+
       - name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2
         id: semantic
@@ -46,26 +46,27 @@ jobs:
       - name: BKP .npmrc Step 2
         run: |
           cp .npmrcbkp .npmrc
+
       - name: Update Pip
         if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install --upgrade pip
-          ls -a
+
       - name: Install Docker Compose
         if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install docker-compose --upgrade
-          ls -a
+
       - name: Install AWS CLI
         if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awscli --upgrade
-          ls -a
+
       - name: Install AWS Elastic Beanstalk CLI
         if: steps.semantic.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awsebcli --upgrade
-          ls -a
+
       - name: Configure AWS Region
         if: steps.semantic.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
@@ -76,7 +77,6 @@ jobs:
         if: steps.semantic.outputs.new_release_published == 'true'
         shell: bash
         run: |
-          ls -a
           if [ ${GITHUB_REF} == "refs/heads/main" ]; then
             echo "##[set-output name=result;]$(echo "production")"
           else
@@ -89,26 +89,6 @@ jobs:
         id: login_ecr
         uses: aws-actions/amazon-ecr-login@v1
 
-      - name: Build, Tag, and Push Image to AWS ECR
-        if: steps.semantic.outputs.new_release_published == 'true'
-        env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
-          docker-compose -f build.docker-compose.yml push
-          ls -a
-      - name: Create ZIP file to Deploy AWS Beanstalk
-        if: steps.semantic.outputs.new_release_published == 'true'
-        env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
-        run: |
-          sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
-          sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
-          zip deploy.zip docker-compose.yml
-          ls -a
       - name: Deploy AWS Beanstalk
         if: steps.semantic.outputs.new_release_published == 'true'
         env:
@@ -116,8 +96,12 @@ jobs:
           AWS_REGION: us-east-1
           APP_NAME: maestro
         run: |
-          [[ -d .elasticbeanstalk/ ]] && rm -R .elasticbeanstalk/
-          printf "1\nn" | eb init $APP_NAME --region $AWS_REGION
           eb use $APP_NAME-$ENV
           echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
-          eb deploy
+          eb deploy
+
+      - name: Remove Docker's Trash
+        continue-on-error: true
+        run: |
+          docker system prune
+          docker rmi -f $(docker images -aq)

.gitignore

@@ -37,10 +37,4 @@ lerna-debug.log*
 # Ignore .env files
 *.env
 
-# Elastic Beanstalk Files
-.elasticbeanstalk/*
-!.elasticbeanstalk/*.cfg.yml
-!.elasticbeanstalk/*.global.yml
-
-
 .env

README.md

@@ -2,3 +2,5 @@
   <image src="./assets/maestro.svg" style="width:10rem">
   <h1 align="center">Maestro</h1>
 </p>
+
+x

docker-compose.yml

@@ -6,6 +6,8 @@ services:
     restart: always
     ports:
       - 3333:3333
+    env_file:
+      - .env
 
   node_exporter:
     image: prom/node-exporter:v1.3.1

package.json

@@ -30,7 +30,7 @@
     "@nestjs/microservices": "^8.4.0",
     "@nestjs/platform-express": "^8.4.0",
     "@nestjs/swagger": "^5.1.5",
-    "@victorradael/protospack": "^1.2.8",
+    "@victorradael/protospack": "^1.3.1",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "grpc": "^1.24.11",

src/clients/inputs/client.config.ts

@@ -10,7 +10,7 @@ export class InputsClientConfiguration {
       options: {
         url: process.env.INFACTORY_URL,
         package: 'input',
-        // credentials: credentials.createSsl(),
+        credentials: credentials.createSsl(),
         protoPath: InputProtofile,
         loader: {
           enums: String,

src/clients/inputs/client.service.ts

@@ -105,11 +105,9 @@ export class InputsClientService implements OnModuleInit {
 
   async update(updateInputDTO:UpdateInputRequest) {
     console.log('InputClientService', 'Update');
-    console.log(updateInputDTO)
     const updateInputResponse = await new Promise((resolve, reject) => {
       this.inputService.Update(updateInputDTO).subscribe({
         next(x) {
-          console.log(x)
           resolve(x);
         },
         error(err) {

src/clients/inputs/interfaces.d.ts

@@ -30,7 +30,7 @@ interface Cron {
 
 export interface ICreateInputRequest {
 
-  cron: Cron;
+  cron: string;
   name: string;
   plugin: string;
   values: Values;
@@ -43,7 +43,7 @@ export interface IIdRequest{
 
 interface UpdateInputRequest {
   id: string;
-  cron: Cron;
+  cron: string;
   name: string;
   plugin: string;
   values: Values;

src/clients/outputs/client.config.ts

@@ -7,9 +7,9 @@ export class OutputsClientConfiguration {
     return {
       transport: Transport.GRPC,
       options: {
-        url: process.env.OUTFACTORY_URL,
+        url: process.env.OTFACTORY_URL,
         package: 'output',
-        // credentials: credentials.createSsl(),
+        credentials: credentials.createSsl(),
         protoPath: OutputProtofile,
         loader: {
           enums: String,

src/clients/outputs/client.service.ts

@@ -1,12 +1,7 @@
 import {
   Controller,
-  Delete,
-  Get,
   Inject,
-  OnModuleInit,
-  Param,
-  Post,
-  Put,
+  OnModuleInit
 } from '@nestjs/common';
 import {
   ClientGrpc,
@@ -33,7 +28,7 @@ export class OutputsClientService implements OnModuleInit {
     console.log('OutputClientService', 'Create');
 
     const createOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.create(createOutputDto).subscribe({
+      this.outputService.Create(createOutputDto).subscribe({
         next(x) {
           resolve(x);
         },
@@ -51,9 +46,8 @@ export class OutputsClientService implements OnModuleInit {
         throw new Error(err);
       });
     
-      const convertedItem = objectCamelToSnake(createOutputResponse["item"])
-      createOutputResponse["item"] = convertedItem
-
+      const convertedItem = objectCamelToSnake(createOutputResponse["output"])
+      createOutputResponse["output"] = convertedItem
     return createOutputResponse;
   }
 
@@ -61,7 +55,7 @@ export class OutputsClientService implements OnModuleInit {
     console.log('OutputClientService', 'FindAll');
 
     const findAllOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.findAll({}).subscribe({
+      this.outputService.FindAll({}).subscribe({
         next(x) {
           resolve(x);
         },
@@ -90,7 +84,7 @@ export class OutputsClientService implements OnModuleInit {
     console.log('OutputClientService', 'FindOne');
 
     const findOneOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.findOne({id}).subscribe({
+      this.outputService.FindOne({id}).subscribe({
         next(x) {
           resolve(x);
         },
@@ -117,7 +111,7 @@ export class OutputsClientService implements OnModuleInit {
     console.log('OutputClientService', 'Update');
 
     const updateOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.update(updateOutPutDTO).subscribe({
+      this.outputService.Update(updateOutPutDTO).subscribe({
         next(x) {
           resolve(x);
         },
@@ -142,7 +136,7 @@ export class OutputsClientService implements OnModuleInit {
     console.log('OutputClientService', 'Remove');
 
     const removeOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.remove({id}).subscribe({
+      this.outputService.Remove({id}).subscribe({
         next(x) {
           resolve(x);
         },

src/clients/pipelines/client.config.ts

@@ -9,7 +9,7 @@ export class PipelinesClientConfiguration {
       options: {
         url: process.env.PIFACTORY_URL,
         package: 'pipeline',
-        // credentials: credentials.createSsl(),
+        credentials: credentials.createSsl(),
         protoPath: PipelineProtofile,
         loader: {
           enums: String,

src/clients/transformations/client.config.ts

@@ -9,7 +9,7 @@ export class TransformationsClientConfiguration {
       options: {
         url: process.env.TRFACTORY_URL,
         package: 'transformation',
-        // credentials: credentials.createSsl(),
+        credentials: credentials.createSsl(),
         protoPath: TransformationProtofile,
         loader: {
           enums: String,

src/clients/transformations/client.service.ts

@@ -16,7 +16,8 @@ import {
 } from '@nestjs/microservices';
 import { ApiTags } from '@nestjs/swagger';
 import { TransformationsServiceInterface } from '@victorradael/protospack';
-import { ICreateTransformationElement, ICreateTransformationsRequest, IdRequest, IUpdateTransformationRequest } from './interfaces';
+import { objectCamelToSnake } from 'src/utils/CaseConverter';
+import { Transformation, ICreateTransformationsRequest, IdRequest, IUpdateTransformationRequest } from './interfaces';
 
 @Controller('transformation')
 export class TransformationsClientService implements OnModuleInit {
@@ -54,7 +55,12 @@ export class TransformationsClientService implements OnModuleInit {
       .catch((err) => {
         throw new Error(err);
       });
-    return createTransformationResponse;
+
+      const convertedTransforms = createTransformationResponse["transformations"].map((tr)=>{
+        return objectCamelToSnake(tr)
+      })
+
+    return {transformations:convertedTransforms};
   }
 
   async findAll() {
@@ -78,8 +84,10 @@ export class TransformationsClientService implements OnModuleInit {
       .catch((err) => {
         throw new Error(err);
       });
-
-    return findAllTransformationResponse;
+    const convertedTransforms = findAllTransformationResponse["transformations"].map((tr)=>{
+      return objectCamelToSnake(tr)
+    })
+    return {transformations:convertedTransforms};
   }
 
   async findOne({id}: IdRequest) {
@@ -104,7 +112,7 @@ export class TransformationsClientService implements OnModuleInit {
         throw new Error(err);
       });
 
-    return findOneTransformationResponse;
+    return objectCamelToSnake(findOneTransformationResponse);
   }
 
   async update(updateTransformationDTO: IUpdateTransformationRequest) {
@@ -129,7 +137,7 @@ export class TransformationsClientService implements OnModuleInit {
         throw new Error(err);
       });
 
-    return updateTransformationResponse;
+    return  objectCamelToSnake(updateTransformationResponse);
   }
 
   async remove({id}: IdRequest) {
@@ -154,6 +162,6 @@ export class TransformationsClientService implements OnModuleInit {
         throw new Error(err);
       });
 
-    return removeTransformationResponse;
+    return objectCamelToSnake(removeTransformationResponse);
   }
 }

src/clients/transformations/interfaces.d.ts

@@ -1,15 +1,18 @@
 export interface ICreateTransformationsRequest {
-    transformations: ICreateTransformationElement[];
+    transformations: Transformation[];
 }
 interface Param {
-    base_column?: string;
-    column_name?: string;
-    n_digits?: number;
-    start_index?: number;
-    randbelow?: number;
-    const?: string;
+    base_column: string;
+    column_name: string;
+    n_digits: number;
+    start_index: number;
+    randbelow: number;
+    const: string;
 }
-export interface ICreateTransformationElement {
+interface Transformation {
+    created_at: string;
+    id: string;
+    client_id: string;
     type: string;
     params: Param[];
     input_source: string;
@@ -21,11 +24,5 @@ export interface IdRequest{
 }
 
 export interface IUpdateTransformationRequest {
-    created_at?: string;
-    id?: string;
-    client_id?: string;
-    params: Param[];
-    type?: string;
-    input_source?: string;
-    table?: string;
+    transformation: Transformation;
 }

src/modules/health/health.controller.ts

@@ -7,6 +7,7 @@ export class HealthController {
 
   @Get()
   check() {
+    console.log(`/health`, 'check');
     return this.healthService.check();
   }
 }

src/modules/health/health.service.ts

@@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common';
 @Injectable()
 export class HealthService {
   check() {
+    console.log('HEALTH');
     return { message: 'Ok' };
   }
 }

src/modules/transformations/transformations.service.ts

@@ -52,8 +52,8 @@ export class TransformationsService {
   async update(id:string, data){
 
     try{    
-
-      const updateTransformationResponse = await this.transformationClient.update({id,...data});
+      data.transformation["id"] = id 
+      const updateTransformationResponse = await this.transformationClient.update({...data});
   
       return updateTransformationResponse