CI: .

Beta -> main

.github/workflows/deploy-manually.yml

@@ -2,37 +2,50 @@ name: Deploy
 on:
   push:
     branches:
-      - alpha
-      - beta
       - main
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Deployment environment'
+        required: true
+        type: choice
+        options:
+          - dev
+          - stg
+          - prd
 
 jobs:
   extract_environment:
     runs-on: ubuntu-latest
     outputs:
       environment: ${{ steps.extract_environment.outputs.environment }}
+    env:
+      EVENT: ${{ github.event_name }}
+      DEPLOY_ENV: ${{ github.event.inputs.environment }}
     steps:
       - name: Extract Environment
         run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
+          if [ ${EVENT} == "workflow_dispatch" ]; then
+            echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
+          elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
             echo "##[set-output name=environment;]$(echo "prd")"
-          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
-            echo "##[set-output name=environment;]$(echo "dev")"
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "##[set-output name=environment;]$(echo "stg")"
           fi
         id: extract_environment
 
   semantic_release:
     runs-on: ubuntu-latest
     outputs:
+      #new_release_published: ${{ steps.semantic.outputs.last_release_version != steps.semantic.outputs.new_release_version }}
       new_release_published: ${{ steps.semantic.outputs.new_release_published }}
       new_release_version: ${{ steps.semantic.outputs.new_release_version }}
+      test: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
+      testt: ${{ (needs.semantic_release.outputs.new_release_published == 'true' && needs.semantic_release.outputs.new_release_version) || '1.0.0' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Semantic Release
+      - if: github.event_name != 'workflow_dispatch'
+        name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2
         id: semantic
         with:
@@ -41,77 +54,58 @@ jobs:
             conventional-changelog-eslint
           branches: |
             [
-              'main',
-              {
-                name: 'alpha',
-                prerelease: true
-              },
-              {
-                name: 'beta',
-                prerelease: true
-              }
+              'main'
             ]
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
   deploy:
+    if: ${{ (github.event_name != 'workflow_dispatch' && needs.semantic_release.outputs.new_release_published == 'true') || 'true' }}
     needs: [extract_environment, semantic_release]
-    runs-on: [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
+    runs-on:
+      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
 
     steps:
+      - name: print
+        env:
+          EVENT: ${{ github.event_name }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.test }}
+          IMAGE_TAGG: ${{ needs.semantic_release.outputs.testT }}
+        run: echo ${IMAGE_TAG} ${IMAGE_TAGG} ${EVENT}
+
       - name: Checkout
         uses: actions/checkout@v2
 
       - name: Update Pip
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install --upgrade pip
 
       - name: Install Docker Compose
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install docker-compose --upgrade
 
       - name: Install AWS CLI
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awscli --upgrade
 
       - name: Install AWS Elastic Beanstalk CLI
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         run: |
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         uses: aws-actions/configure-aws-credentials@v1
         id: aws
         with:
           aws-region: us-east-1
 
-      - name: Extract Environment
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        shell: bash
-        run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=result;]$(echo "prd")"
-          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
-            echo "##[set-output name=result;]$(echo "dev")"
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "##[set-output name=result;]$(echo "stg")"
-          fi
-        id: extract_environment
-
       - name: Login to AWS ECR
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         id: login_ecr
         uses: aws-actions/amazon-ecr-login@v1
 
       - name: Build, Tag, and Push Image to AWS ECR
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
@@ -120,9 +114,8 @@ jobs:
           docker-compose -f build.docker-compose.yml push
 
       - name: Create ZIP file to Deploy AWS Beanstalk
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
@@ -132,11 +125,10 @@ jobs:
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk
-        if: needs.semantic_release.outputs.new_release_published == 'true'
         env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
           AWS_REGION: us-east-1
-          APP_NAME: maestro
+          APP_NAME: ${{ github.event.repository.name }}
         run: |
           eb use $APP_NAME-$ENV
           echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml

package.json

@@ -31,7 +31,7 @@
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^2.3.0",
+    "@victorradael/protospack": "2.3.0",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "helmet": "^5.0.2",

src/clients/auth/client.config.ts

@@ -10,7 +10,9 @@ export class AuthClient {
       options: {
         url: process.env.DUC_URL,
         package: DucPackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: DucProtoFilePath,
         loader: {
           enums: String,

src/clients/inputs/client.config.ts

@@ -10,7 +10,9 @@ export class InputsClientConfiguration {
       options: {
         url: process.env.INFACTORY_URL,
         package: InputPackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: InputProtoFilePath,
         loader: {
           enums: String,

src/clients/inputs/client.service.ts

@@ -1,6 +1,7 @@
 import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
+  InputCreateResponse,
   InputCreateS3Request,
   InputNewCreateRequest,
   InputService,
@@ -42,11 +43,9 @@ export class InputsClientService implements OnModuleInit {
             console.log('done');
           },
         });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
+    }).catch((err) => {
+      throw new Error(err);
+    });
 
     return createInputResponse;
   }

src/clients/outputs/client.config.ts

@@ -9,7 +9,9 @@ export class OutputsClientConfiguration {
       options: {
         url: process.env.OTFACTORY_URL,
         package: OutputPackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: OutputProtoFilePath,
         loader: {
           enums: String,

src/clients/pipelines/client.config.ts

@@ -12,7 +12,9 @@ export class PipelinesClientConfiguration {
       options: {
         url: process.env.PIFACTORY_URL,
         package: PipelinePackages,
-        credentials: credentials.createSsl(),
+        credentials: process.env.LOCAL_ENV
+          ? undefined
+          : credentials.createSsl(),
         protoPath: PipelineProtoFilePath,
         loader: {
           enums: String,

src/modules/inputs/inputs.service.ts

@@ -14,10 +14,18 @@ export class InputsService {
         data.plugin == 'json' ||
         data.plugin == 'parquet'
       ) {
-        const createInputResponse = await this.inputClient.createS3Inputs(data);
+        const { info, ...input } = data;
+        const inputPayload = this.generateInputS3Payload(input);
+        const createInputResponse = await this.inputClient.createS3Inputs({
+          input: inputPayload,
+          info,
+        });
         return createInputResponse;
       } else {
-        const createInputResponse = await this.inputClient.create(data);
+        const createInputResponse: any = await this.inputClient.create(data);
+        if (createInputResponse?.input?.input_jdbc) {
+          return { input: createInputResponse?.input?.input_jdbc };
+        }
         return createInputResponse;
       }
     } catch (err) {
@@ -91,4 +99,29 @@ export class InputsService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  generateInputS3Payload(payload) {
+    const { credentials, plugin } = payload;
+    if (!credentials) return payload;
+
+    const {
+      client_aws_access_key_id,
+      client_aws_secret_access_key,
+      file_format_params,
+      format_file_params,
+      client_bucket,
+      file_to_extract,
+    } = credentials;
+    const formatedPayload = {
+      plugin,
+      source_bucket: client_bucket,
+      source_prefix: file_to_extract,
+      auth_parameters: {
+        aws_access_key_id: client_aws_access_key_id,
+        aws_secret_access_key: client_aws_secret_access_key,
+      },
+      file_format_params: file_format_params || format_file_params,
+    };
+    return formatedPayload;
+  }
 }