Compare commits

...
18 Commits
Author SHA1 Message Date
rodrigo.zamboni 1ffa9ade51 FIX: CI 2022-04-28 10:53:17 -03:00
Rodrigo Zamboni c91aa57344 Merge pull request #78 from dadosfera/beta
Beta
2022-04-27 17:36:54 -03:00
Rodrigo Zamboni 077a95cdbd Merge pull request #77 from dadosfera/alpha
Alpha
2022-04-27 17:16:09 -03:00
Rodrigo Zamboni f39108772b Merge pull request #76 from dadosfera/NewInput
FIX: fix pipeline activate route method
2022-04-27 16:57:56 -03:00
rodrigo.zamboni cdd8beee8f FIX: fix pipeline activate route method 2022-04-27 16:56:03 -03:00
rodrigo.zamboni 14331ed229 FIX: fix pipeline activate route method 2022-04-27 16:54:48 -03:00
Victor Radael 5f3db0cb43 Merge pull request #75 from dadosfera/beta
Beta -> Main
2022-04-27 11:42:18 -03:00
Rodrigo Zamboni 28f304c47c Merge pull request #74 from dadosfera/alpha
Alpha
2022-04-26 17:41:57 -03:00
Rodrigo Zamboni 0c1b62b2aa Merge pull request #73 from dadosfera/NewInput
FIX: Fix get-columns route method
2022-04-26 17:25:11 -03:00
rodrigo.zamboni 13f7e9d812 FIX: Fix get-columns route method 2022-04-26 17:19:18 -03:00
Gabriel Amorim 841dcfef06 Merge pull request #72 from dadosfera/alpha
Alpha
2022-04-26 16:30:31 -03:00
Gabriel Amorim f41ddc1483 Merge pull request #71 from dadosfera/NewInput
UPDATE: Implemented new Input format
2022-04-26 16:19:19 -03:00
rodrigo.zamboni 0a2c7c0a38 UPDATE: Implemented new Input format 2022-04-26 16:15:54 -03:00
Rodrigo Zamboni 87f4c17ca2 Merge pull request #70 from dadosfera/beta
Beta
2022-04-26 13:50:00 -03:00
Rodrigo Zamboni a89625c1e6 Merge pull request #69 from dadosfera/alpha
Alpha
2022-04-26 11:16:06 -03:00
rodrigo.zamboni 44211d5658 FIX: CI 2022-04-26 11:01:20 -03:00
rodrigo.zamboni d4e11b1935 FIX: CI 2022-04-26 10:48:57 -03:00
rodrigo.zamboni 5c5e940cf7 FIX: CI 2022-04-26 10:40:11 -03:00
10 changed files with 137 additions and 915 deletions
+7
View File
@@ -0,0 +1,7 @@
Resources:
AWSEBAutoScalingLaunchConfiguration:
Properties:
MetadataOptions:
HttpEndpoint: enabled
HttpPutResponseHopLimit: 2
HttpTokens: required
+25 -25
View File
@@ -23,19 +23,15 @@ jobs:
fi fi
id: extract_environment id: extract_environment
deploy: semantic_release:
needs: extract_environment runs-on: ubuntu-latest
runs-on: outputs:
[self-hosted, '${{ needs.extract_environment.outputs.environment }}'] new_release_published: ${{ steps.semantic.outputs.new_release_published }}
new_release_version: ${{ steps.semantic.outputs.new_release_version }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
# Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
- name: BKP .npmrc Step 1
run: |
cp .npmrc .npmrcbkp
- name: Semantic Release - name: Semantic Release
uses: cycjimmy/semantic-release-action@v2 uses: cycjimmy/semantic-release-action@v2
id: semantic id: semantic
@@ -60,40 +56,43 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
# Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file. deploy:
- name: BKP .npmrc Step 2 needs: [extract_environment, semantic_release]
run: | runs-on: [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
cp .npmrcbkp .npmrc
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Update Pip - name: Update Pip
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
run: | run: |
python3 -m pip install --upgrade pip python3 -m pip install --upgrade pip
- name: Install Docker Compose - name: Install Docker Compose
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
run: | run: |
python3 -m pip install docker-compose --upgrade python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI - name: Install AWS CLI
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
run: | run: |
python3 -m pip install awscli --upgrade python3 -m pip install awscli --upgrade
- name: Install AWS Elastic Beanstalk CLI - name: Install AWS Elastic Beanstalk CLI
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
run: | run: |
python3 -m pip install awsebcli --upgrade python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region - name: Configure AWS Region
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
uses: aws-actions/configure-aws-credentials@v1 uses: aws-actions/configure-aws-credentials@v1
id: aws id: aws
with: with:
aws-region: us-east-1 aws-region: us-east-1
- name: Extract Environment - name: Extract Environment
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
shell: bash shell: bash
run: | run: |
if [ ${GITHUB_REF} == "refs/heads/main" ]; then if [ ${GITHUB_REF} == "refs/heads/main" ]; then
@@ -106,15 +105,15 @@ jobs:
id: extract_environment id: extract_environment
- name: Login to AWS ECR - name: Login to AWS ECR
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
id: login_ecr id: login_ecr
uses: aws-actions/amazon-ecr-login@v1 uses: aws-actions/amazon-ecr-login@v1
- name: Build, Tag, and Push Image to AWS ECR - name: Build, Tag, and Push Image to AWS ECR
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
env: env:
ENV: ${{ steps.extract_environment.outputs.result }} ENV: ${{ steps.extract_environment.outputs.result }}
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
run: | run: |
@@ -122,10 +121,10 @@ jobs:
docker-compose -f build.docker-compose.yml push docker-compose -f build.docker-compose.yml push
- name: Create ZIP file to Deploy AWS Beanstalk - name: Create ZIP file to Deploy AWS Beanstalk
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
env: env:
ENV: ${{ steps.extract_environment.outputs.result }} ENV: ${{ steps.extract_environment.outputs.result }}
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: | run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
@@ -134,13 +133,14 @@ jobs:
zip deploy.zip docker-compose.yml -r .ebextensions zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk - name: Deploy AWS Beanstalk
if: steps.semantic.outputs.new_release_published == 'true' if: needs.semantic_release.outputs.new_release_published == 'true'
env: env:
ENV: ${{ steps.extract_environment.outputs.result }} ENV: ${{ steps.extract_environment.outputs.result }}
AWS_REGION: us-east-1 AWS_REGION: us-east-1
APP_NAME: maestro APP_NAME: maestro
run: | run: |
eb use $APP_NAME-$ENV eb use $APP_NAME-$ENV
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
eb deploy eb deploy
- name: Remove Docker's Trash - name: Remove Docker's Trash
+23 -852
View File
File diff suppressed because it is too large Load Diff
-1
View File
@@ -33,7 +33,6 @@
"@victorradael/protospack": "^1.5.5", "@victorradael/protospack": "^1.5.5",
"axios": "^0.25.0", "axios": "^0.25.0",
"dotenv": "^14.2.0", "dotenv": "^14.2.0",
"grpc": "^1.24.11",
"helmet": "^5.0.2", "helmet": "^5.0.2",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5", "jwk-to-pem": "^2.0.5",
+48 -22
View File
@@ -1,16 +1,16 @@
import { OnModuleInit, Inject } from '@nestjs/common'; import { OnModuleInit, Inject } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices'; import { ClientGrpc } from '@nestjs/microservices';
import { InputService } from '@victorradael/protospack'; import {
InputNewCreateRequest,
InputService,
TestConnectionGetColumnsRequest,
TestConnectionRequest,
} from '@victorradael/protospack';
import { import {
objectCamelToSnake, objectCamelToSnake,
objectSnakeToCamel, objectSnakeToCamel,
} from 'src/utils/CaseConverter'; } from 'src/utils/CaseConverter';
import { import { IIdRequest, UpdateInputRequest } from './interfaces';
ICreateInputRequest,
IIdRequest,
ITestConnectionRequest,
UpdateInputRequest,
} from './interfaces';
export class InputsClientService implements OnModuleInit { export class InputsClientService implements OnModuleInit {
private inputService: InputService; private inputService: InputService;
@@ -23,22 +23,24 @@ export class InputsClientService implements OnModuleInit {
this.grpcClient.getService<InputService>('InputService'); this.grpcClient.getService<InputService>('InputService');
} }
async create(createInputDto: ICreateInputRequest) { async create(createInputDto: InputNewCreateRequest) {
console.log('InputClientService', 'Create'); console.log('InputClientService', 'Create');
const createInputResponse = await new Promise((resolve, reject) => { const createInputResponse = await new Promise((resolve, reject) => {
this.inputService.Create(objectSnakeToCamel(createInputDto)).subscribe({ this.inputService
next(x) { .NewCreate(objectSnakeToCamel(createInputDto))
resolve(objectCamelToSnake(x)); .subscribe({
}, next(x) {
error(err) { resolve(objectCamelToSnake(x));
console.log('Observable Error'); },
reject(err); error(err) {
}, console.log('Observable Error');
complete() { reject(err);
console.log('done'); },
}, complete() {
}); console.log('done');
},
});
}) })
.then((res) => res) .then((res) => res)
.catch((err) => { .catch((err) => {
@@ -147,10 +149,10 @@ export class InputsClientService implements OnModuleInit {
return removeInputResponse; return removeInputResponse;
} }
async testConnection(data: ITestConnectionRequest) { async testConnection(data: TestConnectionRequest) {
console.log('InputClientService', 'TestConnection'); console.log('InputClientService', 'TestConnection');
const testConnectionResponse = await new Promise((resolve, reject) => { const testConnectionResponse = await new Promise((resolve, reject) => {
this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({ this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
next(x) { next(x) {
resolve(objectCamelToSnake(x)); resolve(objectCamelToSnake(x));
}, },
@@ -170,4 +172,28 @@ export class InputsClientService implements OnModuleInit {
return testConnectionResponse; return testConnectionResponse;
} }
async getColumns(data: TestConnectionGetColumnsRequest) {
console.log('InputClientService', 'TestConnection/Get-Columns');
const getColumnsResponse = await new Promise((resolve, reject) => {
this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
})
.then((res) => res)
.catch((err) => {
throw new Error(err);
});
return getColumnsResponse;
}
} }
+16 -5
View File
@@ -9,10 +9,8 @@ import {
} from '@nestjs/common'; } from '@nestjs/common';
import { InputsService } from './inputs.service'; import { InputsService } from './inputs.service';
import { InputsClientService } from 'src/clients/inputs/client.service'; import { InputsClientService } from 'src/clients/inputs/client.service';
import { import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
ICreateInputRequest, import { InputNewCreateRequest } from '@victorradael/protospack';
UpdateInputRequest,
} from 'src/clients/inputs/interfaces';
@Controller('inputs') @Controller('inputs')
export class InputsController { export class InputsController {
@@ -28,8 +26,21 @@ export class InputsController {
return response; return response;
} }
@Post('/test-connection/get-columns')
async getColumns(@Body() data) {
console.log(
`/test-connection/get-columns`,
'ON TEST CONNECTION GET COLUMNS ROUTE',
);
const inputService = new InputsService(this.inputsClientService);
const response = await inputService.getColumns(data);
return response;
}
@Post() @Post()
async create(@Body() createInputDto: ICreateInputRequest) { async create(@Body() createInputDto: InputNewCreateRequest) {
console.log(`/input`, 'ON CREATE ROUTE'); console.log(`/input`, 'ON CREATE ROUTE');
const inputService = new InputsService(this.inputsClientService); const inputService = new InputsService(this.inputsClientService);
+14 -6
View File
@@ -1,16 +1,13 @@
import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { InputNewCreateRequest } from '@victorradael/protospack';
import { InputsClientService } from 'src/clients/inputs/client.service'; import { InputsClientService } from 'src/clients/inputs/client.service';
import { import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
ICreateInputRequest,
IIdRequest,
Info,
} from 'src/clients/inputs/interfaces';
@Injectable() @Injectable()
export class InputsService { export class InputsService {
constructor(private inputClient: InputsClientService) {} constructor(private inputClient: InputsClientService) {}
async create(@Body() data: ICreateInputRequest) { async create(@Body() data: InputNewCreateRequest) {
try { try {
const createInputResponse = await this.inputClient.create(data); const createInputResponse = await this.inputClient.create(data);
@@ -74,4 +71,15 @@ export class InputsService {
throw new HttpException(err.message, HttpStatus.NOT_FOUND); throw new HttpException(err.message, HttpStatus.NOT_FOUND);
} }
} }
async getColumns(data) {
try {
const testConnectionGetColumnsResponse =
await this.inputClient.getColumns(data);
return testConnectionGetColumnsResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
} }
@@ -14,13 +14,13 @@ import { PipelinesService } from './pipelines.service';
export class PipelinesController { export class PipelinesController {
constructor(private pipelinesClientService: PipelinesClientService) {} constructor(private pipelinesClientService: PipelinesClientService) {}
@Get('start/:id') @Post('start/:id')
async activate(@Param() params, @Body() body) { async activate(@Param() params, @Body() body) {
const { id } = params; const { id } = params;
const { info } = body; const { info } = body;
console.log( console.log(
process.env.DEV_URL + `/transformation/start/${id}`, process.env.DEV_URL + `/pipeline/start/${id}`,
'ON START PIPELINE ROUTE', 'ON START PIPELINE ROUTE',
); );
+1 -1
View File
File diff suppressed because one or more lines are too long