mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-17 10:49:08 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ffa9ade51 | ||
|
|
c91aa57344 | ||
|
|
077a95cdbd | ||
|
|
f39108772b | ||
|
|
cdd8beee8f | ||
|
|
14331ed229 | ||
|
|
5f3db0cb43 | ||
|
|
28f304c47c | ||
|
|
0c1b62b2aa | ||
|
|
13f7e9d812 | ||
|
|
841dcfef06 | ||
|
|
f41ddc1483 | ||
|
|
0a2c7c0a38 | ||
|
|
87f4c17ca2 | ||
|
|
a89625c1e6 | ||
|
|
44211d5658 | ||
|
|
d4e11b1935 | ||
|
|
5c5e940cf7 |
@@ -0,0 +1,7 @@
|
|||||||
|
Resources:
|
||||||
|
AWSEBAutoScalingLaunchConfiguration:
|
||||||
|
Properties:
|
||||||
|
MetadataOptions:
|
||||||
|
HttpEndpoint: enabled
|
||||||
|
HttpPutResponseHopLimit: 2
|
||||||
|
HttpTokens: required
|
||||||
@@ -23,19 +23,15 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
id: extract_environment
|
id: extract_environment
|
||||||
|
|
||||||
deploy:
|
semantic_release:
|
||||||
needs: extract_environment
|
runs-on: ubuntu-latest
|
||||||
runs-on:
|
outputs:
|
||||||
[self-hosted, '${{ needs.extract_environment.outputs.environment }}']
|
new_release_published: ${{ steps.semantic.outputs.new_release_published }}
|
||||||
|
new_release_version: ${{ steps.semantic.outputs.new_release_version }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
# Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
|
|
||||||
- name: BKP .npmrc Step 1
|
|
||||||
run: |
|
|
||||||
cp .npmrc .npmrcbkp
|
|
||||||
|
|
||||||
- name: Semantic Release
|
- name: Semantic Release
|
||||||
uses: cycjimmy/semantic-release-action@v2
|
uses: cycjimmy/semantic-release-action@v2
|
||||||
id: semantic
|
id: semantic
|
||||||
@@ -60,40 +56,43 @@ jobs:
|
|||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
|
|
||||||
# Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
|
deploy:
|
||||||
- name: BKP .npmrc Step 2
|
needs: [extract_environment, semantic_release]
|
||||||
run: |
|
runs-on: [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
|
||||||
cp .npmrcbkp .npmrc
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
- name: Update Pip
|
- name: Update Pip
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install --upgrade pip
|
python3 -m pip install --upgrade pip
|
||||||
|
|
||||||
- name: Install Docker Compose
|
- name: Install Docker Compose
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install docker-compose --upgrade
|
python3 -m pip install docker-compose --upgrade
|
||||||
|
|
||||||
- name: Install AWS CLI
|
- name: Install AWS CLI
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install awscli --upgrade
|
python3 -m pip install awscli --upgrade
|
||||||
|
|
||||||
- name: Install AWS Elastic Beanstalk CLI
|
- name: Install AWS Elastic Beanstalk CLI
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install awsebcli --upgrade
|
python3 -m pip install awsebcli --upgrade
|
||||||
|
|
||||||
- name: Configure AWS Region
|
- name: Configure AWS Region
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
uses: aws-actions/configure-aws-credentials@v1
|
uses: aws-actions/configure-aws-credentials@v1
|
||||||
id: aws
|
id: aws
|
||||||
with:
|
with:
|
||||||
aws-region: us-east-1
|
aws-region: us-east-1
|
||||||
|
|
||||||
- name: Extract Environment
|
- name: Extract Environment
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
if [ ${GITHUB_REF} == "refs/heads/main" ]; then
|
if [ ${GITHUB_REF} == "refs/heads/main" ]; then
|
||||||
@@ -106,15 +105,15 @@ jobs:
|
|||||||
id: extract_environment
|
id: extract_environment
|
||||||
|
|
||||||
- name: Login to AWS ECR
|
- name: Login to AWS ECR
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
id: login_ecr
|
id: login_ecr
|
||||||
uses: aws-actions/amazon-ecr-login@v1
|
uses: aws-actions/amazon-ecr-login@v1
|
||||||
|
|
||||||
- name: Build, Tag, and Push Image to AWS ECR
|
- name: Build, Tag, and Push Image to AWS ECR
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
env:
|
env:
|
||||||
ENV: ${{ steps.extract_environment.outputs.result }}
|
ENV: ${{ steps.extract_environment.outputs.result }}
|
||||||
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
|
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
|
||||||
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
|
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
|
||||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
@@ -122,10 +121,10 @@ jobs:
|
|||||||
docker-compose -f build.docker-compose.yml push
|
docker-compose -f build.docker-compose.yml push
|
||||||
|
|
||||||
- name: Create ZIP file to Deploy AWS Beanstalk
|
- name: Create ZIP file to Deploy AWS Beanstalk
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
env:
|
env:
|
||||||
ENV: ${{ steps.extract_environment.outputs.result }}
|
ENV: ${{ steps.extract_environment.outputs.result }}
|
||||||
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
|
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
|
||||||
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
|
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
|
||||||
run: |
|
run: |
|
||||||
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
|
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
|
||||||
@@ -134,13 +133,14 @@ jobs:
|
|||||||
zip deploy.zip docker-compose.yml -r .ebextensions
|
zip deploy.zip docker-compose.yml -r .ebextensions
|
||||||
|
|
||||||
- name: Deploy AWS Beanstalk
|
- name: Deploy AWS Beanstalk
|
||||||
if: steps.semantic.outputs.new_release_published == 'true'
|
if: needs.semantic_release.outputs.new_release_published == 'true'
|
||||||
env:
|
env:
|
||||||
ENV: ${{ steps.extract_environment.outputs.result }}
|
ENV: ${{ steps.extract_environment.outputs.result }}
|
||||||
AWS_REGION: us-east-1
|
AWS_REGION: us-east-1
|
||||||
APP_NAME: maestro
|
APP_NAME: maestro
|
||||||
run: |
|
run: |
|
||||||
eb use $APP_NAME-$ENV
|
eb use $APP_NAME-$ENV
|
||||||
|
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
|
||||||
eb deploy
|
eb deploy
|
||||||
|
|
||||||
- name: Remove Docker's Trash
|
- name: Remove Docker's Trash
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<p align="center">
|
<p align="center">
|
||||||
<image src="./assets/maestro.svg" style="width:10rem">
|
<image src="./assets/maestro.svg" style="width:10rem">
|
||||||
<h1 align="center">Maestro</h1>
|
<h1 align="center">Maestro</h1>
|
||||||
</p>
|
</p>
|
||||||
Generated
+23
-852
File diff suppressed because it is too large
Load Diff
@@ -33,7 +33,6 @@
|
|||||||
"@victorradael/protospack": "^1.5.5",
|
"@victorradael/protospack": "^1.5.5",
|
||||||
"axios": "^0.25.0",
|
"axios": "^0.25.0",
|
||||||
"dotenv": "^14.2.0",
|
"dotenv": "^14.2.0",
|
||||||
"grpc": "^1.24.11",
|
|
||||||
"helmet": "^5.0.2",
|
"helmet": "^5.0.2",
|
||||||
"jsonwebtoken": "^8.5.1",
|
"jsonwebtoken": "^8.5.1",
|
||||||
"jwk-to-pem": "^2.0.5",
|
"jwk-to-pem": "^2.0.5",
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import { OnModuleInit, Inject } from '@nestjs/common';
|
import { OnModuleInit, Inject } from '@nestjs/common';
|
||||||
import { ClientGrpc } from '@nestjs/microservices';
|
import { ClientGrpc } from '@nestjs/microservices';
|
||||||
import { InputService } from '@victorradael/protospack';
|
import {
|
||||||
|
InputNewCreateRequest,
|
||||||
|
InputService,
|
||||||
|
TestConnectionGetColumnsRequest,
|
||||||
|
TestConnectionRequest,
|
||||||
|
} from '@victorradael/protospack';
|
||||||
import {
|
import {
|
||||||
objectCamelToSnake,
|
objectCamelToSnake,
|
||||||
objectSnakeToCamel,
|
objectSnakeToCamel,
|
||||||
} from 'src/utils/CaseConverter';
|
} from 'src/utils/CaseConverter';
|
||||||
import {
|
import { IIdRequest, UpdateInputRequest } from './interfaces';
|
||||||
ICreateInputRequest,
|
|
||||||
IIdRequest,
|
|
||||||
ITestConnectionRequest,
|
|
||||||
UpdateInputRequest,
|
|
||||||
} from './interfaces';
|
|
||||||
|
|
||||||
export class InputsClientService implements OnModuleInit {
|
export class InputsClientService implements OnModuleInit {
|
||||||
private inputService: InputService;
|
private inputService: InputService;
|
||||||
@@ -23,22 +23,24 @@ export class InputsClientService implements OnModuleInit {
|
|||||||
this.grpcClient.getService<InputService>('InputService');
|
this.grpcClient.getService<InputService>('InputService');
|
||||||
}
|
}
|
||||||
|
|
||||||
async create(createInputDto: ICreateInputRequest) {
|
async create(createInputDto: InputNewCreateRequest) {
|
||||||
console.log('InputClientService', 'Create');
|
console.log('InputClientService', 'Create');
|
||||||
|
|
||||||
const createInputResponse = await new Promise((resolve, reject) => {
|
const createInputResponse = await new Promise((resolve, reject) => {
|
||||||
this.inputService.Create(objectSnakeToCamel(createInputDto)).subscribe({
|
this.inputService
|
||||||
next(x) {
|
.NewCreate(objectSnakeToCamel(createInputDto))
|
||||||
resolve(objectCamelToSnake(x));
|
.subscribe({
|
||||||
},
|
next(x) {
|
||||||
error(err) {
|
resolve(objectCamelToSnake(x));
|
||||||
console.log('Observable Error');
|
},
|
||||||
reject(err);
|
error(err) {
|
||||||
},
|
console.log('Observable Error');
|
||||||
complete() {
|
reject(err);
|
||||||
console.log('done');
|
},
|
||||||
},
|
complete() {
|
||||||
});
|
console.log('done');
|
||||||
|
},
|
||||||
|
});
|
||||||
})
|
})
|
||||||
.then((res) => res)
|
.then((res) => res)
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
@@ -147,10 +149,10 @@ export class InputsClientService implements OnModuleInit {
|
|||||||
return removeInputResponse;
|
return removeInputResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
async testConnection(data: ITestConnectionRequest) {
|
async testConnection(data: TestConnectionRequest) {
|
||||||
console.log('InputClientService', 'TestConnection');
|
console.log('InputClientService', 'TestConnection');
|
||||||
const testConnectionResponse = await new Promise((resolve, reject) => {
|
const testConnectionResponse = await new Promise((resolve, reject) => {
|
||||||
this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({
|
this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
|
||||||
next(x) {
|
next(x) {
|
||||||
resolve(objectCamelToSnake(x));
|
resolve(objectCamelToSnake(x));
|
||||||
},
|
},
|
||||||
@@ -170,4 +172,28 @@ export class InputsClientService implements OnModuleInit {
|
|||||||
|
|
||||||
return testConnectionResponse;
|
return testConnectionResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getColumns(data: TestConnectionGetColumnsRequest) {
|
||||||
|
console.log('InputClientService', 'TestConnection/Get-Columns');
|
||||||
|
const getColumnsResponse = await new Promise((resolve, reject) => {
|
||||||
|
this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
|
||||||
|
next(x) {
|
||||||
|
resolve(objectCamelToSnake(x));
|
||||||
|
},
|
||||||
|
error(err) {
|
||||||
|
console.log('Observable Error');
|
||||||
|
reject(err);
|
||||||
|
},
|
||||||
|
complete() {
|
||||||
|
console.log('done');
|
||||||
|
},
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.then((res) => res)
|
||||||
|
.catch((err) => {
|
||||||
|
throw new Error(err);
|
||||||
|
});
|
||||||
|
|
||||||
|
return getColumnsResponse;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,10 +9,8 @@ import {
|
|||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { InputsService } from './inputs.service';
|
import { InputsService } from './inputs.service';
|
||||||
import { InputsClientService } from 'src/clients/inputs/client.service';
|
import { InputsClientService } from 'src/clients/inputs/client.service';
|
||||||
import {
|
import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
|
||||||
ICreateInputRequest,
|
import { InputNewCreateRequest } from '@victorradael/protospack';
|
||||||
UpdateInputRequest,
|
|
||||||
} from 'src/clients/inputs/interfaces';
|
|
||||||
|
|
||||||
@Controller('inputs')
|
@Controller('inputs')
|
||||||
export class InputsController {
|
export class InputsController {
|
||||||
@@ -28,8 +26,21 @@ export class InputsController {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Post('/test-connection/get-columns')
|
||||||
|
async getColumns(@Body() data) {
|
||||||
|
console.log(
|
||||||
|
`/test-connection/get-columns`,
|
||||||
|
'ON TEST CONNECTION GET COLUMNS ROUTE',
|
||||||
|
);
|
||||||
|
|
||||||
|
const inputService = new InputsService(this.inputsClientService);
|
||||||
|
const response = await inputService.getColumns(data);
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
@Post()
|
@Post()
|
||||||
async create(@Body() createInputDto: ICreateInputRequest) {
|
async create(@Body() createInputDto: InputNewCreateRequest) {
|
||||||
console.log(`/input`, 'ON CREATE ROUTE');
|
console.log(`/input`, 'ON CREATE ROUTE');
|
||||||
|
|
||||||
const inputService = new InputsService(this.inputsClientService);
|
const inputService = new InputsService(this.inputsClientService);
|
||||||
|
|||||||
@@ -1,16 +1,13 @@
|
|||||||
import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
||||||
|
import { InputNewCreateRequest } from '@victorradael/protospack';
|
||||||
import { InputsClientService } from 'src/clients/inputs/client.service';
|
import { InputsClientService } from 'src/clients/inputs/client.service';
|
||||||
import {
|
import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
|
||||||
ICreateInputRequest,
|
|
||||||
IIdRequest,
|
|
||||||
Info,
|
|
||||||
} from 'src/clients/inputs/interfaces';
|
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class InputsService {
|
export class InputsService {
|
||||||
constructor(private inputClient: InputsClientService) {}
|
constructor(private inputClient: InputsClientService) {}
|
||||||
|
|
||||||
async create(@Body() data: ICreateInputRequest) {
|
async create(@Body() data: InputNewCreateRequest) {
|
||||||
try {
|
try {
|
||||||
const createInputResponse = await this.inputClient.create(data);
|
const createInputResponse = await this.inputClient.create(data);
|
||||||
|
|
||||||
@@ -74,4 +71,15 @@ export class InputsService {
|
|||||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getColumns(data) {
|
||||||
|
try {
|
||||||
|
const testConnectionGetColumnsResponse =
|
||||||
|
await this.inputClient.getColumns(data);
|
||||||
|
|
||||||
|
return testConnectionGetColumnsResponse;
|
||||||
|
} catch (err) {
|
||||||
|
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,13 +14,13 @@ import { PipelinesService } from './pipelines.service';
|
|||||||
export class PipelinesController {
|
export class PipelinesController {
|
||||||
constructor(private pipelinesClientService: PipelinesClientService) {}
|
constructor(private pipelinesClientService: PipelinesClientService) {}
|
||||||
|
|
||||||
@Get('start/:id')
|
@Post('start/:id')
|
||||||
async activate(@Param() params, @Body() body) {
|
async activate(@Param() params, @Body() body) {
|
||||||
const { id } = params;
|
const { id } = params;
|
||||||
const { info } = body;
|
const { info } = body;
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
process.env.DEV_URL + `/transformation/start/${id}`,
|
process.env.DEV_URL + `/pipeline/start/${id}`,
|
||||||
'ON START PIPELINE ROUTE',
|
'ON START PIPELINE ROUTE',
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
+1
-1
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user