From b8492c68408d8ea0df4ed61333165845effeaf82 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:03:55 -0300 Subject: [PATCH 01/22] Adding pre-commit --- .env | 1 - .gitignore | 3 +- .pre-commit-config.yaml | 6 ++++ .prettierrc | 2 +- Dockerfile | 2 +- src/app.module.ts | 12 ++++---- src/enrichment/dto/create-enrichment.dto.ts | 2 +- src/enrichment/enrichment.controller.ts | 2 +- src/enrichment/proto/enrichment.proto | 1 - .../dto/create-input-grpc-client.dto.ts | 8 ++--- .../dto/test-connection-grpc-client.dto.ts | 1 - .../entities/input-grcp-client.entity.ts | 10 +++---- src/inputs/input-grpc-client.module.ts | 2 +- src/inputs/input-rest-client.controller.ts | 12 ++++---- src/inputs/proto/input.proto | 4 +-- src/middlewares/authentication.ts | 4 +-- src/outputs/dto/create-output.dto.ts | 1 - src/outputs/dto/update-output.dto.ts | 2 +- src/pipelines/pipelines.controller.ts | 2 +- src/pipelines/proto/pipeline.proto | 8 ++--- .../dto/create-rest-enrichment.dto.ts | 2 +- .../rest-enrichments.controller.ts | 2 +- .../rest-inputs/dto/HTTPInputValues.dto.ts | 6 ++-- .../rest-inputs/dto/JDBCInputValues.dto.ts | 18 +++++------ .../rest-inputs/dto/create-rest-input.dto.ts | 8 ++--- src/rest/rest-inputs/dto/cron.dto.ts | 30 +++++++++---------- src/rest/rest-inputs/dto/s3InputValues.dto.ts | 14 ++++----- .../dto/salesforceInputValues.dto.ts | 8 ++--- .../rest-inputs/dto/testConnection.dto.ts | 6 ++-- .../rest-inputs/entities/rest-input.entity.ts | 10 +++---- .../rest-inputs/rest-inputs.controller.ts | 12 ++++---- .../dto/create-rest-output.dto.ts | 6 ++-- .../rest-outputs/rest-outputs.controller.ts | 2 +- .../rest-transformations.controller.ts | 10 +++---- .../rest-user/dto/create-rest-user.dto.ts | 12 ++++---- src/rest/rest-user/dto/resendCode.dto.ts | 2 +- src/rest/rest-user/dto/userAuth.dto.ts | 2 +- .../rest-user/dto/userConfirmation.dto.ts | 4 +-- src/rest/rest-user/rest-user.controller.ts | 6 ++-- .../dto/create-transformation.dto.ts | 6 ++-- .../entities/transformation.entity.ts | 10 +++---- .../proto/transformation.proto | 1 - .../transformations.controller.ts | 4 +-- src/users-grpc-client/proto/user.proto | 2 +- src/utils/Authenticate.ts | 8 ++--- swagger.json | 2 +- 46 files changed, 140 insertions(+), 138 deletions(-) delete mode 100644 .env create mode 100644 .pre-commit-config.yaml diff --git a/.env b/.env deleted file mode 100644 index 6544940..0000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -TEST=test \ No newline at end of file diff --git a/.gitignore b/.gitignore index ea48d65..b0dd31b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ lerna-debug.log* !.vscode/launch.json !.vscode/extensions.json -.env \ No newline at end of file +# Ignore .env files +*.env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..893c4d5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/.prettierrc b/.prettierrc index dcb7279..a20502b 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,4 +1,4 @@ { "singleQuote": true, "trailingComma": "all" -} \ No newline at end of file +} diff --git a/Dockerfile b/Dockerfile index 7a190ad..3b435f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,4 @@ COPY . /app/ COPY --from=packages /packages/node_modules /app/node_modules RUN npm run build EXPOSE 3333 -ENTRYPOINT npm run start \ No newline at end of file +ENTRYPOINT npm run start diff --git a/src/app.module.ts b/src/app.module.ts index c77364c..c0de4de 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -21,12 +21,12 @@ import { RestEnrichmentsController } from './rest/rest-enrichments/rest-enrichme import { RestFactoryController } from './rest/rest-factory/rest-factory.controller'; import { HealthModule } from './health/health.module'; @Module({ - imports: [ - RestInputsModule, RestOutputsModule, RestEnrichmentsModule, RestTransformationsModule, RestPipelinesModule, RestUserModule, RestFactoryModule, HealthModule, - // InputGrpcClientModule, + imports: [ + RestInputsModule, RestOutputsModule, RestEnrichmentsModule, RestTransformationsModule, RestPipelinesModule, RestUserModule, RestFactoryModule, HealthModule, + // InputGrpcClientModule, //OutputsModule, - //EnrichmentModule, - //TransformationsModule, + //EnrichmentModule, + //TransformationsModule, //PipelinesModule ], controllers: [], @@ -39,6 +39,6 @@ export class AppModule implements NestModule{ .apply(LoggerMiddleware) .forRoutes(RestInputsController,RestOutputsController,RestTransformationsController,RestPipelinesController,RestEnrichmentsController,RestFactoryController) } - + } diff --git a/src/enrichment/dto/create-enrichment.dto.ts b/src/enrichment/dto/create-enrichment.dto.ts index 026514a..6ec0db3 100644 --- a/src/enrichment/dto/create-enrichment.dto.ts +++ b/src/enrichment/dto/create-enrichment.dto.ts @@ -1,5 +1,5 @@ export class CreateEnrichmentDto { - + type:string; columns:string[] diff --git a/src/enrichment/enrichment.controller.ts b/src/enrichment/enrichment.controller.ts index dc1ab55..da8d8fe 100644 --- a/src/enrichment/enrichment.controller.ts +++ b/src/enrichment/enrichment.controller.ts @@ -9,7 +9,7 @@ import { GrpcClientConfiguration } from './config/grpc-client'; @ApiTags('enrichments') @Controller('enrichment') export class EnrichmentController implements OnModuleInit { - + @Client(GrpcClientConfiguration) diff --git a/src/enrichment/proto/enrichment.proto b/src/enrichment/proto/enrichment.proto index fe35124..85baa06 100644 --- a/src/enrichment/proto/enrichment.proto +++ b/src/enrichment/proto/enrichment.proto @@ -41,4 +41,3 @@ message CreateEnrichmentRequest { message ListResponse { repeated Enrichment enrichments = 2; } - diff --git a/src/inputs/dto/create-input-grpc-client.dto.ts b/src/inputs/dto/create-input-grpc-client.dto.ts index 9dbbb0c..c2c1acd 100644 --- a/src/inputs/dto/create-input-grpc-client.dto.ts +++ b/src/inputs/dto/create-input-grpc-client.dto.ts @@ -2,12 +2,12 @@ export class CreateInputGrpcClientDto { cron:string; - + name:string; - + plugin:string; - + values:string; - + operation:string; } diff --git a/src/inputs/dto/test-connection-grpc-client.dto.ts b/src/inputs/dto/test-connection-grpc-client.dto.ts index dabe6fa..48275c4 100644 --- a/src/inputs/dto/test-connection-grpc-client.dto.ts +++ b/src/inputs/dto/test-connection-grpc-client.dto.ts @@ -3,4 +3,3 @@ export class TestConnectionDTO{ values:string; plugin:string; } - diff --git a/src/inputs/entities/input-grcp-client.entity.ts b/src/inputs/entities/input-grcp-client.entity.ts index eed60d3..4e4fed6 100644 --- a/src/inputs/entities/input-grcp-client.entity.ts +++ b/src/inputs/entities/input-grcp-client.entity.ts @@ -1,14 +1,14 @@ export class InputGrpcClient { id:string; - + cron:string; - + name:string; - + plugin:string; - + values:string; - + operation:string; } diff --git a/src/inputs/input-grpc-client.module.ts b/src/inputs/input-grpc-client.module.ts index c81de90..53dd6ed 100644 --- a/src/inputs/input-grpc-client.module.ts +++ b/src/inputs/input-grpc-client.module.ts @@ -5,7 +5,7 @@ import { InputGrpcClientController } from './input-grpc-client.controller'; @Module({ controllers: [ - InputGrpcClientController, + InputGrpcClientController, // InputRestClientController ], providers: [InputGrpcClientService], diff --git a/src/inputs/input-rest-client.controller.ts b/src/inputs/input-rest-client.controller.ts index 574ec6b..99e379a 100644 --- a/src/inputs/input-rest-client.controller.ts +++ b/src/inputs/input-rest-client.controller.ts @@ -8,7 +8,7 @@ import { UpdateInputGrpcClientDto } from './dto/update-input-grpc-client.dto'; @Controller('input') export class InputRestClientController { - + @Get('/test-connection/') async testConnection(){ console.log(process.env.DEV_URL+`/test-connection`, "ON TEST CONNECTION ROUTE") @@ -16,17 +16,17 @@ export class InputRestClientController { return response }) } - + @Post() async create(@Body() createInputDto: CreateInputGrpcClientDto){ console.log(createInputDto) console.log(process.env.DEV_URL+`/input`, "ON CREATE ROUTE") - + await axios.post(process.env.DEV_URL+'/input',createInputDto).then( (response) => { return response }) } - + @Get() async list(){ console.log(process.env.DEV_URL+`/input`, "ON GET ROUTE") @@ -43,7 +43,7 @@ export class InputRestClientController { return response }) } - + @Put(':id') async update( @@ -60,7 +60,7 @@ export class InputRestClientController { @Delete(':id') async delete(@Param() id: string){ console.log(process.env.DEV_URL+`/input/${id}`, "ON DELETE ROUTE") - + await axios.delete(process.env.DEV_URL+`/input/${id}`).then( (response) => { return response }) diff --git a/src/inputs/proto/input.proto b/src/inputs/proto/input.proto index a683d93..87cf829 100644 --- a/src/inputs/proto/input.proto +++ b/src/inputs/proto/input.proto @@ -58,8 +58,8 @@ message TestConnectionRequest{ } message TestConnectionResponse{ - + // nao sei como vai ser a resposta bool respose = 1; -} \ No newline at end of file +} diff --git a/src/middlewares/authentication.ts b/src/middlewares/authentication.ts index d8f1cc0..cdd1412 100644 --- a/src/middlewares/authentication.ts +++ b/src/middlewares/authentication.ts @@ -3,7 +3,7 @@ import axios from 'axios'; import { NestMiddleware } from '@nestjs/common'; export class LoggerMiddleware implements NestMiddleware{ - + use = async ( request: Request, @@ -27,4 +27,4 @@ export class LoggerMiddleware implements NestMiddleware{ } }; -} \ No newline at end of file +} diff --git a/src/outputs/dto/create-output.dto.ts b/src/outputs/dto/create-output.dto.ts index 9ed9967..ac9bb0f 100644 --- a/src/outputs/dto/create-output.dto.ts +++ b/src/outputs/dto/create-output.dto.ts @@ -5,4 +5,3 @@ export class CreateOutputDto { values:string; operation:string; } - diff --git a/src/outputs/dto/update-output.dto.ts b/src/outputs/dto/update-output.dto.ts index b5befe3..62d92a5 100644 --- a/src/outputs/dto/update-output.dto.ts +++ b/src/outputs/dto/update-output.dto.ts @@ -2,5 +2,5 @@ import { PartialType } from '@nestjs/mapped-types'; import { CreateOutputDto } from './create-output.dto'; export class UpdateOutputDto extends PartialType(CreateOutputDto) { - + } diff --git a/src/pipelines/pipelines.controller.ts b/src/pipelines/pipelines.controller.ts index 8e0acb3..9b568d0 100644 --- a/src/pipelines/pipelines.controller.ts +++ b/src/pipelines/pipelines.controller.ts @@ -7,7 +7,7 @@ import { GrpcClientConfiguration } from './config/grpc-client'; @Controller('pipeline') export class PipelinesController implements OnModuleInit{ - + @Client(GrpcClientConfiguration) private pipelineServiceClient: ClientGrpc; diff --git a/src/pipelines/proto/pipeline.proto b/src/pipelines/proto/pipeline.proto index 58de33f..9e187d7 100644 --- a/src/pipelines/proto/pipeline.proto +++ b/src/pipelines/proto/pipeline.proto @@ -37,7 +37,7 @@ message Input { string values = 5; string operation = 6; } - + message Output { string id = 1; string name = 2; @@ -80,12 +80,12 @@ message ListResponse { message getPipelineLogsMessagesRequest{ string id = 1; - string details = 2; + string details = 2; } message getPipelineLogsMessagesResponseItem{ string date = 1; - string log = 2; + string log = 2; } message getPipelineLogsMessagesResponse{ @@ -94,4 +94,4 @@ message getPipelineLogsMessagesResponse{ message getPipelineStatusResponse{ string status = 1; -} \ No newline at end of file +} diff --git a/src/rest/rest-enrichments/dto/create-rest-enrichment.dto.ts b/src/rest/rest-enrichments/dto/create-rest-enrichment.dto.ts index 85c924b..0821f28 100644 --- a/src/rest/rest-enrichments/dto/create-rest-enrichment.dto.ts +++ b/src/rest/rest-enrichments/dto/create-rest-enrichment.dto.ts @@ -1,5 +1,5 @@ export class CreateRestEnrichmentDto { - + type:string; columns diff --git a/src/rest/rest-enrichments/rest-enrichments.controller.ts b/src/rest/rest-enrichments/rest-enrichments.controller.ts index 52591d7..5a84ad2 100644 --- a/src/rest/rest-enrichments/rest-enrichments.controller.ts +++ b/src/rest/rest-enrichments/rest-enrichments.controller.ts @@ -59,7 +59,7 @@ export class RestEnrichmentsController { @Put(':id') async update(@Param('id') id: string,@Headers() headers, @Body() body: UpdateRestEnrichmentDto) { - + const connectionResponse = await axios.put( `${process.env.DEV_URL}/enrichment/${id}`, body, diff --git a/src/rest/rest-inputs/dto/HTTPInputValues.dto.ts b/src/rest/rest-inputs/dto/HTTPInputValues.dto.ts index 1456e8c..f73642e 100644 --- a/src/rest/rest-inputs/dto/HTTPInputValues.dto.ts +++ b/src/rest/rest-inputs/dto/HTTPInputValues.dto.ts @@ -2,16 +2,16 @@ import { ApiProperty } from "@nestjs/swagger"; export class HTTPInputValues{ - + @ApiProperty({ description:"Choosen url", example:"http://testroute/dadosfera/inputs/values" }) urls:string; - + @ApiProperty({ description:"Schedule name", example:"Test_Schedule" }) schedule:string -} \ No newline at end of file +} diff --git a/src/rest/rest-inputs/dto/JDBCInputValues.dto.ts b/src/rest/rest-inputs/dto/JDBCInputValues.dto.ts index 047e4b3..c7bad27 100644 --- a/src/rest/rest-inputs/dto/JDBCInputValues.dto.ts +++ b/src/rest/rest-inputs/dto/JDBCInputValues.dto.ts @@ -2,52 +2,52 @@ import { ApiProperty } from "@nestjs/swagger"; export class JDBCInputvalues{ - + @ApiProperty({ description:"JDBC user", example:"airflow" }) jdbc_user: string; - + @ApiProperty({ description:"JDBC password", example:"password123" }) jdbc_password: string; - + @ApiProperty({ description:"Database", example:"airflow" }) database: string; - + @ApiProperty({ description:"Connection endpoint", example:"dadosfera-postgres-db.test123.us-east" }) endpoint: string; - + @ApiProperty({ description:"Schema name", example:"cadastros" }) schema?:string; - + @ApiProperty({ description:"Tables names", example:["pessoas", "clients"] }) table: string[]; - + @ApiProperty({ description:"Port", example:"5432" }) port: string; - + @ApiProperty({ description:"Engine name", example:"postgres" }) engine: string; -} \ No newline at end of file +} diff --git a/src/rest/rest-inputs/dto/create-rest-input.dto.ts b/src/rest/rest-inputs/dto/create-rest-input.dto.ts index 5c5ce4f..84ce3a3 100644 --- a/src/rest/rest-inputs/dto/create-rest-input.dto.ts +++ b/src/rest/rest-inputs/dto/create-rest-input.dto.ts @@ -29,19 +29,19 @@ export class CreateRestInputDto { } }) cron:CronDTO; - + @ApiProperty({ description:"Name of the input", example:"Test_Input" }) name:string; - + @ApiProperty({ description:"Plugin", example:"s3" }) plugin:string; - + @ApiProperty({ description:"Values for choosen input", example:{ @@ -55,7 +55,7 @@ export class CreateRestInputDto { } }) values: SalesforceInputValuesDTO|HTTPInputValues|S3InputValues|JDBCInputvalues; - + @ApiProperty({ description:"Operation name", example:"input" diff --git a/src/rest/rest-inputs/dto/cron.dto.ts b/src/rest/rest-inputs/dto/cron.dto.ts index 4fdf787..9220775 100644 --- a/src/rest/rest-inputs/dto/cron.dto.ts +++ b/src/rest/rest-inputs/dto/cron.dto.ts @@ -14,89 +14,89 @@ export class CronDTO{ example:true }) hour_interval: boolean; - + @ApiProperty({ description:"Boolean for hour resourse", example:true }) hour_resourse: boolean; - + @ApiProperty({ description:"Value for hour resourse", example:3 }) hour_resourse_value: number; - + @ApiProperty({ description:"Month day to the cron", example:["3"] }) month_day: string[]; - + @ApiProperty({ description:"Boolean for month day interval", example:true }) month_day_interval: boolean; - + @ApiProperty({ description:"Boolean for month day resourse", example:false }) month_day_resourse: boolean; - + @ApiProperty({ description:"Value for month day resourse", example:null }) month_day_resourse_value:number|null; - + @ApiProperty({ description:"Month to the cron", example:["3"] }) month: string[]; - + @ApiProperty({ description:"Boolean from month interval", example:false }) month_interval: boolean; - + @ApiProperty({ description:"Boolean from month resourse", example:false }) month_resourse: boolean; - + @ApiProperty({ description:"Value to month resourse", example:null }) month_resourse_value:number|null; - + @ApiProperty({ description:"Week days to the cron", example:[] }) week_days: string[]; - + @ApiProperty({ description:"Boolean for the week days", example:false }) week_days_interval: boolean; - + @ApiProperty({ description:"Boolean for week day resourse", example:false }) week_days_resourse: boolean; - + @ApiProperty({ description:"Hours to the cron", example:null }) week_days_resourse_value:number|null -} \ No newline at end of file +} diff --git a/src/rest/rest-inputs/dto/s3InputValues.dto.ts b/src/rest/rest-inputs/dto/s3InputValues.dto.ts index 39cfc64..93d371a 100644 --- a/src/rest/rest-inputs/dto/s3InputValues.dto.ts +++ b/src/rest/rest-inputs/dto/s3InputValues.dto.ts @@ -14,37 +14,37 @@ export class S3InputValues{ example:"dadosfera-test" }) target_bucket?: string; - + @ApiProperty({ description:"client bucket path", example:"dadosfera-test" }) client_bucket: string; - + @ApiProperty({ description:"Format to the file", example:"csv" }) format_file: string; - + @ApiProperty({ description:"Clients AWS Access Key Id", example:"AKIBN5B19D3NGZI" }) client_aws_access_key_id: string; - + @ApiProperty({ description:"Clients AWS Secret Access Key Id", example:"ct5R1EjrVC78bvaJ636gG90O5NqOrOUoq4Ri" }) client_aws_secret_access_key : string; - + @ApiProperty({ description:"file params format", example:{sep: "," , header: "true"} }) format_file_params?: { - sep: string, + sep: string, header: string } -} \ No newline at end of file +} diff --git a/src/rest/rest-inputs/dto/salesforceInputValues.dto.ts b/src/rest/rest-inputs/dto/salesforceInputValues.dto.ts index 98049d2..6f6998e 100644 --- a/src/rest/rest-inputs/dto/salesforceInputValues.dto.ts +++ b/src/rest/rest-inputs/dto/salesforceInputValues.dto.ts @@ -8,20 +8,20 @@ export class SalesforceInputValuesDTO{ example:"dadosfera@devs.com" }) username_salesforce: string; - + @ApiProperty({ description:"Salesforce password", example:"Salesforce@test123" })password_salesforce: string; - + @ApiProperty({ description:"Salesforce Security Token", example:"A67Tdg7qHUvhZDOFjWwtPMjmQ" })security_token_salesforce: string; - + @ApiProperty({ description:"Salesforce Query", example:"SELECT * FROM Users" })query_salesforce: string; -} \ No newline at end of file +} diff --git a/src/rest/rest-inputs/dto/testConnection.dto.ts b/src/rest/rest-inputs/dto/testConnection.dto.ts index dc50e4d..0e6d73e 100644 --- a/src/rest/rest-inputs/dto/testConnection.dto.ts +++ b/src/rest/rest-inputs/dto/testConnection.dto.ts @@ -18,7 +18,7 @@ export class testConnectionDTO{ description:"Plugin name", example:{ plugin: "mysql", - values: { + values: { jdbc_user: "test", jdbc_password: "test1234", database: "users", @@ -29,6 +29,6 @@ export class testConnectionDTO{ } } }) - + values:HTTPInputValues|SalesforceInputValuesDTO|S3InputValues|JDBCInputvalues -} \ No newline at end of file +} diff --git a/src/rest/rest-inputs/entities/rest-input.entity.ts b/src/rest/rest-inputs/entities/rest-input.entity.ts index ca8cc69..2525e65 100644 --- a/src/rest/rest-inputs/entities/rest-input.entity.ts +++ b/src/rest/rest-inputs/entities/rest-input.entity.ts @@ -1,14 +1,14 @@ export class RestInput { id:string; - + cron:string; - + name:string; - + plugin:string; - + values:string; - + operation:string; } diff --git a/src/rest/rest-inputs/rest-inputs.controller.ts b/src/rest/rest-inputs/rest-inputs.controller.ts index cba1835..9439ca6 100644 --- a/src/rest/rest-inputs/rest-inputs.controller.ts +++ b/src/rest/rest-inputs/rest-inputs.controller.ts @@ -28,7 +28,7 @@ export class RestInputsController { } ) - + return connectionResponse.data } @@ -46,7 +46,7 @@ export class RestInputsController { } ) - + return connectionResponse.data } @@ -62,7 +62,7 @@ export class RestInputsController { } ) - + return connectionResponse.data } @@ -78,7 +78,7 @@ export class RestInputsController { } ) - + return connectionResponse.data } @@ -100,7 +100,7 @@ export class RestInputsController { } ) - + return connectionResponse.data } @@ -116,7 +116,7 @@ export class RestInputsController { } ) - + return connectionResponse.data } diff --git a/src/rest/rest-outputs/dto/create-rest-output.dto.ts b/src/rest/rest-outputs/dto/create-rest-output.dto.ts index a47256c..9dc126e 100644 --- a/src/rest/rest-outputs/dto/create-rest-output.dto.ts +++ b/src/rest/rest-outputs/dto/create-rest-output.dto.ts @@ -7,13 +7,13 @@ export class CreateRestOutputDto { example:"Test Output" }) name:string; - + @ApiProperty({ description:"Plugin name", example:"Test Plugin" }) plugin:string; - + @ApiProperty({ description:"Plugin values", example:{ @@ -23,7 +23,7 @@ export class CreateRestOutputDto { values:{ prefix:string }; - + @ApiProperty({ description:"Operation name", example:"output" diff --git a/src/rest/rest-outputs/rest-outputs.controller.ts b/src/rest/rest-outputs/rest-outputs.controller.ts index d32c209..1de86fe 100644 --- a/src/rest/rest-outputs/rest-outputs.controller.ts +++ b/src/rest/rest-outputs/rest-outputs.controller.ts @@ -76,7 +76,7 @@ export class RestOutputsController { @Delete(':id') async remove(@Param('id') id: string,@Headers() headers) { - + const connectionResponse = await axios.delete( `${process.env.DEV_URL}/output/${id}`, { diff --git a/src/rest/rest-transformations/rest-transformations.controller.ts b/src/rest/rest-transformations/rest-transformations.controller.ts index d83c0f0..edc0577 100644 --- a/src/rest/rest-transformations/rest-transformations.controller.ts +++ b/src/rest/rest-transformations/rest-transformations.controller.ts @@ -25,7 +25,7 @@ export class RestTransformationsController { } ) - + return connectionResponse.data } @@ -41,7 +41,7 @@ export class RestTransformationsController { } ) - + return connectionResponse.data } @@ -57,7 +57,7 @@ export class RestTransformationsController { } ) - + return connectionResponse.data } @@ -75,7 +75,7 @@ export class RestTransformationsController { } ) - + return connectionResponse.data } @@ -91,7 +91,7 @@ export class RestTransformationsController { } ) - + return connectionResponse.data } } diff --git a/src/rest/rest-user/dto/create-rest-user.dto.ts b/src/rest/rest-user/dto/create-rest-user.dto.ts index fdec23d..bc5f892 100644 --- a/src/rest/rest-user/dto/create-rest-user.dto.ts +++ b/src/rest/rest-user/dto/create-rest-user.dto.ts @@ -1,37 +1,37 @@ import { ApiProperty } from "@nestjs/swagger"; export class CreateRestUserDto { - + @ApiProperty({ description:"User Email", example:"dev@dadosfera.ai" }) - email: string; + email: string; @ApiProperty({ description:"User Password", example:"dadospassword123" }) password: string; - + @ApiProperty({ description:"User Username", example:"dev_fera" }) username: string; - + @ApiProperty({ description:"User name", example:"Fulano da Silva" }) name: string; - + @ApiProperty({ description:"Customer id", example:"dadosfera" }) customer_id: string; - + @ApiProperty({ description:"User rules", example:"['admin']" diff --git a/src/rest/rest-user/dto/resendCode.dto.ts b/src/rest/rest-user/dto/resendCode.dto.ts index 83f7ce7..dd00c8d 100644 --- a/src/rest/rest-user/dto/resendCode.dto.ts +++ b/src/rest/rest-user/dto/resendCode.dto.ts @@ -7,4 +7,4 @@ export class ResendCodeDTO{ example:"dev_fera" }) username:string -} \ No newline at end of file +} diff --git a/src/rest/rest-user/dto/userAuth.dto.ts b/src/rest/rest-user/dto/userAuth.dto.ts index 01b941c..d8f3b1d 100644 --- a/src/rest/rest-user/dto/userAuth.dto.ts +++ b/src/rest/rest-user/dto/userAuth.dto.ts @@ -15,4 +15,4 @@ export class UserAuthDTO{ example:"dadospassword123" }) password:string; -} \ No newline at end of file +} diff --git a/src/rest/rest-user/dto/userConfirmation.dto.ts b/src/rest/rest-user/dto/userConfirmation.dto.ts index 8b38ce7..8c95261 100644 --- a/src/rest/rest-user/dto/userConfirmation.dto.ts +++ b/src/rest/rest-user/dto/userConfirmation.dto.ts @@ -8,10 +8,10 @@ export class UserConfirmationDTO{ }) username:string; - + @ApiProperty({ description:"Received Confirmation Code", example:"315246" }) confirmationCode:string; -} \ No newline at end of file +} diff --git a/src/rest/rest-user/rest-user.controller.ts b/src/rest/rest-user/rest-user.controller.ts index e30f5f1..022a690 100644 --- a/src/rest/rest-user/rest-user.controller.ts +++ b/src/rest/rest-user/rest-user.controller.ts @@ -33,7 +33,7 @@ export class RestUserController { @Post('/auth') async auth(@Body() body: UserAuthDTO) { - + const connectionResponse = await axios.post( `${process.env.DEV_URL}/user/auth`, body, @@ -50,7 +50,7 @@ export class RestUserController { @Post('/confirm') async confirm(@Body() body: UserConfirmationDTO) { - + const connectionResponse = await axios.post( `${process.env.DEV_URL}/user/confirm`, body, @@ -81,5 +81,5 @@ export class RestUserController { return connectionResponse.data } - + } diff --git a/src/transformations/dto/create-transformation.dto.ts b/src/transformations/dto/create-transformation.dto.ts index ec9cea1..1e2f42e 100644 --- a/src/transformations/dto/create-transformation.dto.ts +++ b/src/transformations/dto/create-transformation.dto.ts @@ -2,10 +2,10 @@ export class CreateTransformationDto { inputSource:string; - + table:string; - + type:string; - + params:string; } diff --git a/src/transformations/entities/transformation.entity.ts b/src/transformations/entities/transformation.entity.ts index 9ac2de3..6f067a7 100644 --- a/src/transformations/entities/transformation.entity.ts +++ b/src/transformations/entities/transformation.entity.ts @@ -1,14 +1,14 @@ export class Transformation { id:string; - + inputSource:string; - + table:string; - + type:string; - + params:string; - + } diff --git a/src/transformations/proto/transformation.proto b/src/transformations/proto/transformation.proto index e4658be..55f553e 100644 --- a/src/transformations/proto/transformation.proto +++ b/src/transformations/proto/transformation.proto @@ -47,4 +47,3 @@ message CreateTransformationRequest { message ListResponse { repeated Transformation transformations = 2; } - diff --git a/src/transformations/transformations.controller.ts b/src/transformations/transformations.controller.ts index cccf4b4..e67a2b9 100644 --- a/src/transformations/transformations.controller.ts +++ b/src/transformations/transformations.controller.ts @@ -8,7 +8,7 @@ import { GrpcClientConfiguration } from './config/grpc-client'; @Controller('transformation') export class TransformationsController implements OnModuleInit { - + @Client(GrpcClientConfiguration) private transformationServiceClient: ClientGrpc; @@ -21,7 +21,7 @@ export class TransformationsController implements OnModuleInit { @Post() create(@Payload() createTransformationDto: CreateTransformationDto[]) { - + return createTransformationDto.map( transf => this.transformationService.create(transf)) } diff --git a/src/users-grpc-client/proto/user.proto b/src/users-grpc-client/proto/user.proto index 7bdb82d..3ce8dad 100644 --- a/src/users-grpc-client/proto/user.proto +++ b/src/users-grpc-client/proto/user.proto @@ -34,4 +34,4 @@ message UserResponse { message ShowResponse { repeated UserResponse users = 1; -} \ No newline at end of file +} diff --git a/src/utils/Authenticate.ts b/src/utils/Authenticate.ts index 6327378..e9b4627 100644 --- a/src/utils/Authenticate.ts +++ b/src/utils/Authenticate.ts @@ -5,16 +5,16 @@ export class AuthenticateController{ private username:string; private password:string; - + constructor(){ this.username = process.env.AUTH_USERNAME this.password = process.env.AUTH_PASSWORD } - + async authenticate():Promise{ - + const response = await axios.post(process.env.AUTH_ROUTE, { username:this.username, @@ -26,4 +26,4 @@ export class AuthenticateController{ } -} \ No newline at end of file +} diff --git a/swagger.json b/swagger.json index 227d4bf..1ed2c5f 100644 --- a/swagger.json +++ b/swagger.json @@ -1 +1 @@ -{"openapi":"3.0.0","paths":{"/input/test-connection":{"post":{"operationId":"RestInputsController_testConnection","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/testConnectionDTO"}}}},"responses":{"201":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]}},"/input":{"post":{"operationId":"RestInputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestInputDto"}}}},"responses":{"201":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]},"get":{"operationId":"RestInputsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]}},"/input/{id}":{"get":{"operationId":"RestInputsController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]},"put":{"operationId":"RestInputsController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestInputDto"}}}},"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestInputsController_delete","parameters":[],"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]}},"/output":{"post":{"operationId":"RestOutputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestOutputDto"}}}},"responses":{"201":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]},"get":{"operationId":"RestOutputsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]}},"/output/{id}":{"get":{"operationId":"RestOutputsController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]},"put":{"operationId":"RestOutputsController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestOutputDto"}}}},"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestOutputsController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]}},"/enrichment":{"post":{"operationId":"RestEnrichmentsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestEnrichmentDto"}}}},"responses":{"201":{"description":""}},"tags":["Enrichments"]},"get":{"operationId":"RestEnrichmentsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Enrichments"]}},"/enrichment/{id}":{"get":{"operationId":"RestEnrichmentsController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Enrichments"]},"put":{"operationId":"RestEnrichmentsController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestEnrichmentDto"}}}},"responses":{"200":{"description":""}},"tags":["Enrichments"]},"delete":{"operationId":"RestEnrichmentsController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Enrichments"]}},"/transformation":{"post":{"operationId":"RestTransformationsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestTransformationDto"}}}},"responses":{"201":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]},"get":{"operationId":"RestTransformationsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]}},"/transformation/{id}":{"get":{"operationId":"RestTransformationsController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]},"put":{"operationId":"RestTransformationsController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestTransformationDto"}}}},"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestTransformationsController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]}},"/pipeline":{"post":{"operationId":"RestPipelinesController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestPipelineDto"}}}},"responses":{"201":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]},"get":{"operationId":"RestPipelinesController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/pipeline/{id}/logs/details":{"get":{"operationId":"RestPipelinesController_showLogs","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/pipeline/{id}/status":{"get":{"operationId":"RestPipelinesController_showStatus","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/pipeline/{id}":{"get":{"operationId":"RestPipelinesController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]},"put":{"operationId":"RestPipelinesController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestPipelineDto"}}}},"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestPipelinesController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/user/register":{"post":{"operationId":"RestUserController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestUserDto"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/user/auth":{"post":{"operationId":"RestUserController_auth","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserAuthDTO"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/user/confirm":{"post":{"operationId":"RestUserController_confirm","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserConfirmationDTO"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/user/resendCode":{"post":{"operationId":"RestUserController_resendCode","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResendCodeDTO"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/factory/start":{"get":{"operationId":"RestFactoryController_startDagFactory","parameters":[],"responses":{"200":{"description":""}},"tags":["Dag Factory"],"security":[{"bearer":[]}]}},"/factory/pipeline_params":{"get":{"operationId":"RestFactoryController_sendJSONSettingsToDagFactory","parameters":[],"responses":{"200":{"description":""}},"tags":["Dag Factory"],"security":[{"bearer":[]}]}},"/factory/start/{pipelineId}":{"get":{"operationId":"RestFactoryController_startDag","parameters":[{"name":"pipelineId","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Dag Factory"],"security":[{"bearer":[]}]}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{"testConnectionDTO":{"type":"object","properties":{"plugin":{"type":"string","description":"Input Plugin Name","example":"mysql"},"values":{"type":"object","description":"Plugin name","example":{"plugin":"mysql","values":{"jdbc_user":"test","jdbc_password":"test1234","database":"users","endpoint":"dadosfera.test123.us-east-1.rds.amazonaws.com","schema":"user","port":"5432","engine":"mysql"}}}},"required":["plugin","values"]},"CronDTO":{"type":"object","properties":{"hours":{"description":"Hours to the cron","example":["3","7"],"type":"array","items":{"type":"string"}},"hour_interval":{"type":"boolean","description":"Boolean for hour interval","example":true},"hour_resourse":{"type":"boolean","description":"Boolean for hour resourse","example":true},"hour_resourse_value":{"type":"number","description":"Value for hour resourse","example":3},"month_day":{"description":"Month day to the cron","example":["3"],"type":"array","items":{"type":"string"}},"month_day_interval":{"type":"boolean","description":"Boolean for month day interval","example":true},"month_day_resourse":{"type":"boolean","description":"Boolean for month day resourse","example":false},"month_day_resourse_value":{"type":"number","description":"Value for month day resourse","example":null},"month":{"description":"Month to the cron","example":["3"],"type":"array","items":{"type":"string"}},"month_interval":{"type":"boolean","description":"Boolean from month interval","example":false},"month_resourse":{"type":"boolean","description":"Boolean from month resourse","example":false},"month_resourse_value":{"type":"number","description":"Value to month resourse","example":null},"week_days":{"description":"Week days to the cron","example":[],"type":"array","items":{"type":"string"}},"week_days_interval":{"type":"boolean","description":"Boolean for the week days","example":false},"week_days_resourse":{"type":"boolean","description":"Boolean for week day resourse","example":false},"week_days_resourse_value":{"type":"number","description":"Hours to the cron","example":null}},"required":["hours","hour_interval","hour_resourse","hour_resourse_value","month_day","month_day_interval","month_day_resourse","month_day_resourse_value","month","month_interval","month_resourse","month_resourse_value","week_days","week_days_interval","week_days_resourse","week_days_resourse_value"]},"CreateRestInputDto":{"type":"object","properties":{"cron":{"description":"Input CRON","example":{"hours":["3","7"],"hour_interval":true,"hour_resourse":true,"hour_resourse_value":3,"month_day":["3"],"month_day_interval":true,"month_day_resourse":false,"month_day_resourse_value":null,"month":["3"],"month_interval":false,"month_resourse":false,"month_resourse_value":null,"week_days":[],"week_days_interval":false,"week_days_resourse":false,"week_days_resourse_value":null},"allOf":[{"$ref":"#/components/schemas/CronDTO"}]},"name":{"type":"string","description":"Name of the input","example":"Test_Input"},"plugin":{"type":"string","description":"Plugin","example":"s3"},"values":{"type":"object","description":"Values for choosen input","example":{"jdbc_user":"user123","jdbc_password":"password123","database":"default_database","endpoint":"dadosfera-postgres-db.test123.us-east-1.rds.amazonaws.com","table":"test_table","port":"5432","engine":"postgresql"}},"operation":{"type":"string","description":"Operation name","example":"input"}},"required":["cron","name","plugin","values","operation"]},"UpdateRestInputDto":{"type":"object","properties":{"cron":{"description":"Input CRON","example":{"hours":["3","7"],"hour_interval":true,"hour_resourse":true,"hour_resourse_value":3,"month_day":["3"],"month_day_interval":true,"month_day_resourse":false,"month_day_resourse_value":null,"month":["3"],"month_interval":false,"month_resourse":false,"month_resourse_value":null,"week_days":[],"week_days_interval":false,"week_days_resourse":false,"week_days_resourse_value":null},"allOf":[{"$ref":"#/components/schemas/CronDTO"}]},"name":{"type":"string","description":"Name of the input","example":"Test_Input"},"plugin":{"type":"string","description":"Plugin","example":"s3"},"values":{"type":"object","description":"Values for choosen input","example":{"jdbc_user":"user123","jdbc_password":"password123","database":"default_database","endpoint":"dadosfera-postgres-db.test123.us-east-1.rds.amazonaws.com","table":"test_table","port":"5432","engine":"postgresql"}},"operation":{"type":"string","description":"Operation name","example":"input"}}},"CreateRestOutputDto":{"type":"object","properties":{"name":{"type":"string","description":"Output name","example":"Test Output"},"plugin":{"type":"string","description":"Plugin name","example":"Test Plugin"},"values":{"type":"object","description":"Plugin values","example":{"prefix":"test"}},"operation":{"type":"string","description":"Operation name","example":"output"}},"required":["name","plugin","values","operation"]},"UpdateRestOutputDto":{"type":"object","properties":{"name":{"type":"string","description":"Output name","example":"Test Output"},"plugin":{"type":"string","description":"Plugin name","example":"Test Plugin"},"values":{"type":"object","description":"Plugin values","example":{"prefix":"test"}},"operation":{"type":"string","description":"Operation name","example":"output"}}},"CreateRestEnrichmentDto":{"type":"object","properties":{}},"UpdateRestEnrichmentDto":{"type":"object","properties":{}},"CreateRestTransformationDto":{"type":"object","properties":{"transformations":{"type":"object","description":"Array of transformations","example":[{"type":"hide_full_name","params":[{"base_column":"full_name","column_name":"hiden_full_name"}]},{"type":"hide_cpf","params":[{"base_column":"cpf","column_name":"cpf_hashed"}]}]}},"required":["transformations"]},"UpdateRestTransformationDto":{"type":"object","properties":{"transformations":{"type":"object","description":"Array of transformations","example":[{"type":"hide_full_name","params":[{"base_column":"full_name","column_name":"hiden_full_name"}]},{"type":"hide_cpf","params":[{"base_column":"cpf","column_name":"cpf_hashed"}]}]}}},"CreateRestPipelineDto":{"type":"object","properties":{"input":{"type":"object","description":"Pipeline Input","example":{"id":"ac8f828a-708d-4f28-a5b4-0170eaf96843"}},"transformations":{"description":"Pipeline Transformations","example":[{"id":"b2579a1a-2874-47f6-b609-7568a2930468"},{"id":"6bac1a1a-3037-474f-8211-c3ceb8ea04e8"}],"type":"array","items":{"type":"string"}},"output":{"type":"object","description":"Pipeline Output","example":{"id":"06f9ce35-d9c0-4f95-a49b-c09ea00b8e48"}},"tags":{"description":"Pipeline Tags","example":[],"type":"array","items":{"type":"string"}},"name":{"type":"string","description":"Pipeline Name","example":"Dadosfera Pipeline"},"description":{"type":"string","description":"Pipeline Description","example":"Example pipeline"}},"required":["input","transformations","output","tags","name","description"]},"UpdateRestPipelineDto":{"type":"object","properties":{"input":{"type":"object","description":"Pipeline Input","example":{"id":"ac8f828a-708d-4f28-a5b4-0170eaf96843"}},"transformations":{"description":"Pipeline Transformations","example":[{"id":"b2579a1a-2874-47f6-b609-7568a2930468"},{"id":"6bac1a1a-3037-474f-8211-c3ceb8ea04e8"}],"type":"array","items":{"type":"string"}},"output":{"type":"object","description":"Pipeline Output","example":{"id":"06f9ce35-d9c0-4f95-a49b-c09ea00b8e48"}},"tags":{"description":"Pipeline Tags","example":[],"type":"array","items":{"type":"string"}},"name":{"type":"string","description":"Pipeline Name","example":"Dadosfera Pipeline"},"description":{"type":"string","description":"Pipeline Description","example":"Example pipeline"}}},"CreateRestUserDto":{"type":"object","properties":{"email":{"type":"string","description":"User Email","example":"dev@dadosfera.ai"},"password":{"type":"string","description":"User Password","example":"dadospassword123"},"username":{"type":"string","description":"User Username","example":"dev_fera"},"name":{"type":"string","description":"User name","example":"Fulano da Silva"},"customer_id":{"type":"string","description":"Customer id","example":"dadosfera"},"rules":{"type":"string","description":"User rules","example":"['admin']"}},"required":["email","password","username","name","customer_id","rules"]},"UserAuthDTO":{"type":"object","properties":{"username":{"type":"string","description":"User username","example":"dev_fera"},"password":{"type":"string","description":"User password","example":"dadospassword123"}},"required":["username","password"]},"UserConfirmationDTO":{"type":"object","properties":{"username":{"type":"string","description":"User username","example":"dev_fera"},"confirmationCode":{"type":"string","description":"Received Confirmation Code","example":"315246"}},"required":["username","confirmationCode"]},"ResendCodeDTO":{"type":"object","properties":{"username":{"type":"string","description":"User username","example":"dev_fera"}},"required":["username"]}}}} \ No newline at end of file +{"openapi":"3.0.0","paths":{"/input/test-connection":{"post":{"operationId":"RestInputsController_testConnection","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/testConnectionDTO"}}}},"responses":{"201":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]}},"/input":{"post":{"operationId":"RestInputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestInputDto"}}}},"responses":{"201":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]},"get":{"operationId":"RestInputsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]}},"/input/{id}":{"get":{"operationId":"RestInputsController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]},"put":{"operationId":"RestInputsController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestInputDto"}}}},"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestInputsController_delete","parameters":[],"responses":{"200":{"description":""}},"tags":["Inputs"],"security":[{"bearer":[]}]}},"/output":{"post":{"operationId":"RestOutputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestOutputDto"}}}},"responses":{"201":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]},"get":{"operationId":"RestOutputsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]}},"/output/{id}":{"get":{"operationId":"RestOutputsController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]},"put":{"operationId":"RestOutputsController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestOutputDto"}}}},"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestOutputsController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Outputs"],"security":[{"bearer":[]}]}},"/enrichment":{"post":{"operationId":"RestEnrichmentsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestEnrichmentDto"}}}},"responses":{"201":{"description":""}},"tags":["Enrichments"]},"get":{"operationId":"RestEnrichmentsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Enrichments"]}},"/enrichment/{id}":{"get":{"operationId":"RestEnrichmentsController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Enrichments"]},"put":{"operationId":"RestEnrichmentsController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestEnrichmentDto"}}}},"responses":{"200":{"description":""}},"tags":["Enrichments"]},"delete":{"operationId":"RestEnrichmentsController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Enrichments"]}},"/transformation":{"post":{"operationId":"RestTransformationsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestTransformationDto"}}}},"responses":{"201":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]},"get":{"operationId":"RestTransformationsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]}},"/transformation/{id}":{"get":{"operationId":"RestTransformationsController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]},"put":{"operationId":"RestTransformationsController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestTransformationDto"}}}},"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestTransformationsController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Transformations"],"security":[{"bearer":[]}]}},"/pipeline":{"post":{"operationId":"RestPipelinesController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestPipelineDto"}}}},"responses":{"201":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]},"get":{"operationId":"RestPipelinesController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/pipeline/{id}/logs/details":{"get":{"operationId":"RestPipelinesController_showLogs","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/pipeline/{id}/status":{"get":{"operationId":"RestPipelinesController_showStatus","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/pipeline/{id}":{"get":{"operationId":"RestPipelinesController_show","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]},"put":{"operationId":"RestPipelinesController_update","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateRestPipelineDto"}}}},"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]},"delete":{"operationId":"RestPipelinesController_remove","parameters":[{"name":"id","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Pipelines"],"security":[{"bearer":[]}]}},"/user/register":{"post":{"operationId":"RestUserController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateRestUserDto"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/user/auth":{"post":{"operationId":"RestUserController_auth","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserAuthDTO"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/user/confirm":{"post":{"operationId":"RestUserController_confirm","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserConfirmationDTO"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/user/resendCode":{"post":{"operationId":"RestUserController_resendCode","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResendCodeDTO"}}}},"responses":{"201":{"description":""}},"tags":["Users"]}},"/factory/start":{"get":{"operationId":"RestFactoryController_startDagFactory","parameters":[],"responses":{"200":{"description":""}},"tags":["Dag Factory"],"security":[{"bearer":[]}]}},"/factory/pipeline_params":{"get":{"operationId":"RestFactoryController_sendJSONSettingsToDagFactory","parameters":[],"responses":{"200":{"description":""}},"tags":["Dag Factory"],"security":[{"bearer":[]}]}},"/factory/start/{pipelineId}":{"get":{"operationId":"RestFactoryController_startDag","parameters":[{"name":"pipelineId","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":""}},"tags":["Dag Factory"],"security":[{"bearer":[]}]}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{"testConnectionDTO":{"type":"object","properties":{"plugin":{"type":"string","description":"Input Plugin Name","example":"mysql"},"values":{"type":"object","description":"Plugin name","example":{"plugin":"mysql","values":{"jdbc_user":"test","jdbc_password":"test1234","database":"users","endpoint":"dadosfera.test123.us-east-1.rds.amazonaws.com","schema":"user","port":"5432","engine":"mysql"}}}},"required":["plugin","values"]},"CronDTO":{"type":"object","properties":{"hours":{"description":"Hours to the cron","example":["3","7"],"type":"array","items":{"type":"string"}},"hour_interval":{"type":"boolean","description":"Boolean for hour interval","example":true},"hour_resourse":{"type":"boolean","description":"Boolean for hour resourse","example":true},"hour_resourse_value":{"type":"number","description":"Value for hour resourse","example":3},"month_day":{"description":"Month day to the cron","example":["3"],"type":"array","items":{"type":"string"}},"month_day_interval":{"type":"boolean","description":"Boolean for month day interval","example":true},"month_day_resourse":{"type":"boolean","description":"Boolean for month day resourse","example":false},"month_day_resourse_value":{"type":"number","description":"Value for month day resourse","example":null},"month":{"description":"Month to the cron","example":["3"],"type":"array","items":{"type":"string"}},"month_interval":{"type":"boolean","description":"Boolean from month interval","example":false},"month_resourse":{"type":"boolean","description":"Boolean from month resourse","example":false},"month_resourse_value":{"type":"number","description":"Value to month resourse","example":null},"week_days":{"description":"Week days to the cron","example":[],"type":"array","items":{"type":"string"}},"week_days_interval":{"type":"boolean","description":"Boolean for the week days","example":false},"week_days_resourse":{"type":"boolean","description":"Boolean for week day resourse","example":false},"week_days_resourse_value":{"type":"number","description":"Hours to the cron","example":null}},"required":["hours","hour_interval","hour_resourse","hour_resourse_value","month_day","month_day_interval","month_day_resourse","month_day_resourse_value","month","month_interval","month_resourse","month_resourse_value","week_days","week_days_interval","week_days_resourse","week_days_resourse_value"]},"CreateRestInputDto":{"type":"object","properties":{"cron":{"description":"Input CRON","example":{"hours":["3","7"],"hour_interval":true,"hour_resourse":true,"hour_resourse_value":3,"month_day":["3"],"month_day_interval":true,"month_day_resourse":false,"month_day_resourse_value":null,"month":["3"],"month_interval":false,"month_resourse":false,"month_resourse_value":null,"week_days":[],"week_days_interval":false,"week_days_resourse":false,"week_days_resourse_value":null},"allOf":[{"$ref":"#/components/schemas/CronDTO"}]},"name":{"type":"string","description":"Name of the input","example":"Test_Input"},"plugin":{"type":"string","description":"Plugin","example":"s3"},"values":{"type":"object","description":"Values for choosen input","example":{"jdbc_user":"user123","jdbc_password":"password123","database":"default_database","endpoint":"dadosfera-postgres-db.test123.us-east-1.rds.amazonaws.com","table":"test_table","port":"5432","engine":"postgresql"}},"operation":{"type":"string","description":"Operation name","example":"input"}},"required":["cron","name","plugin","values","operation"]},"UpdateRestInputDto":{"type":"object","properties":{"cron":{"description":"Input CRON","example":{"hours":["3","7"],"hour_interval":true,"hour_resourse":true,"hour_resourse_value":3,"month_day":["3"],"month_day_interval":true,"month_day_resourse":false,"month_day_resourse_value":null,"month":["3"],"month_interval":false,"month_resourse":false,"month_resourse_value":null,"week_days":[],"week_days_interval":false,"week_days_resourse":false,"week_days_resourse_value":null},"allOf":[{"$ref":"#/components/schemas/CronDTO"}]},"name":{"type":"string","description":"Name of the input","example":"Test_Input"},"plugin":{"type":"string","description":"Plugin","example":"s3"},"values":{"type":"object","description":"Values for choosen input","example":{"jdbc_user":"user123","jdbc_password":"password123","database":"default_database","endpoint":"dadosfera-postgres-db.test123.us-east-1.rds.amazonaws.com","table":"test_table","port":"5432","engine":"postgresql"}},"operation":{"type":"string","description":"Operation name","example":"input"}}},"CreateRestOutputDto":{"type":"object","properties":{"name":{"type":"string","description":"Output name","example":"Test Output"},"plugin":{"type":"string","description":"Plugin name","example":"Test Plugin"},"values":{"type":"object","description":"Plugin values","example":{"prefix":"test"}},"operation":{"type":"string","description":"Operation name","example":"output"}},"required":["name","plugin","values","operation"]},"UpdateRestOutputDto":{"type":"object","properties":{"name":{"type":"string","description":"Output name","example":"Test Output"},"plugin":{"type":"string","description":"Plugin name","example":"Test Plugin"},"values":{"type":"object","description":"Plugin values","example":{"prefix":"test"}},"operation":{"type":"string","description":"Operation name","example":"output"}}},"CreateRestEnrichmentDto":{"type":"object","properties":{}},"UpdateRestEnrichmentDto":{"type":"object","properties":{}},"CreateRestTransformationDto":{"type":"object","properties":{"transformations":{"type":"object","description":"Array of transformations","example":[{"type":"hide_full_name","params":[{"base_column":"full_name","column_name":"hiden_full_name"}]},{"type":"hide_cpf","params":[{"base_column":"cpf","column_name":"cpf_hashed"}]}]}},"required":["transformations"]},"UpdateRestTransformationDto":{"type":"object","properties":{"transformations":{"type":"object","description":"Array of transformations","example":[{"type":"hide_full_name","params":[{"base_column":"full_name","column_name":"hiden_full_name"}]},{"type":"hide_cpf","params":[{"base_column":"cpf","column_name":"cpf_hashed"}]}]}}},"CreateRestPipelineDto":{"type":"object","properties":{"input":{"type":"object","description":"Pipeline Input","example":{"id":"ac8f828a-708d-4f28-a5b4-0170eaf96843"}},"transformations":{"description":"Pipeline Transformations","example":[{"id":"b2579a1a-2874-47f6-b609-7568a2930468"},{"id":"6bac1a1a-3037-474f-8211-c3ceb8ea04e8"}],"type":"array","items":{"type":"string"}},"output":{"type":"object","description":"Pipeline Output","example":{"id":"06f9ce35-d9c0-4f95-a49b-c09ea00b8e48"}},"tags":{"description":"Pipeline Tags","example":[],"type":"array","items":{"type":"string"}},"name":{"type":"string","description":"Pipeline Name","example":"Dadosfera Pipeline"},"description":{"type":"string","description":"Pipeline Description","example":"Example pipeline"}},"required":["input","transformations","output","tags","name","description"]},"UpdateRestPipelineDto":{"type":"object","properties":{"input":{"type":"object","description":"Pipeline Input","example":{"id":"ac8f828a-708d-4f28-a5b4-0170eaf96843"}},"transformations":{"description":"Pipeline Transformations","example":[{"id":"b2579a1a-2874-47f6-b609-7568a2930468"},{"id":"6bac1a1a-3037-474f-8211-c3ceb8ea04e8"}],"type":"array","items":{"type":"string"}},"output":{"type":"object","description":"Pipeline Output","example":{"id":"06f9ce35-d9c0-4f95-a49b-c09ea00b8e48"}},"tags":{"description":"Pipeline Tags","example":[],"type":"array","items":{"type":"string"}},"name":{"type":"string","description":"Pipeline Name","example":"Dadosfera Pipeline"},"description":{"type":"string","description":"Pipeline Description","example":"Example pipeline"}}},"CreateRestUserDto":{"type":"object","properties":{"email":{"type":"string","description":"User Email","example":"dev@dadosfera.ai"},"password":{"type":"string","description":"User Password","example":"dadospassword123"},"username":{"type":"string","description":"User Username","example":"dev_fera"},"name":{"type":"string","description":"User name","example":"Fulano da Silva"},"customer_id":{"type":"string","description":"Customer id","example":"dadosfera"},"rules":{"type":"string","description":"User rules","example":"['admin']"}},"required":["email","password","username","name","customer_id","rules"]},"UserAuthDTO":{"type":"object","properties":{"username":{"type":"string","description":"User username","example":"dev_fera"},"password":{"type":"string","description":"User password","example":"dadospassword123"}},"required":["username","password"]},"UserConfirmationDTO":{"type":"object","properties":{"username":{"type":"string","description":"User username","example":"dev_fera"},"confirmationCode":{"type":"string","description":"Received Confirmation Code","example":"315246"}},"required":["username","confirmationCode"]},"ResendCodeDTO":{"type":"object","properties":{"username":{"type":"string","description":"User username","example":"dev_fera"}},"required":["username"]}}}} From cb555b56b30ad8ec1a013458533ec5326b705143 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:18:37 -0300 Subject: [PATCH 02/22] Adding github workflow --- .github/workflows/deploy.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..82f0425 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,34 @@ +name: Deploy +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Configure AWS Region + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-east-1 + + - name: Install AWS CLI + run: | + python3 -m pip install awscli + + - name: Login to AWS ECR + id: login_ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, Tag, and Push Image to AWS ECR + env: + ECR_REGISTRY: ${{ steps.login_ecr.outputs.registry }} + ECR_REPOSITORY: microservices/maestro + IMAGE_TAG: latest + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG From d92cd8d69e570b193a989eab0a224981d9e75c04 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:20:45 -0300 Subject: [PATCH 03/22] Update CI/CD --- .github/workflows/deploy.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 82f0425..80cd6b4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,6 +2,8 @@ name: Deploy on: push: branches: + - dev + - stg - main jobs: @@ -11,6 +13,16 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Extract Environment + shell: bash + run: | + if [ ${GITHUB_REF} == "refs/heads/main" ]; then + echo "##[set-output name=result;]$(echo "prd")" + else + echo "##[set-output name=result;]$(echo ${GITHUB_REF#refs/heads/})" + fi + id: extract_environment + - name: Configure AWS Region uses: aws-actions/configure-aws-credentials@v1 with: @@ -27,7 +39,7 @@ jobs: - name: Build, Tag, and Push Image to AWS ECR env: ECR_REGISTRY: ${{ steps.login_ecr.outputs.registry }} - ECR_REPOSITORY: microservices/maestro + ECR_REPOSITORY: microservices/maestro_${{ steps.extract_environment.outputs.result }} IMAGE_TAG: latest run: | docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . From 13081d21c43d48214a2798d5b447190a838bde19 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:23:10 -0300 Subject: [PATCH 04/22] Update Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b435f3..978e9e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM node:14 as packages -WORKDIR /packages +FROM node:14 as node_modules +WORKDIR /node_modules COPY package.json . RUN npm install -FROM node:alpine +FROM node:14-alpine WORKDIR /app COPY . /app/ -COPY --from=packages /packages/node_modules /app/node_modules +COPY --from=packages /node_modules /app/node_modules RUN npm run build EXPOSE 3333 ENTRYPOINT npm run start From ea79c4da590ba5d251382e4cdcc9e898b076a639 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:25:05 -0300 Subject: [PATCH 05/22] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 978e9e2..13fd9e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN npm install FROM node:14-alpine WORKDIR /app COPY . /app/ -COPY --from=packages /node_modules /app/node_modules +COPY --from=node_modules /node_modules /app/node_modules RUN npm run build EXPOSE 3333 ENTRYPOINT npm run start From 2229ef9f9db7cd222d2a52f653ba18ccfd3b8cb0 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:50:08 -0300 Subject: [PATCH 06/22] Update Dockerfile --- .github/workflows/deploy.yml | 30 +++++++++++++++++------------- Dockerfile | 8 ++++---- docker-compose.yml | 28 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 docker-compose.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 80cd6b4..b45a4f3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,6 +13,19 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Install Docker Compose + run: | + python3 -m pip install docker-compose + + - name: Install AWS CLI + run: | + python3 -m pip install awscli + . + - name: Configure AWS Region + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-east-1 + - name: Extract Environment shell: bash run: | @@ -23,24 +36,15 @@ jobs: fi id: extract_environment - - name: Configure AWS Region - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-east-1 - - - name: Install AWS CLI - run: | - python3 -m pip install awscli - - name: Login to AWS ECR id: login_ecr uses: aws-actions/amazon-ecr-login@v1 - name: Build, Tag, and Push Image to AWS ECR env: - ECR_REGISTRY: ${{ steps.login_ecr.outputs.registry }} - ECR_REPOSITORY: microservices/maestro_${{ steps.extract_environment.outputs.result }} IMAGE_TAG: latest + ENV: ${{ steps.extract_environment.outputs.result }} run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + export ENV=$ENV + docker-compose build + docker-compose push diff --git a/Dockerfile b/Dockerfile index 13fd9e2..3b435f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM node:14 as node_modules -WORKDIR /node_modules +FROM node:14 as packages +WORKDIR /packages COPY package.json . RUN npm install -FROM node:14-alpine +FROM node:alpine WORKDIR /app COPY . /app/ -COPY --from=node_modules /node_modules /app/node_modules +COPY --from=packages /packages/node_modules /app/node_modules RUN npm run build EXPOSE 3333 ENTRYPOINT npm run start diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9218f95 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.8" +services: + maestro: + build: . + image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:latest + container_name: maestro + restart: always + ports: + - 3333:3333 + + node_exporter: + image: prom/node-exporter:v1.3.1 + container_name: node_exporter + restart: always + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' + - '--collector.systemd' + - '--collector.processes' + network_mode: host + pid: host From 3193354bea083cdff1d434fa4b9dd6a357081021 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 10:56:57 -0300 Subject: [PATCH 07/22] Update CI/CD --- .github/workflows/deploy.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b45a4f3..8ef2a7f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,14 +13,22 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Update Pip + run: | + python3 -m pip install --upgrade pip + - name: Install Docker Compose run: | - python3 -m pip install docker-compose + python3 -m pip install docker-compose --upgrade - name: Install AWS CLI run: | - python3 -m pip install awscli - . + python3 -m pip install awscli --upgrade + + - name: Install AWS Elastic Beanstalk ClI + run: | + python3 -m pip install awsebcli --upgrade + - name: Configure AWS Region uses: aws-actions/configure-aws-credentials@v1 with: @@ -42,8 +50,8 @@ jobs: - name: Build, Tag, and Push Image to AWS ECR env: - IMAGE_TAG: latest ENV: ${{ steps.extract_environment.outputs.result }} + IMAGE_TAG: latest run: | export ENV=$ENV docker-compose build From 57768ded8984ed1c76e0ea87454557701745b7ec Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 12:14:27 -0300 Subject: [PATCH 08/22] Adding Deploy AWS Beanstalk --- .github/workflows/deploy.yml | 19 +++++++++++++++++++ .gitignore | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8ef2a7f..5baa30d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -56,3 +56,22 @@ jobs: export ENV=$ENV docker-compose build docker-compose push + + - name: Create ZIP file to Deploy AWS Beanstalk + env: + ENV: ${{ steps.extract_environment.outputs.result }} + run: | + sed -i -e 's/${ENV}/$ENV/g' docker-compose.yml + zip deploy.zip docker-compose.yml + + - name: Deploy AWS Beanstalk + env: + ENV: ${{ steps.extract_environment.outputs.result }} + AWS_REGION: us-east-1 + APP_NAME: maestro + run: | + ENVIRONMENT_NAME=$APP_NAME-$ENV + printf "n" | eb init $APP_NAME --region $AWS_REGION + eb use $ENVIRONMENT_NAME + echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml + eb deploy diff --git a/.gitignore b/.gitignore index b0dd31b..6851d1f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,8 @@ lerna-debug.log* # Ignore .env files *.env + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml From 270b73a2c7b984b5a2f894de6c22168e21159db7 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 12:41:51 -0300 Subject: [PATCH 09/22] Update CI/CD --- .github/workflows/deploy.yml | 4 ++-- build.docker-compose.yml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 build.docker-compose.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5baa30d..d92dc09 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -54,8 +54,8 @@ jobs: IMAGE_TAG: latest run: | export ENV=$ENV - docker-compose build - docker-compose push + docker-compose -f build.docker-compose.yml build + docker-compose -f build.docker-compose.yml push - name: Create ZIP file to Deploy AWS Beanstalk env: diff --git a/build.docker-compose.yml b/build.docker-compose.yml new file mode 100644 index 0000000..f381cd6 --- /dev/null +++ b/build.docker-compose.yml @@ -0,0 +1,5 @@ +version: "3.8" +services: + maestro: + build: . + image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:latest From db858ab65a8a735a52d67ad9f6ef70f582c4ae5b Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 12:42:48 -0300 Subject: [PATCH 10/22] Update environment --- .github/workflows/deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d92dc09..d195763 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,8 +70,7 @@ jobs: AWS_REGION: us-east-1 APP_NAME: maestro run: | - ENVIRONMENT_NAME=$APP_NAME-$ENV printf "n" | eb init $APP_NAME --region $AWS_REGION - eb use $ENVIRONMENT_NAME + eb use $APP_NAME-$ENV echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml eb deploy From 21478cf561ecccefab80ec7103f0a0a9960651f3 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 12:45:59 -0300 Subject: [PATCH 11/22] Removing build flag --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9218f95..514fd97 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ version: "3.8" services: maestro: - build: . image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:latest container_name: maestro restart: always From af209ec4890e14d25ea144f78add1846dddb7117 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 15:17:16 -0300 Subject: [PATCH 12/22] Test eb init --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d195763..244ad07 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -71,6 +71,6 @@ jobs: APP_NAME: maestro run: | printf "n" | eb init $APP_NAME --region $AWS_REGION - eb use $APP_NAME-$ENV - echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml - eb deploy +# eb use $APP_NAME-$ENV +# echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml +# eb deploy From 178f6c56e139d8ac4f28ae7b9eceb2765185e186 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 15:35:28 -0300 Subject: [PATCH 13/22] Using debug flag --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 244ad07..ec926c5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,7 +70,7 @@ jobs: AWS_REGION: us-east-1 APP_NAME: maestro run: | - printf "n" | eb init $APP_NAME --region $AWS_REGION + printf "n" | eb init $APP_NAME --region $AWS_REGION --debug # eb use $APP_NAME-$ENV # echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml # eb deploy From f5d178b035ca11b4ea4daad0e2827f2d6ff6e3d9 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 17:02:29 -0300 Subject: [PATCH 14/22] Enable deploy --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ec926c5..4eb1d28 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -71,6 +71,6 @@ jobs: APP_NAME: maestro run: | printf "n" | eb init $APP_NAME --region $AWS_REGION --debug -# eb use $APP_NAME-$ENV -# echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml -# eb deploy + eb use $APP_NAME-$ENV + echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml + eb deploy From 7346e98ed6738e7934646b7ae6861eb287a18061 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 17:05:39 -0300 Subject: [PATCH 15/22] Removing debug --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4eb1d28..d195763 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,7 +70,7 @@ jobs: AWS_REGION: us-east-1 APP_NAME: maestro run: | - printf "n" | eb init $APP_NAME --region $AWS_REGION --debug + printf "n" | eb init $APP_NAME --region $AWS_REGION eb use $APP_NAME-$ENV echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml eb deploy From 858d73ea5af1e64f71f206624a130327917613de Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Fri, 4 Feb 2022 17:36:42 -0300 Subject: [PATCH 16/22] Fix sed replace --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d195763..45367db 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -61,7 +61,7 @@ jobs: env: ENV: ${{ steps.extract_environment.outputs.result }} run: | - sed -i -e 's/${ENV}/$ENV/g' docker-compose.yml + sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml zip deploy.zip docker-compose.yml - name: Deploy AWS Beanstalk From f34c83509665a86ec512690acfd98718a94a9781 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Mon, 7 Feb 2022 11:45:05 -0300 Subject: [PATCH 17/22] Adding npm_run test --- .github/workflows/test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..dcb99c0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,18 @@ +name: Test +on: + pull_request: + branches: + - dev + - stg + - main + +jobs: + deploy: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: NPM Run Test + run: | + npm run test From be0cda3547a633f2b1d4463da7dd3c51e1bc3d78 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Mon, 7 Feb 2022 11:47:29 -0300 Subject: [PATCH 18/22] NPM Test --- .github/workflows/test.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcb99c0..2818a65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,12 +7,20 @@ on: - main jobs: - deploy: + test: runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v2 + - name: Build + env: + ENV: ${{ steps.extract_environment.outputs.result }} + IMAGE_TAG: latest + run: | + export ENV=$ENV + docker-compose -f build.docker-compose.yml build + - name: NPM Run Test run: | - npm run test + docker-compose run --rm maestro npm run test From e00432c705eff25a3da8c0a9a8cf7a9de691dc4d Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Mon, 7 Feb 2022 12:02:30 -0300 Subject: [PATCH 19/22] Update test build --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2818a65..44685e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,12 +13,16 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Install Docker Compose + run: | + python3 -m pip install docker-compose --upgrade + - name: Build env: ENV: ${{ steps.extract_environment.outputs.result }} IMAGE_TAG: latest run: | - export ENV=$ENV + export ENV=test docker-compose -f build.docker-compose.yml build - name: NPM Run Test From b52549649c31d81374214295e2e114200aa6d36a Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Mon, 7 Feb 2022 12:09:51 -0300 Subject: [PATCH 20/22] Update test build --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 44685e0..c3e4d86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,4 +27,5 @@ jobs: - name: NPM Run Test run: | + export ENV=test docker-compose run --rm maestro npm run test From 57be80b3f8e47458238b86fff59ff7cda6b6da92 Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Mon, 7 Feb 2022 15:08:42 -0300 Subject: [PATCH 21/22] Rename test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c3e4d86..3f78e08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: export ENV=test docker-compose -f build.docker-compose.yml build - - name: NPM Run Test + - name: Run Test run: | export ENV=test docker-compose run --rm maestro npm run test From 6a5294c68bcd0794ecfa52923763468ea48ff7ba Mon Sep 17 00:00:00 2001 From: alexsanderp Date: Tue, 8 Feb 2022 14:29:55 -0300 Subject: [PATCH 22/22] New: Adding semantic versioning --- .github/workflows/deploy.yml | 14 +++++++++++ .releaserc.json | 48 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 .releaserc.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 45367db..5d1e52f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,6 +13,20 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v2 + id: semantic + with: + semantic_version: 16 + extra_plugins: | + @semantic-release/changelog + @semantic-release/git + conventional-changelog-eslint + branches: | + ['main'] + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Update Pip run: | python3 -m pip install --upgrade pip diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 0000000..49fa875 --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,48 @@ +{ + "branch": "dev", + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "eslint", + "releaseRules": [ + { "tag": "Breaking", "release": "major" }, + { "tag": "Build", "release": "minor" }, + { "tag": "Chore", "release": "minor" }, + { "tag": "Fix", "release": "patch" }, + { "tag": "New", "release": "minor" }, + { "tag": "Update", "release": "minor" }, + { "tag": "Upgrade", "release": "minor" } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "eslint", + "releaseRules": [ + { "tag": "Breaking", "release": "major" }, + { "tag": "Build", "release": "minor" }, + { "tag": "Chore", "release": "minor" }, + { "tag": "Fix", "release": "patch" }, + { "tag": "New", "release": "minor" }, + { "tag": "Update", "release": "minor" }, + { "tag": "Upgrade", "release": "minor" } + ] + } + ], + [ + "@semantic-release/changelog", + { + "changelogFile": "CHANGELOG.md" + } + ], + "@semantic-release/github", + [ + "@semantic-release/git", { + "assets": ["package.json", "CHANGELOG.md"], + "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + } + ] + ] +}