From 61752a547f72f04e0e053b2aed2c2cbad34ff04d Mon Sep 17 00:00:00 2001 From: Victor Radael Date: Fri, 12 Aug 2022 12:38:34 -0300 Subject: [PATCH] Pipelines Refactor --- package.json | 1 - src/app.module.ts | 32 +--- src/clients/outputs/client.config.ts | 24 --- src/clients/outputs/client.service.ts | 178 ------------------ src/clients/outputs/interfaces.d.ts | 26 --- src/modules/outputs/outputs.controllers.ts | 123 ------------ src/modules/outputs/outputs.service.ts | 64 ------- .../pipelines/client.service.ts | 6 +- .../pipelines/interfaces.d.ts | 0 src/modules/pipelines/pipelines-client.ts | 29 +++ src/modules/pipelines/pipelines.controller.ts | 2 +- src/modules/pipelines/pipelines.module.ts | 19 ++ src/modules/pipelines/pipelines.service.ts | 4 +- 13 files changed, 59 insertions(+), 449 deletions(-) delete mode 100644 src/clients/outputs/client.config.ts delete mode 100644 src/clients/outputs/client.service.ts delete mode 100644 src/clients/outputs/interfaces.d.ts delete mode 100644 src/modules/outputs/outputs.controllers.ts delete mode 100644 src/modules/outputs/outputs.service.ts rename src/{clients => modules}/pipelines/client.service.ts (97%) rename src/{clients => modules}/pipelines/interfaces.d.ts (100%) create mode 100644 src/modules/pipelines/pipelines-client.ts create mode 100644 src/modules/pipelines/pipelines.module.ts diff --git a/package.json b/package.json index b60c9f5..0dddc3c 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.3", - "@dadosfera/protospack-v2": "3.10.5", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", diff --git a/src/app.module.ts b/src/app.module.ts index 40519f6..33ea38e 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -2,22 +2,14 @@ import { Module } from '@nestjs/common'; import { ClientsModule } from '@nestjs/microservices'; import { ConfigModule } from '@nestjs/config'; import { TransformationsController } from './modules/transformations/transformations.controller'; -import { OutputsController } from './modules/outputs/outputs.controllers'; -import { PipelinesController } from './modules/pipelines/pipelines.controller'; import { HealthController } from './modules/health/health.controller'; import { TransformationsService } from './modules/transformations/transformations.service'; -import { OutputsService } from './modules/outputs/outputs.service'; -import { PipelinesService } from './modules/pipelines/pipelines.service'; import { HealthService } from './modules/health/health.service'; import { TransformationsClientService } from './clients/transformations/client.service'; -import { OutputsClientService } from './clients/outputs/client.service'; -import { PipelinesClientService } from './clients/pipelines/client.service'; -import { OutputsClientConfiguration } from './clients/outputs/client.config'; import { TransformationsClientConfiguration } from './clients/transformations/client.config'; -import { PipelinesClientConfiguration } from './clients/pipelines/client.config'; import { CatalogController } from './modules/catalog/catalog.controller'; import { CatalogService } from './modules/catalog/catalog.service'; import { getOauthSecrets } from './utils/OauthSecrets'; @@ -42,29 +34,18 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test. import { NetworkConfigModule } from './modules/network-config/network-config.module'; import { InputsModule } from './modules/inputs/inputs.module'; import { OauthModule } from './modules/oauth/oauth.module'; +import { PipelinesModule } from './modules/pipelines/pipelines.module'; -const outputClient = new OutputsClientConfiguration(); -const pipelineClient = new PipelinesClientConfiguration(); const transformationClient = new TransformationsClientConfiguration(); @Module({ - controllers: [ - TransformationsController, - OutputsController, - PipelinesController, - HealthController, - CatalogController, - ], + controllers: [TransformationsController, HealthController, CatalogController], providers: [ DadosferaLogger, { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() }, TransformationsService, - OutputsService, - PipelinesService, HealthService, TransformationsClientService, - OutputsClientService, - PipelinesClientService, CatalogService, HubspotStrategy, FacebookStrategy, @@ -87,20 +68,13 @@ const transformationClient = new TransformationsClientConfiguration(); ConnectionModule, NetworkConfigModule, ConnectionTestModule, + PipelinesModule, ClientsModule.register([ { name: 'TRANSFORMATIONS_PACKAGE', ...transformationClient.config(), }, - { - name: 'OUTPUTS_PACKAGE', - ...outputClient.config(), - }, - { - name: 'PIPELINES_PACKAGE', - ...pipelineClient.config(), - }, ]), UsersModule, diff --git a/src/clients/outputs/client.config.ts b/src/clients/outputs/client.config.ts deleted file mode 100644 index 3fac08f..0000000 --- a/src/clients/outputs/client.config.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { OutputPackages, OutputProtoFilePath } from 'protospack'; -import { ClientOptions, Transport } from '@nestjs/microservices'; -import { credentials } from '@grpc/grpc-js'; - -export class OutputsClientConfiguration { - config(): ClientOptions { - return { - transport: Transport.GRPC, - options: { - url: process.env.OTFACTORY_URL, - package: OutputPackages, - credentials: process.env.LOCAL_ENV - ? undefined - : credentials.createSsl(), - protoPath: OutputProtoFilePath, - loader: { - enums: String, - objects: true, - arrays: true, - }, - }, - }; - } -} diff --git a/src/clients/outputs/client.service.ts b/src/clients/outputs/client.service.ts deleted file mode 100644 index 71fbccb..0000000 --- a/src/clients/outputs/client.service.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { Controller, Inject, OnModuleInit } from '@nestjs/common'; -import { ClientGrpc, Payload } from '@nestjs/microservices'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { OutputServicesNames, OutputsServiceInterface } from 'protospack'; -import { - objectCamelToSnake, - objectSnakeToCamel, -} from 'src/utils/CaseConverter'; -import { - ICreateOutputRequest, - IIdRequest, - IOutputUpdateRequest, -} from './interfaces'; - -@Controller('output') -export class OutputsClientService implements OnModuleInit { - private outputService: OutputsServiceInterface; - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - @Inject('OUTPUTS_PACKAGE') private readonly grpcClient: ClientGrpc, - ) { - this.logger = dadosferaLogger.logger; - } - - onModuleInit() { - this.outputService = this.grpcClient.getService( - OutputServicesNames.OutputService, - ); - } - - async create(@Payload() createOutputDto: ICreateOutputRequest) { - this.logger.info('OutputClientService - Create'); - - const createOutputResponse = await new Promise((resolve, reject) => { - this.outputService.Create(objectSnakeToCamel(createOutputDto)).subscribe({ - next(x) { - resolve(x); - }, - error(err) { - reject(err); - }, - complete() { - // console.log('done'); - }, - }); - }) - .then((res) => { - this.logger.info('Done'); - return res; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - const convertedItem = objectCamelToSnake(createOutputResponse['output']); - createOutputResponse['output'] = convertedItem; - return createOutputResponse; - } - - async findAll(data) { - this.logger.info('OutputClientService - FindAll'); - - const findAllOutputResponse = await new Promise((resolve, reject) => { - this.outputService.FindAll(objectSnakeToCamel(data)).subscribe({ - next(x) { - resolve(x); - }, - error(err) { - reject(err); - }, - complete() { - // console.log('done'); - }, - }); - }) - .then((res) => { - this.logger.info('Done'); - return res; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - const convertedOutputs = findAllOutputResponse['outputs'].map((ot) => { - return objectCamelToSnake(ot); - }); - - return { outputs: convertedOutputs }; - } - - async findOne(data: IIdRequest) { - this.logger.info('OutputClientService - FindOne'); - - const findOneOutputResponse = await new Promise((resolve, reject) => { - this.outputService.FindOne(objectSnakeToCamel(data)).subscribe({ - next(x) { - resolve(x); - }, - error(err) { - reject(err); - }, - complete() { - // console.log('done'); - }, - }); - }) - .then((res) => { - this.logger.info('Done'); - return res; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - return objectCamelToSnake(findOneOutputResponse); - } - - async update(updateOutPutDTO: IOutputUpdateRequest) { - this.logger.info('OutputClientService - Update'); - - const updateOutputResponse = await new Promise((resolve, reject) => { - this.outputService.Update(objectSnakeToCamel(updateOutPutDTO)).subscribe({ - next(x) { - resolve(x); - }, - error(err) { - reject(err); - }, - complete() { - // console.log('done'); - }, - }); - }) - .then((res) => { - this.logger.info('Done'); - return res; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - return objectCamelToSnake(updateOutputResponse); - } - - async remove(data: IIdRequest) { - this.logger.info('OutputClientService - Remove'); - - const removeOutputResponse = await new Promise((resolve, reject) => { - this.outputService.Remove(objectSnakeToCamel(data)).subscribe({ - next(x) { - resolve(objectCamelToSnake(x)); - }, - error(err) { - reject(err); - }, - complete() { - // console.log('done'); - }, - }); - }) - .then((res) => { - this.logger.info('Done'); - return res; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - return removeOutputResponse; - } -} diff --git a/src/clients/outputs/interfaces.d.ts b/src/clients/outputs/interfaces.d.ts deleted file mode 100644 index cbee812..0000000 --- a/src/clients/outputs/interfaces.d.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Info } from 'protospack/dist/lib/interfaces'; - -export interface ICreateOutputRequest { - operation: string; - name: string; - plugin: string; - values: Values; -} - -export interface Values { - prefix: string; -} - -export interface IIdRequest { - id: string; - info: Info; -} - -export interface IOutputUpdateRequest { - id: string; - operation: string; - values: Values; - plugin: string; - name: string; - info: Info; -} diff --git a/src/modules/outputs/outputs.controllers.ts b/src/modules/outputs/outputs.controllers.ts deleted file mode 100644 index 588417f..0000000 --- a/src/modules/outputs/outputs.controllers.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { - Body, - Controller, - Delete, - Get, - Inject, - Param, - Post, - Put, -} from '@nestjs/common'; -import { Payload } from '@nestjs/microservices'; -import { ApiTags } from '@nestjs/swagger'; -import { OutputsClientService } from 'src/clients/outputs/client.service'; -import { AuthenticateCondition } from 'src/authentication/authentication.decorator'; -import { PERMISSIONS } from '../../authentication/permissions.enum'; -import { OutputsService } from './outputs.service'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; - -@ApiTags('Outputs') -@Controller('outputs') -@AuthenticateCondition((req, user) => { - let action; - - switch (req.method) { - case 'POST': - action = 'CREATE'; - break; - - case 'PUT': - action = 'UPDATE'; - break; - - default: - action = req.method; - } - - return user.permissions.includes( - PERMISSIONS.OUTPUT.permissions[action].seqid, - ); -}) -export class OutputsController { - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private outputsClientService: OutputsClientService, - ) { - this.logger = dadosferaLogger.logger; - } - - @Post() - async create(@Body() createOutputDto) { - this.logger.info(process.env.DEV_URL + `/output - ON CREATE ROUTE`, { - user: createOutputDto.info.user_id, - customer: createOutputDto.info.customer, - }); - - const outputService = new OutputsService(this.outputsClientService); - const response = await outputService.create(createOutputDto); - - return response; - } - - @Get() - async findAll(@Body() data) { - this.logger.info(process.env.DEV_URL + `/output - ON Find All ROUTE`, { - user: data.info.user_id, - customer: data.info.customer, - }); - - const outputService = new OutputsService(this.outputsClientService); - const response = await outputService.findAll(data); - - return response; - } - - @Get('/:id') - async findOne(@Body() data, @Param() params) { - const { id } = params; - this.logger.info( - process.env.DEV_URL + `/output/${id} - ON Find One ROUTE`, - { - user: data.info.user_id, - customer: data.info.customer, - }, - ); - - const outputService = new OutputsService(this.outputsClientService); - const response = await outputService.findOne({ id, ...data }); - - return response; - } - - @Put(':id') - async update(@Payload() updateOutputDto, @Param() params) { - const { id } = params; - const { info } = updateOutputDto; - this.logger.info(process.env.DEV_URL + `/output/${id} - ON UPDATE ROUTE`, { - user: updateOutputDto.info.user_id, - customer: updateOutputDto.info.customer, - }); - delete updateOutputDto.info; - - const outputService = new OutputsService(this.outputsClientService); - const response = await outputService.update(id, updateOutputDto, info); - - return response; - } - - @Delete(':id') - async delete(@Body() data, @Param() params) { - const { id } = params; - this.logger.info(process.env.DEV_URL + `/output/${id} - ON DELETE ROUTE`, { - user: data.info.user_id, - customer: data.info.customer, - }); - - const outputService = new OutputsService(this.outputsClientService); - const response = await outputService.remove({ id, ...data }); - - return response; - } -} diff --git a/src/modules/outputs/outputs.service.ts b/src/modules/outputs/outputs.service.ts deleted file mode 100644 index 8326da1..0000000 --- a/src/modules/outputs/outputs.service.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; -import { IIdRequest } from 'src/clients/outputs/interfaces'; -import { OutputsClientService } from 'src/clients/outputs/client.service'; -import { Info } from 'protospack/dist/lib/interfaces'; - -@Injectable() -export class OutputsService { - constructor(private outputClient: OutputsClientService) {} - - async create(createOutputDto) { - try { - const createOutputResponse = await this.outputClient.create( - createOutputDto, - ); - - return createOutputResponse; - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async findOne(data: IIdRequest) { - try { - const findOneOutputResponse = await this.outputClient.findOne(data); - return findOneOutputResponse; - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async findAll(data) { - try { - const findAllOutputResponse = await this.outputClient.findAll(data); - - return findAllOutputResponse; - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async update(id: string, data, info: Info) { - try { - const updateOutputResponse = await this.outputClient.update({ - id, - info, - ...data, - }); - - return updateOutputResponse; - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - - async remove(data: IIdRequest) { - try { - const removeOutputResponse = await this.outputClient.remove(data); - - return removeOutputResponse; - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } -} diff --git a/src/clients/pipelines/client.service.ts b/src/modules/pipelines/client.service.ts similarity index 97% rename from src/clients/pipelines/client.service.ts rename to src/modules/pipelines/client.service.ts index e0631d9..f65cbcc 100644 --- a/src/clients/pipelines/client.service.ts +++ b/src/modules/pipelines/client.service.ts @@ -11,13 +11,16 @@ import { objectCamelToSnake, objectSnakeToCamel, } from 'src/utils/CaseConverter'; + import { ICreatePipelineDto, IGetPipelineLogsRequest, IIdRequest, IUpdatePipelineRequest, } from './interfaces'; + import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { PipelinesClientConfiguration } from './pipelines-client'; export class PipelinesClientService implements OnModuleInit { private pipelineService: PipelinesServiceInterface; @@ -25,7 +28,8 @@ export class PipelinesClientService implements OnModuleInit { constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, - @Inject('PIPELINES_PACKAGE') private readonly grpcClient: ClientGrpc, + @Inject(PipelinesClientConfiguration.name) + private readonly grpcClient: ClientGrpc, ) { this.logger = dadosferaLogger.logger; } diff --git a/src/clients/pipelines/interfaces.d.ts b/src/modules/pipelines/interfaces.d.ts similarity index 100% rename from src/clients/pipelines/interfaces.d.ts rename to src/modules/pipelines/interfaces.d.ts diff --git a/src/modules/pipelines/pipelines-client.ts b/src/modules/pipelines/pipelines-client.ts new file mode 100644 index 0000000..f57357c --- /dev/null +++ b/src/modules/pipelines/pipelines-client.ts @@ -0,0 +1,29 @@ +import { + ClientsProviderAsyncOptions, + GrpcOptions, + Transport, +} from '@nestjs/microservices'; +import { PipelinePackages, PipelineProtoFilePath } from 'protospack'; +import { credentials } from '@grpc/grpc-js'; + +export class PipelinesClientConfiguration { + public name = 'PipelinesClientConfiguration'; + private config: GrpcOptions = { + transport: Transport.GRPC, + options: { + url: process.env.PIFACTORY_URL, + package: PipelinePackages, + credentials: process.env.LOCAL_ENV ? undefined : credentials.createSsl(), + protoPath: PipelineProtoFilePath, + loader: { + enums: String, + objects: true, + arrays: true, + }, + }, + }; + providerOptions: ClientsProviderAsyncOptions = { + name: this.name, + ...this.config, + }; +} diff --git a/src/modules/pipelines/pipelines.controller.ts b/src/modules/pipelines/pipelines.controller.ts index 108bb60..6e3cc13 100644 --- a/src/modules/pipelines/pipelines.controller.ts +++ b/src/modules/pipelines/pipelines.controller.ts @@ -9,7 +9,7 @@ import { Put, } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; -import { PipelinesClientService } from 'src/clients/pipelines/client.service'; +import { PipelinesClientService } from './client.service'; import { AuthenticateCondition } from 'src/authentication/authentication.decorator'; import { PERMISSIONS } from '../../authentication/permissions.enum'; import { PipelinesService } from './pipelines.service'; diff --git a/src/modules/pipelines/pipelines.module.ts b/src/modules/pipelines/pipelines.module.ts new file mode 100644 index 0000000..192dcab --- /dev/null +++ b/src/modules/pipelines/pipelines.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { ClientsModule } from '@nestjs/microservices'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { PipelinesController } from './pipelines.controller'; +import { PipelinesService } from './pipelines.service'; + +import { PipelinesClientConfiguration } from './pipelines-client'; +import { PipelinesClientService } from './client.service'; + +const client = new PipelinesClientConfiguration(); + +@Module({ + imports: [ClientsModule.register([client.providerOptions])], + controllers: [PipelinesController], + providers: [PipelinesService, PipelinesClientService, DadosferaLogger], + exports: [PipelinesService], +}) +export class PipelinesModule {} diff --git a/src/modules/pipelines/pipelines.service.ts b/src/modules/pipelines/pipelines.service.ts index 31a2c30..c4d50b7 100644 --- a/src/modules/pipelines/pipelines.service.ts +++ b/src/modules/pipelines/pipelines.service.ts @@ -1,8 +1,8 @@ import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { DecodeGrpcStruct } from 'protospack'; import { Info } from 'protospack/dist/lib/interfaces'; -import { PipelinesClientService } from 'src/clients/pipelines/client.service'; -import { IIdRequest } from 'src/clients/pipelines/interfaces'; +import { PipelinesClientService } from './client.service'; +import { IIdRequest } from './interfaces'; import { objectCamelToSnake } from 'src/utils/CaseConverter'; @Injectable()