Pipelines Refactor

package.json

@@ -26,7 +26,6 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
-    "@dadosfera/protospack-v2": "3.10.5",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/app.module.ts

@@ -2,22 +2,14 @@ import { Module } from '@nestjs/common';
 import { ClientsModule } from '@nestjs/microservices';
 import { ConfigModule } from '@nestjs/config';
 import { TransformationsController } from './modules/transformations/transformations.controller';
-import { OutputsController } from './modules/outputs/outputs.controllers';
-import { PipelinesController } from './modules/pipelines/pipelines.controller';
 import { HealthController } from './modules/health/health.controller';
 
 import { TransformationsService } from './modules/transformations/transformations.service';
-import { OutputsService } from './modules/outputs/outputs.service';
-import { PipelinesService } from './modules/pipelines/pipelines.service';
 import { HealthService } from './modules/health/health.service';
 
 import { TransformationsClientService } from './clients/transformations/client.service';
-import { OutputsClientService } from './clients/outputs/client.service';
-import { PipelinesClientService } from './clients/pipelines/client.service';
 
-import { OutputsClientConfiguration } from './clients/outputs/client.config';
 import { TransformationsClientConfiguration } from './clients/transformations/client.config';
-import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
 import { CatalogController } from './modules/catalog/catalog.controller';
 import { CatalogService } from './modules/catalog/catalog.service';
 import { getOauthSecrets } from './utils/OauthSecrets';
@@ -42,29 +34,18 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
 import { NetworkConfigModule } from './modules/network-config/network-config.module';
 import { InputsModule } from './modules/inputs/inputs.module';
 import { OauthModule } from './modules/oauth/oauth.module';
+import { PipelinesModule } from './modules/pipelines/pipelines.module';
 
-const outputClient = new OutputsClientConfiguration();
-const pipelineClient = new PipelinesClientConfiguration();
 const transformationClient = new TransformationsClientConfiguration();
 
 @Module({
-  controllers: [
-    TransformationsController,
-    OutputsController,
-    PipelinesController,
-    HealthController,
-    CatalogController,
-  ],
+  controllers: [TransformationsController, HealthController, CatalogController],
   providers: [
     DadosferaLogger,
     { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
     TransformationsService,
-    OutputsService,
-    PipelinesService,
     HealthService,
     TransformationsClientService,
-    OutputsClientService,
-    PipelinesClientService,
     CatalogService,
     HubspotStrategy,
     FacebookStrategy,
@@ -87,20 +68,13 @@ const transformationClient = new TransformationsClientConfiguration();
     ConnectionModule,
     NetworkConfigModule,
     ConnectionTestModule,
+    PipelinesModule,
 
     ClientsModule.register([
       {
         name: 'TRANSFORMATIONS_PACKAGE',
         ...transformationClient.config(),
       },
-      {
-        name: 'OUTPUTS_PACKAGE',
-        ...outputClient.config(),
-      },
-      {
-        name: 'PIPELINES_PACKAGE',
-        ...pipelineClient.config(),
-      },
     ]),
 
     UsersModule,

src/clients/outputs/client.config.ts

@@ -1,24 +0,0 @@
-import { OutputPackages, OutputProtoFilePath } from 'protospack';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import { credentials } from '@grpc/grpc-js';
-
-export class OutputsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.OTFACTORY_URL,
-        package: OutputPackages,
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
-        protoPath: OutputProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/outputs/client.service.ts

@@ -1,178 +0,0 @@
-import { Controller, Inject, OnModuleInit } from '@nestjs/common';
-import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { OutputServicesNames, OutputsServiceInterface } from 'protospack';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import {
-  ICreateOutputRequest,
-  IIdRequest,
-  IOutputUpdateRequest,
-} from './interfaces';
-
-@Controller('output')
-export class OutputsClientService implements OnModuleInit {
-  private outputService: OutputsServiceInterface;
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject('OUTPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
-      OutputServicesNames.OutputService,
-    );
-  }
-
-  async create(@Payload() createOutputDto: ICreateOutputRequest) {
-    this.logger.info('OutputClientService - Create');
-
-    const createOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Create(objectSnakeToCamel(createOutputDto)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    const convertedItem = objectCamelToSnake(createOutputResponse['output']);
-    createOutputResponse['output'] = convertedItem;
-    return createOutputResponse;
-  }
-
-  async findAll(data) {
-    this.logger.info('OutputClientService - FindAll');
-
-    const findAllOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    const convertedOutputs = findAllOutputResponse['outputs'].map((ot) => {
-      return objectCamelToSnake(ot);
-    });
-
-    return { outputs: convertedOutputs };
-  }
-
-  async findOne(data: IIdRequest) {
-    this.logger.info('OutputClientService - FindOne');
-
-    const findOneOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(findOneOutputResponse);
-  }
-
-  async update(updateOutPutDTO: IOutputUpdateRequest) {
-    this.logger.info('OutputClientService - Update');
-
-    const updateOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Update(objectSnakeToCamel(updateOutPutDTO)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(updateOutputResponse);
-  }
-
-  async remove(data: IIdRequest) {
-    this.logger.info('OutputClientService - Remove');
-
-    const removeOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Remove(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return removeOutputResponse;
-  }
-}

src/clients/outputs/interfaces.d.ts

@@ -1,26 +0,0 @@
-import { Info } from 'protospack/dist/lib/interfaces';
-
-export interface ICreateOutputRequest {
-  operation: string;
-  name: string;
-  plugin: string;
-  values: Values;
-}
-
-export interface Values {
-  prefix: string;
-}
-
-export interface IIdRequest {
-  id: string;
-  info: Info;
-}
-
-export interface IOutputUpdateRequest {
-  id: string;
-  operation: string;
-  values: Values;
-  plugin: string;
-  name: string;
-  info: Info;
-}

src/modules/outputs/outputs.controllers.ts

@@ -1,123 +0,0 @@
-import {
-  Body,
-  Controller,
-  Delete,
-  Get,
-  Inject,
-  Param,
-  Post,
-  Put,
-} from '@nestjs/common';
-import { Payload } from '@nestjs/microservices';
-import { ApiTags } from '@nestjs/swagger';
-import { OutputsClientService } from 'src/clients/outputs/client.service';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
-import { PERMISSIONS } from '../../authentication/permissions.enum';
-import { OutputsService } from './outputs.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-@ApiTags('Outputs')
-@Controller('outputs')
-@AuthenticateCondition((req, user) => {
-  let action;
-
-  switch (req.method) {
-    case 'POST':
-      action = 'CREATE';
-      break;
-
-    case 'PUT':
-      action = 'UPDATE';
-      break;
-
-    default:
-      action = req.method;
-  }
-
-  return user.permissions.includes(
-    PERMISSIONS.OUTPUT.permissions[action].seqid,
-  );
-})
-export class OutputsController {
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private outputsClientService: OutputsClientService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  @Post()
-  async create(@Body() createOutputDto) {
-    this.logger.info(process.env.DEV_URL + `/output - ON CREATE ROUTE`, {
-      user: createOutputDto.info.user_id,
-      customer: createOutputDto.info.customer,
-    });
-
-    const outputService = new OutputsService(this.outputsClientService);
-    const response = await outputService.create(createOutputDto);
-
-    return response;
-  }
-
-  @Get()
-  async findAll(@Body() data) {
-    this.logger.info(process.env.DEV_URL + `/output - ON Find All ROUTE`, {
-      user: data.info.user_id,
-      customer: data.info.customer,
-    });
-
-    const outputService = new OutputsService(this.outputsClientService);
-    const response = await outputService.findAll(data);
-
-    return response;
-  }
-
-  @Get('/:id')
-  async findOne(@Body() data, @Param() params) {
-    const { id } = params;
-    this.logger.info(
-      process.env.DEV_URL + `/output/${id} - ON Find One ROUTE`,
-      {
-        user: data.info.user_id,
-        customer: data.info.customer,
-      },
-    );
-
-    const outputService = new OutputsService(this.outputsClientService);
-    const response = await outputService.findOne({ id, ...data });
-
-    return response;
-  }
-
-  @Put(':id')
-  async update(@Payload() updateOutputDto, @Param() params) {
-    const { id } = params;
-    const { info } = updateOutputDto;
-    this.logger.info(process.env.DEV_URL + `/output/${id} - ON UPDATE ROUTE`, {
-      user: updateOutputDto.info.user_id,
-      customer: updateOutputDto.info.customer,
-    });
-    delete updateOutputDto.info;
-
-    const outputService = new OutputsService(this.outputsClientService);
-    const response = await outputService.update(id, updateOutputDto, info);
-
-    return response;
-  }
-
-  @Delete(':id')
-  async delete(@Body() data, @Param() params) {
-    const { id } = params;
-    this.logger.info(process.env.DEV_URL + `/output/${id} - ON DELETE ROUTE`, {
-      user: data.info.user_id,
-      customer: data.info.customer,
-    });
-
-    const outputService = new OutputsService(this.outputsClientService);
-    const response = await outputService.remove({ id, ...data });
-
-    return response;
-  }
-}

src/modules/outputs/outputs.service.ts

@@ -1,64 +0,0 @@
-import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
-import { IIdRequest } from 'src/clients/outputs/interfaces';
-import { OutputsClientService } from 'src/clients/outputs/client.service';
-import { Info } from 'protospack/dist/lib/interfaces';
-
-@Injectable()
-export class OutputsService {
-  constructor(private outputClient: OutputsClientService) {}
-
-  async create(createOutputDto) {
-    try {
-      const createOutputResponse = await this.outputClient.create(
-        createOutputDto,
-      );
-
-      return createOutputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
-  async findOne(data: IIdRequest) {
-    try {
-      const findOneOutputResponse = await this.outputClient.findOne(data);
-      return findOneOutputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
-  async findAll(data) {
-    try {
-      const findAllOutputResponse = await this.outputClient.findAll(data);
-
-      return findAllOutputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
-  async update(id: string, data, info: Info) {
-    try {
-      const updateOutputResponse = await this.outputClient.update({
-        id,
-        info,
-        ...data,
-      });
-
-      return updateOutputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
-  async remove(data: IIdRequest) {
-    try {
-      const removeOutputResponse = await this.outputClient.remove(data);
-
-      return removeOutputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-}

src/modules/pipelines/client.service.ts

@@ -11,13 +11,16 @@ import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
+
 import {
   ICreatePipelineDto,
   IGetPipelineLogsRequest,
   IIdRequest,
   IUpdatePipelineRequest,
 } from './interfaces';
+
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { PipelinesClientConfiguration } from './pipelines-client';
 
 export class PipelinesClientService implements OnModuleInit {
   private pipelineService: PipelinesServiceInterface;
@@ -25,7 +28,8 @@ export class PipelinesClientService implements OnModuleInit {
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
-    @Inject('PIPELINES_PACKAGE') private readonly grpcClient: ClientGrpc,
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
   ) {
     this.logger = dadosferaLogger.logger;
   }

src/modules/pipelines/pipelines-client.ts

@@ -0,0 +1,29 @@
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { PipelinePackages, PipelineProtoFilePath } from 'protospack';
+import { credentials } from '@grpc/grpc-js';
+
+export class PipelinesClientConfiguration {
+  public name = 'PipelinesClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.PIFACTORY_URL,
+      package: PipelinePackages,
+      credentials: process.env.LOCAL_ENV ? undefined : credentials.createSsl(),
+      protoPath: PipelineProtoFilePath,
+      loader: {
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/pipelines/pipelines.controller.ts

@@ -9,7 +9,7 @@ import {
   Put,
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
-import { PipelinesClientService } from 'src/clients/pipelines/client.service';
+import { PipelinesClientService } from './client.service';
 import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
 import { PERMISSIONS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';

src/modules/pipelines/pipelines.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { PipelinesController } from './pipelines.controller';
+import { PipelinesService } from './pipelines.service';
+
+import { PipelinesClientConfiguration } from './pipelines-client';
+import { PipelinesClientService } from './client.service';
+
+const client = new PipelinesClientConfiguration();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [PipelinesController],
+  providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
+  exports: [PipelinesService],
+})
+export class PipelinesModule {}

src/modules/pipelines/pipelines.service.ts

@@ -1,8 +1,8 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
 import { DecodeGrpcStruct } from 'protospack';
 import { Info } from 'protospack/dist/lib/interfaces';
-import { PipelinesClientService } from 'src/clients/pipelines/client.service';
-import { IIdRequest } from 'src/clients/pipelines/interfaces';
+import { PipelinesClientService } from './client.service';
+import { IIdRequest } from './interfaces';
 import { objectCamelToSnake } from 'src/utils/CaseConverter';
 
 @Injectable()