Merge Input Refactor

jest.config.ts

@@ -0,0 +1,18 @@
+import type { Config } from '@jest/types';
+const config: Config.InitialOptions = {
+  moduleFileExtensions: ['ts', 'js', 'json'],
+  modulePaths: ['<rootDir>'],
+  roots: ['<rootDir>/src/', '<rootDir>/test/'],
+  testRegex: '.*\\.(test|spec)\\.[jt]s$',
+  transform: { '\\.[jt]s$': 'ts-jest' },
+  collectCoverageFrom: ['**/*.[jt]s'],
+  coverageDirectory: 'coverage',
+  coveragePathIgnorePatterns: [
+    '<rootDir>/node_modules/',
+    '.*\\.module\\.[jt]s$',
+  ],
+  // moduleDirectories: ['node_modules'], // default is already 'node_modules'
+  // rootDir: '.', //No need
+  // testEnvironment: 'node', //Defaults to 'node'
+};
+export default config;

package-lock.json

@@ -23,7 +23,7 @@
         "@nestjs/platform-express": "^8.4.7",
         "@nestjs/schedule": "^1.1.0",
         "@nestjs/swagger": "^5.2.1",
-        "axios": "^0.25.0",
+        "axios": "^0.27.2",
         "class-transformer": "^0.5.1",
         "class-validator": "^0.13.2",
         "cron-parser": "^4.4.0",
@@ -2429,26 +2429,6 @@
         }
       }
     },
-    "node_modules/@nestjs/common/node_modules/axios": {
-      "version": "0.27.2",
-      "license": "MIT",
-      "dependencies": {
-        "follow-redirects": "^1.14.9",
-        "form-data": "^4.0.0"
-      }
-    },
-    "node_modules/@nestjs/common/node_modules/form-data": {
-      "version": "4.0.0",
-      "license": "MIT",
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/@nestjs/config": {
       "version": "1.2.1",
       "license": "MIT",
@@ -3785,10 +3765,25 @@
       }
     },
     "node_modules/axios": {
-      "version": "0.25.0",
-      "license": "MIT",
+      "version": "0.27.2",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/axios/-/axios-0.27.2.tgz",
+      "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
       "dependencies": {
-        "follow-redirects": "^1.14.7"
+        "follow-redirects": "^1.14.9",
+        "form-data": "^4.0.0"
+      }
+    },
+    "node_modules/axios/node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
       }
     },
     "node_modules/babel-jest": {
@@ -12783,23 +12778,6 @@
         "iterare": "1.2.1",
         "tslib": "2.4.0",
         "uuid": "8.3.2"
-      },
-      "dependencies": {
-        "axios": {
-          "version": "0.27.2",
-          "requires": {
-            "follow-redirects": "^1.14.9",
-            "form-data": "^4.0.0"
-          }
-        },
-        "form-data": {
-          "version": "4.0.0",
-          "requires": {
-            "asynckit": "^0.4.0",
-            "combined-stream": "^1.0.8",
-            "mime-types": "^2.1.12"
-          }
-        }
       }
     },
     "@nestjs/config": {
@@ -13713,9 +13691,24 @@
       "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="
     },
     "axios": {
-      "version": "0.25.0",
+      "version": "0.27.2",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/axios/-/axios-0.27.2.tgz",
+      "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
       "requires": {
-        "follow-redirects": "^1.14.7"
+        "follow-redirects": "^1.14.9",
+        "form-data": "^4.0.0"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "4.0.0",
+          "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/form-data/-/form-data-4.0.0.tgz",
+          "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
       }
     },
     "babel-jest": {

package.json

@@ -25,6 +25,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
+    "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
+    "@dadosfera/protospack-v2": "3.10.5",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",
@@ -36,11 +38,10 @@
     "@nestjs/platform-express": "^8.4.7",
     "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
-    "axios": "^0.25.0",
+    "axios": "^0.27.2",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",
     "cron-parser": "^4.4.0",
-    "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.36.0",
     "helmet": "^5.1.0",
@@ -91,28 +92,5 @@
     "ts-node": "^10.8.1",
     "tsconfig-paths": "^3.14.1",
     "typescript": "^4.6.3"
-  },
-  "jest": {
-    "moduleFileExtensions": [
-      "js",
-      "json",
-      "ts"
-    ],
-    "rootDir": ".",
-    "modulePaths": [
-      "<rootDir>"
-    ],
-    "moduleDirectories": [
-      "node_modules"
-    ],
-    "testRegex": ".*\\.spec\\.ts$",
-    "transform": {
-      "^.+\\.(t|j)s$": "ts-jest"
-    },
-    "collectCoverageFrom": [
-      "**/*.(t|j)s"
-    ],
-    "coverageDirectory": "../coverage",
-    "testEnvironment": "node"
   }
 }

src/app.module.ts

@@ -1,30 +1,25 @@
 import { Module } from '@nestjs/common';
 import { ClientsModule } from '@nestjs/microservices';
 import { ConfigModule } from '@nestjs/config';
-import { InputsController } from './modules/inputs/inputs.controller';
 import { TransformationsController } from './modules/transformations/transformations.controller';
 import { OutputsController } from './modules/outputs/outputs.controllers';
 import { PipelinesController } from './modules/pipelines/pipelines.controller';
 import { HealthController } from './modules/health/health.controller';
 
-import { InputsService } from './modules/inputs/inputs.service';
 import { TransformationsService } from './modules/transformations/transformations.service';
 import { OutputsService } from './modules/outputs/outputs.service';
 import { PipelinesService } from './modules/pipelines/pipelines.service';
 import { HealthService } from './modules/health/health.service';
 
-import { InputsClientService } from './clients/inputs/client.service';
 import { TransformationsClientService } from './clients/transformations/client.service';
 import { OutputsClientService } from './clients/outputs/client.service';
 import { PipelinesClientService } from './clients/pipelines/client.service';
 
 import { OutputsClientConfiguration } from './clients/outputs/client.config';
 import { TransformationsClientConfiguration } from './clients/transformations/client.config';
-import { InputsClientConfiguration } from './clients/inputs/client.config';
 import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
 import { CatalogController } from './modules/catalog/catalog.controller';
 import { CatalogService } from './modules/catalog/catalog.service';
-import { OauthController } from './modules/oauth/oauth.controller';
 import { getOauthSecrets } from './utils/OauthSecrets';
 import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot-strategy';
 import { FacebookStrategy } from './modules/oauth/passport-strategies/facebook-strategy';
@@ -45,31 +40,28 @@ import { TermsOfUseModule } from './modules/termsOfUse/termsOfUse.module';
 import { ConnectionModule } from './modules/connection/connection.module';
 import { ConnectionTestModule } from './modules/connection-test/connection-test.module';
 import { NetworkConfigModule } from './modules/network-config/network-config.module';
+import { InputsModule } from './modules/inputs/inputs.module';
+import { OauthModule } from './modules/oauth/oauth.module';
 
-const inputClient = new InputsClientConfiguration();
 const outputClient = new OutputsClientConfiguration();
 const pipelineClient = new PipelinesClientConfiguration();
 const transformationClient = new TransformationsClientConfiguration();
 
 @Module({
   controllers: [
-    InputsController,
     TransformationsController,
     OutputsController,
     PipelinesController,
     HealthController,
     CatalogController,
-    OauthController,
   ],
   providers: [
     DadosferaLogger,
     { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
-    InputsService,
     TransformationsService,
     OutputsService,
     PipelinesService,
     HealthService,
-    InputsClientService,
     TransformationsClientService,
     OutputsClientService,
     PipelinesClientService,
@@ -97,10 +89,6 @@ const transformationClient = new TransformationsClientConfiguration();
     ConnectionTestModule,
 
     ClientsModule.register([
-      {
-        name: 'INPUTS_PACKAGE',
-        ...inputClient.config(),
-      },
       {
         name: 'TRANSFORMATIONS_PACKAGE',
         ...transformationClient.config(),
@@ -118,6 +106,10 @@ const transformationClient = new TransformationsClientConfiguration();
     UsersModule,
 
     RolesModule,
+
+    InputsModule,
+
+    OauthModule,
   ],
 })
 export class AppModule {}

src/clients/inputs/client.config.ts

@@ -1,25 +0,0 @@
-import { credentials } from '@grpc/grpc-js';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-
-import { InputPackages, InputProtoFilePath } from 'protospack';
-
-export class InputsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.INFACTORY_URL,
-        package: InputPackages,
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
-        protoPath: InputProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/inputs/client.service.ts

@@ -1,310 +0,0 @@
-import {
-  OnModuleInit,
-  Inject,
-  HttpException,
-  HttpStatus,
-  NotFoundException,
-  InternalServerErrorException,
-} from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import {
-  DecodeGrpcStruct,
-  EncodeJsonToGrpcStruct,
-  InputCreateS3Request,
-  InputNewCreateRequest,
-  InputService,
-  TestConnectionGetColumnsRequest,
-  TestConnectionRequest,
-} from 'protospack';
-import axios, { AxiosRequestConfig } from 'axios';
-import { lastValueFrom } from 'rxjs';
-import { InputModel } from 'src/modules/inputs/dtos/input.model';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import { mustache } from 'src/utils/mustache';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { OauthSecrets } from 'src/utils/OauthSecrets';
-import { URLSearchParams } from 'url';
-import { IIdRequest, UpdateInputRequest } from './interfaces';
-
-export class InputsClientService implements OnModuleInit {
-  private inputService: InputService;
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-    @Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  async onModuleInit() {
-    this.inputService =
-      this.grpcClient.getService<InputService>('InputService');
-  }
-
-  async newCreate(createInputDto: InputNewCreateRequest) {
-    this.logger.info('InputClientService - Create');
-
-    const input = await lastValueFrom(
-      this.inputService.NewCreate(objectSnakeToCamel(createInputDto)),
-    );
-    objectCamelToSnake(input);
-    return input;
-  }
-
-  async createGeneric(createInputGeneric) {
-    this.logger.info('InputClientService - Create');
-    const { info, ...data } = createInputGeneric;
-    // data.credentials = await this.getInputTokens(data);
-    if (data.options) delete data.options;
-    if (data.credentials.oauth_code) delete data.credentials.oauth_code;
-    const grpcPayload = {
-      input: EncodeJsonToGrpcStruct(data),
-      info,
-    };
-    objectSnakeToCamel(grpcPayload);
-
-    const structReturn = await lastValueFrom(
-      this.inputService.Create(grpcPayload),
-    ).catch((err: { details: string }) => {
-      if (err.details === 'Item Not found!') {
-        throw new NotFoundException('Input not found');
-      }
-      throw new InternalServerErrorException(err.details);
-    });
-    objectCamelToSnake(structReturn);
-    const inputCreated = DecodeGrpcStruct(structReturn.input);
-    return { input: inputCreated };
-  }
-
-  async createS3Inputs(createInputDto: InputCreateS3Request) {
-    this.logger.info('InputClientService - Create');
-    objectSnakeToCamel(createInputDto);
-    const createInputResponse = await lastValueFrom(
-      this.inputService.CreateS3(createInputDto),
-    ).catch((e) => {
-      throw new HttpException(e.details, 500);
-    });
-    this.logger.info('Done');
-    objectCamelToSnake(createInputResponse);
-    return createInputResponse;
-  }
-
-  async findOne(data: IIdRequest) {
-    this.logger.info('InputClientService - FindOne');
-    const findOneInputResponse = await lastValueFrom(
-      this.inputService.FindOne(objectSnakeToCamel(data)),
-    ).catch((e) => {
-      this.logger.error(e.details);
-      throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
-    });
-    objectCamelToSnake(findOneInputResponse);
-    return findOneInputResponse;
-  }
-
-  async findAll(data) {
-    this.logger.info('InputClientService - FindAll');
-    const findAllInputResponse = await lastValueFrom(
-      this.inputService.FindAll(objectSnakeToCamel(data)),
-    ).catch((e) => {
-      this.logger.error(e.details);
-      throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
-    });
-    findAllInputResponse.inputs.forEach((input) => objectCamelToSnake(input));
-
-    return findAllInputResponse;
-  }
-
-  async update(updateInputDTO: UpdateInputRequest) {
-    this.logger.info('InputClientService - Update');
-    const updateInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Update(objectSnakeToCamel(updateInputDTO)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return updateInputResponse;
-  }
-
-  async remove(idRequest: IIdRequest) {
-    this.logger.info('InputClientService - Remove');
-
-    const removeInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Remove(objectSnakeToCamel(idRequest)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return removeInputResponse;
-  }
-
-  async testConnection(data: TestConnectionRequest) {
-    this.logger.info('InputClientService - TestConnection');
-    const testConnectionResponse = await new Promise((resolve, reject) => {
-      this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return testConnectionResponse;
-  }
-
-  async getColumns(data: TestConnectionGetColumnsRequest) {
-    this.logger.info('InputClientService - TestConnection/Get-Columns');
-    const getColumnsResponse = await new Promise((resolve, reject) => {
-      this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return getColumnsResponse;
-  }
-  async getAuthSecrets(plugin) {
-    switch (plugin) {
-      case 'hubspot':
-        return {
-          client_id: this.oauthSecrets.hubspot.client_id || '',
-          client_secret: this.oauthSecrets.hubspot.client_secret || '',
-        };
-    }
-    return {};
-  }
-  async getInputTokens(input: InputModel) {
-    const { credentials } = input;
-    let credentialsTokens = {};
-    switch (credentials.connection_type) {
-      case 'oauth':
-        credentialsTokens = await this.getOauthTokens(input);
-        break;
-    }
-    return { ...credentials, ...credentialsTokens };
-  }
-  async getOauthTokens(input: InputModel) {
-    const tokens: Record<string, any> = {};
-    const { options, credentials } = input;
-    const { oauth } = options;
-    const { get_tokens_url_params, get_tokens_set_response } = oauth;
-    const secrets = await this.getAuthSecrets(input.plugin);
-    const get_tokens_url_params_string = mustache(get_tokens_url_params, {
-      ...secrets,
-      ...credentials,
-    });
-    let get_tokens_url_params_obj: Record<string, any> = {};
-    try {
-      get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
-    } catch (error) {
-      throw new HttpException('Erro transformando get_tokens_url_params', 400);
-    }
-    let params = {};
-    switch (oauth.content_type) {
-      case 'application/x-www-form-urlencoded':
-        params = new URLSearchParams(get_tokens_url_params_obj);
-        params = params.toString();
-        break;
-      default:
-        break;
-    }
-    const axiosRequestConfig: AxiosRequestConfig = {
-      url: oauth.get_tokens_url,
-      method: oauth.get_tokens_method || 'POST',
-      data: params,
-      headers: { 'content-type': oauth.content_type || 'application/json' },
-    };
-
-    const { data } = await axios(axiosRequestConfig).catch((err) => {
-      this.logger.error(err.response.data);
-      throw new HttpException(err.response.data.message, err.response.status);
-    });
-    for (const key in get_tokens_set_response) {
-      const responseKey = get_tokens_set_response[key];
-      tokens[key] = data[responseKey];
-    }
-    const { redirect_uri } = get_tokens_url_params_obj;
-    tokens.redirect_uri = redirect_uri;
-    return tokens;
-  }
-
-  async getAvailableEntities(data) {
-    this.logger.info('InputClientService - GetAvailableEntities');
-    objectSnakeToCamel(data);
-    const response = await lastValueFrom(
-      this.inputService.GetAvailableEntities(data),
-    ).catch((err) => {
-      this.logger.info(err.details);
-      if (err.details && err.details.includes('400'))
-        throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
-      throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
-    });
-    objectCamelToSnake(response);
-
-    return response;
-  }
-}

src/clients/outputs/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 export interface ICreateOutputRequest {
   operation: string;
   name: string;
@@ -9,11 +11,6 @@ export interface Values {
   prefix: string;
 }
 
-interface Info {
-  user_id: string;
-  customer_id: string;
-}
-
 export interface IIdRequest {
   id: string;
   info: Info;

src/clients/pipelines/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 export interface ICreatePipelineDto {
   input: IdRequest;
   transformations: IdRequest[];
@@ -8,11 +10,6 @@ export interface ICreatePipelineDto {
   info: Info;
 }
 
-export interface Info {
-  customer_id: sting;
-  user_id: string;
-}
-
 export interface IdRequest {
   id: string;
 }

src/clients/transformations/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 export interface ICreateTransformationsRequest {
   transformations: Transformation[];
   info: Info;
@@ -11,11 +13,6 @@ interface Param {
   const: string;
 }
 
-export interface Info {
-  customer_id: string;
-  user_id: string;
-}
-
 interface Transformation {
   created_at: string;
   id: string;

src/modules/connection-test/connection-test.controller.spec.ts

@@ -1,25 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { ConnectionTestController } from './connection-test.controller';
-import { ConnectionTestService } from './connection-test.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { ConnectionTestClientConfiguration } from './connection-test-client.config';
-import { ClientsModule } from '@nestjs/microservices';
-
-describe('ConnectionTestController', () => {
-  let controller: ConnectionTestController;
-
-  beforeEach(async () => {
-    const client = new ConnectionTestClientConfiguration();
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [ConnectionTestController],
-      providers: [ConnectionTestService, DadosferaLogger],
-      imports: [ClientsModule.register([client.providerOptions])],
-    }).compile();
-
-    controller = module.get<ConnectionTestController>(ConnectionTestController);
-  });
-
-  it('should be defined', () => {
-    expect(controller).toBeDefined();
-  });
-});

src/modules/connection-test/connection-test.service.spec.ts

@@ -1,25 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { ConnectionTestController } from './connection-test.controller';
-import { ConnectionTestService } from './connection-test.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { ConnectionTestClientConfiguration } from './connection-test-client.config';
-import { ClientsModule } from '@nestjs/microservices';
-
-describe('ConnectionTestService', () => {
-  let service: ConnectionTestService;
-
-  beforeEach(async () => {
-    const client = new ConnectionTestClientConfiguration();
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [ConnectionTestController],
-      providers: [ConnectionTestService, DadosferaLogger],
-      imports: [ClientsModule.register([client.providerOptions])],
-    }).compile();
-
-    service = module.get<ConnectionTestService>(ConnectionTestService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

src/modules/inputs/dtos/input.model.ts

@@ -1,6 +1,5 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
-import { Info } from 'src/clients/inputs/interfaces';
-import { Method } from 'axios';
+import { Info } from 'protospack/dist/lib/interfaces';
 
 export class InputModel {
   category: string;
@@ -13,15 +12,6 @@ export class InputModel {
     oauth_code: string;
     start_date: string;
   };
-  options?: {
-    oauth?: {
-      get_tokens_url: string;
-      get_tokens_method?: Method;
-      get_tokens_url_params: string;
-      get_tokens_set_response: Record<string, string>;
-      content_type: string;
-    };
-  };
 }
 
 class GoogleAnalyticsClientSecrets {

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 interface Values {
   jdbc_user: string;
   jdbc_password: string;
@@ -28,13 +30,6 @@ interface Cron {
   week_days_resourse_value: number;
 }
 
-export interface Info {
-  customer_id: string;
-  user_id: string;
-  customer: string;
-  customer_tier: string;
-}
-
 export interface ICreateInputRequest {
   cron: string;
   name: string;

src/modules/inputs/inputs-client.config.ts

@@ -0,0 +1,28 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { InputPackages, InputProtoFilePath } from 'protospack';
+
+export class InputsGrpcClient {
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: InputPackages,
+      credentials: process.env.LOCAL_ENV ? undefined : credentials.createSsl(),
+      protoPath: InputProtoFilePath,
+      loader: {
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: 'InputsGrpcClient',
+    ...this.config,
+  };
+}

src/modules/inputs/inputs.controller.ts

@@ -10,8 +10,6 @@ import {
 } from '@nestjs/common';
 import { InputsService } from './inputs.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { InputsClientService } from 'src/clients/inputs/client.service';
-import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
 import { PERMISSIONS } from '../../authentication/permissions.enum';
 import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
 import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
@@ -25,7 +23,7 @@ import {
   TestConnectionReq,
   TestConnectionRes,
 } from './dtos/input.model';
-import { User } from 'src/authentication/user.decorator';
+import { UpdateInputRequest } from './dtos/old_interfaces';
 
 @ApiTags('inputs')
 @Controller('inputs')
@@ -59,15 +57,13 @@ import { User } from 'src/authentication/user.decorator';
   return user.permissions.includes(PERMISSIONS.INPUT.permissions[action].seqid);
 })
 export class InputsController {
-  inputService: InputsService;
   logger: DadosferaLogger;
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
-    private inputsClientService: InputsClientService,
+    private inputService: InputsService,
   ) {
     this.logger = dadosferaLogger.logger;
-    this.inputService = new InputsService(this.inputsClientService);
   }
 
   @Get('available-entities/:plugin')

src/modules/inputs/inputs.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { InputsGrpcClient } from './inputs-client.config';
+import { InputsController } from './inputs.controller';
+import { InputsService } from './inputs.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+const client = new InputsGrpcClient();
+
+@Module({
+  controllers: [InputsController],
+  providers: [InputsService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions])],
+  exports: [InputsService],
+})
+export class InputsModule {}

src/modules/inputs/inputs.service.ts

@@ -1,13 +1,247 @@
-import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import {
+  Body,
+  ForbiddenException,
+  HttpException,
+  HttpStatus,
+  Inject,
+  Injectable,
+  InternalServerErrorException,
+  NotFoundException,
+} from '@nestjs/common';
 import { Timeout } from '@nestjs/schedule';
-import { DecodeGrpcStruct } from 'protospack';
+import {
+  DecodeGrpcStruct,
+  EncodeJsonToGrpcStruct,
+  InputCreateS3Request,
+  InputNewCreateRequest,
+  InputService,
+  InputServicesNames,
+  TestConnectionGetColumnsRequest,
+  TestConnectionRequest,
+} from 'protospack';
 import CronParser, { CronExpression } from 'cron-parser';
-import { InputsClientService } from 'src/clients/inputs/client.service';
-import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
+import { Info } from 'protospack/dist/lib/interfaces';
+import { ClientGrpc } from '@nestjs/microservices';
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+import { lastValueFrom } from 'rxjs';
+import {
+  objectSnakeToCamel,
+  objectCamelToSnake,
+} from 'src/utils/CaseConverter';
+import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
 
 @Injectable()
+
+//TODO refactor this class to remove OLD_inputClient
 export class InputsService {
-  constructor(private inputClient: InputsClientService) {}
+  logger: DadosferaLogger;
+  inputService: InputService;
+  OLD_inputClient = {
+    newCreate: async (createInputDto: InputNewCreateRequest) => {
+      this.logger.info('InputClientService - Create');
+
+      const input = await lastValueFrom(
+        this.inputService.NewCreate(objectSnakeToCamel(createInputDto)),
+      );
+      objectCamelToSnake(input);
+      return input;
+    },
+    createGeneric: async (createInputGeneric) => {
+      this.logger.info('InputClientService - Create');
+      const { info, ...data } = createInputGeneric;
+      if (data.options) delete data.options;
+      if (data.credentials.oauth_code) delete data.credentials.oauth_code;
+      const grpcPayload = {
+        input: EncodeJsonToGrpcStruct(data),
+        info,
+      };
+      objectSnakeToCamel(grpcPayload);
+
+      const structReturn = await lastValueFrom(
+        this.inputService.Create(grpcPayload),
+      ).catch((err: { details: string }) => {
+        if (err.details === 'Item Not found!') {
+          throw new NotFoundException('Input not found');
+        }
+        throw new InternalServerErrorException(err.details);
+      });
+      objectCamelToSnake(structReturn);
+      const inputCreated = DecodeGrpcStruct(structReturn.input);
+      return { input: inputCreated };
+    },
+    createS3Inputs: async (createInputDto: InputCreateS3Request) => {
+      this.logger.info('InputClientService - Create');
+      objectSnakeToCamel(createInputDto);
+      const createInputResponse = await lastValueFrom(
+        this.inputService.CreateS3(createInputDto),
+      ).catch((e) => {
+        throw new HttpException(e.details, 500);
+      });
+      this.logger.info('Done');
+      objectCamelToSnake(createInputResponse);
+      return createInputResponse;
+    },
+    findOne: async (data: IIdRequest) => {
+      this.logger.info('InputClientService - FindOne');
+      const findOneInputResponse = await lastValueFrom(
+        this.inputService.FindOne(objectSnakeToCamel(data)),
+      ).catch((e) => {
+        this.logger.error(e.details);
+        throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
+      });
+      objectCamelToSnake(findOneInputResponse);
+      return findOneInputResponse;
+    },
+    findAll: async (data) => {
+      this.logger.info('InputClientService - FindAll');
+      const findAllInputResponse = await lastValueFrom(
+        this.inputService.FindAll(objectSnakeToCamel(data)),
+      ).catch((e) => {
+        this.logger.error(e.details);
+        throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
+      });
+      findAllInputResponse.inputs.forEach((input) => objectCamelToSnake(input));
+
+      return findAllInputResponse;
+    },
+    update: async (updateInputDTO: UpdateInputRequest) => {
+      this.logger.info('InputClientService - Update');
+      const updateInputResponse = await new Promise((resolve, reject) => {
+        this.inputService.Update(objectSnakeToCamel(updateInputDTO)).subscribe({
+          next(x) {
+            resolve(objectCamelToSnake(x));
+          },
+          error(err) {
+            reject(err);
+          },
+          complete() {
+            // console.log('done');
+          },
+        });
+      })
+        .then((res) => {
+          this.logger.info('Done');
+          return res;
+        })
+        .catch((err) => {
+          this.logger.error(err.message);
+          throw new Error(err);
+        });
+
+      return updateInputResponse;
+    },
+    remove: async (idRequest: IIdRequest) => {
+      this.logger.info('InputClientService - Remove');
+
+      const removeInputResponse = await new Promise((resolve, reject) => {
+        this.inputService.Remove(objectSnakeToCamel(idRequest)).subscribe({
+          next(x) {
+            resolve(objectCamelToSnake(x));
+          },
+          error(err) {
+            reject(err);
+          },
+          complete() {
+            // console.log('done');
+          },
+        });
+      })
+        .then((res) => {
+          this.logger.info('Done');
+          return res;
+        })
+        .catch((err) => {
+          this.logger.error(err.message);
+          throw new Error(err);
+        });
+
+      return removeInputResponse;
+    },
+    testConnection: async (data: TestConnectionRequest) => {
+      this.logger.info('InputClientService - TestConnection');
+      const testConnectionResponse = await new Promise((resolve, reject) => {
+        this.inputService
+          .NewTestConnection(objectSnakeToCamel(data))
+          .subscribe({
+            next(x) {
+              resolve(objectCamelToSnake(x));
+            },
+            error(err) {
+              reject(err);
+            },
+            complete() {
+              // console.log('done');
+            },
+          });
+      })
+        .then((res) => {
+          this.logger.info('Done');
+          return res;
+        })
+        .catch((err) => {
+          this.logger.error(err.message);
+          throw new Error(err);
+        });
+
+      return testConnectionResponse;
+    },
+    getColumns: async (data: TestConnectionGetColumnsRequest) => {
+      this.logger.info('InputClientService - TestConnection/Get-Columns');
+      const getColumnsResponse = await new Promise((resolve, reject) => {
+        this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
+          next(x) {
+            resolve(objectCamelToSnake(x));
+          },
+          error(err) {
+            reject(err);
+          },
+          complete() {
+            // console.log('done');
+          },
+        });
+      })
+        .then((res) => {
+          this.logger.info('Done');
+          return res;
+        })
+        .catch((err) => {
+          this.logger.error(err.message);
+          throw new Error(err);
+        });
+
+      return getColumnsResponse;
+    },
+    getAvailableEntities: async (data) => {
+      this.logger.info('InputClientService - GetAvailableEntities');
+      objectSnakeToCamel(data);
+      const response = await lastValueFrom(
+        this.inputService.GetAvailableEntities(data),
+      ).catch((err) => {
+        this.logger.info(err.details);
+        if (err.details && err.details.includes('400'))
+          throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
+        throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
+      });
+      objectCamelToSnake(response);
+
+      return response;
+    },
+  };
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject('InputsGrpcClient') private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  async onModuleInit() {
+    this.inputService = this.grpcClient.getService<InputService>(
+      InputServicesNames.InputService,
+    );
+  }
+
   secondsInADay = 60 * 60 * 24;
   secondsInAnHour = 60 * 60;
 
@@ -37,14 +271,12 @@ export class InputsService {
     const afterNextDate = interval.next().toDate();
     const secondsApart = this.getDifferenceInSeconds(nextDate, afterNextDate);
     if (customer_tier === 'BASIC' && secondsApart < this.secondsInADay) {
-      throw new HttpException(
+      throw new ForbiddenException(
         'Intervalo de tempo não pode ser inferior a um dia.',
-        HttpStatus.FORBIDDEN,
       );
     } else if (secondsApart < this.secondsInAnHour) {
-      throw new HttpException(
+      throw new ForbiddenException(
         'Intervalo de tempo não pode ser inferior a uma hora.',
-        HttpStatus.FORBIDDEN,
       );
     }
   }
@@ -57,7 +289,7 @@ export class InputsService {
       case 'parquet': {
         const { info, ...input } = data;
         const inputPayload = this.generateInputS3Payload(input);
-        response = await this.inputClient.createS3Inputs({
+        response = await this.OLD_inputClient.createS3Inputs({
           input: inputPayload,
           info,
         });
@@ -67,10 +299,10 @@ export class InputsService {
       case 'mysql':
       case 'postgresql':
       case 'sqlserver':
-        response = await this.inputClient.newCreate(data);
+        response = await this.OLD_inputClient.newCreate(data);
         break;
       default:
-        response = await this.inputClient.createGeneric(data);
+        response = await this.OLD_inputClient.createGeneric(data);
         break;
     }
     const adjustedInput = this.adjustInputPayload(response.input);
@@ -79,16 +311,18 @@ export class InputsService {
   async reCreate(data) {
     const { info, cron } = data;
     if (cron) this.validateCron({ info, cron });
-    const response = await this.inputClient.createGeneric(data);
+    const response = await this.OLD_inputClient.createGeneric(data);
     const adjustedInput = this.adjustInputPayload(response.input);
     return { ...response, input: adjustedInput };
   }
   async getAvailableEntities(data): Promise<{ entities: string[] }> {
-    return await this.inputClient.getAvailableEntities(data);
+    return await this.OLD_inputClient.getAvailableEntities(data);
   }
   async findAll(body) {
     try {
-      const findAllInputResponse: any = await this.inputClient.findAll(body);
+      const findAllInputResponse: any = await this.OLD_inputClient.findAll(
+        body,
+      );
       if (findAllInputResponse?.inputs?.length) {
         findAllInputResponse.inputs = findAllInputResponse.inputs.map((input) =>
           this.adjustInputPayload(input),
@@ -102,7 +336,7 @@ export class InputsService {
 
   async findOne(idRequest: IIdRequest) {
     try {
-      const findOneInputResponse: any = await this.inputClient.findOne(
+      const findOneInputResponse: any = await this.OLD_inputClient.findOne(
         idRequest,
       );
       findOneInputResponse.input = this.adjustInputPayload(
@@ -117,7 +351,7 @@ export class InputsService {
   async update(id: string, data, info: Info) {
     this.validateCron({ ...data, info });
     try {
-      const updateInputResponse: any = await this.inputClient.update({
+      const updateInputResponse: any = await this.OLD_inputClient.update({
         id,
         info,
         ...data,
@@ -134,7 +368,7 @@ export class InputsService {
 
   async remove(idRequest: IIdRequest) {
     try {
-      const removeInputResponse = await this.inputClient.remove(idRequest);
+      const removeInputResponse = await this.OLD_inputClient.remove(idRequest);
 
       return removeInputResponse;
     } catch (err) {
@@ -145,9 +379,8 @@ export class InputsService {
   @Timeout(60000 * 10) // Timeout set for 10 minutes
   async testConnection(data) {
     try {
-      const testConnectionInputResponse = await this.inputClient.testConnection(
-        data,
-      );
+      const testConnectionInputResponse =
+        await this.OLD_inputClient.testConnection(data);
 
       return testConnectionInputResponse;
     } catch (err) {
@@ -158,7 +391,7 @@ export class InputsService {
   async getColumns(data) {
     try {
       const testConnectionGetColumnsResponse =
-        await this.inputClient.getColumns(data);
+        await this.OLD_inputClient.getColumns(data);
 
       return testConnectionGetColumnsResponse;
     } catch (err) {

src/modules/network-config/network-config-client.config.ts

@@ -6,7 +6,7 @@ import {
 } from '@nestjs/microservices';
 import { NetworkConfig } from '@dadosfera/protospack-v2';
 
-export class NetworkConfigClientConfiguration {
+export class NetworkConfigGrpcClient {
   private config: GrpcOptions = {
     transport: Transport.GRPC,
     options: {
@@ -15,9 +15,8 @@ export class NetworkConfigClientConfiguration {
         NetworkConfig.ProtoPackages.ReadPackage,
         NetworkConfig.ProtoPackages.WritePackage,
       ],
-      credentials: undefined,
-      // credentials:
-      //   process.env.ENV === 'local' ? undefined : credentials.createSsl(),
+      credentials:
+        process.env.ENV === 'local' ? undefined : credentials.createSsl(),
       protoPath: [
         NetworkConfig.ProtoPaths.ReadFilePath,
         NetworkConfig.ProtoPaths.WriteFilePath,

src/modules/network-config/network-config.controller.spec.ts

@@ -1,25 +0,0 @@
-import { ClientsModule } from '@nestjs/microservices';
-import { Test, TestingModule } from '@nestjs/testing';
-import { NetworkConfigClientConfiguration } from './network-config-client.config';
-import { NetworkConfigController } from './network-config.controller';
-import { NetworkConfigService } from './network-config.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-describe('NetworkConfigController', () => {
-  let controller: NetworkConfigController;
-
-  beforeEach(async () => {
-    const client = new NetworkConfigClientConfiguration();
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [NetworkConfigController],
-      providers: [NetworkConfigService, DadosferaLogger],
-      imports: [ClientsModule.register([client.providerOptions])],
-    }).compile();
-
-    controller = module.get<NetworkConfigController>(NetworkConfigController);
-  });
-
-  it('should be defined', () => {
-    expect(controller).toBeDefined();
-  });
-});

src/modules/network-config/network-config.module.ts

@@ -3,8 +3,8 @@ import { NetworkConfigController } from './network-config.controller';
 import { NetworkConfigService } from './network-config.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { ClientsModule } from '@nestjs/microservices';
-import { NetworkConfigClientConfiguration } from './network-config-client.config';
-const client = new NetworkConfigClientConfiguration();
+import { NetworkConfigGrpcClient } from './network-config-client.config';
+const client = new NetworkConfigGrpcClient();
 
 @Module({
   controllers: [NetworkConfigController],

src/modules/network-config/network-config.service.spec.ts

@@ -1,25 +0,0 @@
-import { ClientsModule } from '@nestjs/microservices';
-import { Test, TestingModule } from '@nestjs/testing';
-import { NetworkConfigClientConfiguration } from './network-config-client.config';
-import { NetworkConfigController } from './network-config.controller';
-import { NetworkConfigService } from './network-config.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-describe('NetworkConfigService', () => {
-  let service: NetworkConfigService;
-
-  beforeEach(async () => {
-    const client = new NetworkConfigClientConfiguration();
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [NetworkConfigController],
-      providers: [NetworkConfigService, DadosferaLogger],
-      imports: [ClientsModule.register([client.providerOptions])],
-    }).compile();
-
-    service = module.get<NetworkConfigService>(NetworkConfigService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

src/modules/network-config/network-config.service.ts

@@ -91,7 +91,11 @@ export class NetworkConfigService {
     file: Express.Multer.File,
     { customer_name, customer_id },
   ): Promise<CreateNetworkConfigRes> {
-    const { description, name, type, network_type_id, plugin } = body;
+    const { description, name, network_type_id } = body;
+    let { type, plugin } = body;
+
+    type = type || 'vpn';
+    plugin = plugin || 'openvpn';
 
     const OVPN_FILE_STRING = Buffer.from(file.buffer).toString();
 

src/modules/oauth/oauth.controller.ts

@@ -1,17 +1,14 @@
 import { Controller, Get, Redirect, Req, UseGuards } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import { AuthGuard } from '@nestjs/passport';
-import { InputsClientService } from 'src/clients/inputs/client.service';
 import { InputsService } from '../inputs/inputs.service';
 import { Request } from 'express';
 
 @ApiTags('oauth')
 @Controller('oauth')
 export class OauthController {
-  inputService: InputsService;
   frontendRedirectUri = '';
-  constructor(private inputsClientService: InputsClientService) {
-    this.inputService = new InputsService(this.inputsClientService);
+  constructor(private inputService: InputsService) {
     switch (process.env.ENV) {
       case 'dev':
       case 'stg':

src/modules/oauth/oauth.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { InputsModule } from '../inputs/inputs.module';
+import { OauthController } from './oauth.controller';
+
+@Module({
+  controllers: [OauthController],
+  imports: [InputsModule],
+})
+export class OauthModule {}

src/modules/outputs/outputs.service.ts

@@ -1,6 +1,7 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
-import { IIdRequest, Info } from 'src/clients/outputs/interfaces';
+import { IIdRequest } from 'src/clients/outputs/interfaces';
 import { OutputsClientService } from 'src/clients/outputs/client.service';
+import { Info } from 'protospack/dist/lib/interfaces';
 
 @Injectable()
 export class OutputsService {

src/modules/pipelines/pipelines.service.ts

@@ -1,7 +1,8 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
 import { DecodeGrpcStruct } from 'protospack';
+import { Info } from 'protospack/dist/lib/interfaces';
 import { PipelinesClientService } from 'src/clients/pipelines/client.service';
-import { IIdRequest, Info } from 'src/clients/pipelines/interfaces';
+import { IIdRequest } from 'src/clients/pipelines/interfaces';
 import { objectCamelToSnake } from 'src/utils/CaseConverter';
 
 @Injectable()