REFACTOR: inputs inside its own module

package-lock.json

@@ -23,7 +23,7 @@
         "@nestjs/platform-express": "^8.4.7",
         "@nestjs/schedule": "^1.1.0",
         "@nestjs/swagger": "^5.2.1",
-        "axios": "^0.25.0",
+        "axios": "^0.27.2",
         "class-transformer": "^0.5.1",
         "class-validator": "^0.13.2",
         "cron-parser": "^4.4.0",
@@ -2423,26 +2423,6 @@
         }
       }
     },
-    "node_modules/@nestjs/common/node_modules/axios": {
-      "version": "0.27.2",
-      "license": "MIT",
-      "dependencies": {
-        "follow-redirects": "^1.14.9",
-        "form-data": "^4.0.0"
-      }
-    },
-    "node_modules/@nestjs/common/node_modules/form-data": {
-      "version": "4.0.0",
-      "license": "MIT",
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/@nestjs/config": {
       "version": "1.2.1",
       "license": "MIT",
@@ -3780,10 +3760,25 @@
       }
     },
     "node_modules/axios": {
-      "version": "0.25.0",
-      "license": "MIT",
+      "version": "0.27.2",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/axios/-/axios-0.27.2.tgz",
+      "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
       "dependencies": {
-        "follow-redirects": "^1.14.7"
+        "follow-redirects": "^1.14.9",
+        "form-data": "^4.0.0"
+      }
+    },
+    "node_modules/axios/node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
       }
     },
     "node_modules/babel-jest": {
@@ -12760,23 +12755,6 @@
         "iterare": "1.2.1",
         "tslib": "2.4.0",
         "uuid": "8.3.2"
-      },
-      "dependencies": {
-        "axios": {
-          "version": "0.27.2",
-          "requires": {
-            "follow-redirects": "^1.14.9",
-            "form-data": "^4.0.0"
-          }
-        },
-        "form-data": {
-          "version": "4.0.0",
-          "requires": {
-            "asynckit": "^0.4.0",
-            "combined-stream": "^1.0.8",
-            "mime-types": "^2.1.12"
-          }
-        }
       }
     },
     "@nestjs/config": {
@@ -13692,9 +13670,24 @@
       "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="
     },
     "axios": {
-      "version": "0.25.0",
+      "version": "0.27.2",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/axios/-/axios-0.27.2.tgz",
+      "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
       "requires": {
-        "follow-redirects": "^1.14.7"
+        "follow-redirects": "^1.14.9",
+        "form-data": "^4.0.0"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "4.0.0",
+          "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/form-data/-/form-data-4.0.0.tgz",
+          "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
       }
     },
     "babel-jest": {

package.json

@@ -25,6 +25,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
+    "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
+    "@dadosfera/protospack-v2": "3.10.5",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",
@@ -36,11 +38,10 @@
     "@nestjs/platform-express": "^8.4.7",
     "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
-    "axios": "^0.25.0",
+    "axios": "^0.27.2",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",
     "cron-parser": "^4.4.0",
-    "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.36.0",
     "helmet": "^5.1.0",
@@ -53,7 +54,6 @@
     "passport-hubspot-oauth2": "^1.0.3",
     "passport-mailchimp": "^1.1.0",
     "protospack": "2.5.2",
-    "@dadosfera/protospack-v2": "3.10.5",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",

src/app.module.ts

@@ -1,30 +1,25 @@
 import { Module } from '@nestjs/common';
 import { ClientsModule } from '@nestjs/microservices';
 import { ConfigModule } from '@nestjs/config';
-import { InputsController } from './modules/inputs/inputs.controller';
 import { TransformationsController } from './modules/transformations/transformations.controller';
 import { OutputsController } from './modules/outputs/outputs.controllers';
 import { PipelinesController } from './modules/pipelines/pipelines.controller';
 import { HealthController } from './modules/health/health.controller';
 
-import { InputsService } from './modules/inputs/inputs.service';
 import { TransformationsService } from './modules/transformations/transformations.service';
 import { OutputsService } from './modules/outputs/outputs.service';
 import { PipelinesService } from './modules/pipelines/pipelines.service';
 import { HealthService } from './modules/health/health.service';
 
-import { InputsClientService } from './clients/inputs/client.service';
 import { TransformationsClientService } from './clients/transformations/client.service';
 import { OutputsClientService } from './clients/outputs/client.service';
 import { PipelinesClientService } from './clients/pipelines/client.service';
 
 import { OutputsClientConfiguration } from './clients/outputs/client.config';
 import { TransformationsClientConfiguration } from './clients/transformations/client.config';
-import { InputsClientConfiguration } from './clients/inputs/client.config';
 import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
 import { CatalogController } from './modules/catalog/catalog.controller';
 import { CatalogService } from './modules/catalog/catalog.service';
-import { OauthController } from './modules/oauth/oauth.controller';
 import { getOauthSecrets } from './utils/OauthSecrets';
 import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot-strategy';
 import { FacebookStrategy } from './modules/oauth/passport-strategies/facebook-strategy';
@@ -45,31 +40,28 @@ import { TermsOfUseModule } from './modules/termsOfUse/termsOfUse.module';
 import { ConnectionModule } from './modules/connection/connection.module';
 import { ConnectionTestModule } from './modules/connection-test/connection-test.module';
 import { NetworkConfigModule } from './modules/network-config/network-config.module';
+import { InputsModule } from './modules/inputs/inputs.module';
+import { OauthModule } from './modules/oauth/oauth.module';
 
-const inputClient = new InputsClientConfiguration();
 const outputClient = new OutputsClientConfiguration();
 const pipelineClient = new PipelinesClientConfiguration();
 const transformationClient = new TransformationsClientConfiguration();
 
 @Module({
   controllers: [
-    InputsController,
     TransformationsController,
     OutputsController,
     PipelinesController,
     HealthController,
     CatalogController,
-    OauthController,
   ],
   providers: [
     DadosferaLogger,
     { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
-    InputsService,
     TransformationsService,
     OutputsService,
     PipelinesService,
     HealthService,
-    InputsClientService,
     TransformationsClientService,
     OutputsClientService,
     PipelinesClientService,
@@ -97,10 +89,6 @@ const transformationClient = new TransformationsClientConfiguration();
     ConnectionTestModule,
 
     ClientsModule.register([
-      {
-        name: 'INPUTS_PACKAGE',
-        ...inputClient.config(),
-      },
       {
         name: 'TRANSFORMATIONS_PACKAGE',
         ...transformationClient.config(),
@@ -118,6 +106,10 @@ const transformationClient = new TransformationsClientConfiguration();
     UsersModule,
 
     RolesModule,
+
+    InputsModule,
+
+    OauthModule,
   ],
 })
 export class AppModule {}

src/clients/inputs/client.service.ts

@@ -16,17 +16,12 @@ import {
   TestConnectionGetColumnsRequest,
   TestConnectionRequest,
 } from 'protospack';
-import axios, { AxiosRequestConfig } from 'axios';
 import { lastValueFrom } from 'rxjs';
-import { InputModel } from 'src/modules/inputs/dtos/input.model';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
-import { mustache } from 'src/utils/mustache';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { OauthSecrets } from 'src/utils/OauthSecrets';
-import { URLSearchParams } from 'url';
 import { IIdRequest, UpdateInputRequest } from './interfaces';
 
 export class InputsClientService implements OnModuleInit {
@@ -35,8 +30,7 @@ export class InputsClientService implements OnModuleInit {
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
-    @Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-    @Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
+    @Inject('InputsGrpcClient') private readonly grpcClient: ClientGrpc,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -59,7 +53,6 @@ export class InputsClientService implements OnModuleInit {
   async createGeneric(createInputGeneric) {
     this.logger.info('InputClientService - Create');
     const { info, ...data } = createInputGeneric;
-    // data.credentials = await this.getInputTokens(data);
     if (data.options) delete data.options;
     if (data.credentials.oauth_code) delete data.credentials.oauth_code;
     const grpcPayload = {
@@ -227,70 +220,6 @@ export class InputsClientService implements OnModuleInit {
 
     return getColumnsResponse;
   }
-  async getAuthSecrets(plugin) {
-    switch (plugin) {
-      case 'hubspot':
-        return {
-          client_id: this.oauthSecrets.hubspot.client_id || '',
-          client_secret: this.oauthSecrets.hubspot.client_secret || '',
-        };
-    }
-    return {};
-  }
-  async getInputTokens(input: InputModel) {
-    const { credentials } = input;
-    let credentialsTokens = {};
-    switch (credentials.connection_type) {
-      case 'oauth':
-        credentialsTokens = await this.getOauthTokens(input);
-        break;
-    }
-    return { ...credentials, ...credentialsTokens };
-  }
-  async getOauthTokens(input: InputModel) {
-    const tokens: Record<string, any> = {};
-    const { options, credentials } = input;
-    const { oauth } = options;
-    const { get_tokens_url_params, get_tokens_set_response } = oauth;
-    const secrets = await this.getAuthSecrets(input.plugin);
-    const get_tokens_url_params_string = mustache(get_tokens_url_params, {
-      ...secrets,
-      ...credentials,
-    });
-    let get_tokens_url_params_obj: Record<string, any> = {};
-    try {
-      get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
-    } catch (error) {
-      throw new HttpException('Erro transformando get_tokens_url_params', 400);
-    }
-    let params = {};
-    switch (oauth.content_type) {
-      case 'application/x-www-form-urlencoded':
-        params = new URLSearchParams(get_tokens_url_params_obj);
-        params = params.toString();
-        break;
-      default:
-        break;
-    }
-    const axiosRequestConfig: AxiosRequestConfig = {
-      url: oauth.get_tokens_url,
-      method: oauth.get_tokens_method || 'POST',
-      data: params,
-      headers: { 'content-type': oauth.content_type || 'application/json' },
-    };
-
-    const { data } = await axios(axiosRequestConfig).catch((err) => {
-      this.logger.error(err.response.data);
-      throw new HttpException(err.response.data.message, err.response.status);
-    });
-    for (const key in get_tokens_set_response) {
-      const responseKey = get_tokens_set_response[key];
-      tokens[key] = data[responseKey];
-    }
-    const { redirect_uri } = get_tokens_url_params_obj;
-    tokens.redirect_uri = redirect_uri;
-    return tokens;
-  }
 
   async getAvailableEntities(data) {
     this.logger.info('InputClientService - GetAvailableEntities');

src/clients/inputs/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 interface Values {
   jdbc_user: string;
   jdbc_password: string;
@@ -28,13 +30,6 @@ interface Cron {
   week_days_resourse_value: number;
 }
 
-export interface Info {
-  customer_id: string;
-  user_id: string;
-  customer: string;
-  customer_tier: string;
-}
-
 export interface ICreateInputRequest {
   cron: string;
   name: string;

src/clients/outputs/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 export interface ICreateOutputRequest {
   operation: string;
   name: string;
@@ -9,11 +11,6 @@ export interface Values {
   prefix: string;
 }
 
-interface Info {
-  user_id: string;
-  customer_id: string;
-}
-
 export interface IIdRequest {
   id: string;
   info: Info;

src/clients/pipelines/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 export interface ICreatePipelineDto {
   input: IdRequest;
   transformations: IdRequest[];
@@ -8,11 +10,6 @@ export interface ICreatePipelineDto {
   info: Info;
 }
 
-export interface Info {
-  customer_id: sting;
-  user_id: string;
-}
-
 export interface IdRequest {
   id: string;
 }

src/clients/transformations/interfaces.d.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 export interface ICreateTransformationsRequest {
   transformations: Transformation[];
   info: Info;
@@ -11,11 +13,6 @@ interface Param {
   const: string;
 }
 
-export interface Info {
-  customer_id: string;
-  user_id: string;
-}
-
 interface Transformation {
   created_at: string;
   id: string;

src/modules/inputs/dtos/input.model.ts

@@ -1,6 +1,5 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
-import { Info } from 'src/clients/inputs/interfaces';
-import { Method } from 'axios';
+import { Info } from 'protospack/dist/lib/interfaces';
 
 export class InputModel {
   category: string;
@@ -13,15 +12,6 @@ export class InputModel {
     oauth_code: string;
     start_date: string;
   };
-  options?: {
-    oauth?: {
-      get_tokens_url: string;
-      get_tokens_method?: Method;
-      get_tokens_url_params: string;
-      get_tokens_set_response: Record<string, string>;
-      content_type: string;
-    };
-  };
 }
 
 class GoogleAnalyticsClientSecrets {

src/modules/inputs/inputs-client.config.ts

@@ -0,0 +1,28 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { InputPackages, InputProtoFilePath } from 'protospack';
+
+export class InputsGrpcClient {
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: InputPackages,
+      credentials: process.env.LOCAL_ENV ? undefined : credentials.createSsl(),
+      protoPath: InputProtoFilePath,
+      loader: {
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: 'InputsGrpcClient',
+    ...this.config,
+  };
+}

src/modules/inputs/inputs.module.ts

@@ -0,0 +1,17 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { InputsGrpcClient } from './inputs-client.config';
+import { InputsController } from './inputs.controller';
+import { InputsService } from './inputs.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { InputsClientService } from 'src/clients/inputs/client.service';
+
+const client = new InputsGrpcClient();
+
+@Module({
+  controllers: [InputsController],
+  providers: [InputsService, InputsClientService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions])],
+  exports: [InputsService],
+})
+export class InputsModule {}

src/modules/inputs/inputs.service.ts

@@ -3,7 +3,8 @@ import { Timeout } from '@nestjs/schedule';
 import { DecodeGrpcStruct } from 'protospack';
 import CronParser, { CronExpression } from 'cron-parser';
 import { InputsClientService } from 'src/clients/inputs/client.service';
-import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
+import { IIdRequest } from 'src/clients/inputs/interfaces';
+import { Info } from 'protospack/dist/lib/interfaces';
 
 @Injectable()
 export class InputsService {

src/modules/network-config/network-config-client.config.ts

@@ -6,7 +6,7 @@ import {
 } from '@nestjs/microservices';
 import { NetworkConfig } from '@dadosfera/protospack-v2';
 
-export class NetworkConfigClientConfiguration {
+export class NetworkConfigGrpcClient {
   private config: GrpcOptions = {
     transport: Transport.GRPC,
     options: {
@@ -15,7 +15,6 @@ export class NetworkConfigClientConfiguration {
         NetworkConfig.ProtoPackages.ReadPackage,
         NetworkConfig.ProtoPackages.WritePackage,
       ],
-      // credentials: undefined,
       credentials:
         process.env.ENV === 'local' ? undefined : credentials.createSsl(),
       protoPath: [

src/modules/network-config/network-config.controller.spec.ts

@@ -1,25 +0,0 @@
-import { ClientsModule } from '@nestjs/microservices';
-import { Test, TestingModule } from '@nestjs/testing';
-import { NetworkConfigClientConfiguration } from './network-config-client.config';
-import { NetworkConfigController } from './network-config.controller';
-import { NetworkConfigService } from './network-config.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-describe('NetworkConfigController', () => {
-  let controller: NetworkConfigController;
-
-  beforeEach(async () => {
-    const client = new NetworkConfigClientConfiguration();
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [NetworkConfigController],
-      providers: [NetworkConfigService, DadosferaLogger],
-      imports: [ClientsModule.register([client.providerOptions])],
-    }).compile();
-
-    controller = module.get<NetworkConfigController>(NetworkConfigController);
-  });
-
-  it('should be defined', () => {
-    expect(controller).toBeDefined();
-  });
-});

src/modules/network-config/network-config.module.ts

@@ -3,8 +3,8 @@ import { NetworkConfigController } from './network-config.controller';
 import { NetworkConfigService } from './network-config.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { ClientsModule } from '@nestjs/microservices';
-import { NetworkConfigClientConfiguration } from './network-config-client.config';
-const client = new NetworkConfigClientConfiguration();
+import { NetworkConfigGrpcClient } from './network-config-client.config';
+const client = new NetworkConfigGrpcClient();
 
 @Module({
   controllers: [NetworkConfigController],

src/modules/network-config/network-config.service.spec.ts

@@ -1,25 +0,0 @@
-import { ClientsModule } from '@nestjs/microservices';
-import { Test, TestingModule } from '@nestjs/testing';
-import { NetworkConfigClientConfiguration } from './network-config-client.config';
-import { NetworkConfigController } from './network-config.controller';
-import { NetworkConfigService } from './network-config.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-describe('NetworkConfigService', () => {
-  let service: NetworkConfigService;
-
-  beforeEach(async () => {
-    const client = new NetworkConfigClientConfiguration();
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [NetworkConfigController],
-      providers: [NetworkConfigService, DadosferaLogger],
-      imports: [ClientsModule.register([client.providerOptions])],
-    }).compile();
-
-    service = module.get<NetworkConfigService>(NetworkConfigService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

src/modules/oauth/oauth.controller.ts

@@ -8,10 +8,8 @@ import { Request } from 'express';
 @ApiTags('oauth')
 @Controller('oauth')
 export class OauthController {
-  inputService: InputsService;
   frontendRedirectUri = '';
-  constructor(private inputsClientService: InputsClientService) {
-    this.inputService = new InputsService(this.inputsClientService);
+  constructor(private inputService: InputsService) {
     switch (process.env.ENV) {
       case 'dev':
       case 'stg':

src/modules/oauth/oauth.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { InputsModule } from '../inputs/inputs.module';
+import { OauthController } from './oauth.controller';
+
+@Module({
+  controllers: [OauthController],
+  imports: [InputsModule],
+})
+export class OauthModule {}

src/modules/outputs/outputs.service.ts

@@ -1,6 +1,7 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
-import { IIdRequest, Info } from 'src/clients/outputs/interfaces';
+import { IIdRequest } from 'src/clients/outputs/interfaces';
 import { OutputsClientService } from 'src/clients/outputs/client.service';
+import { Info } from 'protospack/dist/lib/interfaces';
 
 @Injectable()
 export class OutputsService {

src/modules/pipelines/pipelines.service.ts

@@ -1,7 +1,8 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
 import { DecodeGrpcStruct } from 'protospack';
+import { Info } from 'protospack/dist/lib/interfaces';
 import { PipelinesClientService } from 'src/clients/pipelines/client.service';
-import { IIdRequest, Info } from 'src/clients/pipelines/interfaces';
+import { IIdRequest } from 'src/clients/pipelines/interfaces';
 import { objectCamelToSnake } from 'src/utils/CaseConverter';
 
 @Injectable()