FIX: Use passport for oauth

Merge pull request #105 from dadosfera/feature/passport-oauth

.github/workflows/deploy-manually.yml

@@ -134,7 +134,7 @@ jobs:
           eb deploy
 
       - name: Remove Docker's Trash
-        continue-on-error: true
+        if: always()
         run: |
-          docker system prune
-          docker rmi -f $(docker images -aq)
+          docker system prune --volumes -a -f
+          docker system df

.github/workflows/test.yml

@@ -40,7 +40,7 @@ jobs:
           docker-compose -f build.docker-compose.yml run -e NPM_TOKEN=${NPM_TOKEN} --rm --entrypoint="npm run test" maestro
 
       - name: Remove Docker's Trash
-        continue-on-error: true
+        if: always()
         run: |
-          docker system prune
-          docker rmi -f $(docker images -aq)
+          docker system prune --volumes -a -f
+          docker system df

.nvmrc

@@ -1 +1 @@
-18
+18.3.0

Dockerfile

@@ -1,7 +1,8 @@
-FROM node:14.15.4-alpine3.12 as packages
+FROM node:18.3.0-alpine3.15 as packages
 WORKDIR /packages
 ARG NPM_TOKEN
 COPY package.json .
+COPY package-lock.json .
 COPY .npmrc .
 RUN npm install
 RUN rm -f ./.npmrc 

package.json

@@ -5,6 +5,9 @@
   "author": "",
   "private": true,
   "license": "UNLICENSED",
+  "engines": {
+    "node": "18.3.0"
+  },
   "scripts": {
     "prebuild": "rimraf dist",
     "build": "nest build",
@@ -21,6 +24,7 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@grpc/grpc-js": "^1.5.10",
     "@grpc/proto-loader": "^0.6.9",
     "@nestjs/common": "^8.4.3",
@@ -28,16 +32,20 @@
     "@nestjs/core": "^8.4.3",
     "@nestjs/mapped-types": "*",
     "@nestjs/microservices": "^8.4.3",
+    "@nestjs/passport": "^8.2.2",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "2.4.2-1",
+    "@victorradael/protospack": "2.5.0",
     "axios": "^0.25.0",
     "cron-parser": "^4.4.0",
     "dotenv": "^14.2.0",
     "helmet": "^5.0.2",
     "jsonwebtoken": "^8.5.1",
     "jwk-to-pem": "^2.0.5",
+    "passport": "^0.6.0",
+    "passport-hubspot": "^0.1.0",
+    "passport-hubspot-oauth2": "^1.0.3",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",

src/app.module.ts

@@ -31,6 +31,9 @@ import { InputsClientConfiguration } from './clients/inputs/client.config';
 import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
 import { CatalogController } from './modules/catalog/catalog.controller';
 import { CatalogService } from './modules/catalog/catalog.service';
+import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot';
+import { OauthController } from './modules/oauth/oauth.controller';
+import { getOauthSecrets } from './utils/OauthSecrets';
 
 const authClient = new AuthClient();
 const inputClient = new InputsClientConfiguration();
@@ -47,8 +50,10 @@ const transformationClient = new TransformationsClientConfiguration();
     AuthController,
     HealthController,
     CatalogController,
+    OauthController,
   ],
   providers: [
+    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
     InputsService,
     TransformationsService,
     OutputsService,
@@ -61,6 +66,7 @@ const transformationClient = new TransformationsClientConfiguration();
     PipelinesClientService,
     AuthClientService,
     CatalogService,
+    HubspotStrategy,
   ],
   imports: [
     ConfigModule.forRoot({

src/clients/inputs/client.service.ts

@@ -1,126 +1,112 @@
-import { OnModuleInit, Inject } from '@nestjs/common';
+import {
+  OnModuleInit,
+  Inject,
+  HttpException,
+  HttpStatus,
+  NotFoundException,
+  InternalServerErrorException,
+} from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
-  InputCreateResponse,
+  DecodeGrpcStruct,
+  EncodeJsonToGrpcStruct,
   InputCreateS3Request,
   InputNewCreateRequest,
   InputService,
   TestConnectionGetColumnsRequest,
   TestConnectionRequest,
 } from '@victorradael/protospack';
+import axios, { AxiosRequestConfig } from 'axios';
+import { lastValueFrom } from 'rxjs';
+import { InputModel } from 'src/modules/inputs/dtos/input.model';
 import {
   objectCamelToSnake,
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
+import { mustache } from 'src/utils/mustache';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+import { URLSearchParams } from 'url';
 import { IIdRequest, UpdateInputRequest } from './interfaces';
 
 export class InputsClientService implements OnModuleInit {
   private inputService: InputService;
   constructor(
     @Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
+    @Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
   ) {}
 
-  onModuleInit() {
+  async onModuleInit() {
     this.inputService =
       this.grpcClient.getService<InputService>('InputService');
   }
 
-  async create(createInputDto: InputNewCreateRequest) {
+  async newCreate(createInputDto: InputNewCreateRequest) {
     console.log('InputClientService', 'Create');
 
-    const createInputResponse = await new Promise((resolve, reject) => {
-      this.inputService
-        .NewCreate(objectSnakeToCamel(createInputDto))
-        .subscribe({
-          next(x) {
-            resolve(objectCamelToSnake(x));
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-    }).catch((err) => {
-      throw new Error(err);
-    });
+    const input = await lastValueFrom(
+      this.inputService.NewCreate(objectSnakeToCamel(createInputDto)),
+    );
+    objectCamelToSnake(input);
+    return input;
+  }
 
-    return createInputResponse;
+  async createGeneric(createInputGeneric) {
+    console.log('InputClientService', 'Create');
+    const { info, ...data } = createInputGeneric;
+    // data.credentials = await this.getInputTokens(data);
+    if (data.options) delete data.options;
+    if (data.credentials.oauth_code) delete data.credentials.oauth_code;
+    const grpcPayload = {
+      input: EncodeJsonToGrpcStruct(data),
+      info,
+    };
+    objectSnakeToCamel(grpcPayload);
+
+    const structReturn = await lastValueFrom(
+      this.inputService.Create(grpcPayload),
+    ).catch((err) => {
+      if (err.details === 'Item Not found!') {
+        throw new NotFoundException('Input not found');
+      }
+      throw new InternalServerErrorException(err.details);
+    });
+    objectCamelToSnake(structReturn);
+    const inputCreated = DecodeGrpcStruct(structReturn.input);
+    return { input: inputCreated };
   }
 
   async createS3Inputs(createInputDto: InputCreateS3Request) {
     console.log('InputClientService', 'Create');
-
-    const createInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.CreateS3(objectSnakeToCamel(createInputDto)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
+    objectSnakeToCamel(createInputDto);
+    const createInputResponse = await lastValueFrom(
+      this.inputService.CreateS3(createInputDto),
+    ).catch((e) => {
+      throw new HttpException(e.details, 500);
+    });
+    console.log('done');
+    objectCamelToSnake(createInputResponse);
     return createInputResponse;
   }
 
   async findOne(data: IIdRequest) {
     console.log('InputClientService', 'FindOne');
-
-    const findOneInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
+    const findOneInputResponse = await lastValueFrom(
+      this.inputService.FindOne(objectSnakeToCamel(data)),
+    ).catch((e) => {
+      throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
+    });
+    objectCamelToSnake(findOneInputResponse);
     return findOneInputResponse;
   }
 
   async findAll(data) {
     console.log('InputClientService', 'FindAll');
-
-    const findAllInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
+    const findAllInputResponse = await lastValueFrom(
+      this.inputService.FindAll(objectSnakeToCamel(data)),
+    ).catch((e) => {
+      throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
+    });
+    findAllInputResponse.inputs.forEach((input) => objectCamelToSnake(input));
 
     return findAllInputResponse;
   }
@@ -221,4 +207,84 @@ export class InputsClientService implements OnModuleInit {
 
     return getColumnsResponse;
   }
+  async getAuthSecrets(plugin) {
+    switch (plugin) {
+      case 'hubspot':
+        return {
+          client_id: this.oauthSecrets.hubspot.client_id || '',
+          client_secret: this.oauthSecrets.hubspot.client_secret || '',
+        };
+    }
+    return {};
+  }
+  async getInputTokens(input: InputModel) {
+    const { credentials } = input;
+    let credentialsTokens = {};
+    switch (credentials.connection_type) {
+      case 'oauth':
+        credentialsTokens = await this.getOauthTokens(input);
+        break;
+    }
+    return { ...credentials, ...credentialsTokens };
+  }
+  async getOauthTokens(input: InputModel) {
+    const tokens: Record<string, any> = {};
+    const { options, credentials } = input;
+    const { oauth } = options;
+    const { get_tokens_url_params, get_tokens_set_response } = oauth;
+    const secrets = await this.getAuthSecrets(input.plugin);
+    const get_tokens_url_params_string = mustache(get_tokens_url_params, {
+      ...secrets,
+      ...credentials,
+    });
+    let get_tokens_url_params_obj: Record<string, any> = {};
+    try {
+      get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
+    } catch (error) {
+      throw new HttpException('Erro transformando get_tokens_url_params', 400);
+    }
+    let params = {};
+    switch (oauth.content_type) {
+      case 'application/x-www-form-urlencoded':
+        params = new URLSearchParams(get_tokens_url_params_obj);
+        params = params.toString();
+        break;
+      default:
+        break;
+    }
+    const axiosRequestConfig: AxiosRequestConfig = {
+      url: oauth.get_tokens_url,
+      method: oauth.get_tokens_method || 'POST',
+      data: params,
+      headers: { 'content-type': oauth.content_type || 'application/json' },
+    };
+
+    const { data } = await axios(axiosRequestConfig).catch((err) => {
+      console.log(err.response.data);
+      throw new HttpException(err.response.data.message, err.response.status);
+    });
+    for (const key in get_tokens_set_response) {
+      const responseKey = get_tokens_set_response[key];
+      tokens[key] = data[responseKey];
+    }
+    const { redirect_uri } = get_tokens_url_params_obj;
+    tokens.redirect_uri = redirect_uri;
+    return tokens;
+  }
+
+  async getAvailableEntities(data) {
+    console.log('InputClientService', 'GetAvailableEntities');
+    objectSnakeToCamel(data);
+    const response = await lastValueFrom(
+      this.inputService.GetAvailableEntities(data),
+    ).catch((err) => {
+      console.log(err);
+      if (err.details && err.details.includes('400'))
+        throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
+      throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
+    });
+    objectCamelToSnake(response);
+
+    return response;
+  }
 }

src/main.ts

@@ -26,7 +26,8 @@ async function bootstrap() {
 
   const document = SwaggerModule.createDocument(app, config);
   writeFileSync('./swagger.json', JSON.stringify(document));
-  SwaggerModule.setup('api', app, document);
+  if (process.env.ENV != 'stg' && process.env.ENV != 'prd')
+    SwaggerModule.setup('api', app, document);
 
   await app.listen(3333);
 }

src/modules/inputs/dtos/input.model.ts

@@ -0,0 +1,206 @@
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { Method } from 'axios';
+
+export class InputModel {
+  category: string;
+  plugin: string;
+  name: string;
+  cron?: string;
+  credentials?: {
+    connection_type: string;
+    oauth: string;
+    oauth_code: string;
+    start_date: string;
+  };
+  options?: {
+    oauth?: {
+      get_tokens_url: string;
+      get_tokens_method?: Method;
+      get_tokens_url_params: string;
+      get_tokens_set_response: Record<string, string>;
+      content_type: string;
+    };
+  };
+}
+
+class GoogleAnalyticsClientSecrets {
+  @ApiProperty()
+  type: string;
+  @ApiProperty()
+  project_id: string;
+  @ApiProperty()
+  private_key_id: string;
+  @ApiProperty()
+  private_key: string;
+  @ApiProperty()
+  client_email: string;
+  @ApiProperty()
+  client_id: string;
+  @ApiProperty()
+  auth_uri: string;
+  @ApiProperty()
+  token_uri: string;
+  @ApiProperty()
+  auth_provider_x509_cert_url: string;
+  @ApiProperty()
+  client_x509_cert_url: string;
+}
+class OauthObject {
+  @ApiPropertyOptional()
+  get_tokens_url: string;
+  @ApiPropertyOptional()
+  get_tokens_url_params: string;
+  @ApiPropertyOptional()
+  get_tokens_set_response: Record<string, any>;
+  @ApiPropertyOptional()
+  content_type: string;
+}
+class InputOptions {
+  @ApiPropertyOptional()
+  oauth: OauthObject;
+  @ApiPropertyOptional()
+  skip_select_columns: true;
+  @ApiPropertyOptional()
+  skip_select_entities: false;
+  @ApiPropertyOptional()
+  skip_transformation: true;
+}
+
+class CredentialsJdbc {
+  @ApiPropertyOptional()
+  jdbc_user: string;
+  @ApiPropertyOptional()
+  jdbc_password: string;
+  @ApiPropertyOptional()
+  database: string;
+  @ApiPropertyOptional()
+  endpoint: string;
+  @ApiPropertyOptional()
+  port: string;
+  @ApiPropertyOptional()
+  engine: string;
+  @ApiPropertyOptional()
+  schema: string;
+}
+class FileFormatParams {
+  @ApiPropertyOptional()
+  file_format: string;
+  @ApiPropertyOptional()
+  encoding: string;
+}
+
+class Credentials extends CredentialsJdbc {
+  @ApiProperty()
+  connection_type: string;
+
+  @ApiPropertyOptional()
+  client_aws_access_key_id: string;
+  @ApiPropertyOptional()
+  client_aws_secret_access_key: string;
+  @ApiPropertyOptional()
+  client_bucket: string;
+  @ApiPropertyOptional()
+  file_to_extract: string;
+  @ApiPropertyOptional()
+  file_format_params: FileFormatParams;
+
+  @ApiPropertyOptional()
+  view_id: string;
+  @ApiPropertyOptional()
+  client_secrets: GoogleAnalyticsClientSecrets;
+  @ApiPropertyOptional()
+  start_date: string;
+  @ApiPropertyOptional()
+  end_date: string;
+
+  @ApiPropertyOptional()
+  oauth_code: string;
+}
+class Column {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  type: string;
+}
+class TableColumns {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  columns: Column[];
+  @ApiProperty()
+  references: Column[];
+}
+export class GetAvailableEntitiesReq {
+  @ApiProperty()
+  plugin: string;
+}
+
+export class GetAvailableEntitiesRes {
+  @ApiProperty()
+  entities: string[];
+}
+
+export class TestConnectionGetColumnsReq {
+  @ApiProperty()
+  plugin: string;
+  @ApiProperty()
+  tables: string[];
+  @ApiPropertyOptional()
+  credentials: CredentialsJdbc;
+  @ApiPropertyOptional()
+  id: string;
+}
+export class TestConnectionGetColumnsRes {
+  @ApiProperty()
+  tables: TableColumns[];
+}
+
+export class TestConnectionReq {
+  @ApiProperty()
+  plugin: string;
+  @ApiProperty()
+  credentials: CredentialsJdbc;
+}
+export class TestConnectionRes {
+  @ApiProperty()
+  connection_state: boolean;
+  @ApiProperty()
+  total_entities: number;
+  @ApiProperty()
+  database_tables: string[];
+}
+export class CreateInputReq {
+  @ApiPropertyOptional()
+  id: string;
+  @ApiProperty()
+  plugin: string;
+  @ApiProperty()
+  category: string;
+  @ApiProperty()
+  credentials: Credentials;
+  @ApiProperty()
+  name: string;
+  @ApiPropertyOptional()
+  options: InputOptions;
+}
+
+export class Input {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  category: string;
+  @ApiProperty()
+  plugin: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  cron: string;
+  @ApiProperty()
+  credentials: Credentials;
+  @ApiProperty()
+  client_id: string;
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  updated_at: string;
+}

src/modules/inputs/inputs.controller.ts

@@ -10,41 +10,77 @@ import {
 import { InputsService } from './inputs.service';
 import { InputsClientService } from 'src/clients/inputs/client.service';
 import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
-import { InputNewCreateRequest } from '@victorradael/protospack';
+import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import {
+  CreateInputReq,
+  GetAvailableEntitiesReq,
+  GetAvailableEntitiesRes,
+  Input,
+  TestConnectionGetColumnsReq,
+  TestConnectionGetColumnsRes,
+  TestConnectionReq,
+  TestConnectionRes,
+} from './dtos/input.model';
 
+@ApiTags('inputs')
 @Controller('inputs')
 export class InputsController {
-  constructor(private inputsClientService: InputsClientService) {}
+  inputService: InputsService;
+  constructor(private inputsClientService: InputsClientService) {
+    this.inputService = new InputsService(this.inputsClientService);
+  }
+
+  @Get('available-entities/:plugin')
+  @ApiOkResponse({ type: GetAvailableEntitiesRes })
+  async getAvailableEntities(
+    @Body() body,
+    @Param() params: GetAvailableEntitiesReq,
+  ) {
+    const { info } = body;
+    const { plugin } = params;
+    return await this.inputService.getAvailableEntities({ info, plugin });
+  }
 
   @Post('/test-connection')
-  async testConnection(@Body() data) {
+  @ApiOkResponse({ type: TestConnectionRes })
+  async testConnection(@Body() data: TestConnectionReq) {
     console.log(`/test-connection`, 'ON TEST CONNECTION ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.testConnection(data);
+    const response = await this.inputService.testConnection(data);
 
     return response;
   }
 
   @Post('/test-connection/get-columns')
-  async getColumns(@Body() data) {
+  @ApiOkResponse({ type: TestConnectionGetColumnsRes })
+  async getColumns(@Body() data: TestConnectionGetColumnsReq) {
     console.log(
       `/test-connection/get-columns`,
       'ON TEST CONNECTION GET COLUMNS ROUTE',
     );
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.getColumns(data);
+    const response = await this.inputService.getColumns(data);
 
     return response;
   }
 
   @Post()
-  async create(@Body() createInputDto: InputNewCreateRequest) {
+  @ApiOkResponse({ type: Input })
+  async create(@Body() createInputDto: CreateInputReq) {
     console.log(`/input`, 'ON CREATE ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.create(createInputDto);
+    const response = await this.inputService.create(createInputDto);
+
+    return response;
+  }
+
+  @Post(':id')
+  @ApiOkResponse({ type: Input })
+  async reCreate(@Param('id') id: string, @Body() input: CreateInputReq) {
+    if (!input.id) input.id = id;
+    console.log(`POST /${input.id}`, 'ON RECREATE ROUTE');
+
+    const response = await this.inputService.reCreate(input);
 
     return response;
   }
@@ -53,9 +89,7 @@ export class InputsController {
   async findAll(@Body() body) {
     console.log(`/input`, 'ON FIND ALL ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.findAll(body);
+    const response = await this.inputService.findAll(body);
 
     return response;
   }
@@ -64,9 +98,8 @@ export class InputsController {
   async findOne(@Body() body, @Param() params) {
     const { id } = params;
     console.log(`/input/${id}`, 'ON FIND ONE ROUTE');
-    const inputService = new InputsService(this.inputsClientService);
 
-    const response = await inputService.findOne({ id, ...body });
+    const response = await this.inputService.findOne({ id, ...body });
 
     return response;
   }
@@ -78,9 +111,8 @@ export class InputsController {
     delete updateInputDto.info;
 
     console.log(`/input/${id}`, 'ON UPDATE ROUTE');
-    const inputService = new InputsService(this.inputsClientService);
 
-    const response = await inputService.update(id, updateInputDto, info);
+    const response = await this.inputService.update(id, updateInputDto, info);
 
     return response;
   }
@@ -91,9 +123,7 @@ export class InputsController {
 
     console.log(`/input/${id}`, 'ON DELETE ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.remove({ id, ...data });
+    const response = await this.inputService.remove({ id, ...data });
 
     return response;
   }

src/modules/inputs/inputs.service.ts

@@ -1,5 +1,6 @@
 import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
 import { Timeout } from '@nestjs/schedule';
+import { DecodeGrpcStruct } from '@victorradael/protospack';
 import CronParser, { CronExpression } from 'cron-parser';
 import { InputsClientService } from 'src/clients/inputs/client.service';
 import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
@@ -11,6 +12,8 @@ export class InputsService {
   secondsInAnHour = 60 * 60;
 
   adjustInputPayload(payload) {
+    if (payload?.input_generic) return DecodeGrpcStruct(payload.input_generic);
+    if (!payload.input_s3 && !payload.input_jdbc) return payload;
     return payload?.input_s3 || payload?.input_jdbc;
   }
   getDifferenceInSeconds(date1: Date, date2: Date) {
@@ -47,31 +50,41 @@ export class InputsService {
   }
   async create(@Body() data) {
     this.validateCron(data);
-    try {
-      if (
-        data.plugin == 'csv' ||
-        data.plugin == 'json' ||
-        data.plugin == 'parquet'
-      ) {
+    let response;
+    switch (data.plugin.toLowerCase()) {
+      case 'csv':
+      case 'json':
+      case 'parquet':
         const { info, ...input } = data;
         const inputPayload = this.generateInputS3Payload(input);
-        const createInputResponse = await this.inputClient.createS3Inputs({
+        response = await this.inputClient.createS3Inputs({
           input: inputPayload,
           info,
         });
-        return createInputResponse;
-      } else {
-        const createInputResponse: any = await this.inputClient.create(data);
-        if (createInputResponse?.input?.input_jdbc) {
-          return { input: createInputResponse?.input?.input_jdbc };
-        }
-        return createInputResponse;
-      }
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+        break;
+      case 'oracle':
+      case 'mysql':
+      case 'postgresql':
+      case 'sqlserver':
+        response = await this.inputClient.newCreate(data);
+        break;
+      default:
+        response = await this.inputClient.createGeneric(data);
+        break;
     }
+    const adjustedInput = this.adjustInputPayload(response.input);
+    return { ...response, input: adjustedInput };
+  }
+  async reCreate(data) {
+    const { info, cron } = data;
+    if (cron) this.validateCron({ info, cron });
+    const response = await this.inputClient.createGeneric(data);
+    const adjustedInput = this.adjustInputPayload(response.input);
+    return { ...response, input: adjustedInput };
+  }
+  async getAvailableEntities(data): Promise<{ entities: string[] }> {
+    return await this.inputClient.getAvailableEntities(data);
   }
-
   async findAll(body) {
     try {
       const findAllInputResponse: any = await this.inputClient.findAll(body);
@@ -153,7 +166,7 @@ export class InputsService {
   }
 
   generateInputS3Payload(payload) {
-    const { credentials, plugin } = payload;
+    const { credentials, plugin, cron } = payload;
     if (!credentials) return payload;
 
     const {
@@ -164,8 +177,10 @@ export class InputsService {
       client_bucket,
       file_to_extract,
     } = credentials;
+
     const formatedPayload = {
       plugin,
+      cron,
       source_bucket: client_bucket,
       source_prefix: file_to_extract,
       auth_parameters: {

src/modules/oauth/oauth.controller.ts

@@ -0,0 +1,54 @@
+import {
+  BadRequestException,
+  Controller,
+  Get,
+  Query,
+  Redirect,
+  Req,
+  UseGuards,
+} from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { AuthGuard } from '@nestjs/passport';
+import { InputsClientService } from 'src/clients/inputs/client.service';
+import { InputsService } from '../inputs/inputs.service';
+
+@ApiTags('oauth')
+@Controller('oauth')
+export class OauthController {
+  inputService: InputsService;
+  frontendRedirectUri = '';
+  constructor(private inputsClientService: InputsClientService) {
+    this.inputService = new InputsService(this.inputsClientService);
+    switch (process.env.ENV) {
+      case 'dev':
+      case 'stg':
+        this.frontendRedirectUri = `https://app.${process.env.ENV}.dadosfera.ai/coletix/auth-callback`;
+        break;
+      case 'prd':
+        this.frontendRedirectUri = `https://app.dadosfera.ai/coletix/auth-callback`;
+        break;
+      default:
+        this.frontendRedirectUri = `http://localhost:4200/coletix/auth-callback`;
+    }
+  }
+  @Get('hubspot')
+  @UseGuards(AuthGuard('hubspot'))
+  async oauthHubspot() {
+    return true;
+  }
+
+  @Get('hubspot/callback')
+  @UseGuards(AuthGuard('hubspot'))
+  @Redirect()
+  async oauthHubspotCallback(@Req() req, @Query('state') customer_id) {
+    const { authInfo } = req;
+    if (!authInfo) throw new BadRequestException('Oauth tokens not found');
+    const { accessToken, refreshToken } = authInfo;
+    const response = await this.inputService.create({
+      info: { customer_id, customer: 'customer', user_id: 'empty' },
+      plugin: 'hubspot',
+      credentials: { access_token: accessToken, refresh_token: refreshToken },
+    });
+    return { url: `${this.frontendRedirectUri}?input_id=${response.input.id}` };
+  }
+}

src/modules/oauth/passport-strategies/hubspot.ts

@@ -0,0 +1,33 @@
+import { Strategy } from 'passport-hubspot-oauth2';
+import { PassportStrategy } from '@nestjs/passport';
+import { Inject, Injectable } from '@nestjs/common';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+
+@Injectable()
+export class HubspotStrategy extends PassportStrategy(Strategy) {
+  constructor(
+    @Inject('OAUTH_SECRETS')
+    private readonly oauthSecrets: OauthSecrets,
+  ) {
+    super(
+      {
+        clientID: oauthSecrets.hubspot.client_id,
+        clientSecret: oauthSecrets.hubspot.client_secret,
+        callbackURL: oauthSecrets.hubspot.redirect_uri,
+        redirectUri: oauthSecrets.hubspot.redirect_uri,
+        scope:
+          'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
+        passReqToCallback: true,
+      },
+      (accessToken, refreshToken, tokenInfo, profile, done) => {
+        return done(null, profile, { accessToken, refreshToken, tokenInfo });
+      },
+    );
+  }
+
+  authenticate(req, options) {
+    const { customer_id } = req.query;
+    options.state = customer_id;
+    super.authenticate(req, options);
+  }
+}

src/modules/pipelines/pipelines.service.ts

@@ -1,4 +1,5 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import { DecodeGrpcStruct } from '@victorradael/protospack';
 import { PipelinesClientService } from 'src/clients/pipelines/client.service';
 import { IIdRequest, Info } from 'src/clients/pipelines/interfaces';
 import { objectCamelToSnake } from 'src/utils/CaseConverter';
@@ -8,7 +9,11 @@ export class PipelinesService {
   constructor(private pipelineClient: PipelinesClientService) {}
 
   adjustPayload(payload) {
-    payload.input = payload.input?.input_s3 || payload.input?.input_jdbc;
+    if (payload.input?.input_generic) {
+      payload.input = DecodeGrpcStruct(payload.input?.input_generic);
+    } else {
+      payload.input = payload.input?.input_s3 || payload.input?.input_jdbc;
+    }
   }
 
   async create(createPipelineDto) {

src/utils/OauthSecrets.ts

@@ -0,0 +1,31 @@
+import {
+  SecretsManagerClient,
+  GetSecretValueCommand,
+} from '@aws-sdk/client-secrets-manager';
+
+class OauthSecretsObject {
+  client_id = '';
+  client_secret = '';
+  redirect_uri = '';
+}
+// @Injectable()
+export class OauthSecrets {
+  hubspot = new OauthSecretsObject();
+  google = new OauthSecretsObject();
+  mailchimp = new OauthSecretsObject();
+}
+export async function getOauthSecrets() {
+  const secrets = new OauthSecrets();
+  const path = process.env.SM_OAUTH_PATH;
+  const secretsManagerClient = new SecretsManagerClient({});
+  for (const key in secrets) {
+    const getSecretComand = new GetSecretValueCommand({
+      SecretId: `${path}/${key}`,
+    });
+    const res = await secretsManagerClient
+      .send(getSecretComand)
+      .catch(() => null);
+    if (res) secrets[key] = JSON.parse(res.SecretString);
+  }
+  return secrets;
+}

src/utils/mustache.ts

@@ -0,0 +1,7 @@
+export const mustache = (string: string, data: Record<string, any> = {}) => {
+  return Object.entries(data).reduce((res, [key, value]) => {
+    const mainRe = new RegExp(`(?<!\\\\){{\\s*${key}\\s*}}`, 'g');
+    const escapeRe = new RegExp(`\\\\({{\\s*${key}\\s*}})`, 'g');
+    return res.replace(mainRe, value.toString()).replace(escapeRe, '$1');
+  }, string);
+};