FIX: Use passport for oauth

Merge pull request #105 from dadosfera/feature/passport-oauth

.github/workflows/test.yml

@@ -40,7 +40,7 @@ jobs:
           docker-compose -f build.docker-compose.yml run -e NPM_TOKEN=${NPM_TOKEN} --rm --entrypoint="npm run test" maestro
 
       - name: Remove Docker's Trash
-        continue-on-error: true
+        if: always()
         run: |
-          docker system prune
-          docker rmi -f $(docker images -aq)
+          docker system prune --volumes -a -f
+          docker system df

package.json

@@ -24,6 +24,7 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@grpc/grpc-js": "^1.5.10",
     "@grpc/proto-loader": "^0.6.9",
     "@nestjs/common": "^8.4.3",
@@ -31,6 +32,7 @@
     "@nestjs/core": "^8.4.3",
     "@nestjs/mapped-types": "*",
     "@nestjs/microservices": "^8.4.3",
+    "@nestjs/passport": "^8.2.2",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/schedule": "^1.1.0",
     "@nestjs/swagger": "^5.2.1",
@@ -41,6 +43,9 @@
     "helmet": "^5.0.2",
     "jsonwebtoken": "^8.5.1",
     "jwk-to-pem": "^2.0.5",
+    "passport": "^0.6.0",
+    "passport-hubspot": "^0.1.0",
+    "passport-hubspot-oauth2": "^1.0.3",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",

src/app.module.ts

@@ -31,6 +31,9 @@ import { InputsClientConfiguration } from './clients/inputs/client.config';
 import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
 import { CatalogController } from './modules/catalog/catalog.controller';
 import { CatalogService } from './modules/catalog/catalog.service';
+import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot';
+import { OauthController } from './modules/oauth/oauth.controller';
+import { getOauthSecrets } from './utils/OauthSecrets';
 
 const authClient = new AuthClient();
 const inputClient = new InputsClientConfiguration();
@@ -47,8 +50,10 @@ const transformationClient = new TransformationsClientConfiguration();
     AuthController,
     HealthController,
     CatalogController,
+    OauthController,
   ],
   providers: [
+    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
     InputsService,
     TransformationsService,
     OutputsService,
@@ -61,6 +66,7 @@ const transformationClient = new TransformationsClientConfiguration();
     PipelinesClientService,
     AuthClientService,
     CatalogService,
+    HubspotStrategy,
   ],
   imports: [
     ConfigModule.forRoot({

src/clients/inputs/client.service.ts

@@ -3,6 +3,8 @@ import {
   Inject,
   HttpException,
   HttpStatus,
+  NotFoundException,
+  InternalServerErrorException,
 } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
@@ -14,7 +16,7 @@ import {
   TestConnectionGetColumnsRequest,
   TestConnectionRequest,
 } from '@victorradael/protospack';
-import axios from 'axios';
+import axios, { AxiosRequestConfig } from 'axios';
 import { lastValueFrom } from 'rxjs';
 import { InputModel } from 'src/modules/inputs/dtos/input.model';
 import {
@@ -22,15 +24,18 @@ import {
   objectSnakeToCamel,
 } from 'src/utils/CaseConverter';
 import { mustache } from 'src/utils/mustache';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+import { URLSearchParams } from 'url';
 import { IIdRequest, UpdateInputRequest } from './interfaces';
 
 export class InputsClientService implements OnModuleInit {
   private inputService: InputService;
   constructor(
     @Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
+    @Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
   ) {}
 
-  onModuleInit() {
+  async onModuleInit() {
     this.inputService =
       this.grpcClient.getService<InputService>('InputService');
   }
@@ -48,7 +53,7 @@ export class InputsClientService implements OnModuleInit {
   async createGeneric(createInputGeneric) {
     console.log('InputClientService', 'Create');
     const { info, ...data } = createInputGeneric;
-    data.credentials = await this.getInputTokens(data);
+    // data.credentials = await this.getInputTokens(data);
     if (data.options) delete data.options;
     if (data.credentials.oauth_code) delete data.credentials.oauth_code;
     const grpcPayload = {
@@ -59,7 +64,12 @@ export class InputsClientService implements OnModuleInit {
 
     const structReturn = await lastValueFrom(
       this.inputService.Create(grpcPayload),
-    );
+    ).catch((err) => {
+      if (err.details === 'Item Not found!') {
+        throw new NotFoundException('Input not found');
+      }
+      throw new InternalServerErrorException(err.details);
+    });
     objectCamelToSnake(structReturn);
     const inputCreated = DecodeGrpcStruct(structReturn.input);
     return { input: inputCreated };
@@ -201,64 +211,66 @@ export class InputsClientService implements OnModuleInit {
     switch (plugin) {
       case 'hubspot':
         return {
-          client_id: process.env.HUBSPOT_CLIENT_ID || '',
-          client_secret: process.env.HUBSPOT_CLIENT_SECRET || '',
+          client_id: this.oauthSecrets.hubspot.client_id || '',
+          client_secret: this.oauthSecrets.hubspot.client_secret || '',
         };
     }
     return {};
   }
   async getInputTokens(input: InputModel) {
-    const { credentials, options } = input;
-    const credentialsTokens: any = {};
+    const { credentials } = input;
+    let credentialsTokens = {};
     switch (credentials.connection_type) {
       case 'oauth':
-        const { oauth } = options;
-        const { get_tokens_url_params, get_tokens_set_response } = oauth;
-        const secrets = await this.getAuthSecrets(input.plugin);
-        if (oauth.content_type === 'application/x-www-form-urlencoded') {
-          const requestParams = new URLSearchParams();
-          const get_tokens_url_params_string = mustache(get_tokens_url_params, {
-            ...secrets,
-            ...credentials,
-          });
-          let get_tokens_url_params_obj: Record<string, any> = {};
-          try {
-            get_tokens_url_params_obj = JSON.parse(
-              get_tokens_url_params_string,
-            );
-          } catch (error) {
-            throw new HttpException(
-              'Erro transformando get_tokens_url_params',
-              400,
-            );
-          }
-          const { redirect_uri } = get_tokens_url_params_obj;
-          for (const key in get_tokens_url_params_obj) {
-            const value = get_tokens_url_params_obj[key];
-            requestParams.append(key, value);
-          }
-          const { data } = await axios
-            .post(options.oauth.get_tokens_url, requestParams)
-            .catch((err) => {
-              console.log(err.response.data);
-              throw new HttpException(
-                err.response.data.message,
-                err.response.status,
-              );
-            });
-          for (const key in get_tokens_set_response) {
-            const responseKey = get_tokens_set_response[key];
-            credentialsTokens[key] = data[responseKey];
-          }
-          credentialsTokens.redirect_uri = redirect_uri;
-        }
+        credentialsTokens = await this.getOauthTokens(input);
         break;
     }
     return { ...credentials, ...credentialsTokens };
   }
-  // getOauthTokens(input){
+  async getOauthTokens(input: InputModel) {
+    const tokens: Record<string, any> = {};
+    const { options, credentials } = input;
+    const { oauth } = options;
+    const { get_tokens_url_params, get_tokens_set_response } = oauth;
+    const secrets = await this.getAuthSecrets(input.plugin);
+    const get_tokens_url_params_string = mustache(get_tokens_url_params, {
+      ...secrets,
+      ...credentials,
+    });
+    let get_tokens_url_params_obj: Record<string, any> = {};
+    try {
+      get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
+    } catch (error) {
+      throw new HttpException('Erro transformando get_tokens_url_params', 400);
+    }
+    let params = {};
+    switch (oauth.content_type) {
+      case 'application/x-www-form-urlencoded':
+        params = new URLSearchParams(get_tokens_url_params_obj);
+        params = params.toString();
+        break;
+      default:
+        break;
+    }
+    const axiosRequestConfig: AxiosRequestConfig = {
+      url: oauth.get_tokens_url,
+      method: oauth.get_tokens_method || 'POST',
+      data: params,
+      headers: { 'content-type': oauth.content_type || 'application/json' },
+    };
 
-  // }
+    const { data } = await axios(axiosRequestConfig).catch((err) => {
+      console.log(err.response.data);
+      throw new HttpException(err.response.data.message, err.response.status);
+    });
+    for (const key in get_tokens_set_response) {
+      const responseKey = get_tokens_set_response[key];
+      tokens[key] = data[responseKey];
+    }
+    const { redirect_uri } = get_tokens_url_params_obj;
+    tokens.redirect_uri = redirect_uri;
+    return tokens;
+  }
 
   async getAvailableEntities(data) {
     console.log('InputClientService', 'GetAvailableEntities');

src/main.ts

@@ -26,7 +26,8 @@ async function bootstrap() {
 
   const document = SwaggerModule.createDocument(app, config);
   writeFileSync('./swagger.json', JSON.stringify(document));
-  SwaggerModule.setup('api', app, document);
+  if (process.env.ENV != 'stg' && process.env.ENV != 'prd')
+    SwaggerModule.setup('api', app, document);
 
   await app.listen(3333);
 }

src/modules/inputs/dtos/input.model.ts

@@ -1,4 +1,5 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { Method } from 'axios';
 
 export class InputModel {
   category: string;
@@ -14,6 +15,7 @@ export class InputModel {
   options?: {
     oauth?: {
       get_tokens_url: string;
+      get_tokens_method?: Method;
       get_tokens_url_params: string;
       get_tokens_set_response: Record<string, string>;
       content_type: string;
@@ -168,6 +170,8 @@ export class TestConnectionRes {
   database_tables: string[];
 }
 export class CreateInputReq {
+  @ApiPropertyOptional()
+  id: string;
   @ApiProperty()
   plugin: string;
   @ApiProperty()

src/modules/inputs/inputs.controller.ts

@@ -46,8 +46,7 @@ export class InputsController {
   async testConnection(@Body() data: TestConnectionReq) {
     console.log(`/test-connection`, 'ON TEST CONNECTION ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.testConnection(data);
+    const response = await this.inputService.testConnection(data);
 
     return response;
   }
@@ -60,8 +59,7 @@ export class InputsController {
       'ON TEST CONNECTION GET COLUMNS ROUTE',
     );
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.getColumns(data);
+    const response = await this.inputService.getColumns(data);
 
     return response;
   }
@@ -71,8 +69,18 @@ export class InputsController {
   async create(@Body() createInputDto: CreateInputReq) {
     console.log(`/input`, 'ON CREATE ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.create(createInputDto);
+    const response = await this.inputService.create(createInputDto);
+
+    return response;
+  }
+
+  @Post(':id')
+  @ApiOkResponse({ type: Input })
+  async reCreate(@Param('id') id: string, @Body() input: CreateInputReq) {
+    if (!input.id) input.id = id;
+    console.log(`POST /${input.id}`, 'ON RECREATE ROUTE');
+
+    const response = await this.inputService.reCreate(input);
 
     return response;
   }
@@ -81,9 +89,7 @@ export class InputsController {
   async findAll(@Body() body) {
     console.log(`/input`, 'ON FIND ALL ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.findAll(body);
+    const response = await this.inputService.findAll(body);
 
     return response;
   }
@@ -92,9 +98,8 @@ export class InputsController {
   async findOne(@Body() body, @Param() params) {
     const { id } = params;
     console.log(`/input/${id}`, 'ON FIND ONE ROUTE');
-    const inputService = new InputsService(this.inputsClientService);
 
-    const response = await inputService.findOne({ id, ...body });
+    const response = await this.inputService.findOne({ id, ...body });
 
     return response;
   }
@@ -106,9 +111,8 @@ export class InputsController {
     delete updateInputDto.info;
 
     console.log(`/input/${id}`, 'ON UPDATE ROUTE');
-    const inputService = new InputsService(this.inputsClientService);
 
-    const response = await inputService.update(id, updateInputDto, info);
+    const response = await this.inputService.update(id, updateInputDto, info);
 
     return response;
   }
@@ -119,9 +123,7 @@ export class InputsController {
 
     console.log(`/input/${id}`, 'ON DELETE ROUTE');
 
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.remove({ id, ...data });
+    const response = await this.inputService.remove({ id, ...data });
 
     return response;
   }

src/modules/inputs/inputs.service.ts

@@ -75,6 +75,13 @@ export class InputsService {
     const adjustedInput = this.adjustInputPayload(response.input);
     return { ...response, input: adjustedInput };
   }
+  async reCreate(data) {
+    const { info, cron } = data;
+    if (cron) this.validateCron({ info, cron });
+    const response = await this.inputClient.createGeneric(data);
+    const adjustedInput = this.adjustInputPayload(response.input);
+    return { ...response, input: adjustedInput };
+  }
   async getAvailableEntities(data): Promise<{ entities: string[] }> {
     return await this.inputClient.getAvailableEntities(data);
   }

src/modules/oauth/oauth.controller.ts

@@ -0,0 +1,54 @@
+import {
+  BadRequestException,
+  Controller,
+  Get,
+  Query,
+  Redirect,
+  Req,
+  UseGuards,
+} from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { AuthGuard } from '@nestjs/passport';
+import { InputsClientService } from 'src/clients/inputs/client.service';
+import { InputsService } from '../inputs/inputs.service';
+
+@ApiTags('oauth')
+@Controller('oauth')
+export class OauthController {
+  inputService: InputsService;
+  frontendRedirectUri = '';
+  constructor(private inputsClientService: InputsClientService) {
+    this.inputService = new InputsService(this.inputsClientService);
+    switch (process.env.ENV) {
+      case 'dev':
+      case 'stg':
+        this.frontendRedirectUri = `https://app.${process.env.ENV}.dadosfera.ai/coletix/auth-callback`;
+        break;
+      case 'prd':
+        this.frontendRedirectUri = `https://app.dadosfera.ai/coletix/auth-callback`;
+        break;
+      default:
+        this.frontendRedirectUri = `http://localhost:4200/coletix/auth-callback`;
+    }
+  }
+  @Get('hubspot')
+  @UseGuards(AuthGuard('hubspot'))
+  async oauthHubspot() {
+    return true;
+  }
+
+  @Get('hubspot/callback')
+  @UseGuards(AuthGuard('hubspot'))
+  @Redirect()
+  async oauthHubspotCallback(@Req() req, @Query('state') customer_id) {
+    const { authInfo } = req;
+    if (!authInfo) throw new BadRequestException('Oauth tokens not found');
+    const { accessToken, refreshToken } = authInfo;
+    const response = await this.inputService.create({
+      info: { customer_id, customer: 'customer', user_id: 'empty' },
+      plugin: 'hubspot',
+      credentials: { access_token: accessToken, refresh_token: refreshToken },
+    });
+    return { url: `${this.frontendRedirectUri}?input_id=${response.input.id}` };
+  }
+}

src/modules/oauth/passport-strategies/hubspot.ts

@@ -0,0 +1,33 @@
+import { Strategy } from 'passport-hubspot-oauth2';
+import { PassportStrategy } from '@nestjs/passport';
+import { Inject, Injectable } from '@nestjs/common';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+
+@Injectable()
+export class HubspotStrategy extends PassportStrategy(Strategy) {
+  constructor(
+    @Inject('OAUTH_SECRETS')
+    private readonly oauthSecrets: OauthSecrets,
+  ) {
+    super(
+      {
+        clientID: oauthSecrets.hubspot.client_id,
+        clientSecret: oauthSecrets.hubspot.client_secret,
+        callbackURL: oauthSecrets.hubspot.redirect_uri,
+        redirectUri: oauthSecrets.hubspot.redirect_uri,
+        scope:
+          'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
+        passReqToCallback: true,
+      },
+      (accessToken, refreshToken, tokenInfo, profile, done) => {
+        return done(null, profile, { accessToken, refreshToken, tokenInfo });
+      },
+    );
+  }
+
+  authenticate(req, options) {
+    const { customer_id } = req.query;
+    options.state = customer_id;
+    super.authenticate(req, options);
+  }
+}

src/utils/OauthSecrets.ts

@@ -0,0 +1,31 @@
+import {
+  SecretsManagerClient,
+  GetSecretValueCommand,
+} from '@aws-sdk/client-secrets-manager';
+
+class OauthSecretsObject {
+  client_id = '';
+  client_secret = '';
+  redirect_uri = '';
+}
+// @Injectable()
+export class OauthSecrets {
+  hubspot = new OauthSecretsObject();
+  google = new OauthSecretsObject();
+  mailchimp = new OauthSecretsObject();
+}
+export async function getOauthSecrets() {
+  const secrets = new OauthSecrets();
+  const path = process.env.SM_OAUTH_PATH;
+  const secretsManagerClient = new SecretsManagerClient({});
+  for (const key in secrets) {
+    const getSecretComand = new GetSecretValueCommand({
+      SecretId: `${path}/${key}`,
+    });
+    const res = await secretsManagerClient
+      .send(getSecretComand)
+      .catch(() => null);
+    if (res) secrets[key] = JSON.parse(res.SecretString);
+  }
+  return secrets;
+}