REFACTOR: inputs inside its own module

This commit is contained in:
Gabriel Rosa
2022-08-11 14:28:10 -03:00
parent 430e09c159
commit 117704cd8c
20 changed files with 119 additions and 225 deletions
+6 -14
View File
@@ -1,30 +1,25 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { ConfigModule } from '@nestjs/config';
import { InputsController } from './modules/inputs/inputs.controller';
import { TransformationsController } from './modules/transformations/transformations.controller';
import { OutputsController } from './modules/outputs/outputs.controllers';
import { PipelinesController } from './modules/pipelines/pipelines.controller';
import { HealthController } from './modules/health/health.controller';
import { InputsService } from './modules/inputs/inputs.service';
import { TransformationsService } from './modules/transformations/transformations.service';
import { OutputsService } from './modules/outputs/outputs.service';
import { PipelinesService } from './modules/pipelines/pipelines.service';
import { HealthService } from './modules/health/health.service';
import { InputsClientService } from './clients/inputs/client.service';
import { TransformationsClientService } from './clients/transformations/client.service';
import { OutputsClientService } from './clients/outputs/client.service';
import { PipelinesClientService } from './clients/pipelines/client.service';
import { OutputsClientConfiguration } from './clients/outputs/client.config';
import { TransformationsClientConfiguration } from './clients/transformations/client.config';
import { InputsClientConfiguration } from './clients/inputs/client.config';
import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
import { CatalogController } from './modules/catalog/catalog.controller';
import { CatalogService } from './modules/catalog/catalog.service';
import { OauthController } from './modules/oauth/oauth.controller';
import { getOauthSecrets } from './utils/OauthSecrets';
import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot-strategy';
import { FacebookStrategy } from './modules/oauth/passport-strategies/facebook-strategy';
@@ -45,31 +40,28 @@ import { TermsOfUseModule } from './modules/termsOfUse/termsOfUse.module';
import { ConnectionModule } from './modules/connection/connection.module';
import { ConnectionTestModule } from './modules/connection-test/connection-test.module';
import { NetworkConfigModule } from './modules/network-config/network-config.module';
import { InputsModule } from './modules/inputs/inputs.module';
import { OauthModule } from './modules/oauth/oauth.module';
const inputClient = new InputsClientConfiguration();
const outputClient = new OutputsClientConfiguration();
const pipelineClient = new PipelinesClientConfiguration();
const transformationClient = new TransformationsClientConfiguration();
@Module({
controllers: [
InputsController,
TransformationsController,
OutputsController,
PipelinesController,
HealthController,
CatalogController,
OauthController,
],
providers: [
DadosferaLogger,
{ provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
InputsService,
TransformationsService,
OutputsService,
PipelinesService,
HealthService,
InputsClientService,
TransformationsClientService,
OutputsClientService,
PipelinesClientService,
@@ -97,10 +89,6 @@ const transformationClient = new TransformationsClientConfiguration();
ConnectionTestModule,
ClientsModule.register([
{
name: 'INPUTS_PACKAGE',
...inputClient.config(),
},
{
name: 'TRANSFORMATIONS_PACKAGE',
...transformationClient.config(),
@@ -118,6 +106,10 @@ const transformationClient = new TransformationsClientConfiguration();
UsersModule,
RolesModule,
InputsModule,
OauthModule,
],
})
export class AppModule {}
+1 -72
View File
@@ -16,17 +16,12 @@ import {
TestConnectionGetColumnsRequest,
TestConnectionRequest,
} from 'protospack';
import axios, { AxiosRequestConfig } from 'axios';
import { lastValueFrom } from 'rxjs';
import { InputModel } from 'src/modules/inputs/dtos/input.model';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import { mustache } from 'src/utils/mustache';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { OauthSecrets } from 'src/utils/OauthSecrets';
import { URLSearchParams } from 'url';
import { IIdRequest, UpdateInputRequest } from './interfaces';
export class InputsClientService implements OnModuleInit {
@@ -35,8 +30,7 @@ export class InputsClientService implements OnModuleInit {
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
@Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
@Inject('InputsGrpcClient') private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
@@ -59,7 +53,6 @@ export class InputsClientService implements OnModuleInit {
async createGeneric(createInputGeneric) {
this.logger.info('InputClientService - Create');
const { info, ...data } = createInputGeneric;
// data.credentials = await this.getInputTokens(data);
if (data.options) delete data.options;
if (data.credentials.oauth_code) delete data.credentials.oauth_code;
const grpcPayload = {
@@ -227,70 +220,6 @@ export class InputsClientService implements OnModuleInit {
return getColumnsResponse;
}
async getAuthSecrets(plugin) {
switch (plugin) {
case 'hubspot':
return {
client_id: this.oauthSecrets.hubspot.client_id || '',
client_secret: this.oauthSecrets.hubspot.client_secret || '',
};
}
return {};
}
async getInputTokens(input: InputModel) {
const { credentials } = input;
let credentialsTokens = {};
switch (credentials.connection_type) {
case 'oauth':
credentialsTokens = await this.getOauthTokens(input);
break;
}
return { ...credentials, ...credentialsTokens };
}
async getOauthTokens(input: InputModel) {
const tokens: Record<string, any> = {};
const { options, credentials } = input;
const { oauth } = options;
const { get_tokens_url_params, get_tokens_set_response } = oauth;
const secrets = await this.getAuthSecrets(input.plugin);
const get_tokens_url_params_string = mustache(get_tokens_url_params, {
...secrets,
...credentials,
});
let get_tokens_url_params_obj: Record<string, any> = {};
try {
get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
} catch (error) {
throw new HttpException('Erro transformando get_tokens_url_params', 400);
}
let params = {};
switch (oauth.content_type) {
case 'application/x-www-form-urlencoded':
params = new URLSearchParams(get_tokens_url_params_obj);
params = params.toString();
break;
default:
break;
}
const axiosRequestConfig: AxiosRequestConfig = {
url: oauth.get_tokens_url,
method: oauth.get_tokens_method || 'POST',
data: params,
headers: { 'content-type': oauth.content_type || 'application/json' },
};
const { data } = await axios(axiosRequestConfig).catch((err) => {
this.logger.error(err.response.data);
throw new HttpException(err.response.data.message, err.response.status);
});
for (const key in get_tokens_set_response) {
const responseKey = get_tokens_set_response[key];
tokens[key] = data[responseKey];
}
const { redirect_uri } = get_tokens_url_params_obj;
tokens.redirect_uri = redirect_uri;
return tokens;
}
async getAvailableEntities(data) {
this.logger.info('InputClientService - GetAvailableEntities');
+2 -7
View File
@@ -1,3 +1,5 @@
import { Info } from 'protospack/dist/lib/interfaces';
interface Values {
jdbc_user: string;
jdbc_password: string;
@@ -28,13 +30,6 @@ interface Cron {
week_days_resourse_value: number;
}
export interface Info {
customer_id: string;
user_id: string;
customer: string;
customer_tier: string;
}
export interface ICreateInputRequest {
cron: string;
name: string;
+2 -5
View File
@@ -1,3 +1,5 @@
import { Info } from 'protospack/dist/lib/interfaces';
export interface ICreateOutputRequest {
operation: string;
name: string;
@@ -9,11 +11,6 @@ export interface Values {
prefix: string;
}
interface Info {
user_id: string;
customer_id: string;
}
export interface IIdRequest {
id: string;
info: Info;
+2 -5
View File
@@ -1,3 +1,5 @@
import { Info } from 'protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
transformations: IdRequest[];
@@ -8,11 +10,6 @@ export interface ICreatePipelineDto {
info: Info;
}
export interface Info {
customer_id: sting;
user_id: string;
}
export interface IdRequest {
id: string;
}
+2 -5
View File
@@ -1,3 +1,5 @@
import { Info } from 'protospack/dist/lib/interfaces';
export interface ICreateTransformationsRequest {
transformations: Transformation[];
info: Info;
@@ -11,11 +13,6 @@ interface Param {
const: string;
}
export interface Info {
customer_id: string;
user_id: string;
}
interface Transformation {
created_at: string;
id: string;
+1 -11
View File
@@ -1,6 +1,5 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { Info } from 'src/clients/inputs/interfaces';
import { Method } from 'axios';
import { Info } from 'protospack/dist/lib/interfaces';
export class InputModel {
category: string;
@@ -13,15 +12,6 @@ export class InputModel {
oauth_code: string;
start_date: string;
};
options?: {
oauth?: {
get_tokens_url: string;
get_tokens_method?: Method;
get_tokens_url_params: string;
get_tokens_set_response: Record<string, string>;
content_type: string;
};
};
}
class GoogleAnalyticsClientSecrets {
@@ -0,0 +1,28 @@
import { credentials } from '@grpc/grpc-js';
import {
ClientProviderOptions,
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { InputPackages, InputProtoFilePath } from 'protospack';
export class InputsGrpcClient {
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: InputPackages,
credentials: process.env.LOCAL_ENV ? undefined : credentials.createSsl(),
protoPath: InputProtoFilePath,
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
providerOptions: ClientProviderOptions = {
name: 'InputsGrpcClient',
...this.config,
};
}
+17
View File
@@ -0,0 +1,17 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { InputsGrpcClient } from './inputs-client.config';
import { InputsController } from './inputs.controller';
import { InputsService } from './inputs.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { InputsClientService } from 'src/clients/inputs/client.service';
const client = new InputsGrpcClient();
@Module({
controllers: [InputsController],
providers: [InputsService, InputsClientService, DadosferaLogger],
imports: [ClientsModule.register([client.providerOptions])],
exports: [InputsService],
})
export class InputsModule {}
+2 -1
View File
@@ -3,7 +3,8 @@ import { Timeout } from '@nestjs/schedule';
import { DecodeGrpcStruct } from 'protospack';
import CronParser, { CronExpression } from 'cron-parser';
import { InputsClientService } from 'src/clients/inputs/client.service';
import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
import { IIdRequest } from 'src/clients/inputs/interfaces';
import { Info } from 'protospack/dist/lib/interfaces';
@Injectable()
export class InputsService {
@@ -6,7 +6,7 @@ import {
} from '@nestjs/microservices';
import { NetworkConfig } from '@dadosfera/protospack-v2';
export class NetworkConfigClientConfiguration {
export class NetworkConfigGrpcClient {
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
@@ -15,7 +15,6 @@ export class NetworkConfigClientConfiguration {
NetworkConfig.ProtoPackages.ReadPackage,
NetworkConfig.ProtoPackages.WritePackage,
],
// credentials: undefined,
credentials:
process.env.ENV === 'local' ? undefined : credentials.createSsl(),
protoPath: [
@@ -1,25 +0,0 @@
import { ClientsModule } from '@nestjs/microservices';
import { Test, TestingModule } from '@nestjs/testing';
import { NetworkConfigClientConfiguration } from './network-config-client.config';
import { NetworkConfigController } from './network-config.controller';
import { NetworkConfigService } from './network-config.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
describe('NetworkConfigController', () => {
let controller: NetworkConfigController;
beforeEach(async () => {
const client = new NetworkConfigClientConfiguration();
const module: TestingModule = await Test.createTestingModule({
controllers: [NetworkConfigController],
providers: [NetworkConfigService, DadosferaLogger],
imports: [ClientsModule.register([client.providerOptions])],
}).compile();
controller = module.get<NetworkConfigController>(NetworkConfigController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});
@@ -3,8 +3,8 @@ import { NetworkConfigController } from './network-config.controller';
import { NetworkConfigService } from './network-config.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { NetworkConfigClientConfiguration } from './network-config-client.config';
const client = new NetworkConfigClientConfiguration();
import { NetworkConfigGrpcClient } from './network-config-client.config';
const client = new NetworkConfigGrpcClient();
@Module({
controllers: [NetworkConfigController],
@@ -1,25 +0,0 @@
import { ClientsModule } from '@nestjs/microservices';
import { Test, TestingModule } from '@nestjs/testing';
import { NetworkConfigClientConfiguration } from './network-config-client.config';
import { NetworkConfigController } from './network-config.controller';
import { NetworkConfigService } from './network-config.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
describe('NetworkConfigService', () => {
let service: NetworkConfigService;
beforeEach(async () => {
const client = new NetworkConfigClientConfiguration();
const module: TestingModule = await Test.createTestingModule({
controllers: [NetworkConfigController],
providers: [NetworkConfigService, DadosferaLogger],
imports: [ClientsModule.register([client.providerOptions])],
}).compile();
service = module.get<NetworkConfigService>(NetworkConfigService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
+1 -3
View File
@@ -8,10 +8,8 @@ import { Request } from 'express';
@ApiTags('oauth')
@Controller('oauth')
export class OauthController {
inputService: InputsService;
frontendRedirectUri = '';
constructor(private inputsClientService: InputsClientService) {
this.inputService = new InputsService(this.inputsClientService);
constructor(private inputService: InputsService) {
switch (process.env.ENV) {
case 'dev':
case 'stg':
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { InputsModule } from '../inputs/inputs.module';
import { OauthController } from './oauth.controller';
@Module({
controllers: [OauthController],
imports: [InputsModule],
})
export class OauthModule {}
+2 -1
View File
@@ -1,6 +1,7 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { IIdRequest, Info } from 'src/clients/outputs/interfaces';
import { IIdRequest } from 'src/clients/outputs/interfaces';
import { OutputsClientService } from 'src/clients/outputs/client.service';
import { Info } from 'protospack/dist/lib/interfaces';
@Injectable()
export class OutputsService {
+2 -1
View File
@@ -1,7 +1,8 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { DecodeGrpcStruct } from 'protospack';
import { Info } from 'protospack/dist/lib/interfaces';
import { PipelinesClientService } from 'src/clients/pipelines/client.service';
import { IIdRequest, Info } from 'src/clients/pipelines/interfaces';
import { IIdRequest } from 'src/clients/pipelines/interfaces';
import { objectCamelToSnake } from 'src/utils/CaseConverter';
@Injectable()