From 4e2e2964420b5f7654d7472a4be1df68d1ad6758 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 8 Jun 2022 10:31:59 -0300 Subject: [PATCH 01/18] Fixed cron on create S3 --- src/clients/inputs/client.service.ts | 1 - src/modules/inputs/inputs.service.ts | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index dee162e..0ec746d 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -1,7 +1,6 @@ import { OnModuleInit, Inject } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { - InputCreateResponse, InputCreateS3Request, InputNewCreateRequest, InputService, diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 03898bd..222abbb 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -153,7 +153,7 @@ export class InputsService { } generateInputS3Payload(payload) { - const { credentials, plugin } = payload; + const { credentials, plugin, cron } = payload; if (!credentials) return payload; const { @@ -164,8 +164,10 @@ export class InputsService { client_bucket, file_to_extract, } = credentials; + const formatedPayload = { plugin, + cron, source_bucket: client_bucket, source_prefix: file_to_extract, auth_parameters: { From 28e55f8fd909cedc0456f37f4fa15dba45a79b63 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 8 Jun 2022 11:12:51 -0300 Subject: [PATCH 02/18] update node version --- .nvmrc | 2 +- Dockerfile | 3 ++- package-lock.json | 14 +++++++------- package.json | 5 ++++- src/clients/inputs/client.service.ts | 12 ++++++++++++ 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.nvmrc b/.nvmrc index 25bf17f..1667c1a 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18 \ No newline at end of file +18.3.0 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f80c060..6f073d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ -FROM node:14.15.4-alpine3.12 as packages +FROM node:18.3.0-alpine3.15 as packages WORKDIR /packages ARG NPM_TOKEN COPY package.json . +COPY package-lock.json . COPY .npmrc . RUN npm install RUN rm -f ./.npmrc diff --git a/package-lock.json b/package-lock.json index 0ab3ab4..169589b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.2-1", + "@victorradael/protospack": "2.4.5", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2357,9 +2357,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.4.2-1", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.2-1.tgz", - "integrity": "sha512-wdcGaK0DIEJeOHga1br11kq264JNdhdB1G2toy/u6TvAk90wwBKHNs/cZko0p9WFVuqHQOVehoHj7WJbJ3K25Q==", + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.5.tgz", + "integrity": "sha512-yYitBEvWOIhJytwYuPvwOqysgyAK6awHAdEUeRH7s4emlqtY57Vpr0/p9snRQK5ytnxa9PbrgF5AewSe2/VkNA==", "dependencies": { "rxjs": "^7.5.5" } @@ -10932,9 +10932,9 @@ } }, "@victorradael/protospack": { - "version": "2.4.2-1", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.2-1.tgz", - "integrity": "sha512-wdcGaK0DIEJeOHga1br11kq264JNdhdB1G2toy/u6TvAk90wwBKHNs/cZko0p9WFVuqHQOVehoHj7WJbJ3K25Q==", + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.5.tgz", + "integrity": "sha512-yYitBEvWOIhJytwYuPvwOqysgyAK6awHAdEUeRH7s4emlqtY57Vpr0/p9snRQK5ytnxa9PbrgF5AewSe2/VkNA==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index 5889917..70b5a2e 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,9 @@ "author": "", "private": true, "license": "UNLICENSED", + "engines": { + "node": "18.3.0" + }, "scripts": { "prebuild": "rimraf dist", "build": "nest build", @@ -31,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.2-1", + "@victorradael/protospack": "2.4.5", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index 0ec746d..33f2e63 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -1,12 +1,15 @@ import { OnModuleInit, Inject } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { + DecodeGrpcStruct, + EncodeJsonToGrpcStruct, InputCreateS3Request, InputNewCreateRequest, InputService, TestConnectionGetColumnsRequest, TestConnectionRequest, } from '@victorradael/protospack'; +import { lastValueFrom } from 'rxjs'; import { objectCamelToSnake, objectSnakeToCamel, @@ -49,6 +52,15 @@ export class InputsClientService implements OnModuleInit { return createInputResponse; } + async createGeneric(createInputGeneric) { + console.log('InputClientService', 'Create'); + const grpcPayload = EncodeJsonToGrpcStruct(createInputGeneric); + const structPayload = await lastValueFrom( + this.inputService.Create(objectSnakeToCamel(grpcPayload)), + ); + return DecodeGrpcStruct(structPayload.input); + } + async createS3Inputs(createInputDto: InputCreateS3Request) { console.log('InputClientService', 'Create'); From 0342386a7615ab1a071c39c111c9677c3515f50e Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Thu, 9 Jun 2022 17:14:11 -0300 Subject: [PATCH 03/18] oauth almost complete --- package-lock.json | 17 +++-- package.json | 2 +- src/clients/inputs/client.service.ts | 90 +++++++++++++++++++------- src/modules/inputs/dtos/input.model.ts | 20 ++++++ src/modules/inputs/inputs.service.ts | 33 +++++----- src/utils/mustache.ts | 7 ++ 6 files changed, 121 insertions(+), 48 deletions(-) create mode 100644 src/modules/inputs/dtos/input.model.ts create mode 100644 src/utils/mustache.ts diff --git a/package-lock.json b/package-lock.json index 169589b..c1c0ef5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.5", + "@victorradael/protospack": "^2.4.6", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -56,6 +56,9 @@ "ts-node": "^10.7.0", "tsconfig-paths": "^3.14.1", "typescript": "^4.6.3" + }, + "engines": { + "node": "18.3.0" } }, "node_modules/@ampproject/remapping": { @@ -2357,9 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.5.tgz", - "integrity": "sha512-yYitBEvWOIhJytwYuPvwOqysgyAK6awHAdEUeRH7s4emlqtY57Vpr0/p9snRQK5ytnxa9PbrgF5AewSe2/VkNA==", + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.6.tgz", + "integrity": "sha512-IShD6T7CcgPbHHXGlYhglHPppQMY6hSf9wrs0SC8VWVuPLuMzrxTRBmFIteqnKpMOoYpzo5egteVwXAixuyhJQ==", "dependencies": { "rxjs": "^7.5.5" } @@ -10932,9 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.5.tgz", - "integrity": "sha512-yYitBEvWOIhJytwYuPvwOqysgyAK6awHAdEUeRH7s4emlqtY57Vpr0/p9snRQK5ytnxa9PbrgF5AewSe2/VkNA==", + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.6.tgz", + "integrity": "sha512-IShD6T7CcgPbHHXGlYhglHPppQMY6hSf9wrs0SC8VWVuPLuMzrxTRBmFIteqnKpMOoYpzo5egteVwXAixuyhJQ==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index 70b5a2e..bfdaf4e 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.5", + "@victorradael/protospack": "2.4.6", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index 33f2e63..58e0285 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -1,4 +1,4 @@ -import { OnModuleInit, Inject } from '@nestjs/common'; +import { OnModuleInit, Inject, HttpException } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DecodeGrpcStruct, @@ -9,11 +9,14 @@ import { TestConnectionGetColumnsRequest, TestConnectionRequest, } from '@victorradael/protospack'; +import axios from 'axios'; import { lastValueFrom } from 'rxjs'; +import { InputModel } from 'src/modules/inputs/dtos/input.model'; import { objectCamelToSnake, objectSnakeToCamel, } from 'src/utils/CaseConverter'; +import { mustache } from 'src/utils/mustache'; import { IIdRequest, UpdateInputRequest } from './interfaces'; export class InputsClientService implements OnModuleInit { @@ -27,34 +30,22 @@ export class InputsClientService implements OnModuleInit { this.grpcClient.getService('InputService'); } - async create(createInputDto: InputNewCreateRequest) { + async newCreate(createInputDto: InputNewCreateRequest) { console.log('InputClientService', 'Create'); - const createInputResponse = await new Promise((resolve, reject) => { - this.inputService - .NewCreate(objectSnakeToCamel(createInputDto)) - .subscribe({ - next(x) { - resolve(objectCamelToSnake(x)); - }, - error(err) { - console.log('Observable Error'); - reject(err); - }, - complete() { - console.log('done'); - }, - }); - }).catch((err) => { - throw new Error(err); - }); - - return createInputResponse; + return await lastValueFrom( + this.inputService.NewCreate(objectSnakeToCamel(createInputDto)), + ); } async createGeneric(createInputGeneric) { console.log('InputClientService', 'Create'); - const grpcPayload = EncodeJsonToGrpcStruct(createInputGeneric); + const { info, ...data } = createInputGeneric; + createInputGeneric.credentials = await this.getInputTokens(data); + const grpcPayload = { + input: EncodeJsonToGrpcStruct(createInputGeneric), + info, + }; const structPayload = await lastValueFrom( this.inputService.Create(objectSnakeToCamel(grpcPayload)), ); @@ -232,4 +223,57 @@ export class InputsClientService implements OnModuleInit { return getColumnsResponse; } + async getAuthSecrets(plugin) { + switch (plugin) { + case 'hubspot': + return { + client_id: process.env.HUBSPOT_CLIENT_ID, + client_secret: process.env.HUBSPOT_CLIENT_SECRET, + }; + } + return {}; + } + async getInputTokens(input: InputModel) { + const { credentials, options } = input; + const credentialsTokens = {}; + switch (credentials.connection_type) { + case 'oauth': + const { oauth } = options; + const { get_tokens_url_params, get_tokens_set_response } = oauth; + const secrets = await this.getAuthSecrets(input.plugin); + if (oauth.content_type === 'application/x-www-form-urlencoded') { + const requestParams = new URLSearchParams(); + const get_tokens_url_params_string = mustache(get_tokens_url_params, { + ...secrets, + ...credentials, + }); + const get_tokens_url_params_obj = JSON.parse( + get_tokens_url_params_string, + ); + for (const key in get_tokens_url_params_obj) { + const value = get_tokens_url_params_obj[key]; + requestParams.append(key, value); + } + // const { data } = await axios + // .post(options.oauth.get_tokens_url, requestParams) + // .catch((err) => { + // console.log(err.response.data); + // throw new HttpException( + // err.response.data.message, + // err.response.status, + // ); + // }); + const data = { refresh_token: 'refresh', access_token: 'access' }; + for (const key in get_tokens_set_response) { + const responseKey = get_tokens_set_response[key]; + credentialsTokens[key] = data[responseKey]; + } + } + break; + } + return { ...credentials, ...credentialsTokens }; + } + // getOauthTokens(input){ + + // } } diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts new file mode 100644 index 0000000..4bc5b47 --- /dev/null +++ b/src/modules/inputs/dtos/input.model.ts @@ -0,0 +1,20 @@ +export class InputModel { + category: string; + plugin: string; + name: string; + cron?: string; + credentials?: { + connection_type: string; + oauth: string; + oauth_code: string; + start_date: string; + }; + options?: { + oauth?: { + get_tokens_url: string; + get_tokens_url_params: string; + get_tokens_set_response: Record; + content_type: string; + }; + }; +} diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 222abbb..0ec4c18 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -47,28 +47,27 @@ export class InputsService { } async create(@Body() data) { this.validateCron(data); - try { - if ( - data.plugin == 'csv' || - data.plugin == 'json' || - data.plugin == 'parquet' - ) { + let response; + switch (data.plugin.toLowerCase()) { + case 'csv': + case 'json': + case 'parquet': const { info, ...input } = data; const inputPayload = this.generateInputS3Payload(input); - const createInputResponse = await this.inputClient.createS3Inputs({ + response = await this.inputClient.createS3Inputs({ input: inputPayload, info, }); - return createInputResponse; - } else { - const createInputResponse: any = await this.inputClient.create(data); - if (createInputResponse?.input?.input_jdbc) { - return { input: createInputResponse?.input?.input_jdbc }; - } - return createInputResponse; - } - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); + return response; + case 'oracle': + case 'mysql': + case 'postgresql': + case 'sqlserver': + response = await this.inputClient.newCreate(data); + return { input: response?.input?.input_jdbc }; + default: + response = await this.inputClient.createGeneric(data); + return response; } } diff --git a/src/utils/mustache.ts b/src/utils/mustache.ts new file mode 100644 index 0000000..0f6064a --- /dev/null +++ b/src/utils/mustache.ts @@ -0,0 +1,7 @@ +export const mustache = (string: string, data: Record = {}) => { + return Object.entries(data).reduce((res, [key, value]) => { + const mainRe = new RegExp(`(? Date: Mon, 13 Jun 2022 10:49:27 -0300 Subject: [PATCH 04/18] removing info that won't be saved on database --- package-lock.json | 14 +++++++------- package.json | 2 +- src/clients/inputs/client.service.ts | 26 ++++++++++++++------------ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index c1c0ef5..b16425b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "^2.4.6", + "@victorradael/protospack": "2.4.8", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2360,9 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.6.tgz", - "integrity": "sha512-IShD6T7CcgPbHHXGlYhglHPppQMY6hSf9wrs0SC8VWVuPLuMzrxTRBmFIteqnKpMOoYpzo5egteVwXAixuyhJQ==", + "version": "2.4.8", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.8.tgz", + "integrity": "sha512-iwqhytreFlPo6VwsK9LB+Po822178FDZwXn5z5Qk4LvuIsaBPhltmG6IWYY1rZShsdV3qRpiv9DzgiPT1W2edw==", "dependencies": { "rxjs": "^7.5.5" } @@ -10935,9 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.6.tgz", - "integrity": "sha512-IShD6T7CcgPbHHXGlYhglHPppQMY6hSf9wrs0SC8VWVuPLuMzrxTRBmFIteqnKpMOoYpzo5egteVwXAixuyhJQ==", + "version": "2.4.8", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.8.tgz", + "integrity": "sha512-iwqhytreFlPo6VwsK9LB+Po822178FDZwXn5z5Qk4LvuIsaBPhltmG6IWYY1rZShsdV3qRpiv9DzgiPT1W2edw==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index bfdaf4e..7c8ee8d 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.6", + "@victorradael/protospack": "2.4.8", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index 58e0285..42d3e2a 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -41,14 +41,17 @@ export class InputsClientService implements OnModuleInit { async createGeneric(createInputGeneric) { console.log('InputClientService', 'Create'); const { info, ...data } = createInputGeneric; - createInputGeneric.credentials = await this.getInputTokens(data); + data.credentials = await this.getInputTokens(data); + data.options ? delete data.options : null; + data.credentials.oauth_code ? delete data.credentials.oauth_code : null; const grpcPayload = { - input: EncodeJsonToGrpcStruct(createInputGeneric), + input: EncodeJsonToGrpcStruct(data), info, }; const structPayload = await lastValueFrom( this.inputService.Create(objectSnakeToCamel(grpcPayload)), ); + objectCamelToSnake(structPayload); return DecodeGrpcStruct(structPayload.input); } @@ -254,16 +257,15 @@ export class InputsClientService implements OnModuleInit { const value = get_tokens_url_params_obj[key]; requestParams.append(key, value); } - // const { data } = await axios - // .post(options.oauth.get_tokens_url, requestParams) - // .catch((err) => { - // console.log(err.response.data); - // throw new HttpException( - // err.response.data.message, - // err.response.status, - // ); - // }); - const data = { refresh_token: 'refresh', access_token: 'access' }; + const { data } = await axios + .post(options.oauth.get_tokens_url, requestParams) + .catch((err) => { + console.log(err.response.data); + throw new HttpException( + err.response.data.message, + err.response.status, + ); + }); for (const key in get_tokens_set_response) { const responseKey = get_tokens_set_response[key]; credentialsTokens[key] = data[responseKey]; From 2710159a7350bf9d5a7a541cf58fb7a74afba55d Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Mon, 13 Jun 2022 11:17:41 -0300 Subject: [PATCH 05/18] minor improvements --- src/clients/inputs/client.service.ts | 35 ++++++++++------------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index 42d3e2a..dc8e765 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -42,14 +42,15 @@ export class InputsClientService implements OnModuleInit { console.log('InputClientService', 'Create'); const { info, ...data } = createInputGeneric; data.credentials = await this.getInputTokens(data); - data.options ? delete data.options : null; - data.credentials.oauth_code ? delete data.credentials.oauth_code : null; + if (data.options) delete data.options; + if (data.credentials.oauth_code) delete data.credentials.oauth_code; const grpcPayload = { input: EncodeJsonToGrpcStruct(data), info, }; + const structPayload = await lastValueFrom( - this.inputService.Create(objectSnakeToCamel(grpcPayload)), + this.inputService.Create(grpcPayload), ); objectCamelToSnake(structPayload); return DecodeGrpcStruct(structPayload.input); @@ -57,26 +58,14 @@ export class InputsClientService implements OnModuleInit { async createS3Inputs(createInputDto: InputCreateS3Request) { console.log('InputClientService', 'Create'); - - const createInputResponse = await new Promise((resolve, reject) => { - this.inputService.CreateS3(objectSnakeToCamel(createInputDto)).subscribe({ - next(x) { - resolve(objectCamelToSnake(x)); - }, - error(err) { - console.log('Observable Error'); - reject(err); - }, - complete() { - console.log('done'); - }, - }); - }) - .then((res) => res) - .catch((err) => { - throw new Error(err); - }); - + objectSnakeToCamel(createInputDto); + const createInputResponse = await lastValueFrom( + this.inputService.CreateS3(createInputDto), + ).catch((e) => { + throw new HttpException(e.details, 500); + }); + console.log('done'); + objectCamelToSnake(createInputResponse); return createInputResponse; } From 4dfe6c387072dfea1601a6c95f8fc7dbb9898dfd Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Mon, 13 Jun 2022 13:12:01 -0300 Subject: [PATCH 06/18] fallback for env vars --- src/clients/inputs/client.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index dc8e765..3dbe549 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -219,8 +219,8 @@ export class InputsClientService implements OnModuleInit { switch (plugin) { case 'hubspot': return { - client_id: process.env.HUBSPOT_CLIENT_ID, - client_secret: process.env.HUBSPOT_CLIENT_SECRET, + client_id: process.env.HUBSPOT_CLIENT_ID || '', + client_secret: process.env.HUBSPOT_CLIENT_SECRET || '', }; } return {}; From 852ddbf88beab140fd64420ad2713e18cf7992e1 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Mon, 13 Jun 2022 15:55:02 -0300 Subject: [PATCH 07/18] transforming to camelCase before sending to in-factory --- src/clients/inputs/client.service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index 3dbe549..ebf65f4 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -48,6 +48,7 @@ export class InputsClientService implements OnModuleInit { input: EncodeJsonToGrpcStruct(data), info, }; + objectSnakeToCamel(grpcPayload); const structPayload = await lastValueFrom( this.inputService.Create(grpcPayload), From 305daeeeab364b6b6501566a5b03603723023935 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Mon, 13 Jun 2022 16:50:38 -0300 Subject: [PATCH 08/18] update protospack --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index b16425b..c837df0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.8", + "@victorradael/protospack": "2.4.9", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2360,9 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.4.8", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.8.tgz", - "integrity": "sha512-iwqhytreFlPo6VwsK9LB+Po822178FDZwXn5z5Qk4LvuIsaBPhltmG6IWYY1rZShsdV3qRpiv9DzgiPT1W2edw==", + "version": "2.4.9", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.9.tgz", + "integrity": "sha512-pFa+Qwa/nQLshR5zu/gPhvYAtIPrGONRB+RNhsh67tqle6kAvj41kSRFzZU0YbSDfYqf7Oyi/+PYk1Y1JNz9UQ==", "dependencies": { "rxjs": "^7.5.5" } @@ -10935,9 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "2.4.8", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.8.tgz", - "integrity": "sha512-iwqhytreFlPo6VwsK9LB+Po822178FDZwXn5z5Qk4LvuIsaBPhltmG6IWYY1rZShsdV3qRpiv9DzgiPT1W2edw==", + "version": "2.4.9", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.9.tgz", + "integrity": "sha512-pFa+Qwa/nQLshR5zu/gPhvYAtIPrGONRB+RNhsh67tqle6kAvj41kSRFzZU0YbSDfYqf7Oyi/+PYk1Y1JNz9UQ==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index 7c8ee8d..4cce4e2 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.8", + "@victorradael/protospack": "2.4.9", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", From ff663e7166749fb970456d6da6c6b18dc3abdc7e Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Tue, 14 Jun 2022 11:35:09 -0300 Subject: [PATCH 09/18] update protospack --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index c837df0..7954441 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.9", + "@victorradael/protospack": "2.5.0-4", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2360,9 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.4.9", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.9.tgz", - "integrity": "sha512-pFa+Qwa/nQLshR5zu/gPhvYAtIPrGONRB+RNhsh67tqle6kAvj41kSRFzZU0YbSDfYqf7Oyi/+PYk1Y1JNz9UQ==", + "version": "2.5.0-4", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-4.tgz", + "integrity": "sha512-WgcWv2VPST3IaJsuz9yrSgC+zxsV5rjVB/kAlgB0QAkKvtvS23uQBqWrbvyG4VcKQXG1rGiIwoQ6Fyf23NWG4g==", "dependencies": { "rxjs": "^7.5.5" } @@ -10935,9 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "2.4.9", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.4.9.tgz", - "integrity": "sha512-pFa+Qwa/nQLshR5zu/gPhvYAtIPrGONRB+RNhsh67tqle6kAvj41kSRFzZU0YbSDfYqf7Oyi/+PYk1Y1JNz9UQ==", + "version": "2.5.0-4", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-4.tgz", + "integrity": "sha512-WgcWv2VPST3IaJsuz9yrSgC+zxsV5rjVB/kAlgB0QAkKvtvS23uQBqWrbvyG4VcKQXG1rGiIwoQ6Fyf23NWG4g==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index 4cce4e2..bd00327 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.4.9", + "@victorradael/protospack": "2.5.0-4", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", From 383741f48c5a9b4c437eb2b82bdd34d7b9bb92bf Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Tue, 14 Jun 2022 14:01:40 -0300 Subject: [PATCH 10/18] get entities --- src/clients/inputs/client.service.ts | 23 ++++++++++++++++++++++- src/modules/inputs/inputs.controller.ts | 11 ++++++++++- src/modules/inputs/inputs.service.ts | 5 ++++- swagger.json | 2 +- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index ebf65f4..c13cf14 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -1,4 +1,9 @@ -import { OnModuleInit, Inject, HttpException } from '@nestjs/common'; +import { + OnModuleInit, + Inject, + HttpException, + HttpStatus, +} from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DecodeGrpcStruct, @@ -268,4 +273,20 @@ export class InputsClientService implements OnModuleInit { // getOauthTokens(input){ // } + + async getAvailableEntities(data) { + console.log('InputClientService', 'GetAvailableEntities'); + objectSnakeToCamel(data); + const response = await lastValueFrom( + this.inputService.GetAvailableEntities(data), + ).catch((err) => { + console.log(err); + if (err.details && err.details.includes('400')) + throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST); + throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR); + }); + objectCamelToSnake(response); + + return response; + } } diff --git a/src/modules/inputs/inputs.controller.ts b/src/modules/inputs/inputs.controller.ts index 918aa1a..3373e0b 100644 --- a/src/modules/inputs/inputs.controller.ts +++ b/src/modules/inputs/inputs.controller.ts @@ -14,7 +14,16 @@ import { InputNewCreateRequest } from '@victorradael/protospack'; @Controller('inputs') export class InputsController { - constructor(private inputsClientService: InputsClientService) {} + inputService: InputsService; + constructor(private inputsClientService: InputsClientService) { + this.inputService = new InputsService(this.inputsClientService); + } + @Get('available-entities/:plugin') + async getAvailableEntities(@Body() body, @Param() params) { + const { info } = body; + const { plugin } = params; + return await this.inputService.getAvailableEntities({ info, plugin }); + } @Post('/test-connection') async testConnection(@Body() data) { diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 0ec4c18..3d8a5c1 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -1,6 +1,7 @@ import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Timeout } from '@nestjs/schedule'; import CronParser, { CronExpression } from 'cron-parser'; +import { lastValueFrom } from 'rxjs'; import { InputsClientService } from 'src/clients/inputs/client.service'; import { IIdRequest, Info } from 'src/clients/inputs/interfaces'; @@ -70,7 +71,9 @@ export class InputsService { return response; } } - + async getAvailableEntities(data): Promise<{ entities: string[] }> { + return await this.inputClient.getAvailableEntities(data); + } async findAll(body) { try { const findAllInputResponse: any = await this.inputClient.findAll(body); diff --git a/swagger.json b/swagger.json index 6fc6bdc..bfdeb79 100644 --- a/swagger.json +++ b/swagger.json @@ -1 +1 @@ -{"openapi":"3.0.0","paths":{"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"201":{"description":""}}}},"/inputs/test-connection/get-columns":{"post":{"operationId":"InputsController_getColumns","parameters":[],"responses":{"201":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"patch":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/start/{id}":{"post":{"operationId":"PipelinesController_activate","parameters":[],"responses":{"201":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn_old","parameters":[],"responses":{"200":{"description":""}}}},"/auth/login":{"post":{"operationId":"AuthController_signIn_login","parameters":[],"responses":{"200":{"description":""}}}},"/auth/sign-in":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"200":{"description":""}}}},"/auth/refresh-access-token":{"post":{"operationId":"AuthController_refreshAccessToken","parameters":[],"responses":{"200":{"description":""}}}},"/auth/enable-totp":{"post":{"operationId":"AuthController_enableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/disable-totp":{"post":{"operationId":"AuthController_disableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/dismiss-totp":{"post":{"operationId":"AuthController_dismissTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-totp":{"post":{"operationId":"AuthController_verifyTotp","parameters":[],"responses":{"200":{"description":""}}}},"/auth/change-password":{"post":{"operationId":"AuthController_changePassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/reset-password":{"post":{"operationId":"AuthController_resetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-reset-password-code":{"post":{"operationId":"AuthController_verifyResetPasswordCode","parameters":[],"responses":{"200":{"description":""}}}},"/auth/confirm-reset-password":{"post":{"operationId":"AuthController_confirmResetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/all":{"get":{"operationId":"CatalogController_catalogAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps":{"get":{"operationId":"CatalogController_dataAppsAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps/{id}":{"delete":{"operationId":"CatalogController_dataAppsOne","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase":{"get":{"operationId":"CatalogController_getAllDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase/{id}":{"get":{"operationId":"CatalogController_getOneDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata":{"get":{"operationId":"CatalogController_getAllTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata":{"get":{"operationId":"CatalogController_getOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-preview":{"get":{"operationId":"CatalogController_getOneDataPreview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-status":{"get":{"operationId":"CatalogController_createDataStatus","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-description":{"get":{"operationId":"CatalogController_getDataDescription","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDescription","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-docs":{"get":{"operationId":"CatalogController_getDataDocs","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDocs","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-rating":{"get":{"operationId":"CatalogController_getDataRating","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataRating","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/summary-rating/{id}":{"get":{"operationId":"CatalogController_getSummaryRating","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-comment":{"get":{"operationId":"CatalogController_getDataComment","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataComment","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-review":{"get":{"operationId":"CatalogController_getDataReview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags":{"post":{"operationId":"CatalogController_createTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_findAllTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags/{id}":{"delete":{"operationId":"CatalogController_deleteTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags":{"post":{"operationId":"CatalogController_createTableTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_getAllTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags/{id}":{"delete":{"operationId":"CatalogController_deleteTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules":{"get":{"operationId":"CatalogController_getTableRules","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules/{id}":{"delete":{"operationId":"CatalogController_deleteTableRules","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}} \ No newline at end of file +{"openapi":"3.0.0","paths":{"/inputs/available-entities/{plugin}":{"get":{"operationId":"InputsController_getAvailableEntities","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"201":{"description":""}}}},"/inputs/test-connection/get-columns":{"post":{"operationId":"InputsController_getColumns","parameters":[],"responses":{"201":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"patch":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/start/{id}":{"post":{"operationId":"PipelinesController_activate","parameters":[],"responses":{"201":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn_old","parameters":[],"responses":{"200":{"description":""}}}},"/auth/login":{"post":{"operationId":"AuthController_signIn_login","parameters":[],"responses":{"200":{"description":""}}}},"/auth/sign-in":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"200":{"description":""}}}},"/auth/refresh-access-token":{"post":{"operationId":"AuthController_refreshAccessToken","parameters":[],"responses":{"200":{"description":""}}}},"/auth/enable-totp":{"post":{"operationId":"AuthController_enableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/disable-totp":{"post":{"operationId":"AuthController_disableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/dismiss-totp":{"post":{"operationId":"AuthController_dismissTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-totp":{"post":{"operationId":"AuthController_verifyTotp","parameters":[],"responses":{"200":{"description":""}}}},"/auth/change-password":{"post":{"operationId":"AuthController_changePassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/reset-password":{"post":{"operationId":"AuthController_resetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-reset-password-code":{"post":{"operationId":"AuthController_verifyResetPasswordCode","parameters":[],"responses":{"200":{"description":""}}}},"/auth/confirm-reset-password":{"post":{"operationId":"AuthController_confirmResetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/all":{"get":{"operationId":"CatalogController_catalogAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps":{"get":{"operationId":"CatalogController_dataAppsAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps/{id}":{"delete":{"operationId":"CatalogController_dataAppsOne","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase":{"get":{"operationId":"CatalogController_getAllDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase/{id}":{"get":{"operationId":"CatalogController_getOneDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata":{"get":{"operationId":"CatalogController_getAllTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata":{"get":{"operationId":"CatalogController_getOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-preview":{"get":{"operationId":"CatalogController_getOneDataPreview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-status":{"get":{"operationId":"CatalogController_createDataStatus","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-description":{"get":{"operationId":"CatalogController_getDataDescription","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDescription","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-docs":{"get":{"operationId":"CatalogController_getDataDocs","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDocs","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-rating":{"get":{"operationId":"CatalogController_getDataRating","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataRating","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/summary-rating/{id}":{"get":{"operationId":"CatalogController_getSummaryRating","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-comment":{"get":{"operationId":"CatalogController_getDataComment","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataComment","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-review":{"get":{"operationId":"CatalogController_getDataReview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags":{"post":{"operationId":"CatalogController_createTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_findAllTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags/{id}":{"delete":{"operationId":"CatalogController_deleteTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags":{"post":{"operationId":"CatalogController_createTableTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_getAllTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags/{id}":{"delete":{"operationId":"CatalogController_deleteTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules":{"get":{"operationId":"CatalogController_getTableRules","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules/{id}":{"delete":{"operationId":"CatalogController_deleteTableRules","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}} \ No newline at end of file From abbe88f45b91fd8bf64842b6afbb4a99b99e78bd Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Tue, 14 Jun 2022 14:02:06 -0300 Subject: [PATCH 11/18] removed unused statement --- src/modules/inputs/inputs.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 3d8a5c1..022e0fc 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -1,7 +1,6 @@ import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Timeout } from '@nestjs/schedule'; import CronParser, { CronExpression } from 'cron-parser'; -import { lastValueFrom } from 'rxjs'; import { InputsClientService } from 'src/clients/inputs/client.service'; import { IIdRequest, Info } from 'src/clients/inputs/interfaces'; From f6c7420f59a2efde5f759e2ad867ae3889d4ecc3 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 15 Jun 2022 08:44:21 -0300 Subject: [PATCH 12/18] finding inputs using generic strategy --- package-lock.json | 12 ++--- package.json | 2 +- src/clients/inputs/client.service.ts | 51 +++++----------------- src/modules/inputs/inputs.service.ts | 2 + src/modules/pipelines/pipelines.service.ts | 7 ++- 5 files changed, 27 insertions(+), 47 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7954441..03a937e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.5.0-4", + "@victorradael/protospack": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2361,8 +2361,9 @@ }, "node_modules/@victorradael/protospack": { "version": "2.5.0-4", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-4.tgz", - "integrity": "sha512-WgcWv2VPST3IaJsuz9yrSgC+zxsV5rjVB/kAlgB0QAkKvtvS23uQBqWrbvyG4VcKQXG1rGiIwoQ6Fyf23NWG4g==", + "resolved": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", + "integrity": "sha512-mtHLoi0/iMcMIlfcaSyEuD6nzh6ZYO61sJXb0oz2W+nZgcDGx9eSQA4gzGLKecVXix6sIuo2ZniUf3Z1BxuuvQ==", + "license": "ISC", "dependencies": { "rxjs": "^7.5.5" } @@ -10935,9 +10936,8 @@ } }, "@victorradael/protospack": { - "version": "2.5.0-4", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-4.tgz", - "integrity": "sha512-WgcWv2VPST3IaJsuz9yrSgC+zxsV5rjVB/kAlgB0QAkKvtvS23uQBqWrbvyG4VcKQXG1rGiIwoQ6Fyf23NWG4g==", + "version": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", + "integrity": "sha512-mtHLoi0/iMcMIlfcaSyEuD6nzh6ZYO61sJXb0oz2W+nZgcDGx9eSQA4gzGLKecVXix6sIuo2ZniUf3Z1BxuuvQ==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index bd00327..e615ea0 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.5.0-4", + "@victorradael/protospack": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index c13cf14..fd86ad2 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -77,50 +77,23 @@ export class InputsClientService implements OnModuleInit { async findOne(data: IIdRequest) { console.log('InputClientService', 'FindOne'); - - const findOneInputResponse = await new Promise((resolve, reject) => { - this.inputService.FindOne(objectSnakeToCamel(data)).subscribe({ - next(x) { - resolve(objectCamelToSnake(x)); - }, - error(err) { - console.log('Observable Error'); - reject(err); - }, - complete() { - console.log('done'); - }, - }); - }) - .then((res) => res) - .catch((err) => { - throw new Error(err); - }); - + const findOneInputResponse = await lastValueFrom( + this.inputService.FindOne(objectSnakeToCamel(data)), + ).catch((e) => { + throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR); + }); + objectCamelToSnake(findOneInputResponse); return findOneInputResponse; } async findAll(data) { console.log('InputClientService', 'FindAll'); - - const findAllInputResponse = await new Promise((resolve, reject) => { - this.inputService.FindAll(objectSnakeToCamel(data)).subscribe({ - next(x) { - resolve(objectCamelToSnake(x)); - }, - error(err) { - console.log('Observable Error'); - reject(err); - }, - complete() { - console.log('done'); - }, - }); - }) - .then((res) => res) - .catch((err) => { - throw new Error(err); - }); + const findAllInputResponse = await lastValueFrom( + this.inputService.FindAll(objectSnakeToCamel(data)), + ).catch((e) => { + throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR); + }); + findAllInputResponse.inputs.forEach((input) => objectCamelToSnake(input)); return findAllInputResponse; } diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 022e0fc..714e35c 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -1,5 +1,6 @@ import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Timeout } from '@nestjs/schedule'; +import { DecodeGrpcStruct } from '@victorradael/protospack'; import CronParser, { CronExpression } from 'cron-parser'; import { InputsClientService } from 'src/clients/inputs/client.service'; import { IIdRequest, Info } from 'src/clients/inputs/interfaces'; @@ -11,6 +12,7 @@ export class InputsService { secondsInAnHour = 60 * 60; adjustInputPayload(payload) { + if (payload?.input_generic) return DecodeGrpcStruct(payload.input_generic); return payload?.input_s3 || payload?.input_jdbc; } getDifferenceInSeconds(date1: Date, date2: Date) { diff --git a/src/modules/pipelines/pipelines.service.ts b/src/modules/pipelines/pipelines.service.ts index 4755a00..f0987f0 100644 --- a/src/modules/pipelines/pipelines.service.ts +++ b/src/modules/pipelines/pipelines.service.ts @@ -1,4 +1,5 @@ import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { DecodeGrpcStruct } from '@victorradael/protospack'; import { PipelinesClientService } from 'src/clients/pipelines/client.service'; import { IIdRequest, Info } from 'src/clients/pipelines/interfaces'; import { objectCamelToSnake } from 'src/utils/CaseConverter'; @@ -8,7 +9,11 @@ export class PipelinesService { constructor(private pipelineClient: PipelinesClientService) {} adjustPayload(payload) { - payload.input = payload.input?.input_s3 || payload.input?.input_jdbc; + if (payload.input?.input_generic) { + payload.input = DecodeGrpcStruct(payload.input?.input_generic); + } else { + payload.input = payload.input?.input_s3 || payload.input?.input_jdbc; + } } async create(createPipelineDto) { From 2ff2aa049f3b7bac357829ef075629d336f3ae40 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 15 Jun 2022 08:52:30 -0300 Subject: [PATCH 13/18] update protospack --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 03a937e..ebe2f4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", + "@victorradael/protospack": "2.5.0-5", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2360,10 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.5.0-4", - "resolved": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", - "integrity": "sha512-mtHLoi0/iMcMIlfcaSyEuD6nzh6ZYO61sJXb0oz2W+nZgcDGx9eSQA4gzGLKecVXix6sIuo2ZniUf3Z1BxuuvQ==", - "license": "ISC", + "version": "2.5.0-5", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-5.tgz", + "integrity": "sha512-neuxFp5FSBBNjHspLXs/hBdlB/9sy3Wh5AR4O7KyB5QBK0MO5iOCdQf+9wAAKM9N+C2JNVzIq/Mdxe5hAMiJLA==", "dependencies": { "rxjs": "^7.5.5" } @@ -10936,8 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", - "integrity": "sha512-mtHLoi0/iMcMIlfcaSyEuD6nzh6ZYO61sJXb0oz2W+nZgcDGx9eSQA4gzGLKecVXix6sIuo2ZniUf3Z1BxuuvQ==", + "version": "2.5.0-5", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-5.tgz", + "integrity": "sha512-neuxFp5FSBBNjHspLXs/hBdlB/9sy3Wh5AR4O7KyB5QBK0MO5iOCdQf+9wAAKM9N+C2JNVzIq/Mdxe5hAMiJLA==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index e615ea0..8713d99 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "file:../protospack/victorradael-protospack-2.5.0-4.tgz", + "@victorradael/protospack": "2.5.0-5", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", From 3bb014e714ffb8a4f33c33d2cac9981551b825c3 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 15 Jun 2022 10:08:03 -0300 Subject: [PATCH 14/18] get redirect_uri from get_tokens_url_params --- src/clients/inputs/client.service.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index fd86ad2..805e3a0 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -218,9 +218,18 @@ export class InputsClientService implements OnModuleInit { ...secrets, ...credentials, }); - const get_tokens_url_params_obj = JSON.parse( - get_tokens_url_params_string, - ); + let get_tokens_url_params_obj: Record = {}; + try { + get_tokens_url_params_obj = JSON.parse( + get_tokens_url_params_string, + ); + } catch (error) { + throw new HttpException( + 'Erro transformando get_tokens_url_params', + 400, + ); + } + const { redirect_uri } = get_tokens_url_params_obj; for (const key in get_tokens_url_params_obj) { const value = get_tokens_url_params_obj[key]; requestParams.append(key, value); @@ -238,6 +247,7 @@ export class InputsClientService implements OnModuleInit { const responseKey = get_tokens_set_response[key]; credentialsTokens[key] = data[responseKey]; } + redirect_uri.redirect_uri = redirect_uri; } break; } From 2f1c71eefe65d322b8e2ae093143757aae5221cf Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 15 Jun 2022 11:14:22 -0300 Subject: [PATCH 15/18] update protospack and better adjustInputPayload --- package-lock.json | 14 +++++++------- package.json | 2 +- src/clients/inputs/client.service.ts | 11 +++++++---- src/modules/inputs/inputs.service.ts | 9 ++++++--- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index ebe2f4a..c96e062 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.5.0-5", + "@victorradael/protospack": "2.5.0-6", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2360,9 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.5.0-5", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-5.tgz", - "integrity": "sha512-neuxFp5FSBBNjHspLXs/hBdlB/9sy3Wh5AR4O7KyB5QBK0MO5iOCdQf+9wAAKM9N+C2JNVzIq/Mdxe5hAMiJLA==", + "version": "2.5.0-6", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-6.tgz", + "integrity": "sha512-3SYQ+VMWRaBi3HAX8A4n+roTX+pVvrhMFqtunChfjzuHlueWsH/ASttCI2DrqVK2yZP6RdO6unKawd8pmeXmJw==", "dependencies": { "rxjs": "^7.5.5" } @@ -10935,9 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "2.5.0-5", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-5.tgz", - "integrity": "sha512-neuxFp5FSBBNjHspLXs/hBdlB/9sy3Wh5AR4O7KyB5QBK0MO5iOCdQf+9wAAKM9N+C2JNVzIq/Mdxe5hAMiJLA==", + "version": "2.5.0-6", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-6.tgz", + "integrity": "sha512-3SYQ+VMWRaBi3HAX8A4n+roTX+pVvrhMFqtunChfjzuHlueWsH/ASttCI2DrqVK2yZP6RdO6unKawd8pmeXmJw==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index 8713d99..9891a51 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.5.0-5", + "@victorradael/protospack": "2.5.0-6", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index 805e3a0..e8b4291 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -38,9 +38,11 @@ export class InputsClientService implements OnModuleInit { async newCreate(createInputDto: InputNewCreateRequest) { console.log('InputClientService', 'Create'); - return await lastValueFrom( + const input = await lastValueFrom( this.inputService.NewCreate(objectSnakeToCamel(createInputDto)), ); + objectCamelToSnake(input); + return input; } async createGeneric(createInputGeneric) { @@ -55,11 +57,12 @@ export class InputsClientService implements OnModuleInit { }; objectSnakeToCamel(grpcPayload); - const structPayload = await lastValueFrom( + const structReturn = await lastValueFrom( this.inputService.Create(grpcPayload), ); - objectCamelToSnake(structPayload); - return DecodeGrpcStruct(structPayload.input); + objectCamelToSnake(structReturn); + const inputCreated = DecodeGrpcStruct(structReturn.input); + return { input: inputCreated }; } async createS3Inputs(createInputDto: InputCreateS3Request) { diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 714e35c..2d41455 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -13,6 +13,7 @@ export class InputsService { adjustInputPayload(payload) { if (payload?.input_generic) return DecodeGrpcStruct(payload.input_generic); + if (!payload.input_s3 && !payload.input_jdbc) return payload; return payload?.input_s3 || payload?.input_jdbc; } getDifferenceInSeconds(date1: Date, date2: Date) { @@ -60,17 +61,19 @@ export class InputsService { input: inputPayload, info, }); - return response; + break; case 'oracle': case 'mysql': case 'postgresql': case 'sqlserver': response = await this.inputClient.newCreate(data); - return { input: response?.input?.input_jdbc }; + break; default: response = await this.inputClient.createGeneric(data); - return response; + break; } + const adjustedInput = this.adjustInputPayload(response.input); + return { ...response, input: adjustedInput }; } async getAvailableEntities(data): Promise<{ entities: string[] }> { return await this.inputClient.getAvailableEntities(data); From d057495b2e7fdb1a871d21200277f09bf1abd7ce Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 15 Jun 2022 15:19:45 -0300 Subject: [PATCH 16/18] FIX: redirect_uri.redirect_uri fixed --- src/clients/inputs/client.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clients/inputs/client.service.ts b/src/clients/inputs/client.service.ts index e8b4291..968bb4b 100644 --- a/src/clients/inputs/client.service.ts +++ b/src/clients/inputs/client.service.ts @@ -209,7 +209,7 @@ export class InputsClientService implements OnModuleInit { } async getInputTokens(input: InputModel) { const { credentials, options } = input; - const credentialsTokens = {}; + const credentialsTokens: any = {}; switch (credentials.connection_type) { case 'oauth': const { oauth } = options; @@ -250,7 +250,7 @@ export class InputsClientService implements OnModuleInit { const responseKey = get_tokens_set_response[key]; credentialsTokens[key] = data[responseKey]; } - redirect_uri.redirect_uri = redirect_uri; + credentialsTokens.redirect_uri = redirect_uri; } break; } From 668c5fe6a5306cee6281d67c6f5b2691c6d66070 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Mon, 20 Jun 2022 11:36:26 -0300 Subject: [PATCH 17/18] WIP: Documentation of input API --- src/modules/inputs/dtos/input.model.ts | 182 ++++++++++++++++++++++++ src/modules/inputs/inputs.controller.ts | 29 +++- swagger.json | 2 +- 3 files changed, 207 insertions(+), 6 deletions(-) diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index 4bc5b47..73f0d19 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -1,3 +1,5 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + export class InputModel { category: string; plugin: string; @@ -18,3 +20,183 @@ export class InputModel { }; }; } + +class GoogleAnalyticsClientSecrets { + @ApiProperty() + type: string; + @ApiProperty() + project_id: string; + @ApiProperty() + private_key_id: string; + @ApiProperty() + private_key: string; + @ApiProperty() + client_email: string; + @ApiProperty() + client_id: string; + @ApiProperty() + auth_uri: string; + @ApiProperty() + token_uri: string; + @ApiProperty() + auth_provider_x509_cert_url: string; + @ApiProperty() + client_x509_cert_url: string; +} +class OauthObject { + @ApiPropertyOptional() + get_tokens_url: string; + @ApiPropertyOptional() + get_tokens_url_params: string; + @ApiPropertyOptional() + get_tokens_set_response: Record; + @ApiPropertyOptional() + content_type: string; +} +class InputOptions { + @ApiPropertyOptional() + oauth: OauthObject; + @ApiPropertyOptional() + skip_select_columns: true; + @ApiPropertyOptional() + skip_select_entities: false; + @ApiPropertyOptional() + skip_transformation: true; +} + +class CredentialsJdbc { + @ApiPropertyOptional() + jdbc_user: string; + @ApiPropertyOptional() + jdbc_password: string; + @ApiPropertyOptional() + database: string; + @ApiPropertyOptional() + endpoint: string; + @ApiPropertyOptional() + port: string; + @ApiPropertyOptional() + engine: string; + @ApiPropertyOptional() + schema: string; +} +class FileFormatParams { + @ApiPropertyOptional() + file_format: string; + @ApiPropertyOptional() + encoding: string; +} + +class Credentials extends CredentialsJdbc { + @ApiProperty() + connection_type: string; + + @ApiPropertyOptional() + client_aws_access_key_id: string; + @ApiPropertyOptional() + client_aws_secret_access_key: string; + @ApiPropertyOptional() + client_bucket: string; + @ApiPropertyOptional() + file_to_extract: string; + @ApiPropertyOptional() + file_format_params: FileFormatParams; + + @ApiPropertyOptional() + view_id: string; + @ApiPropertyOptional() + client_secrets: GoogleAnalyticsClientSecrets; + @ApiPropertyOptional() + start_date: string; + @ApiPropertyOptional() + end_date: string; + + @ApiPropertyOptional() + oauth_code: string; +} +class Column { + @ApiProperty() + name: string; + @ApiProperty() + type: string; +} +class TableColumns { + @ApiProperty() + name: string; + @ApiProperty() + columns: Column[]; + @ApiProperty() + references: Column[]; +} +export class GetAvailableEntitiesReq { + @ApiProperty() + plugin: string; +} + +export class GetAvailableEntitiesRes { + @ApiProperty() + entities: string[]; +} + +export class TestConnectionGetColumnsReq { + @ApiProperty() + plugin: string; + @ApiProperty() + tables: string[]; + @ApiPropertyOptional() + credentials: CredentialsJdbc; + @ApiPropertyOptional() + id: string; +} +export class TestConnectionGetColumnsRes { + @ApiProperty() + tables: TableColumns[]; +} + +export class TestConnectionReq { + @ApiProperty() + plugin: string; + @ApiProperty() + credentials: CredentialsJdbc; +} +export class TestConnectionRes { + @ApiProperty() + connection_state: boolean; + @ApiProperty() + total_entities: number; + @ApiProperty() + database_tables: string[]; +} +export class CreateInputReq { + @ApiProperty() + plugin: string; + @ApiProperty() + category: string; + @ApiProperty() + credentials: Credentials; + @ApiProperty() + name: string; + @ApiPropertyOptional() + options: InputOptions; +} + +export class Input { + @ApiProperty() + id: string; + @ApiProperty() + category: string; + @ApiProperty() + plugin: string; + @ApiProperty() + name: string; + @ApiProperty() + cron: string; + @ApiProperty() + credentials: Credentials; + @ApiProperty() + client_id: string; + @ApiProperty() + created_at: string; + @ApiProperty() + updated_at: string; +} diff --git a/src/modules/inputs/inputs.controller.ts b/src/modules/inputs/inputs.controller.ts index 3373e0b..6ddf90a 100644 --- a/src/modules/inputs/inputs.controller.ts +++ b/src/modules/inputs/inputs.controller.ts @@ -10,23 +10,40 @@ import { import { InputsService } from './inputs.service'; import { InputsClientService } from 'src/clients/inputs/client.service'; import { UpdateInputRequest } from 'src/clients/inputs/interfaces'; -import { InputNewCreateRequest } from '@victorradael/protospack'; +import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { + CreateInputReq, + GetAvailableEntitiesReq, + GetAvailableEntitiesRes, + Input, + TestConnectionGetColumnsReq, + TestConnectionGetColumnsRes, + TestConnectionReq, + TestConnectionRes, +} from './dtos/input.model'; +@ApiTags('inputs') @Controller('inputs') export class InputsController { inputService: InputsService; constructor(private inputsClientService: InputsClientService) { this.inputService = new InputsService(this.inputsClientService); } + @Get('available-entities/:plugin') - async getAvailableEntities(@Body() body, @Param() params) { + @ApiOkResponse({ type: GetAvailableEntitiesRes }) + async getAvailableEntities( + @Body() body, + @Param() params: GetAvailableEntitiesReq, + ) { const { info } = body; const { plugin } = params; return await this.inputService.getAvailableEntities({ info, plugin }); } @Post('/test-connection') - async testConnection(@Body() data) { + @ApiOkResponse({ type: TestConnectionRes }) + async testConnection(@Body() data: TestConnectionReq) { console.log(`/test-connection`, 'ON TEST CONNECTION ROUTE'); const inputService = new InputsService(this.inputsClientService); @@ -36,7 +53,8 @@ export class InputsController { } @Post('/test-connection/get-columns') - async getColumns(@Body() data) { + @ApiOkResponse({ type: TestConnectionGetColumnsRes }) + async getColumns(@Body() data: TestConnectionGetColumnsReq) { console.log( `/test-connection/get-columns`, 'ON TEST CONNECTION GET COLUMNS ROUTE', @@ -49,7 +67,8 @@ export class InputsController { } @Post() - async create(@Body() createInputDto: InputNewCreateRequest) { + @ApiOkResponse({ type: Input }) + async create(@Body() createInputDto: CreateInputReq) { console.log(`/input`, 'ON CREATE ROUTE'); const inputService = new InputsService(this.inputsClientService); diff --git a/swagger.json b/swagger.json index bfdeb79..2a30295 100644 --- a/swagger.json +++ b/swagger.json @@ -1 +1 @@ -{"openapi":"3.0.0","paths":{"/inputs/available-entities/{plugin}":{"get":{"operationId":"InputsController_getAvailableEntities","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"responses":{"201":{"description":""}}}},"/inputs/test-connection/get-columns":{"post":{"operationId":"InputsController_getColumns","parameters":[],"responses":{"201":{"description":""}}}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"patch":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/start/{id}":{"post":{"operationId":"PipelinesController_activate","parameters":[],"responses":{"201":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn_old","parameters":[],"responses":{"200":{"description":""}}}},"/auth/login":{"post":{"operationId":"AuthController_signIn_login","parameters":[],"responses":{"200":{"description":""}}}},"/auth/sign-in":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"200":{"description":""}}}},"/auth/refresh-access-token":{"post":{"operationId":"AuthController_refreshAccessToken","parameters":[],"responses":{"200":{"description":""}}}},"/auth/enable-totp":{"post":{"operationId":"AuthController_enableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/disable-totp":{"post":{"operationId":"AuthController_disableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/dismiss-totp":{"post":{"operationId":"AuthController_dismissTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-totp":{"post":{"operationId":"AuthController_verifyTotp","parameters":[],"responses":{"200":{"description":""}}}},"/auth/change-password":{"post":{"operationId":"AuthController_changePassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/reset-password":{"post":{"operationId":"AuthController_resetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-reset-password-code":{"post":{"operationId":"AuthController_verifyResetPasswordCode","parameters":[],"responses":{"200":{"description":""}}}},"/auth/confirm-reset-password":{"post":{"operationId":"AuthController_confirmResetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/all":{"get":{"operationId":"CatalogController_catalogAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps":{"get":{"operationId":"CatalogController_dataAppsAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps/{id}":{"delete":{"operationId":"CatalogController_dataAppsOne","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase":{"get":{"operationId":"CatalogController_getAllDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase/{id}":{"get":{"operationId":"CatalogController_getOneDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata":{"get":{"operationId":"CatalogController_getAllTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata":{"get":{"operationId":"CatalogController_getOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-preview":{"get":{"operationId":"CatalogController_getOneDataPreview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-status":{"get":{"operationId":"CatalogController_createDataStatus","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-description":{"get":{"operationId":"CatalogController_getDataDescription","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDescription","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-docs":{"get":{"operationId":"CatalogController_getDataDocs","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDocs","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-rating":{"get":{"operationId":"CatalogController_getDataRating","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataRating","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/summary-rating/{id}":{"get":{"operationId":"CatalogController_getSummaryRating","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-comment":{"get":{"operationId":"CatalogController_getDataComment","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataComment","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-review":{"get":{"operationId":"CatalogController_getDataReview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags":{"post":{"operationId":"CatalogController_createTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_findAllTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags/{id}":{"delete":{"operationId":"CatalogController_deleteTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags":{"post":{"operationId":"CatalogController_createTableTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_getAllTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags/{id}":{"delete":{"operationId":"CatalogController_deleteTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules":{"get":{"operationId":"CatalogController_getTableRules","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules/{id}":{"delete":{"operationId":"CatalogController_deleteTableRules","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{}}} \ No newline at end of file +{"openapi":"3.0.0","paths":{"/inputs/available-entities/{plugin}":{"get":{"operationId":"InputsController_getAvailableEntities","parameters":[{"name":"plugin","required":true,"in":"path","schema":{"type":"string"}}],"responses":{"200":{"description":"","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetAvailableEntitiesRes"}}}}},"tags":["inputs"]}},"/inputs/test-connection":{"post":{"operationId":"InputsController_testConnection","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestConnectionReq"}}}},"responses":{"200":{"description":"","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestConnectionRes"}}}}},"tags":["inputs"]}},"/inputs/test-connection/get-columns":{"post":{"operationId":"InputsController_getColumns","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestConnectionGetColumnsReq"}}}},"responses":{"200":{"description":"","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestConnectionGetColumnsRes"}}}}},"tags":["inputs"]}},"/inputs":{"post":{"operationId":"InputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInputReq"}}}},"responses":{"200":{"description":"","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Input"}}}}},"tags":["inputs"]},"get":{"operationId":"InputsController_findAll","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]}},"/inputs/{id}":{"get":{"operationId":"InputsController_findOne","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]},"patch":{"operationId":"InputsController_update","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]},"delete":{"operationId":"InputsController_delete","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]}},"/transformations":{"post":{"operationId":"TransformationsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"TransformationsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/transformations/{id}":{"get":{"operationId":"TransformationsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"TransformationsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"TransformationsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/outputs":{"post":{"operationId":"OutputsController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"OutputsController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/outputs/{id}":{"get":{"operationId":"OutputsController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"OutputsController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"OutputsController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/start/{id}":{"post":{"operationId":"PipelinesController_activate","parameters":[],"responses":{"201":{"description":""}}}},"/pipelines/{id}/status":{"get":{"operationId":"PipelinesController_getPipelineStatus","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}/{details}":{"get":{"operationId":"PipelinesController_getPipelineLogs","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines":{"post":{"operationId":"PipelinesController_create","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"PipelinesController_findAll","parameters":[],"responses":{"200":{"description":""}}}},"/pipelines/{id}":{"get":{"operationId":"PipelinesController_findOne","parameters":[],"responses":{"200":{"description":""}}},"put":{"operationId":"PipelinesController_update","parameters":[],"responses":{"200":{"description":""}}},"delete":{"operationId":"PipelinesController_delete","parameters":[],"responses":{"200":{"description":""}}}},"/auth":{"post":{"operationId":"AuthController_signIn_old","parameters":[],"responses":{"200":{"description":""}}}},"/auth/login":{"post":{"operationId":"AuthController_signIn_login","parameters":[],"responses":{"200":{"description":""}}}},"/auth/sign-in":{"post":{"operationId":"AuthController_signIn","parameters":[],"responses":{"200":{"description":""}}}},"/auth/refresh-access-token":{"post":{"operationId":"AuthController_refreshAccessToken","parameters":[],"responses":{"200":{"description":""}}}},"/auth/enable-totp":{"post":{"operationId":"AuthController_enableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/disable-totp":{"post":{"operationId":"AuthController_disableTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/dismiss-totp":{"post":{"operationId":"AuthController_dismissTotpMFA","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-totp":{"post":{"operationId":"AuthController_verifyTotp","parameters":[],"responses":{"200":{"description":""}}}},"/auth/change-password":{"post":{"operationId":"AuthController_changePassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/reset-password":{"post":{"operationId":"AuthController_resetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/auth/verify-reset-password-code":{"post":{"operationId":"AuthController_verifyResetPasswordCode","parameters":[],"responses":{"200":{"description":""}}}},"/auth/confirm-reset-password":{"post":{"operationId":"AuthController_confirmResetPassword","parameters":[],"responses":{"200":{"description":""}}}},"/health":{"get":{"operationId":"HealthController_check","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/all":{"get":{"operationId":"CatalogController_catalogAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps":{"get":{"operationId":"CatalogController_dataAppsAll","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data_apps/{id}":{"delete":{"operationId":"CatalogController_dataAppsOne","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase":{"get":{"operationId":"CatalogController_getAllDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/dashboard-metabase/{id}":{"get":{"operationId":"CatalogController_getOneDashboardMetabase","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata":{"get":{"operationId":"CatalogController_getAllTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneTableMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata":{"get":{"operationId":"CatalogController_getOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/column-metadata/{id}":{"delete":{"operationId":"CatalogController_deleteOneColumnMetadata","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-preview":{"get":{"operationId":"CatalogController_getOneDataPreview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-status":{"get":{"operationId":"CatalogController_createDataStatus","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-description":{"get":{"operationId":"CatalogController_getDataDescription","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDescription","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-docs":{"get":{"operationId":"CatalogController_getDataDocs","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataDocs","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-rating":{"get":{"operationId":"CatalogController_getDataRating","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataRating","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/summary-rating/{id}":{"get":{"operationId":"CatalogController_getSummaryRating","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/data-comment":{"get":{"operationId":"CatalogController_getDataComment","parameters":[],"responses":{"200":{"description":""}}},"post":{"operationId":"CatalogController_createDataComment","parameters":[],"responses":{"201":{"description":""}}}},"/catalog/data-review":{"get":{"operationId":"CatalogController_getDataReview","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags":{"post":{"operationId":"CatalogController_createTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_findAllTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/tags/{id}":{"delete":{"operationId":"CatalogController_deleteTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags":{"post":{"operationId":"CatalogController_createTableTags","parameters":[],"responses":{"201":{"description":""}}},"get":{"operationId":"CatalogController_getAllTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-tags/{id}":{"delete":{"operationId":"CatalogController_deleteTableTags","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules":{"get":{"operationId":"CatalogController_getTableRules","parameters":[],"responses":{"200":{"description":""}}}},"/catalog/table-rules/{id}":{"delete":{"operationId":"CatalogController_deleteTableRules","parameters":[],"responses":{"200":{"description":""}}}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[],"servers":[],"components":{"securitySchemes":{"bearer":{"scheme":"bearer","bearerFormat":"JWT","type":"http"}},"schemas":{"GetAvailableEntitiesRes":{"type":"object","properties":{"entities":{"type":"array","items":{"type":"string"}}},"required":["entities"]},"CredentialsJdbc":{"type":"object","properties":{"jdbc_user":{"type":"string"},"jdbc_password":{"type":"string"},"database":{"type":"string"},"endpoint":{"type":"string"},"port":{"type":"string"},"engine":{"type":"string"},"schema":{"type":"string"}}},"TestConnectionReq":{"type":"object","properties":{"plugin":{"type":"string"},"credentials":{"$ref":"#/components/schemas/CredentialsJdbc"}},"required":["plugin","credentials"]},"TestConnectionRes":{"type":"object","properties":{"connection_state":{"type":"boolean"},"total_entities":{"type":"number"},"database_tables":{"type":"array","items":{"type":"string"}}},"required":["connection_state","total_entities","database_tables"]},"TestConnectionGetColumnsReq":{"type":"object","properties":{"plugin":{"type":"string"},"tables":{"type":"array","items":{"type":"string"}},"credentials":{"$ref":"#/components/schemas/CredentialsJdbc"},"id":{"type":"string"}},"required":["plugin","tables"]},"TestConnectionGetColumnsRes":{"type":"object","properties":{"tables":{"type":"array","items":{"type":"string"}}},"required":["tables"]},"FileFormatParams":{"type":"object","properties":{"file_format":{"type":"string"},"encoding":{"type":"string"}}},"GoogleAnalyticsClientSecrets":{"type":"object","properties":{"type":{"type":"string"},"project_id":{"type":"string"},"private_key_id":{"type":"string"},"private_key":{"type":"string"},"client_email":{"type":"string"},"client_id":{"type":"string"},"auth_uri":{"type":"string"},"token_uri":{"type":"string"},"auth_provider_x509_cert_url":{"type":"string"},"client_x509_cert_url":{"type":"string"}},"required":["type","project_id","private_key_id","private_key","client_email","client_id","auth_uri","token_uri","auth_provider_x509_cert_url","client_x509_cert_url"]},"Credentials":{"type":"object","properties":{"jdbc_user":{"type":"string"},"jdbc_password":{"type":"string"},"database":{"type":"string"},"endpoint":{"type":"string"},"port":{"type":"string"},"engine":{"type":"string"},"schema":{"type":"string"},"connection_type":{"type":"string"},"client_aws_access_key_id":{"type":"string"},"client_aws_secret_access_key":{"type":"string"},"client_bucket":{"type":"string"},"file_to_extract":{"type":"string"},"file_format_params":{"$ref":"#/components/schemas/FileFormatParams"},"view_id":{"type":"string"},"client_secrets":{"$ref":"#/components/schemas/GoogleAnalyticsClientSecrets"},"start_date":{"type":"string"},"end_date":{"type":"string"},"oauth_code":{"type":"string"}},"required":["connection_type"]},"OauthObject":{"type":"object","properties":{"get_tokens_url":{"type":"string"},"get_tokens_url_params":{"type":"string"},"get_tokens_set_response":{"type":"object"},"content_type":{"type":"string"}}},"InputOptions":{"type":"object","properties":{"oauth":{"$ref":"#/components/schemas/OauthObject"},"skip_select_columns":{"type":"boolean"},"skip_select_entities":{"type":"boolean"},"skip_transformation":{"type":"boolean"}}},"CreateInputReq":{"type":"object","properties":{"plugin":{"type":"string"},"category":{"type":"string"},"credentials":{"$ref":"#/components/schemas/Credentials"},"name":{"type":"string"},"options":{"$ref":"#/components/schemas/InputOptions"}},"required":["plugin","category","credentials","name"]},"Input":{"type":"object","properties":{"id":{"type":"string"},"category":{"type":"string"},"plugin":{"type":"string"},"name":{"type":"string"},"cron":{"type":"string"},"credentials":{"$ref":"#/components/schemas/Credentials"},"client_id":{"type":"string"},"created_at":{"type":"string"},"updated_at":{"type":"string"}},"required":["id","category","plugin","name","cron","credentials","client_id","created_at","updated_at"]}}}} \ No newline at end of file From 6c3eef71817da92f5eb7510b7c910dbb14785f58 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Tue, 21 Jun 2022 10:13:20 -0300 Subject: [PATCH 18/18] update protospack and deploy file --- .github/workflows/deploy-manually.yml | 6 +++--- package-lock.json | 14 +++++++------- package.json | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy-manually.yml b/.github/workflows/deploy-manually.yml index b204e48..dea0b1d 100644 --- a/.github/workflows/deploy-manually.yml +++ b/.github/workflows/deploy-manually.yml @@ -134,7 +134,7 @@ jobs: eb deploy - name: Remove Docker's Trash - continue-on-error: true + if: always() run: | - docker system prune - docker rmi -f $(docker images -aq) + docker system prune --volumes -a -f + docker system df diff --git a/package-lock.json b/package-lock.json index c96e062..52d832f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.5.0-6", + "@victorradael/protospack": "2.5.0", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0", @@ -2360,9 +2360,9 @@ } }, "node_modules/@victorradael/protospack": { - "version": "2.5.0-6", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-6.tgz", - "integrity": "sha512-3SYQ+VMWRaBi3HAX8A4n+roTX+pVvrhMFqtunChfjzuHlueWsH/ASttCI2DrqVK2yZP6RdO6unKawd8pmeXmJw==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0.tgz", + "integrity": "sha512-DORlm3uHAYVH9dlcmwyOxjA4rk1UZWFyYUpD02k0bFfrH+uzSWe8qiPtTgZym9P8U9D8bXYRWeLlroasBj2Rdw==", "dependencies": { "rxjs": "^7.5.5" } @@ -10935,9 +10935,9 @@ } }, "@victorradael/protospack": { - "version": "2.5.0-6", - "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0-6.tgz", - "integrity": "sha512-3SYQ+VMWRaBi3HAX8A4n+roTX+pVvrhMFqtunChfjzuHlueWsH/ASttCI2DrqVK2yZP6RdO6unKawd8pmeXmJw==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@victorradael/protospack/-/protospack-2.5.0.tgz", + "integrity": "sha512-DORlm3uHAYVH9dlcmwyOxjA4rk1UZWFyYUpD02k0bFfrH+uzSWe8qiPtTgZym9P8U9D8bXYRWeLlroasBj2Rdw==", "requires": { "rxjs": "^7.5.5" } diff --git a/package.json b/package.json index 9891a51..b31a45c 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@nestjs/platform-express": "^8.4.3", "@nestjs/schedule": "^1.1.0", "@nestjs/swagger": "^5.2.1", - "@victorradael/protospack": "2.5.0-6", + "@victorradael/protospack": "2.5.0", "axios": "^0.25.0", "cron-parser": "^4.4.0", "dotenv": "^14.2.0",