From 3e3268cb261ad3ae9b12e79214f1245f72602056 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Wed, 16 Nov 2022 10:37:21 -0300 Subject: [PATCH] FIX: New getAvailableEntities payload --- docsfera.json | 500 ++---------------------- package-lock.json | 14 +- package.json | 2 +- src/modules/inputs/dtos/input.model.ts | 166 +------- src/modules/inputs/inputs.controller.ts | 47 +-- src/modules/inputs/inputs.service.ts | 48 +-- 6 files changed, 57 insertions(+), 720 deletions(-) diff --git a/docsfera.json b/docsfera.json index faa111a..f3e27da 100644 --- a/docsfera.json +++ b/docsfera.json @@ -2394,88 +2394,6 @@ ] } }, - "/inputs/test-connection": { - "post": { - "operationId": "InputsController_testConnection", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/TestConnectionReq" - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/TestConnectionRes" - } - } - } - }, - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Inputs" - ] - } - }, - "/inputs/test-connection/get-columns": { - "post": { - "operationId": "InputsController_getColumns", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/TestConnectionGetColumnsReq" - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/TestConnectionGetColumnsRes" - } - } - } - }, - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Inputs" - ] - } - }, "/inputs": { "post": { "operationId": "InputsController_create", @@ -3298,9 +3216,9 @@ } }, "info": { - "title": "Maestro - catalog-permissions", + "title": "Maestro - feat/incremental-table", "description": "Documentation for Maestro gateway", - "version": "14.11.2022 - 16:15", + "version": "16.11.2022 - 9:54", "contact": {} }, "tags": [], @@ -5174,13 +5092,31 @@ "updated_at" ] }, + "AvailableEntity": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "replication_methods": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "replication_methods" + ] + }, "GetAvailableEntitiesRes": { "type": "object", "properties": { "entities": { "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/AvailableEntity" } } }, @@ -5188,99 +5124,6 @@ "entities" ] }, - "CredentialsJdbc": { - "type": "object", - "properties": { - "jdbc_user": { - "type": "string" - }, - "jdbc_password": { - "type": "string" - }, - "database": { - "type": "string" - }, - "endpoint": { - "type": "string" - }, - "port": { - "type": "string" - }, - "engine": { - "type": "string" - }, - "schema": { - "type": "string" - } - } - }, - "TestConnectionReq": { - "type": "object", - "properties": { - "plugin": { - "type": "string" - }, - "credentials": { - "$ref": "#/components/schemas/CredentialsJdbc" - }, - "info": { - "type": "object" - } - }, - "required": [ - "plugin", - "credentials" - ] - }, - "TestConnectionRes": { - "type": "object", - "properties": { - "connection_state": { - "type": "boolean" - }, - "total_entities": { - "type": "number" - }, - "database_tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "connection_state", - "total_entities", - "database_tables" - ] - }, - "TestConnectionGetColumnsReq": { - "type": "object", - "properties": { - "plugin": { - "type": "string" - }, - "tables": { - "type": "array", - "items": { - "type": "string" - } - }, - "credentials": { - "$ref": "#/components/schemas/CredentialsJdbc" - }, - "id": { - "type": "string" - }, - "info": { - "type": "object" - } - }, - "required": [ - "plugin", - "tables" - ] - }, "TableColumns": { "type": "object", "properties": { @@ -5298,203 +5141,16 @@ "items": { "type": "string" } + }, + "type": { + "type": "string" } }, "required": [ "name", "columns", - "references" - ] - }, - "TestConnectionGetColumnsRes": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "$ref": "#/components/schemas/TableColumns" - } - } - }, - "required": [ - "tables" - ] - }, - "FileFormatParams": { - "type": "object", - "properties": { - "sep": { - "type": "string" - }, - "header": { - "type": "boolean" - }, - "encoding": { - "type": "string" - } - }, - "required": [ - "sep", - "header", - "encoding" - ] - }, - "GoogleAnalyticsClientSecrets": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "project_id": { - "type": "string" - }, - "private_key_id": { - "type": "string" - }, - "private_key": { - "type": "string" - }, - "client_email": { - "type": "string" - }, - "client_id": { - "type": "string" - }, - "auth_uri": { - "type": "string" - }, - "token_uri": { - "type": "string" - }, - "auth_provider_x509_cert_url": { - "type": "string" - }, - "client_x509_cert_url": { - "type": "string" - } - }, - "required": [ - "type", - "project_id", - "private_key_id", - "private_key", - "client_email", - "client_id", - "auth_uri", - "token_uri", - "auth_provider_x509_cert_url", - "client_x509_cert_url" - ] - }, - "Credentials": { - "type": "object", - "properties": { - "jdbc_user": { - "type": "string" - }, - "jdbc_password": { - "type": "string" - }, - "database": { - "type": "string" - }, - "endpoint": { - "type": "string" - }, - "port": { - "type": "string" - }, - "engine": { - "type": "string" - }, - "schema": { - "type": "string" - }, - "connection_type": { - "type": "string" - }, - "client_aws_access_key_id": { - "type": "string" - }, - "client_aws_secret_access_key": { - "type": "string" - }, - "client_bucket": { - "type": "string" - }, - "file_to_extract": { - "type": "string" - }, - "file_format_params": { - "$ref": "#/components/schemas/FileFormatParams" - }, - "view_id": { - "type": "string" - }, - "client_secrets": { - "$ref": "#/components/schemas/GoogleAnalyticsClientSecrets" - }, - "start_date": { - "type": "string" - }, - "end_date": { - "type": "string" - }, - "oauth_code": { - "type": "string" - } - }, - "required": [ - "connection_type" - ] - }, - "OauthObject": { - "type": "object", - "properties": { - "get_tokens_url": { - "type": "string" - }, - "get_tokens_url_params": { - "type": "string" - }, - "get_tokens_set_response": { - "type": "object" - }, - "content_type": { - "type": "string" - } - } - }, - "InputOptions": { - "type": "object", - "properties": { - "oauth": { - "$ref": "#/components/schemas/OauthObject" - }, - "skip_select_columns": { - "type": "boolean" - }, - "skip_select_entities": { - "type": "boolean" - }, - "skip_transformation": { - "type": "boolean" - } - } - }, - "AuthParameters": { - "type": "object", - "properties": { - "aws_access_key_id": { - "type": "string" - }, - "aws_secret_access_key": { - "type": "string" - } - }, - "required": [ - "aws_access_key_id", - "aws_secret_access_key" + "references", + "type" ] }, "CreateInputReq": { @@ -5517,58 +5173,6 @@ }, "type": { "type": "string" - }, - "category": { - "type": "string", - "deprecated": true - }, - "credentials": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/Credentials" - } - ] - }, - "options": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/InputOptions" - } - ] - }, - "info": { - "type": "object", - "deprecated": true - }, - "cron": { - "type": "string", - "deprecated": true - }, - "source_bucket": { - "type": "string", - "deprecated": true - }, - "source_prefix": { - "type": "string", - "deprecated": true - }, - "auth_parameters": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/AuthParameters" - } - ] - }, - "file_format_params": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/FileFormatParams" - } - ] } }, "required": [ @@ -5607,58 +5211,6 @@ }, "updated_at": { "type": "string" - }, - "category": { - "type": "string", - "deprecated": true - }, - "credentials": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/Credentials" - } - ] - }, - "options": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/InputOptions" - } - ] - }, - "info": { - "type": "object", - "deprecated": true - }, - "cron": { - "type": "string", - "deprecated": true - }, - "source_bucket": { - "type": "string", - "deprecated": true - }, - "source_prefix": { - "type": "string", - "deprecated": true - }, - "auth_parameters": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/AuthParameters" - } - ] - }, - "file_format_params": { - "deprecated": true, - "allOf": [ - { - "$ref": "#/components/schemas/FileFormatParams" - } - ] } }, "required": [ diff --git a/package-lock.json b/package-lock.json index f8c9084..02c6ff3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.19.4", + "@dadosfera/protospack-v2": "3.20.0-beta.1", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.19.4", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.4.tgz", - "integrity": "sha512-Vb6bdFgQKYHf0kDK9kuMtN+nt3nkaMhJZ3wNVaUmQNxsnpGyARWBL29R2n+YnxS8g23tRwuj331sjg+DY0Fnhg==", + "version": "3.20.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.20.0-beta.1.tgz", + "integrity": "sha512-5OJC8Gvyn20qXXct/xm2HyBiOkKlcXBnUo4bnZkY0leWcJxEiHEo1M7PsXmdXBnL3KufJKmL/STE1nPZjYx7DA==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12330,9 +12330,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.19.4", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.4.tgz", - "integrity": "sha512-Vb6bdFgQKYHf0kDK9kuMtN+nt3nkaMhJZ3wNVaUmQNxsnpGyARWBL29R2n+YnxS8g23tRwuj331sjg+DY0Fnhg==", + "version": "3.20.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.20.0-beta.1.tgz", + "integrity": "sha512-5OJC8Gvyn20qXXct/xm2HyBiOkKlcXBnUo4bnZkY0leWcJxEiHEo1M7PsXmdXBnL3KufJKmL/STE1nPZjYx7DA==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index 03bfb2a..c5b33b9 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.19.4", + "@dadosfera/protospack-v2": "3.20.0-beta.1", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index fd8c5c5..7b40d1e 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -1,106 +1,5 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { Info } from 'protospack/dist/lib/interfaces'; -export class GoogleAnalyticsClientSecrets { - @ApiProperty() - type: string; - @ApiProperty() - project_id: string; - @ApiProperty() - private_key_id: string; - @ApiProperty() - private_key: string; - @ApiProperty() - client_email: string; - @ApiProperty() - client_id: string; - @ApiProperty() - auth_uri: string; - @ApiProperty() - token_uri: string; - @ApiProperty() - auth_provider_x509_cert_url: string; - @ApiProperty() - client_x509_cert_url: string; -} -export class OauthObject { - @ApiPropertyOptional() - get_tokens_url: string; - @ApiPropertyOptional() - get_tokens_url_params: string; - @ApiPropertyOptional() - get_tokens_set_response: Record; - @ApiPropertyOptional() - content_type: string; -} -export class InputOptions { - @ApiPropertyOptional() - oauth: OauthObject; - @ApiPropertyOptional() - skip_select_columns: true; - @ApiPropertyOptional() - skip_select_entities: false; - @ApiPropertyOptional() - skip_transformation: true; -} -export class FileFormatParams { - @ApiProperty() - sep: string; - @ApiProperty() - header: boolean; - @ApiProperty() - encoding: string; -} - -export class AuthParameters { - @ApiProperty() - aws_access_key_id: string; - @ApiProperty() - aws_secret_access_key: string; -} -export class CredentialsJdbc { - @ApiPropertyOptional() - jdbc_user: string; - @ApiPropertyOptional() - jdbc_password: string; - @ApiPropertyOptional() - database: string; - @ApiPropertyOptional() - endpoint: string; - @ApiPropertyOptional() - port: string; - @ApiPropertyOptional() - engine: string; - @ApiPropertyOptional() - schema: string; -} -export class Credentials extends CredentialsJdbc { - @ApiProperty() - connection_type: string; - - @ApiPropertyOptional() - client_aws_access_key_id: string; - @ApiPropertyOptional() - client_aws_secret_access_key: string; - @ApiPropertyOptional() - client_bucket: string; - @ApiPropertyOptional() - file_to_extract: string; - @ApiPropertyOptional() - file_format_params: FileFormatParams; - - @ApiPropertyOptional() - view_id: string; - @ApiPropertyOptional() - client_secrets: GoogleAnalyticsClientSecrets; - @ApiPropertyOptional() - start_date: string; - @ApiPropertyOptional() - end_date: string; - - @ApiPropertyOptional() - oauth_code: string; -} export class Column { @ApiProperty() name: string; @@ -114,6 +13,14 @@ export class TableColumns { columns: string[]; @ApiProperty() references: Column[]; + @ApiProperty() + type: string; +} +export class AvailableEntity { + @ApiProperty() + name: string; + @ApiProperty() + replication_methods: string[]; } export class Input { @@ -133,26 +40,6 @@ export class Input { created_at: string; @ApiProperty() updated_at: string; - - //deprecated attributes (for retro compatibility) - @ApiPropertyOptional({ deprecated: true }) - category: string; - @ApiPropertyOptional({ deprecated: true }) - credentials: Credentials; - @ApiPropertyOptional({ deprecated: true }) - options: InputOptions; - @ApiPropertyOptional({ deprecated: true }) - info: Info; - @ApiPropertyOptional({ deprecated: true }) - cron: string; - @ApiPropertyOptional({ deprecated: true }) - source_bucket: string; - @ApiPropertyOptional({ deprecated: true }) - source_prefix: string; - @ApiPropertyOptional({ deprecated: true }) - auth_parameters: AuthParameters; - @ApiPropertyOptional({ deprecated: true }) - file_format_params: FileFormatParams; } export class GetAvailableEntitiesReq { @ApiProperty() @@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq { } export class GetAvailableEntitiesRes { - @ApiProperty() - entities: string[]; + @ApiProperty({ type: [AvailableEntity] }) + entities: AvailableEntity[]; } -export class TestConnectionGetColumnsReq { - @ApiProperty() - plugin: string; - @ApiProperty() - tables: string[]; - @ApiPropertyOptional() - credentials: CredentialsJdbc; - @ApiPropertyOptional() - id: string; - @ApiPropertyOptional() - info: Info; -} -export class TestConnectionGetColumnsRes { - @ApiProperty({ type: [TableColumns] }) - tables: TableColumns[]; -} - -export class TestConnectionReq { - @ApiProperty() - plugin: string; - @ApiProperty() - credentials: CredentialsJdbc; - @ApiPropertyOptional() - info: Info; -} -export class TestConnectionRes { - @ApiProperty() - connection_state: boolean; - @ApiProperty() - total_entities: number; - @ApiProperty() - database_tables: string[]; -} export class CreateInputReq extends OmitType(Input, [ 'id', 'created_at', diff --git a/src/modules/inputs/inputs.controller.ts b/src/modules/inputs/inputs.controller.ts index 021eb35..4ce5dc6 100644 --- a/src/modules/inputs/inputs.controller.ts +++ b/src/modules/inputs/inputs.controller.ts @@ -19,16 +19,10 @@ import { GetAvailableEntitiesReq, GetAvailableEntitiesRes, Input, - TestConnectionGetColumnsReq, - TestConnectionGetColumnsRes, - TestConnectionReq, - TestConnectionRes, } from './dtos/input.model'; import { UpdateInputRequest } from './dtos/old_interfaces'; import { RequestUser, User } from 'src/authentication/user.decorator'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; -import ErrorBuilder from 'src/utils/ErrorBuilder'; -import ErrorCodes from 'src/utils/errorCodes'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @ApiTags('Inputs') @@ -75,42 +69,19 @@ export class InputsController { @Get('available-entities/:plugin') @ApiOkResponse({ type: GetAvailableEntitiesRes }) async getAvailableEntities( - @Body() body, @Param() params: GetAvailableEntitiesReq, + @User() user: RequestUser, ) { - const { info } = body; const { plugin } = params; - return await this.inputService.getAvailableEntities({ info, plugin }); - } - - @Post('/test-connection') - @ApiOkResponse({ type: TestConnectionRes }) - async testConnection(@Body() data: TestConnectionReq) { - this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, { - user: data.info.user_id, - customer: data.info.customer, - }); - - const response = await this.inputService.testConnection(data); - - return response; - } - - @Post('/test-connection/get-columns') - @ApiOkResponse({ type: TestConnectionGetColumnsRes }) - async getColumns(@Body() data: TestConnectionGetColumnsReq) { - throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED); - this.logger.info( - `/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`, - { - user: data.info.user_id, - customer: data.info.customer, + const availableEntities = await this.inputService.getAvailableEntities({ + info: { + customer: user.customer_name, + customer_id: user.customer_id, + user_id: user.user_id, }, - ); - - const response = await this.inputService.getColumns(data); - - return response; + plugin, + }); + return availableEntities; } @Post() diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index 924ec32..c67e6ef 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter'; import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces'; import { Input } from '@dadosfera/protospack-v2'; import { + GetAvailableEntitiesRequest, InputCreateGenericRequest, InputCreateS3Request, InputNewCreateRequest, - TestConnectionGetColumnsRequest, TestConnectionRequest, } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @@ -88,32 +88,6 @@ export class InputsService { return testConnectionResponse; }, - getColumns: async (data: TestConnectionGetColumnsRequest) => { - this.logger.info('InputClientService - TestConnection/Get-Columns'); - const getColumnsResponse = await new Promise((resolve, reject) => { - this.inputReadService.GetColumns(data).subscribe({ - next(x) { - resolve(objectCamelToSnake(x)); - }, - error(err) { - reject(err); - }, - complete() { - // console.log('done'); - }, - }); - }) - .then((res) => { - this.logger.info('Done'); - return res; - }) - .catch((err) => { - this.logger.error(err.message); - throw new Error(err); - }); - - return getColumnsResponse; - }, getAvailableEntities: async (data) => { this.logger.info('InputClientService - GetAvailableEntities'); data; @@ -188,12 +162,9 @@ export class InputsService { // this.validateCron(data); const { body, info } = data; - if (body.info) delete body.info; - - const inputCreateGenericRequest: any | InputCreateGenericRequest = { + const inputCreateGenericRequest: InputCreateGenericRequest = { input: { ...body, - credentials: body.credentials && JSON.stringify(body.credentials), }, info, }; @@ -205,8 +176,8 @@ export class InputsService { return { input: adjustedInput }; } - async getAvailableEntities(data): Promise<{ entities: string[] }> { - return await this.OLD_inputClient.getAvailableEntities(data); + async getAvailableEntities(data: GetAvailableEntitiesRequest) { + return lastValueFrom(this.inputReadService.GetAvailableEntities(data)); } async findAll(info: Info) { @@ -263,17 +234,6 @@ export class InputsService { } } - async getColumns(data) { - try { - const testConnectionGetColumnsResponse = - await this.OLD_inputClient.getColumns(data); - - return testConnectionGetColumnsResponse; - } catch (err) { - throw new HttpException(err.message, HttpStatus.NOT_FOUND); - } - } - generateInputS3Payload(payload) { const { credentials, plugin, cron } = payload; if (!credentials) return payload;