From 84d64424caf8e24aaf61924c16080f7c70ed0de3 Mon Sep 17 00:00:00 2001 From: iruy-fr Date: Thu, 30 Jul 2026 10:17:28 -0300 Subject: [PATCH] feat: read connection metadata from catalog cache --- deploy/helm-chart/templates/deployment.yaml | 2 + deploy/helm-chart/values-stg.yaml | 1 + .../connection-test.controller.ts | 6 +- .../connection-test/connection-test.module.ts | 7 +- .../connection-test.service.spec.ts | 109 ++++++++++++++++++ .../connection-test.service.ts | 68 ++++++----- .../connection-test/dto/connection-test.ts | 2 + .../connections-api/connections-api.config.ts | 11 ++ .../connections-api/connections-api.module.ts | 10 ++ .../connections-api.service.ts | 87 ++++++++++++++ 10 files changed, 273 insertions(+), 30 deletions(-) create mode 100644 src/modules/connection-test/connection-test.service.spec.ts create mode 100644 src/modules/connections-api/connections-api.config.ts create mode 100644 src/modules/connections-api/connections-api.module.ts create mode 100644 src/modules/connections-api/connections-api.service.ts diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 518fd7b..2cdfb2c 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -111,6 +111,8 @@ spec: value: "{{ .Values.maestro.redis_tls }}" - name: PLATFORM_API_URL value: {{ .Values.maestro.platform_api_url }} + - name: CONNECTIONS_API_URL + value: {{ .Values.maestro.connections_api_url | default "" | quote }} - name: STORAGE_EXPLORER_API_URL value: {{ .Values.maestro.storage_explorer_api_url | quote }} - name: FIREBASE_BASE_URL diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 70d2170..9515b1c 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,6 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com + connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg diff --git a/src/modules/connection-test/connection-test.controller.ts b/src/modules/connection-test/connection-test.controller.ts index f197358..ac3d03f 100644 --- a/src/modules/connection-test/connection-test.controller.ts +++ b/src/modules/connection-test/connection-test.controller.ts @@ -84,7 +84,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListSchemas( body, - user.customer_name, + user, ); } @@ -101,7 +101,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListTables( body, - user.customer_name, + user, ); } @@ -118,7 +118,7 @@ export class ConnectionTestController { }); return this.connectionTestService.getTableMetadata( body, - user.customer_name, + user, ); } } diff --git a/src/modules/connection-test/connection-test.module.ts b/src/modules/connection-test/connection-test.module.ts index 1eb4099..b162429 100644 --- a/src/modules/connection-test/connection-test.module.ts +++ b/src/modules/connection-test/connection-test.module.ts @@ -5,10 +5,15 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { ClientsModule } from '@nestjs/microservices'; import { ConnectionTestClientConfiguration } from './connection-test-client.config'; import { ConnectionModule } from '../connection/connection.module'; +import { ConnectionsApiModule } from '../connections-api/connections-api.module'; const client = new ConnectionTestClientConfiguration(); @Module({ controllers: [ConnectionTestController], providers: [ConnectionTestService, DadosferaLogger], - imports: [ClientsModule.register([client.providerOptions]), ConnectionModule], + imports: [ + ClientsModule.register([client.providerOptions]), + ConnectionModule, + ConnectionsApiModule, + ], }) export class ConnectionTestModule {} diff --git a/src/modules/connection-test/connection-test.service.spec.ts b/src/modules/connection-test/connection-test.service.spec.ts new file mode 100644 index 0000000..a4f71a5 --- /dev/null +++ b/src/modules/connection-test/connection-test.service.spec.ts @@ -0,0 +1,109 @@ +import { ConnectionTestService } from './connection-test.service'; +import { RequestUser } from 'src/decorators/user.decorator'; + +describe('ConnectionTestService catalog cache', () => { + const user: RequestUser = { + user_id: 'user-id', + username: 'user@example.com', + permissions: [], + customer_id: 'customer-id', + customer_name: 'customer-name', + customer_tier: 'standard', + access_token: 'token', + customer_modules: [], + roles: [], + }; + const grpcClient = { getService: jest.fn().mockReturnValue({}) }; + const connectionsService = {}; + const connectionsApiService = { proxy: jest.fn() }; + let service: ConnectionTestService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new ConnectionTestService( + grpcClient as any, + connectionsService as any, + connectionsApiService as any, + ); + }); + + it('keeps the existing schemas response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }], + }); + + await expect( + service.connectionTestListSchemas( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + schema_list: ['analytics', 'public'], + }); + }); + + it('keeps the existing tables response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + tables: [{ table_name: 'customers' }, { table_name: 'orders' }], + }); + + await expect( + service.connectionTestListTables( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + table_list: ['customers', 'orders'], + }); + }); + + it('maps cached columns to the existing table metadata contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + columns: [ + { + column_name: 'id', + data_type: 'bigint', + is_primary_key: true, + }, + ], + }); + + await expect( + service.getTableMetadata( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + table_list: ['customers'], + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + tables_metadata: [ + { + table_name: 'customers', + columns: [ + { + name: 'id', + type: 'bigint', + is_primary_key: true, + }, + ], + references: [], + }, + ], + }); + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'GET', + '/connection_catalog/config-id/schemas/public/tables/customers/columns', + user, + ); + }); +}); diff --git a/src/modules/connection-test/connection-test.service.ts b/src/modules/connection-test/connection-test.service.ts index 3b92e12..af5c10b 100644 --- a/src/modules/connection-test/connection-test.service.ts +++ b/src/modules/connection-test/connection-test.service.ts @@ -21,6 +21,7 @@ import { } from '../connection/dtos/connection'; import { RequestUser } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { ConnectionsApiService } from '../connections-api/connections-api.service'; @Injectable() export class ConnectionTestService { @@ -28,6 +29,7 @@ export class ConnectionTestService { constructor( @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc, private connectionsService: ConnectionClientService, + private connectionsApiService: ConnectionsApiService, ) { this.connectionTestReadClient = grpcClient.getService( @@ -147,45 +149,59 @@ export class ConnectionTestService { } async connectionTestListSchemas( body: ConnectionTestListSchemasReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin } = body; - return lastValueFrom( - this.connectionTestReadClient.ListSchemas({ - connection_id, - customer_name, - plugin, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`, + user, ); + return { + operation_result: true, + schema_list: result.schemas.map((schema) => schema.schema_name), + }; } + async connectionTestListTables( body: ConnectionTestListTablesReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin, schema } = body; - return lastValueFrom( - this.connectionTestReadClient.ListTables({ - connection_id, - customer_name, - plugin, - schema, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}/tables`, + user, ); + return { + operation_result: true, + table_list: result.tables.map((table) => table.table_name), + }; } async getTableMetadata( body: GetTableMetadataReq, - customer_name: string, + user: RequestUser, ): Promise { - const { schema, plugin, table_list, connection_id } = body; - return lastValueFrom( - this.connectionTestReadClient.GetTableMetadata({ - connection_id, - customer_name, - plugin, - schema, - table_list, + const tables_metadata = await Promise.all( + body.table_list.map(async (table_name) => { + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}` + + `/tables/${encodeURIComponent(table_name)}/columns`, + user, + ); + return { + table_name, + columns: result.columns.map((column) => ({ + name: column.column_name, + type: column.data_type, + is_primary_key: column.is_primary_key, + })), + references: [], + }; }), ); + return { operation_result: true, tables_metadata }; } } diff --git a/src/modules/connection-test/dto/connection-test.ts b/src/modules/connection-test/dto/connection-test.ts index b6bb4c4..44120fc 100644 --- a/src/modules/connection-test/dto/connection-test.ts +++ b/src/modules/connection-test/dto/connection-test.ts @@ -7,6 +7,8 @@ export class ColumnDto { name: string; @ApiProperty() type: string; + @ApiProperty() + is_primary_key: boolean; } export class TableMetadataDto { @ApiProperty() diff --git a/src/modules/connections-api/connections-api.config.ts b/src/modules/connections-api/connections-api.config.ts new file mode 100644 index 0000000..0e6e450 --- /dev/null +++ b/src/modules/connections-api/connections-api.config.ts @@ -0,0 +1,11 @@ +export const CONNECTIONS_API_CONFIG = { + getUrl: (): string => { + const url = process.env.CONNECTIONS_API_URL; + if (!url) { + throw new Error('CONNECTIONS_API_URL environment variable is not set'); + } + return url; + }, + region: process.env.AWS_REGION || 'us-east-1', + timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10), +}; diff --git a/src/modules/connections-api/connections-api.module.ts b/src/modules/connections-api/connections-api.module.ts new file mode 100644 index 0000000..1835186 --- /dev/null +++ b/src/modules/connections-api/connections-api.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { ConnectionsApiService } from './connections-api.service'; + +@Module({ + providers: [ConnectionsApiService, DadosferaLogger], + exports: [ConnectionsApiService], +}) +export class ConnectionsApiModule {} diff --git a/src/modules/connections-api/connections-api.service.ts b/src/modules/connections-api/connections-api.service.ts new file mode 100644 index 0000000..81e5c41 --- /dev/null +++ b/src/modules/connections-api/connections-api.service.ts @@ -0,0 +1,87 @@ +import { Injectable, Inject, HttpException } from '@nestjs/common'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import axios, { AxiosResponse, Method } from 'axios'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { RequestUser } from '../../decorators/user.decorator'; +import { CONNECTIONS_API_CONFIG } from './connections-api.config'; + +@Injectable() +export class ConnectionsApiService { + private signer: SignatureV4; + private logger: any; + + constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) { + this.logger = dadosferaLogger.logger; + this.signer = new SignatureV4({ + service: 'execute-api', + region: CONNECTIONS_API_CONFIG.region, + credentials: defaultProvider(), + sha256: Sha256, + }); + } + + async proxy( + method: string, + path: string, + user: RequestUser, + ): Promise { + const baseUrl = CONNECTIONS_API_CONFIG.getUrl(); + const url = new URL(`${baseUrl}${path}`); + const headers: Record = { + host: url.hostname, + 'content-type': 'application/json', + customer_name: user.customer_name || '', + customer_id: user.customer_id || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + 'x-customer-id': user.customer_id || '', + }; + const requestToSign = { + method: method.toUpperCase(), + protocol: url.protocol, + hostname: url.hostname, + port: url.port ? parseInt(url.port, 10) : undefined, + path: url.pathname, + headers, + }; + + try { + const signedRequest = await this.signer.sign(requestToSign); + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers: signedRequest.headers as Record, + timeout: CONNECTIONS_API_CONFIG.timeout, + validateStatus: () => true, + }); + + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + return response.data; + } catch (error) { + this.logger.error('Connections API proxy error', { + error: error.message, + path, + method: method.toUpperCase(), + }); + if (error instanceof HttpException) { + throw error; + } + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + if (error.code === 'ECONNREFUSED') { + throw new HttpException('Connections API service unavailable', 503); + } + if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') { + throw new HttpException('Connections API request timeout', 504); + } + throw new HttpException('Internal server error', 500); + } + } +}