diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 8a67a9c..ae6d383 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -71,6 +71,11 @@ jobs: sudo mv helmfile /usr/local/bin/ helmfile --version + - name: Install Helm Diff plugin + run: | + helm plugin install https://github.com/databus23/helm-diff --version v3.9.3 + helm diff version + - name: Debug Helm env run: | helm env @@ -102,4 +107,5 @@ jobs: - name: Run Helmfile Diff env: ENV: ${{ needs.extract_environment.outputs.environment }} + HELM_PLUGINS: /home/runner/.local/share/helm/plugins run: helmfile -f deploy/helmfiles/${ENV}.yaml diff diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 518fd7b..2cdfb2c 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -111,6 +111,8 @@ spec: value: "{{ .Values.maestro.redis_tls }}" - name: PLATFORM_API_URL value: {{ .Values.maestro.platform_api_url }} + - name: CONNECTIONS_API_URL + value: {{ .Values.maestro.connections_api_url | default "" | quote }} - name: STORAGE_EXPLORER_API_URL value: {{ .Values.maestro.storage_explorer_api_url | quote }} - name: FIREBASE_BASE_URL diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 70d2170..9515b1c 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,6 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com + connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 5042b24..738337a 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -56,6 +56,8 @@ maestro: redis_tls: "true" cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf" firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd + connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com + autoscaling: enabled: false minReplicas: 1 diff --git a/docsfera.json b/docsfera.json index 1b0f444..8dadf56 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4764,9 +4764,6 @@ "Platform API" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -6023,57 +6020,6 @@ "access-token": [] } ] - }, - "patch": { - "operationId": "CatalogController_updateColumnsMetadata", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - }, - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UpdateColumnsMetadataRequest" - } - } - } - }, - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] } }, "/catalog/data-asset/{id}/preview": { @@ -7513,94 +7459,6 @@ ] } }, - "/connection-test/refresh-catalog": { - "post": { - "operationId": "ConnectionTestController_refreshCatalog", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RefreshCatalogReq" - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RefreshCatalogRes" - } - } - } - }, - "202": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RefreshCatalogRes" - } - } - } - } - }, - "tags": [ - "Connection Test" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/connection-test/refresh-catalog/status": { - "post": { - "operationId": "ConnectionTestController_refreshCatalogStatus", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RefreshCatalogStatusReq" - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RefreshCatalogRes" - } - } - } - } - }, - "tags": [ - "Connection Test" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/oauth/hubspot": { "get": { "operationId": "OauthController_oauthHubspot", @@ -11361,35 +11219,6 @@ "columns_metadata" ] }, - "ColumnDescriptionDto": { - "type": "object", - "properties": { - "column_name": { - "type": "string" - }, - "description": { - "type": "string" - } - }, - "required": [ - "column_name", - "description" - ] - }, - "UpdateColumnsMetadataRequest": { - "type": "object", - "properties": { - "columns": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ColumnDescriptionDto" - } - } - }, - "required": [ - "columns" - ] - }, "IData": { "type": "object", "properties": { @@ -11986,15 +11815,11 @@ }, "type": { "type": "string" - }, - "is_primary_key": { - "type": "boolean" } }, "required": [ "name", - "type", - "is_primary_key" + "type" ] }, "TableMetadataDto": { @@ -12040,79 +11865,6 @@ "tables_metadata" ] }, - "RefreshCatalogReq": { - "type": "object", - "properties": { - "connection_id": { - "type": "string" - }, - "plugin": { - "type": "string", - "enum": [ - "oracle", - "mysql", - "postgresql", - "sqlserver" - ] - } - }, - "required": [ - "connection_id", - "plugin" - ] - }, - "RefreshCatalogRes": { - "type": "object", - "properties": { - "operation_result": { - "type": "boolean" - }, - "status": { - "type": "string" - }, - "session_id": { - "type": "string" - }, - "date": { - "type": "string" - } - }, - "required": [ - "operation_result", - "status", - "session_id", - "date" - ] - }, - "RefreshCatalogStatusReq": { - "type": "object", - "properties": { - "connection_id": { - "type": "string" - }, - "plugin": { - "type": "string", - "enum": [ - "oracle", - "mysql", - "postgresql", - "sqlserver" - ] - }, - "session_id": { - "type": "string" - }, - "date": { - "type": "string" - } - }, - "required": [ - "connection_id", - "plugin", - "session_id", - "date" - ] - }, "INote": { "type": "object", "properties": { diff --git a/jest.config.ts b/jest.config.ts index 9442e04..9aea4aa 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -11,6 +11,7 @@ const config: Config.InitialOptions = { '/node_modules/', '.*\\.module\\.[jt]s$', ], + setupFiles: ['/jest.setup.ts'], // moduleDirectories: ['node_modules'], // default is already 'node_modules' // rootDir: '.', //No need // testEnvironment: 'node', //Defaults to 'node' diff --git a/jest.setup.ts b/jest.setup.ts new file mode 100644 index 0000000..c2d29e3 --- /dev/null +++ b/jest.setup.ts @@ -0,0 +1,3 @@ +process.env.DUC_URL="duc:50051" +process.env.INFACTORY_URL="in-factory:50052" +process.env.PIFACTORY_URL="pi-factory:50053" \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5506401..66e5f72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.19", + "@dadosfera/protospack-v2": "3.40.0", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,9 +1735,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.19", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.19.tgz", - "integrity": "sha512-RT7BYWHD2aD945FXzDXHa/xM+sD2B2Ypepy/W/WS+RwKYTSMCfuUXGzK2lXcd9HP9+U+Zjfh90Lkktl++8enjg==", + "version": "3.40.0", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0.tgz", + "integrity": "sha512-tzYAGosFjGUgi3pAvXqv+Eqqjca9tH1kU9kJdRI5d1rLXNimo0fIuPWxWng0C4OBH/2sIgC1WLHseq+DZ6qgdw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index fd16bd9..0642072 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.19", + "@dadosfera/protospack-v2": "3.40.0", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 40eaad9..5eb2b0f 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -37,6 +37,7 @@ import { RequireAllPermissions, } from 'src/decorators/authentication.decorator'; import { AuthClientService } from './auth.service'; +import { UserDTO } from './dtos/login'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter'; import { RequestUser, User } from 'src/decorators/user.decorator'; @@ -486,7 +487,7 @@ export class AuthController { this.logger.info('Authenticating via X-Api-key header'); const { api_key } = await this.apiKeyService.get(apiKey); - const userDto = { + const userDto: UserDTO = { id: api_key.user_id, name: api_key.username, email: api_key.username, @@ -494,7 +495,8 @@ export class AuthController { id: api_key.customer_id, name: api_key.customer_name, tier: api_key.customer_tier, - } + }, + permissions: [], }; return res.status(200).json(userDto); diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 8539d0e..c82aeef 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -447,6 +447,9 @@ export class AuthClientService implements OnModuleInit { name: payload.customer_name, tier: payload.customer_tier, }, + // Raw permission seqids from the JWT. Consumers own the seqid->meaning + // mapping (e.g. Orchest's auth-server); Maestro reports them as-is. + permissions: payload.permissions ?? [], }; return userDto; diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 2e22363..1ea22f0 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -152,5 +152,6 @@ export type UserDTO = { id: string, name: string, tier: string, - } + }, + permissions: number[], } diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 1690e58..2b2bc74 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -285,6 +285,23 @@ export class CatalogController { } + @Get('custom-properties') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getCustomPropertyDefinitions(@User() user: RequestUser) { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + }); + + return this.catalogService.getCustomPropertyDefinitions(metadata); + } + @Get('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, @@ -1094,4 +1111,4 @@ export class CatalogController { this.logger.error(error.message); } } -} \ No newline at end of file +} diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 8fdf270..d50e819 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -120,6 +120,10 @@ class CatalogService implements OnModuleInit { } } + async getCustomPropertyDefinitions(metadata: Metadata) { + return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata)); + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 6b71473..c462deb 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -206,6 +206,27 @@ export class IData { day_opening: number; } + +export enum CustomPropertyType { + TEXT = 'text', + NUMBER = 'number', + DATE = 'date', + BOOLEAN = 'boolean', +} + +export class CustomPropertyDto { + @ApiProperty() + key: string; + @ApiProperty() + value: string; + @ApiProperty({ enum: CustomPropertyType }) + type: CustomPropertyType; + @ApiPropertyOptional() + color?: string; + @ApiPropertyOptional() + emoji?: string; +} + export class IUpdateDataRequest { @ApiProperty() name: string; @@ -219,6 +240,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; + @ApiPropertyOptional({ type: [CustomPropertyDto] }) + custom_properties?: CustomPropertyDto[]; } export class IUpdateCertificationStatusRequest { @@ -400,4 +423,4 @@ export type CreateDataDocsDTO = { docs: string; asset_type: string; -} \ No newline at end of file +} diff --git a/src/modules/connection-test/connection-test.controller.ts b/src/modules/connection-test/connection-test.controller.ts index 029da68..043ab29 100644 --- a/src/modules/connection-test/connection-test.controller.ts +++ b/src/modules/connection-test/connection-test.controller.ts @@ -22,6 +22,9 @@ import { ConnectionTestListTablesRes, GetTableMetadataRes, GetTableMetadataReq, + RefreshCatalogReq, + RefreshCatalogRes, + RefreshCatalogStatusReq, } from './dto/connection-test'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator'; @@ -88,7 +91,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListSchemas( body, - user.customer_name, + user, ); } @@ -105,7 +108,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListTables( body, - user.customer_name, + user, ); } @@ -122,7 +125,38 @@ export class ConnectionTestController { }); return this.connectionTestService.getTableMetadata( body, - user.customer_name, + user, ); } + + @Post('refresh-catalog') + @ApiOkResponse({ type: RefreshCatalogRes }) + @HttpCode(HttpStatus.ACCEPTED) + async refreshCatalog( + @User() user: RequestUser, + @Body(new ValidationPipe()) body: RefreshCatalogReq, + ) { + this.logger.info('/connection-test/refresh-catalog', { + user: user.user_id, + customer: user.customer_name, + connection: body.connection_id, + }); + return this.connectionTestService.refreshCatalog(body, user); + } + + @Post('refresh-catalog/status') + @ApiOkResponse({ type: RefreshCatalogRes }) + @HttpCode(HttpStatus.OK) + async refreshCatalogStatus( + @User() user: RequestUser, + @Body(new ValidationPipe()) body: RefreshCatalogStatusReq, + ) { + this.logger.info('/connection-test/refresh-catalog/status', { + user: user.user_id, + customer: user.customer_name, + connection: body.connection_id, + session: body.session_id, + }); + return this.connectionTestService.refreshCatalogStatus(body, user); + } } diff --git a/src/modules/connection-test/connection-test.module.ts b/src/modules/connection-test/connection-test.module.ts index 1eb4099..c184391 100644 --- a/src/modules/connection-test/connection-test.module.ts +++ b/src/modules/connection-test/connection-test.module.ts @@ -5,10 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { ClientsModule } from '@nestjs/microservices'; import { ConnectionTestClientConfiguration } from './connection-test-client.config'; import { ConnectionModule } from '../connection/connection.module'; +import { ConnectionsApiModule } from '../connections-api/connections-api.module'; +import { PlatformApiModule } from '../platform-api/platform-api.module'; const client = new ConnectionTestClientConfiguration(); @Module({ controllers: [ConnectionTestController], providers: [ConnectionTestService, DadosferaLogger], - imports: [ClientsModule.register([client.providerOptions]), ConnectionModule], + imports: [ + ClientsModule.register([client.providerOptions]), + ConnectionModule, + ConnectionsApiModule, + PlatformApiModule, + ], }) export class ConnectionTestModule {} diff --git a/src/modules/connection-test/connection-test.service.spec.ts b/src/modules/connection-test/connection-test.service.spec.ts new file mode 100644 index 0000000..aed103f --- /dev/null +++ b/src/modules/connection-test/connection-test.service.spec.ts @@ -0,0 +1,205 @@ +import { ConnectionTestService } from './connection-test.service'; +import { RequestUser } from 'src/decorators/user.decorator'; + +describe('ConnectionTestService catalog cache', () => { + const user: RequestUser = { + user_id: 'user-id', + username: 'user@example.com', + permissions: [], + customer_id: 'customer-id', + customer_name: 'customer-name', + customer_tier: 'standard', + access_token: 'token', + customer_modules: [], + roles: [], + }; + const grpcClient = { getService: jest.fn().mockReturnValue({}) }; + const connectionsService = {}; + const connectionsApiService = { proxy: jest.fn() }; + const platformApiService = { proxy: jest.fn() }; + let service: ConnectionTestService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new ConnectionTestService( + grpcClient as any, + connectionsService as any, + connectionsApiService as any, + platformApiService as any, + ); + }); + + it('keeps the existing schemas response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }], + }); + + await expect( + service.connectionTestListSchemas( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + schema_list: ['analytics', 'public'], + }); + }); + + it('keeps the existing tables response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + tables: [{ table_name: 'customers' }, { table_name: 'orders' }], + }); + + await expect( + service.connectionTestListTables( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + table_list: ['customers', 'orders'], + }); + }); + + it('maps cached columns to the existing table metadata contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + columns: [ + { + column_name: 'id', + data_type: 'bigint', + is_primary_key: true, + }, + ], + }); + + await expect( + service.getTableMetadata( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + table_list: ['customers'], + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + tables_metadata: [ + { + table_name: 'customers', + columns: [ + { + name: 'id', + type: 'bigint', + is_primary_key: true, + }, + ], + references: [], + }, + ], + }); + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'GET', + '/connection_catalog/config-id/schemas/public/tables/customers/columns', + user, + ); + }); + + it('submits a catalog refresh without holding the request open', async () => { + platformApiService.proxy.mockResolvedValue({ + session_id: 'session-id', + date: '20260731', + }); + + await expect( + service.refreshCatalog( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + status: 'PENDING', + session_id: 'session-id', + date: '20260731', + }); + + expect(platformApiService.proxy).toHaveBeenCalledWith( + 'POST', + '/connection_test', + user, + { + customer_id: user.customer_name, + plugin: 'postgresql', + task: { + task_type: 'refresh_catalog', + connection: { + provider: 'connection_manager', + config_id: 'config-id', + }, + }, + }, + ); + }); + + it('keeps polling without changing the catalog pointer while pending', async () => { + platformApiService.proxy.mockResolvedValue({ status: 'PENDING' }); + + await expect( + service.refreshCatalogStatus( + { + connection_id: 'config-id', + plugin: 'postgresql', + session_id: 'session-id', + date: '20260731', + }, + user, + ), + ).resolves.toEqual({ + operation_result: false, + status: 'PENDING', + session_id: 'session-id', + date: '20260731', + }); + + expect(connectionsApiService.proxy).not.toHaveBeenCalled(); + }); + + it('publishes the catalog pointer after the refresh finishes', async () => { + platformApiService.proxy.mockResolvedValue({ status: 'DONE' }); + connectionsApiService.proxy.mockResolvedValue({ + last_catalog_refresh_status: 'SUCCESS', + }); + + await expect( + service.refreshCatalogStatus( + { + connection_id: 'config/id', + plugin: 'postgresql', + session_id: 'session-id', + date: '20260731', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + status: 'DONE', + session_id: 'session-id', + date: '20260731', + }); + + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'PUT', + '/connection_config/config%2Fid/catalog_metadata', + user, + { + last_catalog_refresh_status: 'SUCCESS', + last_catalog_connection_test_date: '20260731', + last_catalog_connection_test_session_id: 'session-id', + }, + ); + }); +}); diff --git a/src/modules/connection-test/connection-test.service.ts b/src/modules/connection-test/connection-test.service.ts index 3b92e12..fe4930d 100644 --- a/src/modules/connection-test/connection-test.service.ts +++ b/src/modules/connection-test/connection-test.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { ConnectionTest } from '@dadosfera/protospack-v2'; import { lastValueFrom } from 'rxjs'; @@ -13,6 +13,9 @@ import { ConnectionTestPingRes, GetTableMetadataReq, GetTableMetadataRes, + RefreshCatalogReq, + RefreshCatalogRes, + RefreshCatalogStatusReq, } from './dto/connection-test'; import { ConnectionClientService } from '../connection/client.service'; import { @@ -21,6 +24,8 @@ import { } from '../connection/dtos/connection'; import { RequestUser } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { ConnectionsApiService } from '../connections-api/connections-api.service'; +import { PlatformApiService } from '../platform-api/platform-api.service'; @Injectable() export class ConnectionTestService { @@ -28,6 +33,8 @@ export class ConnectionTestService { constructor( @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc, private connectionsService: ConnectionClientService, + private connectionsApiService: ConnectionsApiService, + private platformApiService: PlatformApiService, ) { this.connectionTestReadClient = grpcClient.getService( @@ -147,45 +154,137 @@ export class ConnectionTestService { } async connectionTestListSchemas( body: ConnectionTestListSchemasReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin } = body; - return lastValueFrom( - this.connectionTestReadClient.ListSchemas({ - connection_id, - customer_name, - plugin, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`, + user, ); + return { + operation_result: true, + schema_list: result.schemas.map((schema) => schema.schema_name), + }; } + async connectionTestListTables( body: ConnectionTestListTablesReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin, schema } = body; - return lastValueFrom( - this.connectionTestReadClient.ListTables({ - connection_id, - customer_name, - plugin, - schema, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}/tables`, + user, ); + return { + operation_result: true, + table_list: result.tables.map((table) => table.table_name), + }; } async getTableMetadata( body: GetTableMetadataReq, - customer_name: string, + user: RequestUser, ): Promise { - const { schema, plugin, table_list, connection_id } = body; - return lastValueFrom( - this.connectionTestReadClient.GetTableMetadata({ - connection_id, - customer_name, - plugin, - schema, - table_list, + const tables_metadata = await Promise.all( + body.table_list.map(async (table_name) => { + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}` + + `/tables/${encodeURIComponent(table_name)}/columns`, + user, + ); + return { + table_name, + columns: result.columns.map((column) => ({ + name: column.column_name, + type: column.data_type, + is_primary_key: column.is_primary_key, + })), + references: [], + }; }), ); + return { operation_result: true, tables_metadata }; + } + + async refreshCatalog( + body: RefreshCatalogReq, + user: RequestUser, + ): Promise { + const task = await this.platformApiService.proxy( + 'POST', + '/connection_test', + user, + { + customer_id: user.customer_name, + plugin: body.plugin, + task: { + task_type: 'refresh_catalog', + connection: { + provider: 'connection_manager', + config_id: body.connection_id, + }, + }, + }, + ); + + if (!task.session_id || !task.date) { + throw new HttpException( + 'Platform API did not return a catalog refresh task identifier', + HttpStatus.BAD_GATEWAY, + ); + } + + return { + operation_result: true, + status: 'PENDING', + session_id: task.session_id, + date: task.date, + }; + } + + async refreshCatalogStatus( + body: RefreshCatalogStatusReq, + user: RequestUser, + ): Promise { + const result = await this.platformApiService.proxy( + 'POST', + '/connection_test/status', + user, + { + session_id: body.session_id, + date: body.date, + }, + ); + + if (result.status === 'DONE') { + await this.connectionsApiService.proxy( + 'PUT', + `/connection_config/${encodeURIComponent( + body.connection_id, + )}/catalog_metadata`, + user, + { + last_catalog_refresh_status: 'SUCCESS', + last_catalog_connection_test_date: body.date, + last_catalog_connection_test_session_id: body.session_id, + }, + ); + } else if (result.status === 'ERROR' || result.status === 'EXPIRED') { + throw new HttpException( + `Catalog refresh finished with status ${result.status}`, + HttpStatus.BAD_GATEWAY, + ); + } + + return { + operation_result: result.status === 'DONE', + status: result.status, + session_id: body.session_id, + date: body.date, + }; } } diff --git a/src/modules/connection-test/dto/connection-test.ts b/src/modules/connection-test/dto/connection-test.ts index b6bb4c4..3496d4c 100644 --- a/src/modules/connection-test/dto/connection-test.ts +++ b/src/modules/connection-test/dto/connection-test.ts @@ -1,5 +1,5 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { IsString, IsOptional } from 'class-validator'; +import { IsIn, IsString, IsOptional } from 'class-validator'; import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection'; import { CreateConnectionDto } from 'src/modules/connection/dtos/connection'; export class ColumnDto { @@ -7,6 +7,8 @@ export class ColumnDto { name: string; @ApiProperty() type: string; + @ApiProperty() + is_primary_key: boolean; } export class TableMetadataDto { @ApiProperty() @@ -131,3 +133,37 @@ export class GetTableMetadataRes { @ApiProperty({ type: [TableMetadataDto] }) tables_metadata: TableMetadataDto[]; } + +export class RefreshCatalogReq { + @ApiProperty() + @IsString() + connection_id: string; + + @ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] }) + @IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver']) + plugin: string; +} + +export class RefreshCatalogStatusReq extends RefreshCatalogReq { + @ApiProperty() + @IsString() + session_id: string; + + @ApiProperty() + @IsString() + date: string; +} + +export class RefreshCatalogRes { + @ApiProperty() + operation_result: boolean; + + @ApiProperty() + status: string; + + @ApiProperty() + session_id: string; + + @ApiProperty() + date: string; +} diff --git a/src/modules/connections-api/connections-api.config.ts b/src/modules/connections-api/connections-api.config.ts new file mode 100644 index 0000000..0e6e450 --- /dev/null +++ b/src/modules/connections-api/connections-api.config.ts @@ -0,0 +1,11 @@ +export const CONNECTIONS_API_CONFIG = { + getUrl: (): string => { + const url = process.env.CONNECTIONS_API_URL; + if (!url) { + throw new Error('CONNECTIONS_API_URL environment variable is not set'); + } + return url; + }, + region: process.env.AWS_REGION || 'us-east-1', + timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10), +}; diff --git a/src/modules/connections-api/connections-api.module.ts b/src/modules/connections-api/connections-api.module.ts new file mode 100644 index 0000000..1835186 --- /dev/null +++ b/src/modules/connections-api/connections-api.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { ConnectionsApiService } from './connections-api.service'; + +@Module({ + providers: [ConnectionsApiService, DadosferaLogger], + exports: [ConnectionsApiService], +}) +export class ConnectionsApiModule {} diff --git a/src/modules/connections-api/connections-api.service.ts b/src/modules/connections-api/connections-api.service.ts new file mode 100644 index 0000000..ae2dfd4 --- /dev/null +++ b/src/modules/connections-api/connections-api.service.ts @@ -0,0 +1,99 @@ +import { Injectable, Inject, HttpException } from '@nestjs/common'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import axios, { AxiosResponse, Method } from 'axios'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { RequestUser } from '../../decorators/user.decorator'; +import { CONNECTIONS_API_CONFIG } from './connections-api.config'; + +@Injectable() +export class ConnectionsApiService { + private signer: SignatureV4; + private logger: any; + + constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) { + this.logger = dadosferaLogger.logger; + this.signer = new SignatureV4({ + service: 'execute-api', + region: CONNECTIONS_API_CONFIG.region, + credentials: defaultProvider(), + sha256: Sha256, + }); + } + + async proxy( + method: string, + path: string, + user: RequestUser, + body?: any, + query?: Record, + ): Promise { + const baseUrl = CONNECTIONS_API_CONFIG.getUrl(); + const url = new URL(`${baseUrl}${path}`); + + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } + const headers: Record = { + host: url.hostname, + 'content-type': 'application/json', + customer_name: user.customer_name || '', + customer_id: user.customer_id || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + 'x-customer-id': user.customer_id || '', + }; + const requestToSign = { + method: method.toUpperCase(), + protocol: url.protocol, + hostname: url.hostname, + port: url.port ? parseInt(url.port, 10) : undefined, + path: url.pathname + url.search, + headers, + body: body ? JSON.stringify(body) : undefined, + }; + + try { + const signedRequest = await this.signer.sign(requestToSign); + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers: signedRequest.headers as Record, + data: body, + timeout: CONNECTIONS_API_CONFIG.timeout, + validateStatus: () => true, + }); + + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + return response.data; + } catch (error) { + this.logger.error('Connections API proxy error', { + error: error.message, + path, + method: method.toUpperCase(), + }); + if (error instanceof HttpException) { + throw error; + } + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + if (error.code === 'ECONNREFUSED') { + throw new HttpException('Connections API service unavailable', 503); + } + if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') { + throw new HttpException('Connections API request timeout', 504); + } + throw new HttpException('Internal server error', 500); + } + } +} diff --git a/src/modules/permissions/permissions.controller.spec.ts b/src/modules/permissions/permissions.controller.spec.ts index 7ab875b..1bc3004 100644 --- a/src/modules/permissions/permissions.controller.spec.ts +++ b/src/modules/permissions/permissions.controller.spec.ts @@ -1,6 +1,6 @@ import { ClientsModule } from '@nestjs/microservices'; import { Test, TestingModule } from '@nestjs/testing'; -// import { DucClient } from 'src/clients/duc/client.config'; +// import { DucClient } from '../duc/client.config' import { PermissionsController } from './permissions.controller'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PermissionsService } from './permissions.service'; @@ -22,7 +22,7 @@ describe.skip('PermissionsController', () => { provide: DadosferaLogger, useValue: { logger }, }, - PermissionsService, + PermissionsService ], }).compile(); diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 503d117..8121242 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -14,7 +14,7 @@ import { NotFoundException, UseGuards, } from '@nestjs/common'; -import { ApiTags, ApiOperation } from '@nestjs/swagger'; +import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { @@ -74,6 +74,10 @@ export class PlatformApiController { return id?.replace(/-/g, '_') || ''; } + private decodePathParam(value: string): string { + return value ? decodeURIComponent(value) : ''; + } + /** * Denormalize ID back to UUID format (replace _ with -). * Used when we receive a normalized ID but need the original UUID. @@ -834,6 +838,40 @@ export class PlatformApiController { ); } + @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') + @ApiOperation({ + summary: 'Get pipeline run jobs', + description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.', + }) + @ApiOkResponse({ + description: 'DB-backed job runs for the selected pipeline run.', + schema: { + type: 'object', + properties: { + jobs: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['jobs'], + }, + }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getPipelineRunJobs( + @Param('pipelineId') pipelineId: string, + @Param('runId') runId: string, + @User() user: RequestUser, + ) { + const normalizedPipelineId = this.normalizePipelineId(pipelineId); + const decodedRunId = this.decodePathParam(runId); + + return this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, + user, + ); + } + // ==================== JOBS - COLUMN EDITING ROUTES ==================== @Put('jobs/:jobId/input') diff --git a/src/modules/release_note/release_note.controller.spec.ts b/src/modules/release_note/release_note.controller.spec.ts index 6086196..af307e7 100644 --- a/src/modules/release_note/release_note.controller.spec.ts +++ b/src/modules/release_note/release_note.controller.spec.ts @@ -5,11 +5,21 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; describe('ReleaseNoteController', () => { let controller: ReleaseNoteController; + const logger = { + info: (...args) => args, + error: (...args) => args, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [ReleaseNoteController], - providers: [ReleaseNoteService, DadosferaLogger], + providers: [ + { + provide: DadosferaLogger, + useValue: { logger }, + }, + ReleaseNoteService + ], }).compile(); controller = module.get(ReleaseNoteController); diff --git a/src/modules/release_note/release_note.service.spec.ts b/src/modules/release_note/release_note.service.spec.ts index 266146d..fa919ae 100644 --- a/src/modules/release_note/release_note.service.spec.ts +++ b/src/modules/release_note/release_note.service.spec.ts @@ -4,10 +4,20 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; describe('ReleaseNoteService', () => { let service: ReleaseNoteService; + const logger = { + info: (...args) => args, + error: (...args) => args, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - providers: [ReleaseNoteService, DadosferaLogger], + providers: [ + { + provide: DadosferaLogger, + useValue: { logger }, + }, + ReleaseNoteService + ], }).compile(); service = module.get(ReleaseNoteService); diff --git a/tsconfig.build.json b/tsconfig.build.json index ca83a6b..0744d9f 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,4 +1,4 @@ { "extends": "./tsconfig.json", - "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"] + "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.*.ts"] }