diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 8a67a9c..ae6d383 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -71,6 +71,11 @@ jobs: sudo mv helmfile /usr/local/bin/ helmfile --version + - name: Install Helm Diff plugin + run: | + helm plugin install https://github.com/databus23/helm-diff --version v3.9.3 + helm diff version + - name: Debug Helm env run: | helm env @@ -102,4 +107,5 @@ jobs: - name: Run Helmfile Diff env: ENV: ${{ needs.extract_environment.outputs.environment }} + HELM_PLUGINS: /home/runner/.local/share/helm/plugins run: helmfile -f deploy/helmfiles/${ENV}.yaml diff diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 518fd7b..2cdfb2c 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -111,6 +111,8 @@ spec: value: "{{ .Values.maestro.redis_tls }}" - name: PLATFORM_API_URL value: {{ .Values.maestro.platform_api_url }} + - name: CONNECTIONS_API_URL + value: {{ .Values.maestro.connections_api_url | default "" | quote }} - name: STORAGE_EXPLORER_API_URL value: {{ .Values.maestro.storage_explorer_api_url | quote }} - name: FIREBASE_BASE_URL diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 70d2170..9515b1c 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,6 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com + connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg diff --git a/docsfera.json b/docsfera.json index b3a85b5..8dadf56 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4714,6 +4714,62 @@ ] } }, + "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunJobs", + "summary": "Get pipeline run jobs", + "description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "DB-backed job runs for the selected pipeline run.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/jobs/{jobId}/input": { "put": { "operationId": "PlatformApiController_updateJobInput", @@ -5662,6 +5718,48 @@ ] } }, + "/catalog/custom-properties": { + "get": { + "operationId": "CatalogController_getCustomPropertyDefinitions", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset/{id}": { "get": { "operationId": "CatalogController_getDataAsset", @@ -11223,6 +11321,37 @@ "docs" ] }, + "CustomPropertyDto": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "text", + "number", + "date", + "boolean" + ] + }, + "color": { + "type": "string" + }, + "emoji": { + "type": "string" + } + }, + "required": [ + "key", + "value", + "type" + ] + }, "IUpdateDataRequest": { "type": "object", "properties": { @@ -11251,6 +11380,12 @@ }, "docs": { "type": "string" + }, + "custom_properties": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomPropertyDto" + } } }, "required": [ diff --git a/jest.config.ts b/jest.config.ts index 9442e04..9aea4aa 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -11,6 +11,7 @@ const config: Config.InitialOptions = { '/node_modules/', '.*\\.module\\.[jt]s$', ], + setupFiles: ['/jest.setup.ts'], // moduleDirectories: ['node_modules'], // default is already 'node_modules' // rootDir: '.', //No need // testEnvironment: 'node', //Defaults to 'node' diff --git a/jest.setup.ts b/jest.setup.ts new file mode 100644 index 0000000..c2d29e3 --- /dev/null +++ b/jest.setup.ts @@ -0,0 +1,3 @@ +process.env.DUC_URL="duc:50051" +process.env.INFACTORY_URL="in-factory:50052" +process.env.PIFACTORY_URL="pi-factory:50053" \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index dcb4ea9..cf5d6e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.14", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,9 +1735,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.10", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz", - "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==", + "version": "3.40.0-beta.14", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz", + "integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index fdbc61e..2c10052 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.10", + "@dadosfera/protospack-v2": "^3.40.0-beta.14", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 40eaad9..5eb2b0f 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -37,6 +37,7 @@ import { RequireAllPermissions, } from 'src/decorators/authentication.decorator'; import { AuthClientService } from './auth.service'; +import { UserDTO } from './dtos/login'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter'; import { RequestUser, User } from 'src/decorators/user.decorator'; @@ -486,7 +487,7 @@ export class AuthController { this.logger.info('Authenticating via X-Api-key header'); const { api_key } = await this.apiKeyService.get(apiKey); - const userDto = { + const userDto: UserDTO = { id: api_key.user_id, name: api_key.username, email: api_key.username, @@ -494,7 +495,8 @@ export class AuthController { id: api_key.customer_id, name: api_key.customer_name, tier: api_key.customer_tier, - } + }, + permissions: [], }; return res.status(200).json(userDto); diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 8539d0e..c82aeef 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -447,6 +447,9 @@ export class AuthClientService implements OnModuleInit { name: payload.customer_name, tier: payload.customer_tier, }, + // Raw permission seqids from the JWT. Consumers own the seqid->meaning + // mapping (e.g. Orchest's auth-server); Maestro reports them as-is. + permissions: payload.permissions ?? [], }; return userDto; diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 2e22363..1ea22f0 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -152,5 +152,6 @@ export type UserDTO = { id: string, name: string, tier: string, - } + }, + permissions: number[], } diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index ef8ab78..b15c6e7 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -283,6 +283,23 @@ export class CatalogController { } + @Get('custom-properties') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getCustomPropertyDefinitions(@User() user: RequestUser) { + const { customer_id, customer_name, user_id, username } = user; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + }); + + return this.catalogService.getCustomPropertyDefinitions(metadata); + } + @Get('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, @@ -1056,4 +1073,4 @@ export class CatalogController { this.logger.error(error.message); } } -} \ No newline at end of file +} diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index f0fced8..e6e9d18 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -120,6 +120,10 @@ class CatalogService implements OnModuleInit { } } + async getCustomPropertyDefinitions(metadata: Metadata) { + return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata)); + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 770b01c..4334de7 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -1,5 +1,10 @@ import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger'; -import { IsEnum } from 'class-validator'; +import { + IsEnum, + IsNotEmpty, + IsOptional, + IsString, +} from 'class-validator'; import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; export enum DataAssetShareType { @@ -198,6 +203,27 @@ export class IData { day_opening: number; } + +export enum CustomPropertyType { + TEXT = 'text', + NUMBER = 'number', + DATE = 'date', + BOOLEAN = 'boolean', +} + +export class CustomPropertyDto { + @ApiProperty() + key: string; + @ApiProperty() + value: string; + @ApiProperty({ enum: CustomPropertyType }) + type: CustomPropertyType; + @ApiPropertyOptional() + color?: string; + @ApiPropertyOptional() + emoji?: string; +} + export class IUpdateDataRequest { @ApiProperty() name: string; @@ -211,6 +237,8 @@ export class IUpdateDataRequest { share_type?: DataAssetShareType; @ApiPropertyOptional() docs?: string; + @ApiPropertyOptional({ type: [CustomPropertyDto] }) + custom_properties?: CustomPropertyDto[]; } export class IUpdateCertificationStatusRequest { @@ -372,4 +400,4 @@ export type CreateDataDocsDTO = { docs: string; asset_type: string; -} \ No newline at end of file +} diff --git a/src/modules/connection-test/connection-test.controller.ts b/src/modules/connection-test/connection-test.controller.ts index 029da68..043ab29 100644 --- a/src/modules/connection-test/connection-test.controller.ts +++ b/src/modules/connection-test/connection-test.controller.ts @@ -22,6 +22,9 @@ import { ConnectionTestListTablesRes, GetTableMetadataRes, GetTableMetadataReq, + RefreshCatalogReq, + RefreshCatalogRes, + RefreshCatalogStatusReq, } from './dto/connection-test'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator'; @@ -88,7 +91,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListSchemas( body, - user.customer_name, + user, ); } @@ -105,7 +108,7 @@ export class ConnectionTestController { }); return this.connectionTestService.connectionTestListTables( body, - user.customer_name, + user, ); } @@ -122,7 +125,38 @@ export class ConnectionTestController { }); return this.connectionTestService.getTableMetadata( body, - user.customer_name, + user, ); } + + @Post('refresh-catalog') + @ApiOkResponse({ type: RefreshCatalogRes }) + @HttpCode(HttpStatus.ACCEPTED) + async refreshCatalog( + @User() user: RequestUser, + @Body(new ValidationPipe()) body: RefreshCatalogReq, + ) { + this.logger.info('/connection-test/refresh-catalog', { + user: user.user_id, + customer: user.customer_name, + connection: body.connection_id, + }); + return this.connectionTestService.refreshCatalog(body, user); + } + + @Post('refresh-catalog/status') + @ApiOkResponse({ type: RefreshCatalogRes }) + @HttpCode(HttpStatus.OK) + async refreshCatalogStatus( + @User() user: RequestUser, + @Body(new ValidationPipe()) body: RefreshCatalogStatusReq, + ) { + this.logger.info('/connection-test/refresh-catalog/status', { + user: user.user_id, + customer: user.customer_name, + connection: body.connection_id, + session: body.session_id, + }); + return this.connectionTestService.refreshCatalogStatus(body, user); + } } diff --git a/src/modules/connection-test/connection-test.module.ts b/src/modules/connection-test/connection-test.module.ts index 1eb4099..c184391 100644 --- a/src/modules/connection-test/connection-test.module.ts +++ b/src/modules/connection-test/connection-test.module.ts @@ -5,10 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { ClientsModule } from '@nestjs/microservices'; import { ConnectionTestClientConfiguration } from './connection-test-client.config'; import { ConnectionModule } from '../connection/connection.module'; +import { ConnectionsApiModule } from '../connections-api/connections-api.module'; +import { PlatformApiModule } from '../platform-api/platform-api.module'; const client = new ConnectionTestClientConfiguration(); @Module({ controllers: [ConnectionTestController], providers: [ConnectionTestService, DadosferaLogger], - imports: [ClientsModule.register([client.providerOptions]), ConnectionModule], + imports: [ + ClientsModule.register([client.providerOptions]), + ConnectionModule, + ConnectionsApiModule, + PlatformApiModule, + ], }) export class ConnectionTestModule {} diff --git a/src/modules/connection-test/connection-test.service.spec.ts b/src/modules/connection-test/connection-test.service.spec.ts new file mode 100644 index 0000000..aed103f --- /dev/null +++ b/src/modules/connection-test/connection-test.service.spec.ts @@ -0,0 +1,205 @@ +import { ConnectionTestService } from './connection-test.service'; +import { RequestUser } from 'src/decorators/user.decorator'; + +describe('ConnectionTestService catalog cache', () => { + const user: RequestUser = { + user_id: 'user-id', + username: 'user@example.com', + permissions: [], + customer_id: 'customer-id', + customer_name: 'customer-name', + customer_tier: 'standard', + access_token: 'token', + customer_modules: [], + roles: [], + }; + const grpcClient = { getService: jest.fn().mockReturnValue({}) }; + const connectionsService = {}; + const connectionsApiService = { proxy: jest.fn() }; + const platformApiService = { proxy: jest.fn() }; + let service: ConnectionTestService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new ConnectionTestService( + grpcClient as any, + connectionsService as any, + connectionsApiService as any, + platformApiService as any, + ); + }); + + it('keeps the existing schemas response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }], + }); + + await expect( + service.connectionTestListSchemas( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + schema_list: ['analytics', 'public'], + }); + }); + + it('keeps the existing tables response contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + tables: [{ table_name: 'customers' }, { table_name: 'orders' }], + }); + + await expect( + service.connectionTestListTables( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + table_list: ['customers', 'orders'], + }); + }); + + it('maps cached columns to the existing table metadata contract', async () => { + connectionsApiService.proxy.mockResolvedValue({ + columns: [ + { + column_name: 'id', + data_type: 'bigint', + is_primary_key: true, + }, + ], + }); + + await expect( + service.getTableMetadata( + { + connection_id: 'config-id', + plugin: 'postgresql', + schema: 'public', + table_list: ['customers'], + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + tables_metadata: [ + { + table_name: 'customers', + columns: [ + { + name: 'id', + type: 'bigint', + is_primary_key: true, + }, + ], + references: [], + }, + ], + }); + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'GET', + '/connection_catalog/config-id/schemas/public/tables/customers/columns', + user, + ); + }); + + it('submits a catalog refresh without holding the request open', async () => { + platformApiService.proxy.mockResolvedValue({ + session_id: 'session-id', + date: '20260731', + }); + + await expect( + service.refreshCatalog( + { connection_id: 'config-id', plugin: 'postgresql' }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + status: 'PENDING', + session_id: 'session-id', + date: '20260731', + }); + + expect(platformApiService.proxy).toHaveBeenCalledWith( + 'POST', + '/connection_test', + user, + { + customer_id: user.customer_name, + plugin: 'postgresql', + task: { + task_type: 'refresh_catalog', + connection: { + provider: 'connection_manager', + config_id: 'config-id', + }, + }, + }, + ); + }); + + it('keeps polling without changing the catalog pointer while pending', async () => { + platformApiService.proxy.mockResolvedValue({ status: 'PENDING' }); + + await expect( + service.refreshCatalogStatus( + { + connection_id: 'config-id', + plugin: 'postgresql', + session_id: 'session-id', + date: '20260731', + }, + user, + ), + ).resolves.toEqual({ + operation_result: false, + status: 'PENDING', + session_id: 'session-id', + date: '20260731', + }); + + expect(connectionsApiService.proxy).not.toHaveBeenCalled(); + }); + + it('publishes the catalog pointer after the refresh finishes', async () => { + platformApiService.proxy.mockResolvedValue({ status: 'DONE' }); + connectionsApiService.proxy.mockResolvedValue({ + last_catalog_refresh_status: 'SUCCESS', + }); + + await expect( + service.refreshCatalogStatus( + { + connection_id: 'config/id', + plugin: 'postgresql', + session_id: 'session-id', + date: '20260731', + }, + user, + ), + ).resolves.toEqual({ + operation_result: true, + status: 'DONE', + session_id: 'session-id', + date: '20260731', + }); + + expect(connectionsApiService.proxy).toHaveBeenCalledWith( + 'PUT', + '/connection_config/config%2Fid/catalog_metadata', + user, + { + last_catalog_refresh_status: 'SUCCESS', + last_catalog_connection_test_date: '20260731', + last_catalog_connection_test_session_id: 'session-id', + }, + ); + }); +}); diff --git a/src/modules/connection-test/connection-test.service.ts b/src/modules/connection-test/connection-test.service.ts index 3b92e12..fe4930d 100644 --- a/src/modules/connection-test/connection-test.service.ts +++ b/src/modules/connection-test/connection-test.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { ConnectionTest } from '@dadosfera/protospack-v2'; import { lastValueFrom } from 'rxjs'; @@ -13,6 +13,9 @@ import { ConnectionTestPingRes, GetTableMetadataReq, GetTableMetadataRes, + RefreshCatalogReq, + RefreshCatalogRes, + RefreshCatalogStatusReq, } from './dto/connection-test'; import { ConnectionClientService } from '../connection/client.service'; import { @@ -21,6 +24,8 @@ import { } from '../connection/dtos/connection'; import { RequestUser } from 'src/decorators/user.decorator'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { ConnectionsApiService } from '../connections-api/connections-api.service'; +import { PlatformApiService } from '../platform-api/platform-api.service'; @Injectable() export class ConnectionTestService { @@ -28,6 +33,8 @@ export class ConnectionTestService { constructor( @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc, private connectionsService: ConnectionClientService, + private connectionsApiService: ConnectionsApiService, + private platformApiService: PlatformApiService, ) { this.connectionTestReadClient = grpcClient.getService( @@ -147,45 +154,137 @@ export class ConnectionTestService { } async connectionTestListSchemas( body: ConnectionTestListSchemasReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin } = body; - return lastValueFrom( - this.connectionTestReadClient.ListSchemas({ - connection_id, - customer_name, - plugin, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`, + user, ); + return { + operation_result: true, + schema_list: result.schemas.map((schema) => schema.schema_name), + }; } + async connectionTestListTables( body: ConnectionTestListTablesReq, - customer_name: string, + user: RequestUser, ): Promise { - const { connection_id, plugin, schema } = body; - return lastValueFrom( - this.connectionTestReadClient.ListTables({ - connection_id, - customer_name, - plugin, - schema, - }), + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}/tables`, + user, ); + return { + operation_result: true, + table_list: result.tables.map((table) => table.table_name), + }; } async getTableMetadata( body: GetTableMetadataReq, - customer_name: string, + user: RequestUser, ): Promise { - const { schema, plugin, table_list, connection_id } = body; - return lastValueFrom( - this.connectionTestReadClient.GetTableMetadata({ - connection_id, - customer_name, - plugin, - schema, - table_list, + const tables_metadata = await Promise.all( + body.table_list.map(async (table_name) => { + const result = await this.connectionsApiService.proxy( + 'GET', + `/connection_catalog/${encodeURIComponent(body.connection_id)}` + + `/schemas/${encodeURIComponent(body.schema)}` + + `/tables/${encodeURIComponent(table_name)}/columns`, + user, + ); + return { + table_name, + columns: result.columns.map((column) => ({ + name: column.column_name, + type: column.data_type, + is_primary_key: column.is_primary_key, + })), + references: [], + }; }), ); + return { operation_result: true, tables_metadata }; + } + + async refreshCatalog( + body: RefreshCatalogReq, + user: RequestUser, + ): Promise { + const task = await this.platformApiService.proxy( + 'POST', + '/connection_test', + user, + { + customer_id: user.customer_name, + plugin: body.plugin, + task: { + task_type: 'refresh_catalog', + connection: { + provider: 'connection_manager', + config_id: body.connection_id, + }, + }, + }, + ); + + if (!task.session_id || !task.date) { + throw new HttpException( + 'Platform API did not return a catalog refresh task identifier', + HttpStatus.BAD_GATEWAY, + ); + } + + return { + operation_result: true, + status: 'PENDING', + session_id: task.session_id, + date: task.date, + }; + } + + async refreshCatalogStatus( + body: RefreshCatalogStatusReq, + user: RequestUser, + ): Promise { + const result = await this.platformApiService.proxy( + 'POST', + '/connection_test/status', + user, + { + session_id: body.session_id, + date: body.date, + }, + ); + + if (result.status === 'DONE') { + await this.connectionsApiService.proxy( + 'PUT', + `/connection_config/${encodeURIComponent( + body.connection_id, + )}/catalog_metadata`, + user, + { + last_catalog_refresh_status: 'SUCCESS', + last_catalog_connection_test_date: body.date, + last_catalog_connection_test_session_id: body.session_id, + }, + ); + } else if (result.status === 'ERROR' || result.status === 'EXPIRED') { + throw new HttpException( + `Catalog refresh finished with status ${result.status}`, + HttpStatus.BAD_GATEWAY, + ); + } + + return { + operation_result: result.status === 'DONE', + status: result.status, + session_id: body.session_id, + date: body.date, + }; } } diff --git a/src/modules/connection-test/dto/connection-test.ts b/src/modules/connection-test/dto/connection-test.ts index b6bb4c4..3496d4c 100644 --- a/src/modules/connection-test/dto/connection-test.ts +++ b/src/modules/connection-test/dto/connection-test.ts @@ -1,5 +1,5 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; -import { IsString, IsOptional } from 'class-validator'; +import { IsIn, IsString, IsOptional } from 'class-validator'; import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection'; import { CreateConnectionDto } from 'src/modules/connection/dtos/connection'; export class ColumnDto { @@ -7,6 +7,8 @@ export class ColumnDto { name: string; @ApiProperty() type: string; + @ApiProperty() + is_primary_key: boolean; } export class TableMetadataDto { @ApiProperty() @@ -131,3 +133,37 @@ export class GetTableMetadataRes { @ApiProperty({ type: [TableMetadataDto] }) tables_metadata: TableMetadataDto[]; } + +export class RefreshCatalogReq { + @ApiProperty() + @IsString() + connection_id: string; + + @ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] }) + @IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver']) + plugin: string; +} + +export class RefreshCatalogStatusReq extends RefreshCatalogReq { + @ApiProperty() + @IsString() + session_id: string; + + @ApiProperty() + @IsString() + date: string; +} + +export class RefreshCatalogRes { + @ApiProperty() + operation_result: boolean; + + @ApiProperty() + status: string; + + @ApiProperty() + session_id: string; + + @ApiProperty() + date: string; +} diff --git a/src/modules/connections-api/connections-api.config.ts b/src/modules/connections-api/connections-api.config.ts new file mode 100644 index 0000000..0e6e450 --- /dev/null +++ b/src/modules/connections-api/connections-api.config.ts @@ -0,0 +1,11 @@ +export const CONNECTIONS_API_CONFIG = { + getUrl: (): string => { + const url = process.env.CONNECTIONS_API_URL; + if (!url) { + throw new Error('CONNECTIONS_API_URL environment variable is not set'); + } + return url; + }, + region: process.env.AWS_REGION || 'us-east-1', + timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10), +}; diff --git a/src/modules/connections-api/connections-api.module.ts b/src/modules/connections-api/connections-api.module.ts new file mode 100644 index 0000000..1835186 --- /dev/null +++ b/src/modules/connections-api/connections-api.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { ConnectionsApiService } from './connections-api.service'; + +@Module({ + providers: [ConnectionsApiService, DadosferaLogger], + exports: [ConnectionsApiService], +}) +export class ConnectionsApiModule {} diff --git a/src/modules/connections-api/connections-api.service.ts b/src/modules/connections-api/connections-api.service.ts new file mode 100644 index 0000000..ae2dfd4 --- /dev/null +++ b/src/modules/connections-api/connections-api.service.ts @@ -0,0 +1,99 @@ +import { Injectable, Inject, HttpException } from '@nestjs/common'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import axios, { AxiosResponse, Method } from 'axios'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { RequestUser } from '../../decorators/user.decorator'; +import { CONNECTIONS_API_CONFIG } from './connections-api.config'; + +@Injectable() +export class ConnectionsApiService { + private signer: SignatureV4; + private logger: any; + + constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) { + this.logger = dadosferaLogger.logger; + this.signer = new SignatureV4({ + service: 'execute-api', + region: CONNECTIONS_API_CONFIG.region, + credentials: defaultProvider(), + sha256: Sha256, + }); + } + + async proxy( + method: string, + path: string, + user: RequestUser, + body?: any, + query?: Record, + ): Promise { + const baseUrl = CONNECTIONS_API_CONFIG.getUrl(); + const url = new URL(`${baseUrl}${path}`); + + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } + const headers: Record = { + host: url.hostname, + 'content-type': 'application/json', + customer_name: user.customer_name || '', + customer_id: user.customer_id || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + 'x-customer-id': user.customer_id || '', + }; + const requestToSign = { + method: method.toUpperCase(), + protocol: url.protocol, + hostname: url.hostname, + port: url.port ? parseInt(url.port, 10) : undefined, + path: url.pathname + url.search, + headers, + body: body ? JSON.stringify(body) : undefined, + }; + + try { + const signedRequest = await this.signer.sign(requestToSign); + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers: signedRequest.headers as Record, + data: body, + timeout: CONNECTIONS_API_CONFIG.timeout, + validateStatus: () => true, + }); + + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + return response.data; + } catch (error) { + this.logger.error('Connections API proxy error', { + error: error.message, + path, + method: method.toUpperCase(), + }); + if (error instanceof HttpException) { + throw error; + } + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + if (error.code === 'ECONNREFUSED') { + throw new HttpException('Connections API service unavailable', 503); + } + if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') { + throw new HttpException('Connections API request timeout', 504); + } + throw new HttpException('Internal server error', 500); + } + } +} diff --git a/src/modules/permissions/permissions.controller.spec.ts b/src/modules/permissions/permissions.controller.spec.ts index 7ab875b..1bc3004 100644 --- a/src/modules/permissions/permissions.controller.spec.ts +++ b/src/modules/permissions/permissions.controller.spec.ts @@ -1,6 +1,6 @@ import { ClientsModule } from '@nestjs/microservices'; import { Test, TestingModule } from '@nestjs/testing'; -// import { DucClient } from 'src/clients/duc/client.config'; +// import { DucClient } from '../duc/client.config' import { PermissionsController } from './permissions.controller'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PermissionsService } from './permissions.service'; @@ -22,7 +22,7 @@ describe.skip('PermissionsController', () => { provide: DadosferaLogger, useValue: { logger }, }, - PermissionsService, + PermissionsService ], }).compile(); diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 503d117..8121242 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -14,7 +14,7 @@ import { NotFoundException, UseGuards, } from '@nestjs/common'; -import { ApiTags, ApiOperation } from '@nestjs/swagger'; +import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { @@ -74,6 +74,10 @@ export class PlatformApiController { return id?.replace(/-/g, '_') || ''; } + private decodePathParam(value: string): string { + return value ? decodeURIComponent(value) : ''; + } + /** * Denormalize ID back to UUID format (replace _ with -). * Used when we receive a normalized ID but need the original UUID. @@ -834,6 +838,40 @@ export class PlatformApiController { ); } + @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs') + @ApiOperation({ + summary: 'Get pipeline run jobs', + description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.', + }) + @ApiOkResponse({ + description: 'DB-backed job runs for the selected pipeline run.', + schema: { + type: 'object', + properties: { + jobs: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['jobs'], + }, + }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getPipelineRunJobs( + @Param('pipelineId') pipelineId: string, + @Param('runId') runId: string, + @User() user: RequestUser, + ) { + const normalizedPipelineId = this.normalizePipelineId(pipelineId); + const decodedRunId = this.decodePathParam(runId); + + return this.platformApiService.proxy( + 'GET', + `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`, + user, + ); + } + // ==================== JOBS - COLUMN EDITING ROUTES ==================== @Put('jobs/:jobId/input') diff --git a/src/modules/release_note/release_note.controller.spec.ts b/src/modules/release_note/release_note.controller.spec.ts index 6086196..af307e7 100644 --- a/src/modules/release_note/release_note.controller.spec.ts +++ b/src/modules/release_note/release_note.controller.spec.ts @@ -5,11 +5,21 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; describe('ReleaseNoteController', () => { let controller: ReleaseNoteController; + const logger = { + info: (...args) => args, + error: (...args) => args, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [ReleaseNoteController], - providers: [ReleaseNoteService, DadosferaLogger], + providers: [ + { + provide: DadosferaLogger, + useValue: { logger }, + }, + ReleaseNoteService + ], }).compile(); controller = module.get(ReleaseNoteController); diff --git a/src/modules/release_note/release_note.service.spec.ts b/src/modules/release_note/release_note.service.spec.ts index 266146d..fa919ae 100644 --- a/src/modules/release_note/release_note.service.spec.ts +++ b/src/modules/release_note/release_note.service.spec.ts @@ -4,10 +4,20 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; describe('ReleaseNoteService', () => { let service: ReleaseNoteService; + const logger = { + info: (...args) => args, + error: (...args) => args, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - providers: [ReleaseNoteService, DadosferaLogger], + providers: [ + { + provide: DadosferaLogger, + useValue: { logger }, + }, + ReleaseNoteService + ], }).compile(); service = module.get(ReleaseNoteService);