diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 360aafe..2c1737d 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,6 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com + storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api" hostname: maestro.stg.dadosfera.ai diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 3c8ebcc..4e431f5 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -47,7 +47,7 @@ maestro: open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com - storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api" + storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api" dedicated_proxy: "" restricted_ip: "" redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com" diff --git a/docsfera.json b/docsfera.json index d13bcd6..801658d 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7275,6 +7275,100 @@ ] } }, + "/platform/pipelines/catalog/tables": { + "get": { + "operationId": "PlatformApiController_getAvailableTables", + "summary": "Get all tables available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get all schemas available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate tables and schemas", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/pipeline/{pipelineId}/pipeline_run": { "get": { "operationId": "PlatformApiController_getPipelineRuns", @@ -8440,36 +8534,6 @@ ] } }, - "/storage-explorer/storage/link-to-table": { - "post": { - "operationId": "StorageExplorerController_linkFileToTable", - "summary": "Link an existing file in storage to a table", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/storage-explorer/storage/metadata": { "get": { "operationId": "StorageExplorerController_getFileMetadata", diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index e0d59f2..978f464 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -26,6 +26,14 @@ import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; + +type ValidateTablesDTO = { + tables: Array<{ + table_schema: string, + table_name: string + }> +} + @ApiTags('Platform API') @Controller('platform') export class PlatformApiController { @@ -720,6 +728,57 @@ export class PlatformApiController { ); } + // ==================== Catalog ROUTES ==================== + + @Get('pipelines/catalog/tables') + @ApiOperation({ summary: 'Get all tables available' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getAvailableTables( + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'GET', + '/catalog/tables', + user, + undefined, + query, + ); + } + + @Get('pipelines/catalog/schemas') + @ApiOperation({ summary: 'Get all schemas available' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getAvailableSchemas( + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'GET', + '/catalog/schemas', + user, + undefined, + query, + ); + } + + @Post('pipelines/catalog/tables/validate') + @ApiOperation({ summary: 'Validate tables and schemas' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async validateTableAndSchema( + @User() user: RequestUser, + @Query() query: Record, + @Body() validateTablesDto: ValidateTablesDTO[] + ) { + return this.platformApiService.proxy( + 'POST', + '/catalog/tables/validate', + user, + validateTablesDto, + query, + ); + } + // ==================== PIPELINE RUN ROUTES ==================== @Get('pipeline/:pipelineId/pipeline_run') diff --git a/src/modules/storage-explorer/storage-explorer.config.ts b/src/modules/storage-explorer/storage-explorer.config.ts index 417eb15..aa1a46e 100644 --- a/src/modules/storage-explorer/storage-explorer.config.ts +++ b/src/modules/storage-explorer/storage-explorer.config.ts @@ -1,13 +1,13 @@ export const STORAGE_EXPLORER_CONFIG = { - getUrl: (customerId: string): string => { + getUrl: (customerName: string): string => { const urlTemplate = process.env.STORAGE_EXPLORER_API_URL; if (!urlTemplate) { throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set'); } // Replace {customer_id} placeholder with actual customer ID // For local: http://172.17.0.1:8000/api (no placeholder) - // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api - return urlTemplate.replace('{customer_id}', customerId); + // For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api + return urlTemplate.replace('{customer}', customerName); }, timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10), }; diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index ef6a071..34e94fc 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -3,13 +3,13 @@ import { Get, Post, Put, - Delete, Param, Body, Query, Inject, UseInterceptors, UploadedFiles, + Headers, } from '@nestjs/common'; import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger'; import { FilesInterceptor } from '@nestjs/platform-express'; @@ -47,11 +47,13 @@ export class StorageExplorerController { async validateTableName( @Body() body: any, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'POST', '/tables/validate-name', user, + body, ); } @@ -62,11 +64,13 @@ export class StorageExplorerController { async createTable( @Body() body: any, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'POST', '/tables/', user, + body, ); } @@ -77,11 +81,13 @@ export class StorageExplorerController { async listTables( @Query('page') page: number, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', '/tables/', user, + undefined, { page }, ); @@ -93,6 +99,7 @@ export class StorageExplorerController { async getTable( @Param('tableId') tableId: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', @@ -108,11 +115,13 @@ export class StorageExplorerController { @Param('tableId') tableId: string, @Param('datasetId') datasetId: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'POST', `/tables/${tableId}/datasets/${datasetId}`, user, + ); } @@ -122,11 +131,13 @@ export class StorageExplorerController { async getTableDatasets( @Param('tableId') tableId: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', `/tables/${tableId}/datasets`, user, + ); } @@ -136,11 +147,13 @@ export class StorageExplorerController { async getTableSchema( @Param('tableId') tableId: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', `/tables/${tableId}/schema`, user, + ); } @@ -156,6 +169,7 @@ export class StorageExplorerController { 'POST', `/tables/${tableId}/validate-compatibility/${datasetId}`, user, + ); } @@ -170,11 +184,13 @@ export class StorageExplorerController { @Param('datasetId') datasetId: string, @Query('limit') limit: number, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', `/datasets/${datasetId}/preview`, user, + undefined, { limit }, ); @@ -187,11 +203,13 @@ export class StorageExplorerController { @Param('datasetId') datasetId: string, @Query('force_refresh') forceRefresh: boolean, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', `/datasets/${datasetId}/schema`, user, + undefined, { force_refresh: forceRefresh }, ); @@ -203,11 +221,13 @@ export class StorageExplorerController { async listDatasetsByUpload( @Param('uploadId') uploadId: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', `/datasets/upload/${uploadId}`, user, + ); } @@ -218,11 +238,13 @@ export class StorageExplorerController { @Param('datasetId') datasetId: string, @Body() body: any, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'PUT', `/datasets/${datasetId}/refresh-schema`, user, + body, ); } @@ -239,11 +261,13 @@ export class StorageExplorerController { @Query('limit') limit: number, @Query('folder_path') folderPath: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', '/storage/uploads/history', user, + undefined, { page, limit, folder_path: folderPath }, ); @@ -255,11 +279,13 @@ export class StorageExplorerController { async browseStorage( @Query('path') path: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', '/storage/browse', user, + undefined, { path }, ); @@ -274,6 +300,7 @@ export class StorageExplorerController { @UploadedFiles() files: Array, @Body('folder_path') folderPath: string, @User() user: RequestUser, + ) { // Create FormData to forward files to storage-explorer API const formData = new FormData(); @@ -297,6 +324,7 @@ export class StorageExplorerController { 'POST', '/storage/upload/batch', user, + formData, ); } @@ -307,11 +335,13 @@ export class StorageExplorerController { async createFolder( @Body() body: any, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'POST', '/storage/folder/create', user, + body, ); } @@ -322,11 +352,13 @@ export class StorageExplorerController { async downloadFile( @Query('file_path') filePath: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', '/storage/download', user, + undefined, { file_path: filePath }, ); @@ -338,6 +370,7 @@ export class StorageExplorerController { async getFileMetadata( @Query('file_path') filePath: string, @User() user: RequestUser, + ) { return this.storageExplorerService.proxy( 'GET', diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts index ba149c0..8f4e4b0 100644 --- a/src/modules/storage-explorer/storage-explorer.service.ts +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -20,7 +20,7 @@ export class StorageExplorerService { path: string, user: RequestUser, body?: any, - query?: Record, + query?: Record ): Promise { // Validate customer_id is present for multi-tenant isolation if (!user.customer_id) { @@ -28,7 +28,7 @@ export class StorageExplorerService { } // Get customer-specific storage-explorer URL - const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name); const url = new URL(`${baseUrl}${path}`); // Add query params @@ -113,7 +113,7 @@ export class StorageExplorerService { } // Get customer-specific storage-explorer URL - const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name); const url = new URL(`${baseUrl}${path}`); // Add query params