diff --git a/deploy/helm-chart/values-stg.yaml b/deploy/helm-chart/values-stg.yaml index 360aafe..ed83c67 100644 --- a/deploy/helm-chart/values-stg.yaml +++ b/deploy/helm-chart/values-stg.yaml @@ -9,6 +9,7 @@ maestro: cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36" redis_database: "1" platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com + storage_explorer_api_url: "https://storage-explorer-{customer}.stg.dadosfera.ai/" hostname: maestro.stg.dadosfera.ai diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 3c8ebcc..6420616 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -47,7 +47,7 @@ maestro: open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com - storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api" + storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/" dedicated_proxy: "" restricted_ip: "" redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com" diff --git a/docsfera.json b/docsfera.json index d13bcd6..801658d 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7275,6 +7275,100 @@ ] } }, + "/platform/pipelines/catalog/tables": { + "get": { + "operationId": "PlatformApiController_getAvailableTables", + "summary": "Get all tables available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get all schemas available", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate tables and schemas", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/platform/pipeline/{pipelineId}/pipeline_run": { "get": { "operationId": "PlatformApiController_getPipelineRuns", @@ -8440,36 +8534,6 @@ ] } }, - "/storage-explorer/storage/link-to-table": { - "post": { - "operationId": "StorageExplorerController_linkFileToTable", - "summary": "Link an existing file in storage to a table", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/storage-explorer/storage/metadata": { "get": { "operationId": "StorageExplorerController_getFileMetadata", diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index e0d59f2..978f464 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -26,6 +26,14 @@ import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; + +type ValidateTablesDTO = { + tables: Array<{ + table_schema: string, + table_name: string + }> +} + @ApiTags('Platform API') @Controller('platform') export class PlatformApiController { @@ -720,6 +728,57 @@ export class PlatformApiController { ); } + // ==================== Catalog ROUTES ==================== + + @Get('pipelines/catalog/tables') + @ApiOperation({ summary: 'Get all tables available' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getAvailableTables( + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'GET', + '/catalog/tables', + user, + undefined, + query, + ); + } + + @Get('pipelines/catalog/schemas') + @ApiOperation({ summary: 'Get all schemas available' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) + async getAvailableSchemas( + @User() user: RequestUser, + @Query() query: Record, + ) { + return this.platformApiService.proxy( + 'GET', + '/catalog/schemas', + user, + undefined, + query, + ); + } + + @Post('pipelines/catalog/tables/validate') + @ApiOperation({ summary: 'Validate tables and schemas' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async validateTableAndSchema( + @User() user: RequestUser, + @Query() query: Record, + @Body() validateTablesDto: ValidateTablesDTO[] + ) { + return this.platformApiService.proxy( + 'POST', + '/catalog/tables/validate', + user, + validateTablesDto, + query, + ); + } + // ==================== PIPELINE RUN ROUTES ==================== @Get('pipeline/:pipelineId/pipeline_run') diff --git a/src/modules/storage-explorer/storage-explorer.config.ts b/src/modules/storage-explorer/storage-explorer.config.ts index 417eb15..4a9ff43 100644 --- a/src/modules/storage-explorer/storage-explorer.config.ts +++ b/src/modules/storage-explorer/storage-explorer.config.ts @@ -1,5 +1,5 @@ export const STORAGE_EXPLORER_CONFIG = { - getUrl: (customerId: string): string => { + getUrl: (customerName: string): string => { const urlTemplate = process.env.STORAGE_EXPLORER_API_URL; if (!urlTemplate) { throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set'); @@ -7,7 +7,7 @@ export const STORAGE_EXPLORER_CONFIG = { // Replace {customer_id} placeholder with actual customer ID // For local: http://172.17.0.1:8000/api (no placeholder) // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api - return urlTemplate.replace('{customer_id}', customerId); + return urlTemplate.replace('{customer}', customerName); }, timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10), }; diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts index ba149c0..5ff3bf7 100644 --- a/src/modules/storage-explorer/storage-explorer.service.ts +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -28,7 +28,7 @@ export class StorageExplorerService { } // Get customer-specific storage-explorer URL - const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name); const url = new URL(`${baseUrl}${path}`); // Add query params