Merge pull request #438 from dadosfera/fix/proxy-urls

FIX: storage api

deploy/helm-chart/values-stg.yaml

@@ -9,6 +9,7 @@ maestro:
   cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
+  storage_explorer_api_url: "https://storage-explorer-{customer}.stg.dadosfera.ai/api"
 
 hostname: maestro.stg.dadosfera.ai
 

deploy/helm-chart/values.yaml

@@ -47,7 +47,7 @@ maestro:
   open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
   open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
   platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
-  storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api"
+  storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
   dedicated_proxy: ""
   restricted_ip: ""
   redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"

docsfera.json

@@ -7275,6 +7275,100 @@
           ]
         }
       },
+      "/platform/pipelines/catalog/tables": {
+        "get": {
+          "operationId": "PlatformApiController_getAvailableTables",
+          "summary": "Get all tables available",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Platform API"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/platform/pipelines/catalog/schemas": {
+        "get": {
+          "operationId": "PlatformApiController_getAvailableSchemas",
+          "summary": "Get all schemas available",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Platform API"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/platform/pipelines/catalog/tables/validate": {
+        "post": {
+          "operationId": "PlatformApiController_validateTableAndSchema",
+          "summary": "Validate tables and schemas",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Platform API"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/platform/pipeline/{pipelineId}/pipeline_run": {
         "get": {
           "operationId": "PlatformApiController_getPipelineRuns",
@@ -8440,36 +8534,6 @@
           ]
         }
       },
-      "/storage-explorer/storage/link-to-table": {
-        "post": {
-          "operationId": "StorageExplorerController_linkFileToTable",
-          "summary": "Link an existing file in storage to a table",
-          "parameters": [],
-          "responses": {
-            "201": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Storage Explorer"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
       "/storage-explorer/storage/metadata": {
         "get": {
           "operationId": "StorageExplorerController_getFileMetadata",

src/modules/platform-api/platform-api.controller.ts

@@ -26,6 +26,14 @@ import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
 
+
+type ValidateTablesDTO = {
+  tables: Array<{
+    table_schema: string,
+    table_name: string
+  }>
+}
+
 @ApiTags('Platform API')
 @Controller('platform')
 export class PlatformApiController {
@@ -720,6 +728,57 @@ export class PlatformApiController {
     );
   }
 
+  // ==================== Catalog ROUTES ====================
+
+  @Get('pipelines/catalog/tables')
+  @ApiOperation({ summary: 'Get all tables available' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getAvailableTables(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'GET',
+      '/catalog/tables',
+      user,
+      undefined,
+      query,
+    );
+  }
+
+  @Get('pipelines/catalog/schemas')
+  @ApiOperation({ summary: 'Get all schemas available' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getAvailableSchemas(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'GET',
+      '/catalog/schemas',
+      user,
+      undefined,
+      query,
+    );
+  }
+
+  @Post('pipelines/catalog/tables/validate')
+  @ApiOperation({ summary: 'Validate tables and schemas' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  async validateTableAndSchema(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+    @Body() validateTablesDto: ValidateTablesDTO[]
+  ) {
+    return this.platformApiService.proxy(
+      'POST',
+      '/catalog/tables/validate',
+      user,
+      validateTablesDto,
+      query,
+    );
+  }
+
   // ==================== PIPELINE RUN ROUTES ====================
 
   @Get('pipeline/:pipelineId/pipeline_run')

src/modules/storage-explorer/storage-explorer.config.ts

@@ -1,5 +1,5 @@
 export const STORAGE_EXPLORER_CONFIG = {
-  getUrl: (customerId: string): string => {
+  getUrl: (customerName: string): string => {
     const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
     if (!urlTemplate) {
       throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
@@ -7,7 +7,7 @@ export const STORAGE_EXPLORER_CONFIG = {
     // Replace {customer_id} placeholder with actual customer ID
     // For local: http://172.17.0.1:8000/api (no placeholder)
     // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api
-    return urlTemplate.replace('{customer_id}', customerId);
+    return urlTemplate.replace('{customer}', customerName);
   },
   timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
 };

src/modules/storage-explorer/storage-explorer.service.ts

@@ -28,7 +28,7 @@ export class StorageExplorerService {
     }
 
     // Get customer-specific storage-explorer URL
-    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
     const url = new URL(`${baseUrl}${path}`);
 
     // Add query params