Merge pull request #442 from dadosfera/feat/qualify

FIX: platform routes

deploy/helm-chart/values-stg.yaml

@@ -9,6 +9,7 @@ maestro:
   cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
+  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api"
 
 hostname: maestro.stg.dadosfera.ai
 

deploy/helm-chart/values.yaml

@@ -47,7 +47,7 @@ maestro:
   open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
   open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
   platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
-  storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api"
+  storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
   dedicated_proxy: ""
   restricted_ip: ""
   redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"

docsfera.json

@@ -7275,6 +7275,70 @@
           ]
         }
       },
+      "/platform/pipelines/catalog/schemas": {
+        "get": {
+          "operationId": "PlatformApiController_getAvailableSchemas",
+          "summary": "Get available schemas",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Platform API"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/platform/pipelines/catalog/tables/validate": {
+        "post": {
+          "operationId": "PlatformApiController_validateTableAndSchema",
+          "summary": "Validate Table and Schema",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ValidationTableDTO"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Platform API"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/platform/pipeline/{pipelineId}/pipeline_run": {
         "get": {
           "operationId": "PlatformApiController_getPipelineRuns",
@@ -8440,36 +8504,6 @@
           ]
         }
       },
-      "/storage-explorer/storage/link-to-table": {
-        "post": {
-          "operationId": "StorageExplorerController_linkFileToTable",
-          "summary": "Link an existing file in storage to a table",
-          "parameters": [],
-          "responses": {
-            "201": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Storage Explorer"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
       "/storage-explorer/storage/metadata": {
         "get": {
           "operationId": "StorageExplorerController_getFileMetadata",
@@ -10054,6 +10088,15 @@
                 "type": "string"
               }
             },
+            "identifier_columns": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "destination": {
+              "type": "object"
+            },
             "type": {
               "type": "string"
             }
@@ -10062,6 +10105,8 @@
             "name",
             "columns",
             "references",
+            "identifier_columns",
+            "destination",
             "type"
           ]
         },
@@ -11470,6 +11515,20 @@
           "required": [
             "providers"
           ]
+        },
+        "ValidationTableDTO": {
+          "type": "object",
+          "properties": {
+            "tables": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "tables"
+          ]
         }
       }
     }

package-lock.json

@@ -17,7 +17,7 @@
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+        "@dadosfera/protospack-v2": "3.38.0-beta.27",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1744,9 +1744,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.38.0-beta.26",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
-      "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
+      "version": "3.38.0-beta.27",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz",
+      "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -35,7 +35,7 @@
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+    "@dadosfera/protospack-v2": "3.38.0-beta.27",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/inputs/dtos/input.model.ts

@@ -14,6 +14,13 @@ export class TableColumns {
   @ApiProperty()
   references: Column[];
   @ApiProperty()
+  identifier_columns: string[];
+  @ApiProperty()
+  destination: Record<'raw' | 'qualify', {
+    table_name: string;
+    table_schema: string;
+  }> | null;
+  @ApiProperty()
   type: string;
 }
 export class AvailableEntity {

src/modules/platform-api/platform-api.controller.ts

@@ -25,6 +25,15 @@ import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
+import { ValidationTableDTO } from './platform-api.dto';
+
+
+type ValidateTablesDTO = {
+  tables: Array<{
+    table_schema: string,
+    table_name: string
+  }>
+}
 
 @ApiTags('Platform API')
 @Controller('platform')
@@ -720,6 +729,41 @@ export class PlatformApiController {
     );
   }
 
+  // ==================== PIPELINE VALIDATION ====================
+
+  @Get('pipelines/catalog/schemas')
+  @ApiOperation({ summary: 'Get available schemas' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getAvailableSchemas(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'GET',
+      `/catalog/schemas`,
+      user,
+      undefined,
+      query,
+    );
+  }
+
+  @Post('pipelines/catalog/tables/validate')
+  @ApiOperation({ summary: 'Validate Table and Schema' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async validateTableAndSchema(
+    @Body() payload: ValidationTableDTO,
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'POST',
+      `/catalog/tables/validate`,
+      user,
+      payload,
+      query,
+    );
+  }
+
   // ==================== PIPELINE RUN ROUTES ====================
 
   @Get('pipeline/:pipelineId/pipeline_run')

src/modules/platform-api/platform-api.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ValidationTableDTO {
+    @ApiProperty()
+    tables: Array<{
+        table_name: string;
+        table_schema: string;
+    }>
+}

src/modules/storage-explorer/storage-explorer.config.ts

@@ -1,13 +1,13 @@
 export const STORAGE_EXPLORER_CONFIG = {
-  getUrl: (customerId: string): string => {
+  getUrl: (customerName: string): string => {
     const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
     if (!urlTemplate) {
       throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
     }
     // Replace {customer_id} placeholder with actual customer ID
     // For local: http://172.17.0.1:8000/api (no placeholder)
-    // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api
-    return urlTemplate.replace('{customer_id}', customerId);
+    // For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
+    return urlTemplate.replace('{customer}', customerName);
   },
   timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
 };

src/modules/storage-explorer/storage-explorer.controller.ts

@@ -3,13 +3,13 @@ import {
   Get,
   Post,
   Put,
-  Delete,
   Param,
   Body,
   Query,
   Inject,
   UseInterceptors,
   UploadedFiles,
+  Headers,
 } from '@nestjs/common';
 import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
 import { FilesInterceptor } from '@nestjs/platform-express';
@@ -47,11 +47,13 @@ export class StorageExplorerController {
   async validateTableName(
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/tables/validate-name',
       user,
+      
       body,
     );
   }
@@ -62,11 +64,13 @@ export class StorageExplorerController {
   async createTable(
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/tables/',
       user,
+      
       body,
     );
   }
@@ -77,11 +81,13 @@ export class StorageExplorerController {
   async listTables(
     @Query('page') page: number,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/tables/',
       user,
+      
       undefined,
       { page },
     );
@@ -93,6 +99,7 @@ export class StorageExplorerController {
   async getTable(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
@@ -108,11 +115,13 @@ export class StorageExplorerController {
     @Param('tableId') tableId: string,
     @Param('datasetId') datasetId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       `/tables/${tableId}/datasets/${datasetId}`,
       user,
+      
     );
   }
 
@@ -122,11 +131,13 @@ export class StorageExplorerController {
   async getTableDatasets(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}/datasets`,
       user,
+      
     );
   }
 
@@ -136,11 +147,13 @@ export class StorageExplorerController {
   async getTableSchema(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}/schema`,
       user,
+      
     );
   }
 
@@ -156,6 +169,7 @@ export class StorageExplorerController {
       'POST',
       `/tables/${tableId}/validate-compatibility/${datasetId}`,
       user,
+      
     );
   }
 
@@ -170,11 +184,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Query('limit') limit: number,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/${datasetId}/preview`,
       user,
+      
       undefined,
       { limit },
     );
@@ -187,11 +203,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Query('force_refresh') forceRefresh: boolean,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/${datasetId}/schema`,
       user,
+      
       undefined,
       { force_refresh: forceRefresh },
     );
@@ -203,11 +221,13 @@ export class StorageExplorerController {
   async listDatasetsByUpload(
     @Param('uploadId') uploadId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/upload/${uploadId}`,
       user,
+      
     );
   }
 
@@ -218,11 +238,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'PUT',
       `/datasets/${datasetId}/refresh-schema`,
       user,
+      
       body,
     );
   }
@@ -239,11 +261,13 @@ export class StorageExplorerController {
     @Query('limit') limit: number,
     @Query('folder_path') folderPath: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/uploads/history',
       user,
+      
       undefined,
       { page, limit, folder_path: folderPath },
     );
@@ -255,11 +279,13 @@ export class StorageExplorerController {
   async browseStorage(
     @Query('path') path: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/browse',
       user,
+      
       undefined,
       { path },
     );
@@ -274,6 +300,7 @@ export class StorageExplorerController {
     @UploadedFiles() files: Array<Express.Multer.File>,
     @Body('folder_path') folderPath: string,
     @User() user: RequestUser,
+    
   ) {
     // Create FormData to forward files to storage-explorer API
     const formData = new FormData();
@@ -297,6 +324,7 @@ export class StorageExplorerController {
       'POST',
       '/storage/upload/batch',
       user,
+      
       formData,
     );
   }
@@ -307,11 +335,13 @@ export class StorageExplorerController {
   async createFolder(
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/storage/folder/create',
       user,
+      
       body,
     );
   }
@@ -322,11 +352,13 @@ export class StorageExplorerController {
   async downloadFile(
     @Query('file_path') filePath: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/download',
       user,
+      
       undefined,
       { file_path: filePath },
     );
@@ -338,6 +370,7 @@ export class StorageExplorerController {
   async getFileMetadata(
     @Query('file_path') filePath: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',

src/modules/storage-explorer/storage-explorer.service.ts

@@ -20,7 +20,7 @@ export class StorageExplorerService {
     path: string,
     user: RequestUser,
     body?: any,
-    query?: Record<string, any>,
+    query?: Record<string, any>
   ): Promise<any> {
     // Validate customer_id is present for multi-tenant isolation
     if (!user.customer_id) {
@@ -28,7 +28,7 @@ export class StorageExplorerService {
     }
 
     // Get customer-specific storage-explorer URL
-    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
     const url = new URL(`${baseUrl}${path}`);
 
     // Add query params