Merge pull request #441 from dadosfera/feat/qualify

FEAT: qualify contract

deploy/helm-chart/values-stg.yaml

@@ -9,7 +9,7 @@ maestro:
   cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
-  storage_explorer_api_url: "https://storage-explorer-{customer}.stg.dadosfera.ai/api"
+  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api"
 
 hostname: maestro.stg.dadosfera.ai
 

docsfera.json

@@ -7865,16 +7865,7 @@
         "post": {
           "operationId": "StorageExplorerController_validateTableName",
           "summary": "Validate table name in PostgreSQL and Snowflake",
-          "parameters": [
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
+          "parameters": [],
           "responses": {
             "201": {
               "description": "",
@@ -7904,16 +7895,7 @@
         "post": {
           "operationId": "StorageExplorerController_createTable",
           "summary": "Create a new table",
-          "parameters": [
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
+          "parameters": [],
           "responses": {
             "201": {
               "description": "",
@@ -7949,14 +7931,6 @@
               "schema": {
                 "type": "number"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -7996,14 +7970,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8051,14 +8017,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8098,14 +8056,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8145,14 +8095,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8200,14 +8142,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8255,14 +8189,6 @@
               "schema": {
                 "type": "number"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8310,14 +8236,6 @@
               "schema": {
                 "type": "boolean"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8357,14 +8275,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8404,14 +8314,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8467,14 +8369,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8514,14 +8408,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8553,16 +8439,7 @@
         "post": {
           "operationId": "StorageExplorerController_batchUpload",
           "summary": "Upload multiple files to storage",
-          "parameters": [
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
+          "parameters": [],
           "responses": {
             "201": {
               "description": "",
@@ -8592,16 +8469,7 @@
         "post": {
           "operationId": "StorageExplorerController_createFolder",
           "summary": "Create a new folder in storage",
-          "parameters": [
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
+          "parameters": [],
           "responses": {
             "201": {
               "description": "",
@@ -8639,14 +8507,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -8686,14 +8546,6 @@
               "schema": {
                 "type": "string"
               }
-            },
-            {
-              "name": "Authorization",
-              "required": true,
-              "in": "header",
-              "schema": {
-                "type": "string"
-              }
             }
           ],
           "responses": {
@@ -10266,6 +10118,15 @@
                 "type": "string"
               }
             },
+            "identifier_columns": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "destination": {
+              "type": "object"
+            },
             "type": {
               "type": "string"
             }
@@ -10274,6 +10135,8 @@
             "name",
             "columns",
             "references",
+            "identifier_columns",
+            "destination",
             "type"
           ]
         },

package-lock.json

@@ -17,7 +17,7 @@
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+        "@dadosfera/protospack-v2": "3.38.0-beta.27",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1744,9 +1744,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.38.0-beta.26",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
-      "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
+      "version": "3.38.0-beta.27",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz",
+      "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -35,7 +35,7 @@
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+    "@dadosfera/protospack-v2": "3.38.0-beta.27",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/inputs/dtos/input.model.ts

@@ -14,6 +14,13 @@ export class TableColumns {
   @ApiProperty()
   references: Column[];
   @ApiProperty()
+  identifier_columns: string[];
+  @ApiProperty()
+  destination: Record<'raw' | 'qualify', {
+    table_name: string;
+    table_schema: string;
+  }> | null;
+  @ApiProperty()
   type: string;
 }
 export class AvailableEntity {

src/modules/storage-explorer/storage-explorer.config.ts

@@ -6,7 +6,7 @@ export const STORAGE_EXPLORER_CONFIG = {
     }
     // Replace {customer_id} placeholder with actual customer ID
     // For local: http://172.17.0.1:8000/api (no placeholder)
-    // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api
+    // For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
     return urlTemplate.replace('{customer}', customerName);
   },
   timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),

src/modules/storage-explorer/storage-explorer.controller.ts

@@ -47,13 +47,13 @@ export class StorageExplorerController {
   async validateTableName(
     @Body() body: any,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/tables/validate-name',
       user,
-      token,
+      
       body,
     );
   }
@@ -64,13 +64,13 @@ export class StorageExplorerController {
   async createTable(
     @Body() body: any,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/tables/',
       user,
-      token,
+      
       body,
     );
   }
@@ -81,13 +81,13 @@ export class StorageExplorerController {
   async listTables(
     @Query('page') page: number,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/tables/',
       user,
-      token,
+      
       undefined,
       { page },
     );
@@ -99,13 +99,12 @@ export class StorageExplorerController {
   async getTable(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}`,
       user,
-      token
     );
   }
 
@@ -116,13 +115,13 @@ export class StorageExplorerController {
     @Param('tableId') tableId: string,
     @Param('datasetId') datasetId: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       `/tables/${tableId}/datasets/${datasetId}`,
       user,
-      token
+      
     );
   }
 
@@ -132,13 +131,13 @@ export class StorageExplorerController {
   async getTableDatasets(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}/datasets`,
       user,
-      token
+      
     );
   }
 
@@ -148,13 +147,13 @@ export class StorageExplorerController {
   async getTableSchema(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}/schema`,
       user,
-      token
+      
     );
   }
 
@@ -165,13 +164,12 @@ export class StorageExplorerController {
     @Param('tableId') tableId: string,
     @Param('datasetId') datasetId: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       `/tables/${tableId}/validate-compatibility/${datasetId}`,
       user,
-      token
+      
     );
   }
 
@@ -186,13 +184,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Query('limit') limit: number,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/${datasetId}/preview`,
       user,
-      token,
+      
       undefined,
       { limit },
     );
@@ -205,13 +203,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Query('force_refresh') forceRefresh: boolean,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/${datasetId}/schema`,
       user,
-      token,
+      
       undefined,
       { force_refresh: forceRefresh },
     );
@@ -223,13 +221,13 @@ export class StorageExplorerController {
   async listDatasetsByUpload(
     @Param('uploadId') uploadId: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/upload/${uploadId}`,
       user,
-      token
+      
     );
   }
 
@@ -240,13 +238,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Body() body: any,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'PUT',
       `/datasets/${datasetId}/refresh-schema`,
       user,
-      token,
+      
       body,
     );
   }
@@ -263,13 +261,13 @@ export class StorageExplorerController {
     @Query('limit') limit: number,
     @Query('folder_path') folderPath: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/uploads/history',
       user,
-      token,
+      
       undefined,
       { page, limit, folder_path: folderPath },
     );
@@ -281,13 +279,13 @@ export class StorageExplorerController {
   async browseStorage(
     @Query('path') path: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/browse',
       user,
-      token,
+      
       undefined,
       { path },
     );
@@ -302,7 +300,7 @@ export class StorageExplorerController {
     @UploadedFiles() files: Array<Express.Multer.File>,
     @Body('folder_path') folderPath: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     // Create FormData to forward files to storage-explorer API
     const formData = new FormData();
@@ -326,7 +324,7 @@ export class StorageExplorerController {
       'POST',
       '/storage/upload/batch',
       user,
-      token,
+      
       formData,
     );
   }
@@ -337,13 +335,13 @@ export class StorageExplorerController {
   async createFolder(
     @Body() body: any,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/storage/folder/create',
       user,
-      token,
+      
       body,
     );
   }
@@ -354,13 +352,13 @@ export class StorageExplorerController {
   async downloadFile(
     @Query('file_path') filePath: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/download',
       user,
-      token,
+      
       undefined,
       { file_path: filePath },
     );
@@ -372,13 +370,12 @@ export class StorageExplorerController {
   async getFileMetadata(
     @Query('file_path') filePath: string,
     @User() user: RequestUser,
-    @Headers('Authorization') token: string
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/metadata',
       user,
-      token,
       undefined,
       { file_path: filePath },
     );

src/modules/storage-explorer/storage-explorer.service.ts

@@ -19,7 +19,6 @@ export class StorageExplorerService {
     method: string,
     path: string,
     user: RequestUser,
-    token: string,
     body?: any,
     query?: Record<string, any>
   ): Promise<any> {
@@ -42,7 +41,6 @@ export class StorageExplorerService {
     }
 
     const headers: Record<string, string> = {
-      'Authorization': token,
       'content-type': 'application/json',
     };