Merge pull request #442 from dadosfera/feat/qualify

FIX: platform routes
merge
2026-09-01 04:08:16 +00:00 · 2026-02-05 10:33:39 -03:00 · 2026-02-05 10:32:07 -03:00 · 2026-02-05 10:29:26 -03:00 · 2026-02-04 17:09:19 -03:00 · 2026-02-04 16:50:20 -03:00
9 changed files with 62 additions and 65 deletions
@@ -9,7 +9,7 @@ maestro:
  cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
  redis_database: "1"
  platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
-  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
+  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api"

 hostname: maestro.stg.dadosfera.ai

@@ -7275,37 +7275,10 @@
          ]
        }
      },
-      "/platform/pipelines/catalog/tables": {
-        "get": {
-          "operationId": "PlatformApiController_getAvailableTables",
-          "summary": "Get all tables available",
-          "parameters": [],
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Platform API"
-          ],
-          "security": [
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
      "/platform/pipelines/catalog/schemas": {
        "get": {
          "operationId": "PlatformApiController_getAvailableSchemas",
-          "summary": "Get all schemas available",
+          "summary": "Get available schemas",
          "parameters": [],
          "responses": {
            "200": {
@@ -7332,17 +7305,14 @@
      "/platform/pipelines/catalog/tables/validate": {
        "post": {
          "operationId": "PlatformApiController_validateTableAndSchema",
-          "summary": "Validate tables and schemas",
+          "summary": "Validate Table and Schema",
          "parameters": [],
          "requestBody": {
            "required": true,
            "content": {
              "application/json": {
                "schema": {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
+                  "$ref": "#/components/schemas/ValidationTableDTO"
                }
              }
            }
@@ -10118,6 +10088,15 @@
                "type": "string"
              }
            },
+            "identifier_columns": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "destination": {
+              "type": "object"
+            },
            "type": {
              "type": "string"
            }
@@ -10126,6 +10105,8 @@
            "name",
            "columns",
            "references",
+            "identifier_columns",
+            "destination",
            "type"
          ]
        },
@@ -11534,6 +11515,20 @@
          "required": [
            "providers"
          ]
+        },
+        "ValidationTableDTO": {
+          "type": "object",
+          "properties": {
+            "tables": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "tables"
+          ]
        }
      }
    }
@@ -17,7 +17,7 @@
        "@aws-sdk/signature-v4": "^3.370.0",
        "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
        "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+        "@dadosfera/protospack-v2": "3.38.0-beta.27",
        "@grpc/grpc-js": "^1.9.3",
        "@grpc/proto-loader": "^0.7.9",
        "@nestjs/cli": "^9.5.0",
@@ -1744,9 +1744,10 @@
      }
    },
    "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.38.0-beta.26",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
-      "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
+      "version": "3.38.0-beta.27",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.27.tgz",
+      "integrity": "sha512-lvV3g/SyPLFVfdtxoc/l1fCtl0o/X9Dv/KOdmDAXQyRJRjd3/81V83rwomp3UtI5IYGYuSbREgolH+X3aNYF7w==",
+      "license": "ISC",
      "dependencies": {
        "@grpc/grpc-js": "^1.9.3",
        "rxjs": "^7.5.5"
@@ -35,7 +35,7 @@
    "@aws-sdk/signature-v4": "^3.370.0",
    "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
    "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+    "@dadosfera/protospack-v2": "3.38.0-beta.27",
    "@grpc/grpc-js": "^1.9.3",
    "@grpc/proto-loader": "^0.7.9",
    "@nestjs/cli": "^9.5.0",
@@ -502,7 +502,7 @@ export class AuthController {
    const accessToken = req.cookies['ddf-auth'];
    const refreshToken = req.cookies['ddf-refresh-auth'];
    const userId = req.cookies['ddf-user-id'];
-    const resourceHost = req.headers["x-original-url"] as string || "" ;
+    const resourceHost = req.headers["host"]

    const hasUserSession = Boolean(accessToken) && Boolean(userId);
    this.logger.info('Has User Session: ' + hasUserSession);
@@ -14,6 +14,13 @@ export class TableColumns {
  @ApiProperty()
  references: Column[];
  @ApiProperty()
+  identifier_columns: string[];
+  @ApiProperty()
+  destination: Record<'raw' | 'qualify', {
+    table_name: string;
+    table_schema: string;
+  }> | null;
+  @ApiProperty()
  type: string;
 }
 export class AvailableEntity {
@@ -25,6 +25,7 @@ import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
+import { ValidationTableDTO } from './platform-api.dto';


 type ValidateTablesDTO = {
@@ -728,26 +729,10 @@ export class PlatformApiController {
    );
  }

-  // ==================== Catalog ROUTES ====================
-
-  @Get('pipelines/catalog/tables')
-  @ApiOperation({ summary: 'Get all tables available' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getAvailableTables(
-    @User() user: RequestUser,
-    @Query() query: Record<string, string>,
-  ) {
-    return this.platformApiService.proxy(
-      'GET',
-      '/catalog/tables',
-      user,
-      undefined,
-      query,
-    );
-  }
+  // ==================== PIPELINE VALIDATION ====================

  @Get('pipelines/catalog/schemas')
-  @ApiOperation({ summary: 'Get all schemas available' })
+  @ApiOperation({ summary: 'Get available schemas' })
  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
  async getAvailableSchemas(
    @User() user: RequestUser,
@@ -755,7 +740,7 @@ export class PlatformApiController {
  ) {
    return this.platformApiService.proxy(
      'GET',
-      '/catalog/schemas',
+      `/catalog/schemas`,
      user,
      undefined,
      query,
@@ -763,18 +748,18 @@ export class PlatformApiController {
  }

  @Post('pipelines/catalog/tables/validate')
-  @ApiOperation({ summary: 'Validate tables and schemas' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  @ApiOperation({ summary: 'Validate Table and Schema' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
  async validateTableAndSchema(
+    @Body() payload: ValidationTableDTO,
    @User() user: RequestUser,
    @Query() query: Record<string, string>,
-    @Body() validateTablesDto: ValidateTablesDTO[]
  ) {
    return this.platformApiService.proxy(
      'POST',
-      '/catalog/tables/validate',
+      `/catalog/tables/validate`,
      user,
-      validateTablesDto,
+      payload,
      query,
    );
  }
@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ValidationTableDTO {
+    @ApiProperty()
+    tables: Array<{
+        table_name: string;
+        table_schema: string;
+    }>
+}
@@ -113,7 +113,7 @@ export class StorageExplorerService {
    }

    // Get customer-specific storage-explorer URL
-    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
    const url = new URL(`${baseUrl}${path}`);

    // Add query params
Author	SHA1	Message	Date
Marcos Rodrigues Silva	2dc032e7e7	Merge pull request #442 from dadosfera/feat/qualify FIX: platform routes	2026-02-05 10:33:39 -03:00
marcos-silva-rodrigues	7f5981731f	merge	2026-02-05 10:32:07 -03:00
marcos-silva-rodrigues	66309c7bbe	FIX: platform routes	2026-02-05 10:29:26 -03:00
Marcos Rodrigues Silva	15048eaf8a	Merge pull request #441 from dadosfera/feat/qualify FEAT: qualify contract	2026-02-04 17:09:19 -03:00
marcos-silva-rodrigues	0e169a3cbc	FIX: protospack version	2026-02-04 16:50:20 -03:00
marcos-silva-rodrigues	b5d933eaf3	FEAT: qualify contract	2026-02-04 16:43:49 -03:00
Marcos Rodrigues Silva	6fa9bf861a	Merge pull request #440 from dadosfera/fix/proxy-urls FIX: storage service dns	2026-02-02 16:38:00 -03:00
Marcos Rodrigues Silva	e08734c97f	Merge pull request #439 from dadosfera/fix/proxy-urls FIX: send token by storage	2026-02-02 15:39:00 -03:00
Marcos Rodrigues Silva	54b75ce11b	Merge pull request #438 from dadosfera/fix/proxy-urls FIX: storage api	2026-02-02 14:43:00 -03:00
Marcos Rodrigues Silva	85234fe0dd	Merge pull request #437 from dadosfera/fix/proxy-urls Fix/proxy urls	2026-02-02 14:28:04 -03:00