FIX: update connectionTestListSchemas to derive references from engine allowlist

FIX: sync proto

deploy/helm-chart/values.yaml

@@ -56,6 +56,8 @@ maestro:
   redis_tls: "true"
   cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
   firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
+  connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com
+
 autoscaling:
   enabled: false
   minReplicas: 1

package-lock.json

@@ -16,7 +16,7 @@
         "@aws-sdk/lib-dynamodb": "^3.414.0",
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "^3.40.0-beta.14",
+        "@dadosfera/protospack-v2": "3.40.0",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.40.0-beta.14",
+      "version": "3.40.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0.tgz",
-      "integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
+      "integrity": "sha512-tzYAGosFjGUgi3pAvXqv+Eqqjca9tH1kU9kJdRI5d1rLXNimo0fIuPWxWng0C4OBH/2sIgC1WLHseq+DZ6qgdw==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -34,7 +34,7 @@
     "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "^3.40.0-beta.14",
+    "@dadosfera/protospack-v2": "3.40.0",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/connection-test/connection-test.service.spec.ts

@@ -65,7 +65,7 @@ describe('ConnectionTestService catalog cache', () => {
     });
   });
 
-  it('maps cached columns to the existing table metadata contract', async () => {
+  it('maps cached columns and derives references from the engine allowlist', async () => {
     connectionsApiService.proxy.mockResolvedValue({
       columns: [
         {
@@ -73,6 +73,16 @@ describe('ConnectionTestService catalog cache', () => {
           data_type: 'bigint',
           is_primary_key: true,
         },
+        {
+          column_name: 'name',
+          data_type: 'varchar',
+          is_primary_key: false,
+        },
+      ],
+    });
+    platformApiService.proxy.mockResolvedValue({
+      allowed_datatypes: [
+        { engine: 'postgresql', allowed_datatypes: ['bigint', 'timestamp'] },
       ],
     });
 
@@ -92,13 +102,10 @@ describe('ConnectionTestService catalog cache', () => {
         {
           table_name: 'customers',
           columns: [
-            {
+            { name: 'id', type: 'bigint', is_primary_key: true },
-              name: 'id',
+            { name: 'name', type: 'varchar', is_primary_key: false },
-              type: 'bigint',
-              is_primary_key: true,
-            },
           ],
-          references: [],
+          references: [{ name: 'id', type: 'bigint', is_primary_key: true }],
         },
       ],
     });
@@ -107,6 +114,39 @@ describe('ConnectionTestService catalog cache', () => {
       '/connection_catalog/config-id/schemas/public/tables/customers/columns',
       user,
     );
+    expect(platformApiService.proxy).toHaveBeenCalledWith(
+      'GET',
+      '/jobs/jdbc/configs/allowed_datatypes',
+      user,
+    );
+  });
+
+  it('returns empty references when the allowlist call fails', async () => {
+    connectionsApiService.proxy.mockResolvedValue({
+      columns: [{ column_name: 'id', data_type: 'bigint', is_primary_key: true }],
+    });
+    platformApiService.proxy.mockRejectedValue(new Error('platform down'));
+
+    await expect(
+      service.getTableMetadata(
+        {
+          connection_id: 'config-id',
+          plugin: 'postgresql',
+          schema: 'public',
+          table_list: ['customers'],
+        },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      tables_metadata: [
+        {
+          table_name: 'customers',
+          columns: [{ name: 'id', type: 'bigint', is_primary_key: true }],
+          references: [],
+        },
+      ],
+    });
   });
 
   it('submits a catalog refresh without holding the request open', async () => {

src/modules/connection-test/connection-test.service.ts

@@ -187,6 +187,20 @@ export class ConnectionTestService {
     body: GetTableMetadataReq,
     user: RequestUser,
   ): Promise<GetTableMetadataRes> {
+    // Columns eligible as the incremental reference field are the ones whose
+    // data type is allowed for this engine (e.g. int/date/timestamp). The
+    // allowlist is owned by the platform API, keyed by engine === plugin.
+    const allowedByEngine = await this.platformApiService
+      .proxy('GET', '/jobs/jdbc/configs/allowed_datatypes', user)
+      .catch(() => null);
+    const allowedDataTypes: string[] =
+      allowedByEngine?.allowed_datatypes?.find(
+        (datatypes) => datatypes.engine === body.plugin,
+      )?.allowed_datatypes ?? [];
+    const allowedSet = new Set(
+      allowedDataTypes.map((type) => type.toLowerCase()),
+    );
+
     const tables_metadata = await Promise.all(
       body.table_list.map(async (table_name) => {
         const result = await this.connectionsApiService.proxy(
@@ -196,14 +210,18 @@ export class ConnectionTestService {
             `/tables/${encodeURIComponent(table_name)}/columns`,
           user,
         );
+        const columns = result.columns.map((column) => ({
+          name: column.column_name,
+          type: column.data_type,
+          is_primary_key: column.is_primary_key,
+        }));
+        const references = columns.filter((column) =>
+          allowedSet.has(String(column.type).toLowerCase()),
+        );
         return {
           table_name,
-          columns: result.columns.map((column) => ({
+          columns,
-            name: column.column_name,
+          references,
-            type: column.data_type,
-            is_primary_key: column.is_primary_key,
-          })),
-          references: [],
         };
       }),
     );

tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.*.ts"]
 }