FIX: update connectionTestListSchemas to derive references from engine allowlist

FIX: sync proto

deploy/helm-chart/values.yaml

@@ -56,6 +56,8 @@ maestro:
   redis_tls: "true"
   cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
   firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
+  connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com
+
 autoscaling:
   enabled: false
   minReplicas: 1

docsfera.json

@@ -4764,9 +4764,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6023,57 +6020,6 @@
               "access-token": []
             }
           ]
-        },
-        "patch": {
-          "operationId": "CatalogController_updateColumnsMetadata",
-          "parameters": [
-            {
-              "name": "dadosfera-lang",
-              "in": "header",
-              "required": false,
-              "schema": {
-                "enum": [
-                  "pt-br",
-                  "en-us"
-                ],
-                "type": "string"
-              }
-            },
-            {
-              "name": "id",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "requestBody": {
-            "required": true,
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/UpdateColumnsMetadataRequest"
-                }
-              }
-            }
-          },
-          "responses": {
-            "200": {
-              "description": ""
-            }
-          },
-          "tags": [
-            "Catalog"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
         }
       },
       "/catalog/data-asset/{id}/preview": {
@@ -7513,94 +7459,6 @@
           ]
         }
       },
-      "/connection-test/refresh-catalog": {
-        "post": {
-          "operationId": "ConnectionTestController_refreshCatalog",
-          "parameters": [],
-          "requestBody": {
-            "required": true,
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/RefreshCatalogReq"
-                }
-              }
-            }
-          },
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "$ref": "#/components/schemas/RefreshCatalogRes"
-                  }
-                }
-              }
-            },
-            "202": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "$ref": "#/components/schemas/RefreshCatalogRes"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Connection Test"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
-      "/connection-test/refresh-catalog/status": {
-        "post": {
-          "operationId": "ConnectionTestController_refreshCatalogStatus",
-          "parameters": [],
-          "requestBody": {
-            "required": true,
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/RefreshCatalogStatusReq"
-                }
-              }
-            }
-          },
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "$ref": "#/components/schemas/RefreshCatalogRes"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Connection Test"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
       "/oauth/hubspot": {
         "get": {
           "operationId": "OauthController_oauthHubspot",
@@ -11361,35 +11219,6 @@
             "columns_metadata"
           ]
         },
-        "ColumnDescriptionDto": {
-          "type": "object",
-          "properties": {
-            "column_name": {
-              "type": "string"
-            },
-            "description": {
-              "type": "string"
-            }
-          },
-          "required": [
-            "column_name",
-            "description"
-          ]
-        },
-        "UpdateColumnsMetadataRequest": {
-          "type": "object",
-          "properties": {
-            "columns": {
-              "type": "array",
-              "items": {
-                "$ref": "#/components/schemas/ColumnDescriptionDto"
-              }
-            }
-          },
-          "required": [
-            "columns"
-          ]
-        },
         "IData": {
           "type": "object",
           "properties": {
@@ -11986,15 +11815,11 @@
             },
             "type": {
               "type": "string"
-            },
-            "is_primary_key": {
-              "type": "boolean"
             }
           },
           "required": [
             "name",
-            "type",
-            "is_primary_key"
+            "type"
           ]
         },
         "TableMetadataDto": {
@@ -12040,79 +11865,6 @@
             "tables_metadata"
           ]
         },
-        "RefreshCatalogReq": {
-          "type": "object",
-          "properties": {
-            "connection_id": {
-              "type": "string"
-            },
-            "plugin": {
-              "type": "string",
-              "enum": [
-                "oracle",
-                "mysql",
-                "postgresql",
-                "sqlserver"
-              ]
-            }
-          },
-          "required": [
-            "connection_id",
-            "plugin"
-          ]
-        },
-        "RefreshCatalogRes": {
-          "type": "object",
-          "properties": {
-            "operation_result": {
-              "type": "boolean"
-            },
-            "status": {
-              "type": "string"
-            },
-            "session_id": {
-              "type": "string"
-            },
-            "date": {
-              "type": "string"
-            }
-          },
-          "required": [
-            "operation_result",
-            "status",
-            "session_id",
-            "date"
-          ]
-        },
-        "RefreshCatalogStatusReq": {
-          "type": "object",
-          "properties": {
-            "connection_id": {
-              "type": "string"
-            },
-            "plugin": {
-              "type": "string",
-              "enum": [
-                "oracle",
-                "mysql",
-                "postgresql",
-                "sqlserver"
-              ]
-            },
-            "session_id": {
-              "type": "string"
-            },
-            "date": {
-              "type": "string"
-            }
-          },
-          "required": [
-            "connection_id",
-            "plugin",
-            "session_id",
-            "date"
-          ]
-        },
         "INote": {
           "type": "object",
           "properties": {

jest.config.ts

@@ -11,6 +11,7 @@ const config: Config.InitialOptions = {
     '<rootDir>/node_modules/',
     '.*\\.module\\.[jt]s$',
   ],
+  setupFiles: ['<rootDir>/jest.setup.ts'],
   // moduleDirectories: ['node_modules'], // default is already 'node_modules'
   // rootDir: '.', //No need
   // testEnvironment: 'node', //Defaults to 'node'

jest.setup.ts

@@ -0,0 +1,3 @@
+process.env.DUC_URL="duc:50051"
+process.env.INFACTORY_URL="in-factory:50052"
+process.env.PIFACTORY_URL="pi-factory:50053"

package-lock.json

@@ -16,7 +16,7 @@
         "@aws-sdk/lib-dynamodb": "^3.414.0",
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "^3.40.0-beta.19",
+        "@dadosfera/protospack-v2": "3.40.0",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.40.0-beta.19",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.19.tgz",
-      "integrity": "sha512-RT7BYWHD2aD945FXzDXHa/xM+sD2B2Ypepy/W/WS+RwKYTSMCfuUXGzK2lXcd9HP9+U+Zjfh90Lkktl++8enjg==",
+      "version": "3.40.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0.tgz",
+      "integrity": "sha512-tzYAGosFjGUgi3pAvXqv+Eqqjca9tH1kU9kJdRI5d1rLXNimo0fIuPWxWng0C4OBH/2sIgC1WLHseq+DZ6qgdw==",
       "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",

package.json

@@ -34,7 +34,7 @@
     "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "^3.40.0-beta.19",
+    "@dadosfera/protospack-v2": "3.40.0",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/catalog/catalog.controller.ts

@@ -9,7 +9,6 @@ import {
   Inject,
   NotFoundException,
   Param,
-  Patch,
   Post,
   Put,
   Query,
@@ -52,7 +51,6 @@ import {
   IUpdateDataRequest,
   TriggerCatalogReq,
   TriggerCatalogRes,
-  UpdateColumnsMetadataRequest,
 } from './dtos';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { Language } from 'src/decorators/language.decorator';
@@ -450,42 +448,6 @@ export class CatalogController {
     return { columns_metadata };
   }
 
-  @Patch('data-asset/:id/columns-metadata')
-  @RequireSomePermission(
-    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
-    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
-  )
-  async updateColumnsMetadata(
-    @User() user: RequestUser,
-    @Language() language: LanguageEnum,
-    @Param('id') id: string,
-    @Body(new ValidationPipe()) body: UpdateColumnsMetadataRequest,
-  ): Promise<{ success: boolean }> {
-    const { customer_name, customer_id, user_id, username } = user;
-
-    this.logger.info(`/catalog - update columns metadata`, {
-      user_id,
-      customer_name,
-      columns_count: body.columns.length,
-    });
-
-    const metadata = PackTheMetadata({
-      customer_name,
-      customer_id,
-      user_id,
-      username,
-      language,
-    });
-
-    await this.catalogService.updateColumnsDescriptions(
-      id,
-      body.columns,
-      metadata,
-    );
-
-    return { success: true };
-  }
-
   @Get('data-asset/:id/preview')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,

src/modules/catalog/catalog.service.ts

@@ -467,16 +467,6 @@ class CatalogService implements OnModuleInit {
     return result;
   }
 
-  async updateColumnsDescriptions(
-    id: string,
-    columns: { column_name: string; description: string }[],
-    metadata: Metadata,
-  ) {
-    await lastValueFrom(
-      this.catalogWriteService.UpdateColumnDescriptions({ id, columns }, metadata),
-    );
-  }
-
   async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
     if (body.asset_type === 'table' || body.asset_type === 'view') {
       return this.createDataDocsViaNimbus(body);
@@ -842,8 +832,6 @@ class CatalogService implements OnModuleInit {
               data_asset_id: table_metadata_id.toString(),
               customer_name: customer_name,
               data_asset_type: 'dataset',
-              column_metadata: [],
-              data_preview: '',
             },
           ],
         },

src/modules/catalog/dtos/index.ts

@@ -1,13 +1,10 @@
 import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
 import {
-  ArrayNotEmpty,
-  IsArray,
   IsEnum,
   IsNotEmpty,
+  IsOptional,
   IsString,
-  ValidateNested,
 } from 'class-validator';
-import { Type } from 'class-transformer';
 import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 export enum DataAssetShareType {
@@ -250,26 +247,6 @@ export class IUpdateCertificationStatusRequest {
   certification_status: CertificationStatus;
 }
 
-export class ColumnDescriptionDto {
-  @ApiProperty()
-  @IsString()
-  @IsNotEmpty()
-  column_name: string;
-
-  @ApiProperty()
-  @IsString()
-  description: string;
-}
-
-export class UpdateColumnsMetadataRequest {
-  @ApiProperty({ type: [ColumnDescriptionDto] })
-  @IsArray()
-  @ArrayNotEmpty()
-  @ValidateNested({ each: true })
-  @Type(() => ColumnDescriptionDto)
-  columns: ColumnDescriptionDto[];
-}
-
 export class ICreateDataAsset implements CreateDataAssetRequest {
   @ApiProperty()
   display_name: string;

src/modules/connection-test/connection-test.service.spec.ts

@@ -65,7 +65,7 @@ describe('ConnectionTestService catalog cache', () => {
     });
   });
 
-  it('maps cached columns to the existing table metadata contract', async () => {
+  it('maps cached columns and derives references from the engine allowlist', async () => {
     connectionsApiService.proxy.mockResolvedValue({
       columns: [
         {
@@ -73,6 +73,16 @@ describe('ConnectionTestService catalog cache', () => {
           data_type: 'bigint',
           is_primary_key: true,
         },
+        {
+          column_name: 'name',
+          data_type: 'varchar',
+          is_primary_key: false,
+        },
+      ],
+    });
+    platformApiService.proxy.mockResolvedValue({
+      allowed_datatypes: [
+        { engine: 'postgresql', allowed_datatypes: ['bigint', 'timestamp'] },
       ],
     });
 
@@ -92,13 +102,10 @@ describe('ConnectionTestService catalog cache', () => {
         {
           table_name: 'customers',
           columns: [
-            {
-              name: 'id',
-              type: 'bigint',
-              is_primary_key: true,
-            },
+            { name: 'id', type: 'bigint', is_primary_key: true },
+            { name: 'name', type: 'varchar', is_primary_key: false },
           ],
-          references: [],
+          references: [{ name: 'id', type: 'bigint', is_primary_key: true }],
         },
       ],
     });
@@ -107,6 +114,39 @@ describe('ConnectionTestService catalog cache', () => {
       '/connection_catalog/config-id/schemas/public/tables/customers/columns',
       user,
     );
+    expect(platformApiService.proxy).toHaveBeenCalledWith(
+      'GET',
+      '/jobs/jdbc/configs/allowed_datatypes',
+      user,
+    );
+  });
+
+  it('returns empty references when the allowlist call fails', async () => {
+    connectionsApiService.proxy.mockResolvedValue({
+      columns: [{ column_name: 'id', data_type: 'bigint', is_primary_key: true }],
+    });
+    platformApiService.proxy.mockRejectedValue(new Error('platform down'));
+
+    await expect(
+      service.getTableMetadata(
+        {
+          connection_id: 'config-id',
+          plugin: 'postgresql',
+          schema: 'public',
+          table_list: ['customers'],
+        },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      tables_metadata: [
+        {
+          table_name: 'customers',
+          columns: [{ name: 'id', type: 'bigint', is_primary_key: true }],
+          references: [],
+        },
+      ],
+    });
   });
 
   it('submits a catalog refresh without holding the request open', async () => {

src/modules/connection-test/connection-test.service.ts

@@ -187,6 +187,20 @@ export class ConnectionTestService {
     body: GetTableMetadataReq,
     user: RequestUser,
   ): Promise<GetTableMetadataRes> {
+    // Columns eligible as the incremental reference field are the ones whose
+    // data type is allowed for this engine (e.g. int/date/timestamp). The
+    // allowlist is owned by the platform API, keyed by engine === plugin.
+    const allowedByEngine = await this.platformApiService
+      .proxy('GET', '/jobs/jdbc/configs/allowed_datatypes', user)
+      .catch(() => null);
+    const allowedDataTypes: string[] =
+      allowedByEngine?.allowed_datatypes?.find(
+        (datatypes) => datatypes.engine === body.plugin,
+      )?.allowed_datatypes ?? [];
+    const allowedSet = new Set(
+      allowedDataTypes.map((type) => type.toLowerCase()),
+    );
+
     const tables_metadata = await Promise.all(
       body.table_list.map(async (table_name) => {
         const result = await this.connectionsApiService.proxy(
@@ -196,14 +210,18 @@ export class ConnectionTestService {
             `/tables/${encodeURIComponent(table_name)}/columns`,
           user,
         );
+        const columns = result.columns.map((column) => ({
+          name: column.column_name,
+          type: column.data_type,
+          is_primary_key: column.is_primary_key,
+        }));
+        const references = columns.filter((column) =>
+          allowedSet.has(String(column.type).toLowerCase()),
+        );
         return {
           table_name,
-          columns: result.columns.map((column) => ({
-            name: column.column_name,
-            type: column.data_type,
-            is_primary_key: column.is_primary_key,
-          })),
-          references: [],
+          columns,
+          references,
         };
       }),
     );

src/modules/permissions/permissions.controller.spec.ts

@@ -1,6 +1,6 @@
 import { ClientsModule } from '@nestjs/microservices';
 import { Test, TestingModule } from '@nestjs/testing';
-// import { DucClient } from 'src/clients/duc/client.config';
+// import { DucClient } from '../duc/client.config'
 import { PermissionsController } from './permissions.controller';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PermissionsService } from './permissions.service';
@@ -22,7 +22,7 @@ describe.skip('PermissionsController', () => {
           provide: DadosferaLogger,
           useValue: { logger },
         },
-        PermissionsService,
+        PermissionsService
       ],
     }).compile();
 

src/modules/release_note/release_note.controller.spec.ts

@@ -1,14 +1,25 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteController } from './release_note.controller';
 import { ReleaseNoteService } from './release_note.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteController', () => {
   let controller: ReleaseNoteController;
+  const logger = {
+    info: (...args) => args,
+    error: (...args) => args,
+  };
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
       controllers: [ReleaseNoteController],
-      providers: [ReleaseNoteService],
+      providers: [
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        ReleaseNoteService
+      ],
     }).compile();
 
     controller = module.get<ReleaseNoteController>(ReleaseNoteController);

src/modules/release_note/release_note.service.spec.ts

@@ -1,12 +1,23 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteService } from './release_note.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteService', () => {
   let service: ReleaseNoteService;
+  const logger = {
+    info: (...args) => args,
+    error: (...args) => args,
+  };
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [ReleaseNoteService],
+      providers: [
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        ReleaseNoteService
+      ],
     }).compile();
 
     service = module.get<ReleaseNoteService>(ReleaseNoteService);

src/utils/ErrorBuilder.ts

@@ -315,14 +315,6 @@ export function EnrichErrorCode(code: string) {
           'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
         code,
       };
-    case ErrorCodes.CATALOG.COLUMN_NOT_FOUND:
-      return {
-        statusCode: HttpStatus.NOT_FOUND,
-        error: 'Coluna não encontrada',
-        message:
-          'Uma ou mais colunas informadas não existem neste ativo. Verifique os nomes e tente novamente.',
-        code,
-      };
     case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
       return {
         statusCode: HttpStatus.INTERNAL_SERVER_ERROR,

src/utils/errorCodes.ts

@@ -74,7 +74,6 @@ const CATALOG = {
   DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
   PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
   METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
-  COLUMN_NOT_FOUND: 'CATALOG.COLUMN_NOT_FOUND',
 };
 const IDENTITY_PROVIDER = {
   INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',

tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.*.ts"]
 }