Merge remote-tracking branch 'origin/main' into release/nimbus-elastic-migration

# Conflicts:
#	docsfera.json
#	package-lock.json
#	package.json
#	src/modules/catalog/dtos/index.ts

deploy/helm-chart/values.yaml

@@ -56,6 +56,8 @@ maestro:
   redis_tls: "true"
   cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
   firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
+  connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com
+
 autoscaling:
   enabled: false
   minReplicas: 1

docsfera.json

@@ -4705,6 +4705,9 @@
             "Platform API"
           ],
           "security": [
+            {
+              "access-token": []
+            },
             {
               "access-token": []
             }

jest.config.ts

@@ -11,6 +11,7 @@ const config: Config.InitialOptions = {
     '<rootDir>/node_modules/',
     '.*\\.module\\.[jt]s$',
   ],
+  setupFiles: ['<rootDir>/jest.setup.ts'],
   // moduleDirectories: ['node_modules'], // default is already 'node_modules'
   // rootDir: '.', //No need
   // testEnvironment: 'node', //Defaults to 'node'

jest.setup.ts

@@ -0,0 +1,3 @@
+process.env.DUC_URL="duc:50051"
+process.env.INFACTORY_URL="in-factory:50052"
+process.env.PIFACTORY_URL="pi-factory:50053"

package-lock.json

@@ -16,7 +16,7 @@
         "@aws-sdk/lib-dynamodb": "^3.414.0",
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "^3.40.0-beta.14",
+        "@dadosfera/protospack-v2": "3.40.0",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.40.0-beta.14",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
-      "integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
+      "version": "3.40.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0.tgz",
+      "integrity": "sha512-tzYAGosFjGUgi3pAvXqv+Eqqjca9tH1kU9kJdRI5d1rLXNimo0fIuPWxWng0C4OBH/2sIgC1WLHseq+DZ6qgdw==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -34,7 +34,7 @@
     "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "^3.40.0-beta.14",
+    "@dadosfera/protospack-v2": "3.40.0",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/auth/auth.controller.ts

@@ -37,6 +37,7 @@ import {
   RequireAllPermissions,
 } from 'src/decorators/authentication.decorator';
 import { AuthClientService } from './auth.service';
+import { UserDTO } from './dtos/login';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
 import { RequestUser, User } from 'src/decorators/user.decorator';
@@ -486,7 +487,7 @@ export class AuthController {
       this.logger.info('Authenticating via X-Api-key header');
       const { api_key } = await this.apiKeyService.get(apiKey);
 
-      const userDto = {
+      const userDto: UserDTO = {
         id: api_key.user_id,
         name: api_key.username,
         email: api_key.username,
@@ -494,7 +495,8 @@ export class AuthController {
           id: api_key.customer_id,
           name: api_key.customer_name,
           tier: api_key.customer_tier,
-        }
+        },
+        permissions: [],
       };
 
       return res.status(200).json(userDto);

src/modules/auth/auth.service.ts

@@ -447,6 +447,9 @@ export class AuthClientService implements OnModuleInit {
         name: payload.customer_name,
         tier: payload.customer_tier,
       },
+      // Raw permission seqids from the JWT. Consumers own the seqid->meaning
+      // mapping (e.g. Orchest's auth-server); Maestro reports them as-is.
+      permissions: payload.permissions ?? [],
     };
 
     return userDto;

src/modules/auth/dtos/login.ts

@@ -152,5 +152,6 @@ export type UserDTO = {
     id: string,
     name: string,
     tier: string,
-  }
+  },
+  permissions: number[],
 }

src/modules/catalog/catalog.controller.ts

@@ -9,6 +9,7 @@ import {
   Inject,
   NotFoundException,
   Param,
+  Patch,
   Post,
   Put,
   Query,
@@ -51,6 +52,7 @@ import {
   IUpdateDataRequest,
   TriggerCatalogReq,
   TriggerCatalogRes,
+  UpdateColumnsMetadataRequest,
 } from './dtos';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { Language } from 'src/decorators/language.decorator';
@@ -448,6 +450,42 @@ export class CatalogController {
     return { columns_metadata };
   }
 
+  @Patch('data-asset/:id/columns-metadata')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async updateColumnsMetadata(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @Body(new ValidationPipe()) body: UpdateColumnsMetadataRequest,
+  ): Promise<{ success: boolean }> {
+    const { customer_name, customer_id, user_id, username } = user;
+
+    this.logger.info(`/catalog - update columns metadata`, {
+      user_id,
+      customer_name,
+      columns_count: body.columns.length,
+    });
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      customer_id,
+      user_id,
+      username,
+      language,
+    });
+
+    await this.catalogService.updateColumnsDescriptions(
+      id,
+      body.columns,
+      metadata,
+    );
+
+    return { success: true };
+  }
+
   @Get('data-asset/:id/preview')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,

src/modules/catalog/catalog.service.ts

@@ -467,6 +467,16 @@ class CatalogService implements OnModuleInit {
     return result;
   }
 
+  async updateColumnsDescriptions(
+    id: string,
+    columns: { column_name: string; description: string }[],
+    metadata: Metadata,
+  ) {
+    await lastValueFrom(
+      this.catalogWriteService.UpdateColumnDescriptions({ id, columns }, metadata),
+    );
+  }
+
   async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
     if (body.asset_type === 'table' || body.asset_type === 'view') {
       return this.createDataDocsViaNimbus(body);
@@ -832,6 +842,8 @@ class CatalogService implements OnModuleInit {
               data_asset_id: table_metadata_id.toString(),
               customer_name: customer_name,
               data_asset_type: 'dataset',
+              column_metadata: [],
+              data_preview: '',
             },
           ],
         },

src/modules/catalog/dtos/index.ts

@@ -1,10 +1,13 @@
 import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
 import {
+  ArrayNotEmpty,
+  IsArray,
   IsEnum,
   IsNotEmpty,
-  IsOptional,
   IsString,
+  ValidateNested,
 } from 'class-validator';
+import { Type } from 'class-transformer';
 import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 export enum DataAssetShareType {
@@ -247,6 +250,26 @@ export class IUpdateCertificationStatusRequest {
   certification_status: CertificationStatus;
 }
 
+export class ColumnDescriptionDto {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty()
+  column_name: string;
+
+  @ApiProperty()
+  @IsString()
+  description: string;
+}
+
+export class UpdateColumnsMetadataRequest {
+  @ApiProperty({ type: [ColumnDescriptionDto] })
+  @IsArray()
+  @ArrayNotEmpty()
+  @ValidateNested({ each: true })
+  @Type(() => ColumnDescriptionDto)
+  columns: ColumnDescriptionDto[];
+}
+
 export class ICreateDataAsset implements CreateDataAssetRequest {
   @ApiProperty()
   display_name: string;

src/modules/permissions/permissions.controller.spec.ts

@@ -1,6 +1,6 @@
 import { ClientsModule } from '@nestjs/microservices';
 import { Test, TestingModule } from '@nestjs/testing';
-// import { DucClient } from 'src/clients/duc/client.config';
+// import { DucClient } from '../duc/client.config'
 import { PermissionsController } from './permissions.controller';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PermissionsService } from './permissions.service';
@@ -22,7 +22,7 @@ describe.skip('PermissionsController', () => {
           provide: DadosferaLogger,
           useValue: { logger },
         },
-        PermissionsService,
+        PermissionsService
       ],
     }).compile();
 

src/modules/release_note/release_note.controller.spec.ts

@@ -1,14 +1,25 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteController } from './release_note.controller';
 import { ReleaseNoteService } from './release_note.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteController', () => {
   let controller: ReleaseNoteController;
+  const logger = {
+    info: (...args) => args,
+    error: (...args) => args,
+  };
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
       controllers: [ReleaseNoteController],
-      providers: [ReleaseNoteService],
+      providers: [
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        ReleaseNoteService
+      ],
     }).compile();
 
     controller = module.get<ReleaseNoteController>(ReleaseNoteController);

src/modules/release_note/release_note.service.spec.ts

@@ -1,12 +1,23 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteService } from './release_note.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteService', () => {
   let service: ReleaseNoteService;
+  const logger = {
+    info: (...args) => args,
+    error: (...args) => args,
+  };
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [ReleaseNoteService],
+      providers: [
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        ReleaseNoteService
+      ],
     }).compile();
 
     service = module.get<ReleaseNoteService>(ReleaseNoteService);

src/utils/ErrorBuilder.ts

@@ -315,6 +315,14 @@ export function EnrichErrorCode(code: string) {
           'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
         code,
       };
+    case ErrorCodes.CATALOG.COLUMN_NOT_FOUND:
+      return {
+        statusCode: HttpStatus.NOT_FOUND,
+        error: 'Coluna não encontrada',
+        message:
+          'Uma ou mais colunas informadas não existem neste ativo. Verifique os nomes e tente novamente.',
+        code,
+      };
     case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
       return {
         statusCode: HttpStatus.INTERNAL_SERVER_ERROR,

src/utils/errorCodes.ts

@@ -74,6 +74,7 @@ const CATALOG = {
   DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
   PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
   METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
+  COLUMN_NOT_FOUND: 'CATALOG.COLUMN_NOT_FOUND',
 };
 const IDENTITY_PROVIDER = {
   INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',

tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.*.ts"]
 }