diff --git a/docsfera.json b/docsfera.json index e01df67..1b0f444 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4705,6 +4705,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4761,6 +4764,9 @@ "Platform API" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -6017,6 +6023,57 @@ "access-token": [] } ] + }, + "patch": { + "operationId": "CatalogController_updateColumnsMetadata", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateColumnsMetadataRequest" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] } }, "/catalog/data-asset/{id}/preview": { @@ -7456,6 +7513,94 @@ ] } }, + "/connection-test/refresh-catalog": { + "post": { + "operationId": "ConnectionTestController_refreshCatalog", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RefreshCatalogReq" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RefreshCatalogRes" + } + } + } + }, + "202": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RefreshCatalogRes" + } + } + } + } + }, + "tags": [ + "Connection Test" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/connection-test/refresh-catalog/status": { + "post": { + "operationId": "ConnectionTestController_refreshCatalogStatus", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RefreshCatalogStatusReq" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RefreshCatalogRes" + } + } + } + } + }, + "tags": [ + "Connection Test" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/oauth/hubspot": { "get": { "operationId": "OauthController_oauthHubspot", @@ -11216,6 +11361,35 @@ "columns_metadata" ] }, + "ColumnDescriptionDto": { + "type": "object", + "properties": { + "column_name": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": [ + "column_name", + "description" + ] + }, + "UpdateColumnsMetadataRequest": { + "type": "object", + "properties": { + "columns": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ColumnDescriptionDto" + } + } + }, + "required": [ + "columns" + ] + }, "IData": { "type": "object", "properties": { @@ -11812,11 +11986,15 @@ }, "type": { "type": "string" + }, + "is_primary_key": { + "type": "boolean" } }, "required": [ "name", - "type" + "type", + "is_primary_key" ] }, "TableMetadataDto": { @@ -11862,6 +12040,79 @@ "tables_metadata" ] }, + "RefreshCatalogReq": { + "type": "object", + "properties": { + "connection_id": { + "type": "string" + }, + "plugin": { + "type": "string", + "enum": [ + "oracle", + "mysql", + "postgresql", + "sqlserver" + ] + } + }, + "required": [ + "connection_id", + "plugin" + ] + }, + "RefreshCatalogRes": { + "type": "object", + "properties": { + "operation_result": { + "type": "boolean" + }, + "status": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "date": { + "type": "string" + } + }, + "required": [ + "operation_result", + "status", + "session_id", + "date" + ] + }, + "RefreshCatalogStatusReq": { + "type": "object", + "properties": { + "connection_id": { + "type": "string" + }, + "plugin": { + "type": "string", + "enum": [ + "oracle", + "mysql", + "postgresql", + "sqlserver" + ] + }, + "session_id": { + "type": "string" + }, + "date": { + "type": "string" + } + }, + "required": [ + "connection_id", + "plugin", + "session_id", + "date" + ] + }, "INote": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index cf5d6e5..5506401 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.14", + "@dadosfera/protospack-v2": "^3.40.0-beta.19", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1735,9 +1735,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.40.0-beta.14", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz", - "integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==", + "version": "3.40.0-beta.19", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.19.tgz", + "integrity": "sha512-RT7BYWHD2aD945FXzDXHa/xM+sD2B2Ypepy/W/WS+RwKYTSMCfuUXGzK2lXcd9HP9+U+Zjfh90Lkktl++8enjg==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 2c10052..fd16bd9 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.40.0-beta.14", + "@dadosfera/protospack-v2": "^3.40.0-beta.19", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index b15c6e7..2b2bc74 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -9,6 +9,7 @@ import { Inject, NotFoundException, Param, + Patch, Post, Put, Query, @@ -51,6 +52,7 @@ import { IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, + UpdateColumnsMetadataRequest, } from './dtos'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { Language } from 'src/decorators/language.decorator'; @@ -448,6 +450,42 @@ export class CatalogController { return { columns_metadata }; } + @Patch('data-asset/:id/columns-metadata') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async updateColumnsMetadata( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + @Body(new ValidationPipe()) body: UpdateColumnsMetadataRequest, + ): Promise<{ success: boolean }> { + const { customer_name, customer_id, user_id, username } = user; + + this.logger.info(`/catalog - update columns metadata`, { + user_id, + customer_name, + columns_count: body.columns.length, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + }); + + await this.catalogService.updateColumnsDescriptions( + id, + body.columns, + metadata, + ); + + return { success: true }; + } + @Get('data-asset/:id/preview') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index e6e9d18..d50e819 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -467,6 +467,16 @@ class CatalogService implements OnModuleInit { return result; } + async updateColumnsDescriptions( + id: string, + columns: { column_name: string; description: string }[], + metadata: Metadata, + ) { + await lastValueFrom( + this.catalogWriteService.UpdateColumnDescriptions({ id, columns }, metadata), + ); + } + async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) { if (body.asset_type === 'table' || body.asset_type === 'view') { return this.createDataDocsViaNimbus(body); @@ -832,6 +842,8 @@ class CatalogService implements OnModuleInit { data_asset_id: table_metadata_id.toString(), customer_name: customer_name, data_asset_type: 'dataset', + column_metadata: [], + data_preview: '', }, ], }, diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 4334de7..c462deb 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -1,10 +1,13 @@ import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger'; import { + ArrayNotEmpty, + IsArray, IsEnum, IsNotEmpty, - IsOptional, IsString, + ValidateNested, } from 'class-validator'; +import { Type } from 'class-transformer'; import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; export enum DataAssetShareType { @@ -247,6 +250,26 @@ export class IUpdateCertificationStatusRequest { certification_status: CertificationStatus; } +export class ColumnDescriptionDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + column_name: string; + + @ApiProperty() + @IsString() + description: string; +} + +export class UpdateColumnsMetadataRequest { + @ApiProperty({ type: [ColumnDescriptionDto] }) + @IsArray() + @ArrayNotEmpty() + @ValidateNested({ each: true }) + @Type(() => ColumnDescriptionDto) + columns: ColumnDescriptionDto[]; +} + export class ICreateDataAsset implements CreateDataAssetRequest { @ApiProperty() display_name: string; diff --git a/src/utils/ErrorBuilder.ts b/src/utils/ErrorBuilder.ts index 49171ee..7e48cb5 100644 --- a/src/utils/ErrorBuilder.ts +++ b/src/utils/ErrorBuilder.ts @@ -315,6 +315,14 @@ export function EnrichErrorCode(code: string) { 'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte', code, }; + case ErrorCodes.CATALOG.COLUMN_NOT_FOUND: + return { + statusCode: HttpStatus.NOT_FOUND, + error: 'Coluna não encontrada', + message: + 'Uma ou mais colunas informadas não existem neste ativo. Verifique os nomes e tente novamente.', + code, + }; case ErrorCodes.CATALOG.PREVIEW_TOO_BIG: return { statusCode: HttpStatus.INTERNAL_SERVER_ERROR, diff --git a/src/utils/errorCodes.ts b/src/utils/errorCodes.ts index a5856bd..c6f41f9 100644 --- a/src/utils/errorCodes.ts +++ b/src/utils/errorCodes.ts @@ -74,6 +74,7 @@ const CATALOG = { DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND', PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG', METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG', + COLUMN_NOT_FOUND: 'CATALOG.COLUMN_NOT_FOUND', }; const IDENTITY_PROVIDER = { INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',