gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-08-31 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #514 from dadosfera/feat/migration-nimbus-data-beta

Browse Source 
Feat/migration nimbus data beta
This commit is contained in:
vinicius gadea
2026-08-25 17:27:45 -03:00
committed by GitHub
parent bf0314f5b1 ba609a9494
commit c1f5c013ee
8 changed files with 341 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docsfera.json
+252 -1
View File
@@ -4705,6 +4705,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4761,6 +4764,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6017,6 +6023,57 @@
"access-token": []
}
]
},
"patch": {
"operationId": "CatalogController_updateColumnsMetadata",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UpdateColumnsMetadataRequest"
}
}
}
},
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/data-asset/{id}/preview": {
@@ -7456,6 +7513,94 @@
]
}
},
"/connection-test/refresh-catalog": {
"post": {
"operationId": "ConnectionTestController_refreshCatalog",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RefreshCatalogReq"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RefreshCatalogRes"
}
}
}
},
"202": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RefreshCatalogRes"
}
}
}
}
},
"tags": [
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/connection-test/refresh-catalog/status": {
"post": {
"operationId": "ConnectionTestController_refreshCatalogStatus",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RefreshCatalogStatusReq"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RefreshCatalogRes"
}
}
}
}
},
"tags": [
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/oauth/hubspot": {
"get": {
"operationId": "OauthController_oauthHubspot",
@@ -11216,6 +11361,35 @@
"columns_metadata"
]
},
"ColumnDescriptionDto": {
"type": "object",
"properties": {
"column_name": {
"type": "string"
},
"description": {
"type": "string"
}
},
"required": [
"column_name",
"description"
]
},
"UpdateColumnsMetadataRequest": {
"type": "object",
"properties": {
"columns": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ColumnDescriptionDto"
}
}
},
"required": [
"columns"
]
},
"IData": {
"type": "object",
"properties": {
@@ -11812,11 +11986,15 @@
},
"type": {
"type": "string"
},
"is_primary_key": {
"type": "boolean"
}
},
"required": [
"name",
"type"
"type",
"is_primary_key"
]
},
"TableMetadataDto": {
@@ -11862,6 +12040,79 @@
"tables_metadata"
]
},
"RefreshCatalogReq": {
"type": "object",
"properties": {
"connection_id": {
"type": "string"
},
"plugin": {
"type": "string",
"enum": [
"oracle",
"mysql",
"postgresql",
"sqlserver"
]
}
},
"required": [
"connection_id",
"plugin"
]
},
"RefreshCatalogRes": {
"type": "object",
"properties": {
"operation_result": {
"type": "boolean"
},
"status": {
"type": "string"
},
"session_id": {
"type": "string"
},
"date": {
"type": "string"
}
},
"required": [
"operation_result",
"status",
"session_id",
"date"
]
},
"RefreshCatalogStatusReq": {
"type": "object",
"properties": {
"connection_id": {
"type": "string"
},
"plugin": {
"type": "string",
"enum": [
"oracle",
"mysql",
"postgresql",
"sqlserver"
]
},
"session_id": {
"type": "string"
},
"date": {
"type": "string"
}
},
"required": [
"connection_id",
"plugin",
"session_id",
"date"
]
},
"INote": {
"type": "object",
"properties": {
package-lock.json
Generated
+5 -4
View File
@@ -16,7 +16,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
"@dadosfera/protospack-v2": "^3.40.0-beta.19",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,10 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.14",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
"version": "3.40.0-beta.19",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.19.tgz",
"integrity": "sha512-RT7BYWHD2aD945FXzDXHa/xM+sD2B2Ypepy/W/WS+RwKYTSMCfuUXGzK2lXcd9HP9+U+Zjfh90Lkktl++8enjg==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
package.json
+1 -1
View File
@@ -34,7 +34,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
"@dadosfera/protospack-v2": "^3.40.0-beta.19",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
src/modules/catalog/catalog.controller.ts
+38
View File
@@ -9,6 +9,7 @@ import {
Inject,
NotFoundException,
Param,
Patch,
Post,
Put,
Query,
@@ -51,6 +52,7 @@ import {
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
UpdateColumnsMetadataRequest,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
@@ -448,6 +450,42 @@ export class CatalogController {
return { columns_metadata };
}
@Patch('data-asset/:id/columns-metadata')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async updateColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Body(new ValidationPipe()) body: UpdateColumnsMetadataRequest,
): Promise<{ success: boolean }> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - update columns metadata`, {
user_id,
customer_name,
columns_count: body.columns.length,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
await this.catalogService.updateColumnsDescriptions(
id,
body.columns,
metadata,
);
return { success: true };
}
@Get('data-asset/:id/preview')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
src/modules/catalog/catalog.service.ts
+12
View File
@@ -467,6 +467,16 @@ class CatalogService implements OnModuleInit {
return result;
}
async updateColumnsDescriptions(
id: string,
columns: { column_name: string; description: string }[],
metadata: Metadata,
) {
await lastValueFrom(
this.catalogWriteService.UpdateColumnDescriptions({ id, columns }, metadata),
);
}
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
if (body.asset_type === 'table' || body.asset_type === 'view') {
return this.createDataDocsViaNimbus(body);
@@ -832,6 +842,8 @@ class CatalogService implements OnModuleInit {
data_asset_id: table_metadata_id.toString(),
customer_name: customer_name,
data_asset_type: 'dataset',
column_metadata: [],
data_preview: '',
},
],
},
src/modules/catalog/dtos/index.ts
+24 -1
View File
@@ -1,10 +1,13 @@
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
import {
ArrayNotEmpty,
IsArray,
IsEnum,
IsNotEmpty,
IsOptional,
IsString,
ValidateNested,
} from 'class-validator';
import { Type } from 'class-transformer';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export enum DataAssetShareType {
@@ -247,6 +250,26 @@ export class IUpdateCertificationStatusRequest {
certification_status: CertificationStatus;
}
export class ColumnDescriptionDto {
@ApiProperty()
@IsString()
@IsNotEmpty()
column_name: string;
@ApiProperty()
@IsString()
description: string;
}
export class UpdateColumnsMetadataRequest {
@ApiProperty({ type: [ColumnDescriptionDto] })
@IsArray()
@ArrayNotEmpty()
@ValidateNested({ each: true })
@Type(() => ColumnDescriptionDto)
columns: ColumnDescriptionDto[];
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
src/utils/ErrorBuilder.ts
+8
View File
@@ -315,6 +315,14 @@ export function EnrichErrorCode(code: string) {
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.CATALOG.COLUMN_NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Coluna não encontrada',
message:
'Uma ou mais colunas informadas não existem neste ativo. Verifique os nomes e tente novamente.',
code,
};
case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
return {
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
src/utils/errorCodes.ts
+1
View File
@@ -74,6 +74,7 @@ const CATALOG = {
DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
COLUMN_NOT_FOUND: 'CATALOG.COLUMN_NOT_FOUND',
};
const IDENTITY_PROVIDER = {
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',