From b10459cb15e706446d0c79c83a93182e0394b6f6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 17:51:14 -0300 Subject: [PATCH 1/7] FIX: maestro custom hostname to unimed --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index c426fa9..421bb64 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -33,7 +33,7 @@ charts: - name: maestro.duc_url value: duc.dadosfera.ai - name: hostname - value: maestro.dadosfera.ai + value: maestro-unimed.dadosfera.ai - name: maestro.pi_factory_url value: pi-factory.dadosfera.ai - name: maestro.in_factory_url From 6910a2f04cd7a754f034a7ee21a26bff641e46b9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 18:38:20 -0300 Subject: [PATCH 2/7] FIX: correct helm value --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 421bb64..3c42dde 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -49,5 +49,5 @@ charts: value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd - name: replicaCount value: 1 - - name: restricted-ip + - name: maestro.restricted_ip value: "177.52.172.0/24" From 5d8e908339386b4727303ec534322d7726f871e5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 24 Jun 2025 10:26:50 -0300 Subject: [PATCH 3/7] FIX: add nginx proxy timeout 5 min --- maestro/templates/ingress.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index ccd8360..1594baa 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -3,6 +3,9 @@ kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; From bdd371ffe88b3490eff5e460533fba997028381f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 25 Jun 2025 10:58:18 -0300 Subject: [PATCH 4/7] FIX: add dadosfera vpn ip in unimed chart --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 3c42dde..15a9241 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -50,4 +50,4 @@ charts: - name: replicaCount value: 1 - name: maestro.restricted_ip - value: "177.52.172.0/24" + value: "177.52.172.0/24,57.151.113.140/30" From d638d518f8e6f9e85bc6b35ae220d0e1019cf188 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 25 Jun 2025 11:06:41 -0300 Subject: [PATCH 5/7] FIX: change unimed chart name --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 15a9241..b44aaf3 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -23,7 +23,7 @@ charts: - name: replicaCount value: 2 - - name: unimed + - name: unimed-maestro chart: ../maestro values: - ../maestro/values.yaml From d68e436a8bd4ec495d71f9b3da830f33caf2fa47 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 5 Jul 2025 09:38:16 -0300 Subject: [PATCH 6/7] FEAT: export users in csv --- docsfera.json | 35 +++++++++++++++++++++++++++ src/modules/users/dtos/entities.ts | 11 +++++++++ src/modules/users/users.controller.ts | 23 ++++++++++++++++++ src/modules/users/users.service.ts | 30 +++++++++++++++++++++++ 4 files changed, 99 insertions(+) diff --git a/docsfera.json b/docsfera.json index b6e8be8..77461b3 100644 --- a/docsfera.json +++ b/docsfera.json @@ -969,6 +969,41 @@ ] } }, + "/users/download": { + "get": { + "operationId": "UsersController_downloadUsersInCsv", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Users" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/users/hierarchies": { "get": { "operationId": "UsersController_getAllHierarchies", diff --git a/src/modules/users/dtos/entities.ts b/src/modules/users/dtos/entities.ts index 9272e52..8e8b852 100644 --- a/src/modules/users/dtos/entities.ts +++ b/src/modules/users/dtos/entities.ts @@ -174,3 +174,14 @@ export class GetAllDepartmentsRes { @ApiProperty() departments: string[]; } + + +export interface UserReporter { + name: string; + email: string; + mfaStatus: string; + status: string; + lastLogin: string; + createdAt: string; + updatedAt: string; +} diff --git a/src/modules/users/users.controller.ts b/src/modules/users/users.controller.ts index 9e00f7c..c80317a 100644 --- a/src/modules/users/users.controller.ts +++ b/src/modules/users/users.controller.ts @@ -12,6 +12,7 @@ import { Post, Put, Query, + Res, UseFilters, } from '@nestjs/common'; import { @@ -24,6 +25,7 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { Authenticated, RequireAllPermissions, + RequireSomePermission, } from 'src/decorators/authentication.decorator'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; @@ -52,6 +54,7 @@ import { UpdateUserRes, } from './dtos/entities'; import { UsersService } from './users.service'; +import { Response } from 'express'; @ApiInternalOnlyController() @ApiTags('Users') @@ -80,6 +83,26 @@ export class UsersController { return await this.userService.findAllUsersByCustomerId(user.customer_id); } + @Get('/download') + @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + async downloadUsersInCsv( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Res() res: Response + ) { + this.logger.info('downloadUsersInCsv'); + this.userService.setLanguage(language); + const { + file, + filename + } = await this.userService.downloadUsersInCsv(user.customer_id); + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', 'text/csv'); + + res.end(file); + } + @Get('hierarchies') @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse({ type: GetAllHierarchiesRes }) diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index e6717fa..0f52284 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -21,6 +21,7 @@ import { IUserByCustomer, SetUserRolesReq, UpdateUserReq, + UserReporter, } from './dtos/entities'; import { RolesService } from '../roles/roles.service'; import { HIERARCHIES } from './hierarchies'; @@ -29,6 +30,7 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces import { EnrichErrorCode } from 'src/utils/ErrorBuilder'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; +import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; @Injectable() export class UsersService implements OnModuleInit { @@ -78,6 +80,34 @@ export class UsersService implements OnModuleInit { }; } + async downloadUsersInCsv(customerId: string) { + const { users } = await lastValueFrom( + this.usersClientService.UserFindAllByCustomerId({ customerId }), + ); + + const formatUsers: UserReporter[] = users.map(user => ({ + createdAt: user.createdAt, + email: user.email, + lastLogin: user.lastLogin, + mfaStatus: user.mfaStatus, + name: user.name, + status: user.status, + updatedAt: user.updatedAt + })) + + const parser = ParserBuilder.build('csv'); + + const file = await parser.parse(formatUsers); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `dadosfera-users-${timestamp}.csv`; + + return { + file, + filename + } + } + async findOneById(id: string): Promise<{ user: IUserByCustomer }> { const { user } = await lastValueFrom( this.usersClientService.UserFindOneById({ id }), From c98a5ff896a401a4e74dc56bc1975307e8778c24 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 5 Jul 2025 11:21:05 -0300 Subject: [PATCH 7/7] FEAT: endpoint to dowload assets list --- docsfera.json | 79 +++++++++++++++++++++++ src/modules/catalog/catalog.controller.ts | 46 +++++++++++++ src/modules/catalog/catalog.service.ts | 29 +++++++++ src/modules/catalog/dtos/index.ts | 8 +++ src/modules/users/users.service.ts | 2 +- 5 files changed, 163 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 77461b3..65c1901 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3909,6 +3909,85 @@ ] } }, + "/catalog/download": { + "get": { + "operationId": "CatalogController_dowloadAsserts", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "size", + "required": false, + "in": "query", + "description": "Número de registros a ser retornado", + "schema": { + "minimum": 1, + "maximum": 10000, + "type": "number" + } + }, + { + "name": "page", + "required": false, + "in": "query", + "description": "Usado para paginação, em conjunto com `size`", + "schema": { + "minimum": 1, + "type": "number" + } + }, + { + "name": "sort_by", + "required": false, + "in": "query", + "description": "Campo do data asset para usar na ordenação. Padrão: `created_at` ", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "required": false, + "in": "query", + "description": "Tipo de ordenação - `asc`: crescente; `desc`: decrescente ", + "schema": { + "default": "asc", + "enum": [ + "asc", + "desc" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset": { "get": { "operationId": "CatalogController_findByPipelineAndObject", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 09daa16..fe83be0 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -117,6 +117,52 @@ export class CatalogController { return res; } + @Get('/download') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async dowloadAsserts( + @User() user: RequestUser, + @Query() query: ICatalogAllRequest, + @Res() res: Response + ) { + const { user_id, customer_name, customer_id, username, permissions } = user; + this.logger.info(`/catalog/download - searchCatalog`, { + user_id, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + + const roles = await this.catalogService.getUserRolesIds(user_id); + + const metadata = PackTheMetadata({ + user_id, + customer_id, + customer_name, + username, + roles, + is_data_manager, + }); + + const { + file, + filename + } = await this.catalogService.downloadAssets( + query, + metadata, + customer_id, + ); + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', 'text/csv'); + + res.end(file); + } + @ApiInternalOnlyEndpoint() @Get('data-asset') async findByPipelineAndObject(@User() user: RequestUser, @Query() query) { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index a3db9cc..fcfb9e7 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -25,6 +25,7 @@ import { UsersService } from '../users/users.service'; import { RolesService } from '../roles/roles.service'; import { Metadata } from '@grpc/grpc-js'; import { + AssetReporter, BatchRemoveRlsRulesRequest, IUpdateDataRequest, TriggerCatalogReq, @@ -240,6 +241,34 @@ class CatalogService implements OnModuleInit { return { data_assets: response, total }; } + async downloadAssets( + query: Record, + metadata: Metadata, + customer_id: string, + ) { + const data = await this.searchDataAssets(query, metadata, customer_id); + + const formatData = data.data_assets.map(asset => ({ + id: asset.id, + display_name: asset.display_name, + data_asset_type: asset.data_asset_type, + created_at: asset.created_at, + tags: '[' + asset.tags.join(', ') + ']' + })) + + const parser = ParserBuilder.build('csv'); + + const file = await parser.parse(formatData); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `dadosfera_assets_${timestamp}.csv`; + + return { + file, + filename + } + } + async getOneDataAsset(data: { id: string; customer_id: string; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index b869e5e..ef12e78 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -320,3 +320,11 @@ export class BatchRemoveRlsRulesRequest { @ApiPropertyOptional() id_rls?: string; } + +export type AssetReporter = { + id: string; + display_name: string; + data_asset_type: string; + created_at: string; + tags: string; +} diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index 0f52284..3057010 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -100,7 +100,7 @@ export class UsersService implements OnModuleInit { const file = await parser.parse(formatUsers); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filename = `dadosfera-users-${timestamp}.csv`; + const filename = `dadosfera_users_${timestamp}.csv`; return { file,