From d94c5dee4ea536c7b7e87b9817676b844fb36291 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 6 May 2025 10:10:54 -0300 Subject: [PATCH 01/31] FEAT: init public routes to get data asset by public embed --- docsfera.json | 713 ++++++++---------- src/modules/auth/auth.controller.ts | 2 +- src/modules/catalog/catalog.module.ts | 7 +- src/modules/catalog/catalog.service.ts | 143 +++- .../{ => controllers}/catalog.controller.ts | 302 ++------ .../controllers/catalog.public.controller.ts | 184 +++++ .../connection-test.service.ts | 2 +- .../connection/connection.controller.ts | 2 +- src/modules/customers/customers.controller.ts | 2 +- src/modules/customers/customers.service.ts | 10 + .../network-policy.controller.ts | 2 +- src/modules/oauth/oauth.controller.ts | 2 +- src/modules/open-data/open-data.controller.ts | 2 +- .../pipelinesV2/pipelines.controller.ts | 2 +- src/modules/users/users.controller.ts | 2 +- ... PackTheMetadata.ts => PackTheMetadata.ts} | 0 16 files changed, 724 insertions(+), 653 deletions(-) rename src/modules/catalog/{ => controllers}/catalog.controller.ts (80%) create mode 100644 src/modules/catalog/controllers/catalog.public.controller.ts rename src/utils/{ PackTheMetadata.ts => PackTheMetadata.ts} (100%) diff --git a/docsfera.json b/docsfera.json index b8d9671..2fd799a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3873,9 +3873,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -3906,11 +3903,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - } ] } }, @@ -3947,9 +3939,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -3981,7 +3970,7 @@ } }, { - "name": "shared", + "name": "c", "required": true, "in": "query", "schema": { @@ -3996,14 +3985,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } ] }, "put": { @@ -4056,9 +4037,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4104,9 +4082,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4147,9 +4122,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4179,6 +4151,14 @@ "schema": { "type": "string" } + }, + { + "name": "c", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -4195,14 +4175,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } ] } }, @@ -4229,6 +4201,14 @@ "schema": { "type": "string" } + }, + { + "name": "c", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -4245,14 +4225,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } ] } }, @@ -4279,6 +4251,14 @@ "schema": { "type": "string" } + }, + { + "name": "c", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -4295,14 +4275,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } ] }, "post": { @@ -4345,9 +4317,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4386,11 +4355,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - } ] } }, @@ -4426,11 +4390,6 @@ }, "tags": [ "Catalog" - ], - "security": [ - { - "access-token": [] - } ] } }, @@ -4485,9 +4444,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4543,9 +4499,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4595,9 +4548,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4645,9 +4595,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4687,9 +4634,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4730,9 +4674,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4779,9 +4720,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4829,9 +4767,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4877,9 +4812,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4919,9 +4851,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4954,9 +4883,6 @@ "Catalog" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -4996,6 +4922,238 @@ "tags": [ "Catalog" ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/links": { + "get": { + "operationId": "CustomersController_getCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinksResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_setCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinkRequest" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/token": { + "get": { + "operationId": "CustomersController_getCustomerToken", + "parameters": [ + { + "name": "exp", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/monitoring-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMonitoringDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/logs-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], "security": [ { "access-token": [] @@ -5519,241 +5677,6 @@ ] } }, - "/customers/{id}/mfa": { - "post": { - "operationId": "CustomersController_enableMfaEnforce", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EnforceMfa" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/links": { - "get": { - "operationId": "CustomersController_getCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinksResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_setCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinkRequest" - } - } - } - }, - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/token": { - "get": { - "operationId": "CustomersController_getCustomerToken", - "parameters": [ - { - "name": "exp", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/monitoring-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMonitoringDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/logs-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/access-dashboard": { - "get": { - "operationId": "CustomersController_getAccessDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/open-data/sharing-ocean-data": { "post": { "operationId": "OpenDataController_createUser", @@ -8435,6 +8358,67 @@ "updated_by" ] }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLink": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, "ConnectionTestPingReq": { "type": "object", "properties": { @@ -8729,67 +8713,6 @@ "tags" ] }, - "EnforceMfa": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - }, - "required": [ - "enabled" - ] - }, - "CustomerLink": { - "type": "object", - "properties": { - "href": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "iconSrc": { - "type": "string" - } - }, - "required": [ - "href", - "name", - "description" - ] - }, - "CustomerLinksResponse": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, "CustomerThemeRequest": { "type": "object", "properties": { diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 226036a..9489563 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -45,7 +45,7 @@ import { AuthSignInRes, BulkEditRequest, } from './dtos/login'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { AuthGuard } from '@nestjs/passport'; import { Request } from 'express'; import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes'; diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index 45cc06e..d990344 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -2,13 +2,15 @@ import { Module } from '@nestjs/common'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { CatalogController } from './catalog.controller'; +import { CatalogController } from './controllers/catalog.controller'; import { CatalogService } from './catalog.service'; import { CatalogClientConfiguration } from './catalog-client'; import { ClientsModule } from '@nestjs/microservices'; import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { UsersModule } from '../users/users.module'; import { RolesModule } from '../roles/roles.module'; +import { CustomersModule } from '../customers/customers.module'; +import { CatalogPublicController } from './controllers/catalog.public.controller'; const client = new CatalogClientConfiguration(); @@ -18,8 +20,9 @@ const client = new CatalogClientConfiguration(); OldPipelineModule, UsersModule, RolesModule, + CustomersModule ], - controllers: [CatalogController], + controllers: [CatalogController, CatalogPublicController], providers: [CatalogService, DadosferaLogger], exports: [CatalogService], }) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index a3db9cc..f7a2ed0 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -12,9 +12,11 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces'; import { BadRequestException, + ForbiddenException, HttpException, HttpStatus, Inject, + NotFoundException, OnModuleInit, } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; @@ -43,6 +45,11 @@ import { CsvParser } from 'src/utils/FileParser/csv-parser'; import { PDFParser } from 'src/utils/FileParser/pdf-parser'; import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; +import { RequestUser } from 'src/decorators/user.decorator'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { CustomersService } from '../customers/customers.service'; +import { LanguageEnum } from 'src/utils/languages.enum'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -56,6 +63,7 @@ class CatalogService implements OnModuleInit { private readonly grpcClient: ClientGrpc, private readonly userService: UsersService, private readonly roleService: RolesService, + private readonly customerService: CustomersService ) { this.logger = dadosferaLogger.logger; } @@ -240,6 +248,77 @@ class CatalogService implements OnModuleInit { return { data_assets: response, total }; } + async getOneDataAssetPrivate(user: RequestUser, id: string) { + const { username, user_id, customer_id, customer_name, permissions } = user; + + this.logger.info("getOneDataAssetPrivate: " + user_id + " - " + id) + + this.logger.info(`data: `, { + username, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + + const is_get_catalog = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, + ); + let has_permission = false; + const metadata = PackTheMetadata({ + username, + user_id, + customer_id, + customer_name, + }); + + const user_roles = await this.getUserRolesIds(user_id); + + const { data_asset } = await this.getOneDataAsset({ + customer_id, + id, + metadata, + }); + + has_permission = + is_data_manager || + is_get_catalog || + data_asset?.owner === username || + (user_roles as Array).some((r) => + data_asset.p_roles.includes(r), + ) || + data_asset.p_users.includes(user_id); + + + if (!has_permission) + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + delete data_asset.p_roles; + delete data_asset.p_users; + + return { data_asset }; + } + + async getOneDataAssetPublic(customerId: string, id: string) { + this.logger.info("getOneDataAssetPublic: " + customerId + " - " + id) + const metadata = await this.createMetadataToPublicAccess(customerId); + const { data_asset } = await this.getOneDataAsset({ + customer_id: customerId, + id, + metadata, + }); + this.logger.info("found asset: " + JSON.stringify(data_asset)) + delete data_asset.p_roles; + delete data_asset.p_users; + data_asset.share_type = 'public'; + if (data_asset.share_type !== 'public') + throw new NotFoundException(); + + return { data_asset }; + } + async getOneDataAsset(data: { id: string; customer_id: string; @@ -479,27 +558,26 @@ class CatalogService implements OnModuleInit { async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; - this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); - this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); - this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); + this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, { ...body.logMetadata }); + this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); + this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, { ...body.logMetadata }); try { - const { data, status } = await axios.post(endpoint, {...body.table_metadata}); + const { data, status } = await axios.post(endpoint, { ...body.table_metadata }); this.logger.info( `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); return data.id; } catch (error) { this.logger.error( - `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${ - error.response?.status - } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata}); + `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${error.response?.status + } because of ${JSON.stringify(error.response?.data) || error.message}`, { ...body.logMetadata }); throw new Error(error.response?.data?.message || error.message); } } @@ -509,51 +587,49 @@ class CatalogService implements OnModuleInit { this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; - + try { - this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); - this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); - this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); + this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, { ...body.logMetadata }); + this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); + this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, { ...body.logMetadata }); const { data, status } = await axios.post(endpoint, body.column_metadata); this.logger.info( `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); return data.map((column) => column.id); } catch (error) { this.logger.error( - `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${ - error.response?.status - } because of ${error.response?.data || error.message}`, {...body.logMetadata}); + `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${error.response?.status + } because of ${error.response?.data || error.message}`, { ...body.logMetadata }); throw new Error(error.response?.data?.message || error.message); } } async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; - this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); - this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); - this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); + this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, { ...body.logMetadata }); + this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); + this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, { ...body.logMetadata }); try { const { data, status } = await axios.post(endpoint, body.data_preview); this.logger.info( `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); return data.id; } catch (error) { this.logger.error( - `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${ - error.response?.status + `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${error.response?.status } because of ${error.response?.data || error.message}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); throw new Error(error.response?.data?.message || error.message); } @@ -582,6 +658,23 @@ class CatalogService implements OnModuleInit { ); return res; } + + async createMetadataToPublicAccess(customerId?: string, language?: LanguageEnum) { + if (!customerId) { + throw new NotFoundException(); + } + + const { + customer + } = await this.customerService.getCustomer(customerId); + + return PackTheMetadata({ + customer_id: customer.id, + customer_name: customer.name, + language, + }) + } + } export { CatalogService }; diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/controllers/catalog.controller.ts similarity index 80% rename from src/modules/catalog/catalog.controller.ts rename to src/modules/catalog/controllers/catalog.controller.ts index 09daa16..ca5d88a 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/controllers/catalog.controller.ts @@ -28,11 +28,14 @@ import { RequireAllPermissions, RequireModule, RequireSomePermission, -} from '../../decorators/authentication.decorator'; -import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; -import { CatalogService } from './catalog.service'; +} from '../../../decorators/authentication.decorator'; +import { + DADOSFERA_MODULES_KEYS, + PERMISSIONS_GROUPS, +} from '../../../authentication/permissions.enum'; +import { CatalogService } from '../catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { BatchRemoveRlsRulesRequest, @@ -49,7 +52,7 @@ import { IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, -} from './dtos'; +} from '../dtos'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; @@ -195,209 +198,6 @@ export class CatalogController { return res; } - @Get('data-asset/:id') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAsset( - @User() user: RequestUser, - @Param('id') id: string, - @Query('shared') shared?: 'true', - ) { - const { username, user_id, customer_id, customer_name, permissions } = user; - - this.logger.info(`GET /data-asset/${id}`, { - username, - customer_name, - }); - - const is_data_manager = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - let has_permission = false; - const metadata = PackTheMetadata({ - username, - user_id, - customer_id, - customer_name, - }); - - const user_roles = await this.catalogService.getUserRolesIds(user_id); - - const { data_asset } = await this.catalogService.getOneDataAsset({ - customer_id, - id, - metadata, - }); - has_permission = - is_data_manager || - data_asset?.owner === username || - (user_roles as Array).some((r) => data_asset.p_roles.includes(r)) || - data_asset.p_users.includes(user_id); - if ( - shared === 'true' && - (data_asset.share_type === undefined || data_asset.share_type === 'none') - ) - throw new NotFoundException(); - if (!has_permission) - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - delete data_asset.p_roles; - delete data_asset.p_users; - - return { data_asset }; - } - - @ApiInternalOnlyEndpoint() - @Get('data-asset/rls/:id') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAssetRls( - @User() user: RequestUser, - @Headers() headers, - @Param('id') id: string, - ) { - const { username, user_id, customer_id, customer_name, permissions } = user; - - this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { - username, - customer_name, - }); - - const is_data_manager = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - let has_permission = false; - const metadata = PackTheMetadata({ - username, - user_id, - customer_id, - customer_name, - }); - - const user_roles = await this.catalogService.getUserRolesIds(user_id); - - const { data_asset } = await this.catalogService.getOneDataAsset({ - customer_id, - id, - metadata, - }); - - if (data_asset?.owner === username) has_permission = true; - - for (const role of user_roles) { - if (data_asset.p_roles.includes(role)) has_permission = true; - } - - if (data_asset.p_users.includes(user_id)) has_permission = true; - - if (is_data_manager || has_permission) { - delete data_asset.p_roles; - delete data_asset.p_users; - return { data_asset }; - } - - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - } - - @Get('data-asset/:id/columns-metadata') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAssetColumnsMetadata( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - ): Promise { - const { customer_name, customer_id, user_id, username } = user; - - this.logger.info(`/catalog - columns-metadata`, { - user_id, - customer_name, - }); - - const metadata = PackTheMetadata({ - customer_name, - customer_id, - user_id, - username, - language, - }); - - const columns_metadata = - await this.catalogService.getDatasetColumnsMetadata(id, metadata); - - return { columns_metadata }; - } - - @Get('data-asset/:id/preview') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAssetPreview( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - ): Promise { - const { customer_name, customer_id, user_id, username, customer_modules } = user; - - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { - user_id, - customer_name, - }); - - const metadata = PackTheMetadata({ - customer_name, - customer_id, - user_id, - username, - language, - is_mask: customer_modules.some(mod => mod === 'pii') - }); - - const preview = await this.catalogService.getDatasetPreview(id, metadata); - - return { preview }; - } - - @Get('data-asset/:id/docs') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAssetDocs( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - ): Promise { - const { customer_name, customer_id, user_id, username } = user; - - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { - user_id, - customer_name, - }); - - const metadata = PackTheMetadata({ - customer_name, - customer_id, - user_id, - username, - language, - }); - - const docs = await this.catalogService.getDataDocs(id, metadata); - - return { docs }; - } - @Put('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, @@ -648,6 +448,62 @@ export class CatalogController { return response; } + @ApiInternalOnlyEndpoint() + @Get('data-asset/rls/:id') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetRls( + @User() user: RequestUser, + @Headers() headers, + @Param('id') id: string, + ) { + const { username, user_id, customer_id, customer_name, permissions } = user; + + this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { + username, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + let has_permission = false; + const metadata = PackTheMetadata({ + username, + user_id, + customer_id, + customer_name, + }); + + const user_roles = await this.catalogService.getUserRolesIds(user_id); + + const { data_asset } = await this.catalogService.getOneDataAsset({ + customer_id, + id, + metadata, + }); + + if (data_asset?.owner === username) has_permission = true; + + for (const role of user_roles) { + if (data_asset.p_roles.includes(role)) has_permission = true; + } + + if (data_asset.p_users.includes(user_id)) has_permission = true; + + if (is_data_manager || has_permission) { + delete data_asset.p_roles; + delete data_asset.p_users; + return { data_asset }; + } + + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + } + @Post('rls-rule') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) { @@ -859,29 +715,31 @@ export class CatalogController { } @Get('pii-reporter') - @RequireSomePermission( - PERMISSIONS_GROUPS.USERS.permissions.ADMIN - ) - @RequireModule( - DADOSFERA_MODULES_KEYS.PII - ) - async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") { + @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @RequireModule(DADOSFERA_MODULES_KEYS.PII) + async getPiiReporter( + @User() user: RequestUser, + @Res() res: Response, + @Query('type') contentType: TypeParser = 'pdf', + ) { this.logger.info('GET pii-reporter'); const metadata = PackTheMetadata(user); try { - const { - file, - filename, - type - } = await this.catalogService.getPiiReporter(metadata, contentType); - res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + const { file, filename, type } = await this.catalogService.getPiiReporter( + metadata, + contentType, + ); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${filename}"`, + ); res.setHeader('Content-Type', type); // use res.end to send buffer return res.end(file); } catch (error) { - console.error(error) + console.error(error); this.logger.error(error.message); } } diff --git a/src/modules/catalog/controllers/catalog.public.controller.ts b/src/modules/catalog/controllers/catalog.public.controller.ts new file mode 100644 index 0000000..6d3be81 --- /dev/null +++ b/src/modules/catalog/controllers/catalog.public.controller.ts @@ -0,0 +1,184 @@ +import { + Controller, + ForbiddenException, + Get, + Headers, + Inject, + Param, + Query, + UseFilters, +} from '@nestjs/common'; +import { + ApiHeaders, + ApiTags, +} from '@nestjs/swagger'; +import { + RequireSomePermission, +} from '../../../decorators/authentication.decorator'; +import { + PERMISSIONS_GROUPS, +} from '../../../authentication/permissions.enum'; +import { CatalogService } from '../catalog.service'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { + IColumnsMetadataResponse, + IDocsResponse, + IPreviewResponse, + +} from '../dtos'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { Language } from 'src/decorators/language.decorator'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; + + +@ApiTags('Catalog') +@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) +@Controller('catalog') +@UseFilters(new GrpcToHttpExceptionFilter()) +export class CatalogPublicController { + logger: DadosferaLogger; + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private catalogService: CatalogService, + ) { + this.logger = dadosferaLogger.logger; + } + + private async createAccessToPublicAssets( + user?: RequestUser, + customerId?: string, + language?: LanguageEnum, + ) { + if (user) { + const { customer_name, customer_id, user_id, username } = user; + return PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + }); + } else { + return await this.catalogService.createMetadataToPublicAccess( + customerId, + language, + ); + } + } + + private checkDefaultPermission(user: RequestUser) { + const is_data_manager = user.permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + + const is_get = user.permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, + ); + + if (is_data_manager || is_get) { + return; + } + + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + } + + @Get('data-asset/:id') + async getDataAsset( + @User() user: RequestUser, + @Param('id') id: string, + @Query('c') customerId?: string, + ) { + this.logger.info(`GET /data-asset/${id}`); + try { + if (user) { + return await this.catalogService.getOneDataAssetPrivate(user, id); + } else { + return await this.catalogService.getOneDataAssetPublic(customerId, id); + } + } catch (error) { + this.logger.error(error.message); + throw error; + } + } + + + + @Get('data-asset/:id/columns-metadata') + async getDataAssetColumnsMetadata( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + @Query('c') customerId: string, + ): Promise { + const metadata = await this.createAccessToPublicAssets( + user, + customerId, + language, + ); + this.logger.info(`/catalog - columns-metadata`); + + const columns_metadata = + await this.catalogService.getDatasetColumnsMetadata(id, metadata); + + if (user) { + this.checkDefaultPermission(user); + } + + return { columns_metadata }; + } + + @Get('data-asset/:id/preview') + async getDataAssetPreview( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + @Query('c') customerId: string, + ): Promise { + this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`); + + const metadata = await this.createAccessToPublicAssets( + user, + customerId, + language, + ); + + const preview = await this.catalogService.getDatasetPreview(id, metadata); + + if (user) { + this.checkDefaultPermission(user); + } + + return { preview }; + } + + @Get('data-asset/:id/docs') + async getDataAssetDocs( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + @Query('c') customerId: string, + ): Promise { + this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`); + + const metadata = await this.createAccessToPublicAssets( + user, + customerId, + language, + ); + + const docs = await this.catalogService.getDataDocs(id, metadata); + + if (user) { + this.checkDefaultPermission(user); + } + + return { docs }; + } + +} diff --git a/src/modules/connection-test/connection-test.service.ts b/src/modules/connection-test/connection-test.service.ts index 310555a..3b92e12 100644 --- a/src/modules/connection-test/connection-test.service.ts +++ b/src/modules/connection-test/connection-test.service.ts @@ -20,7 +20,7 @@ import { DatabaseConnectionPropertiesDto, } from '../connection/dtos/connection'; import { RequestUser } from 'src/decorators/user.decorator'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; @Injectable() export class ConnectionTestService { diff --git a/src/modules/connection/connection.controller.ts b/src/modules/connection/connection.controller.ts index 7c4275a..194e9aa 100644 --- a/src/modules/connection/connection.controller.ts +++ b/src/modules/connection/connection.controller.ts @@ -28,7 +28,7 @@ import { UpdateConnectionDto, } from './dtos/connection'; import { CreateConnectionDto } from './dtos/connection'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index bcab557..2606ebd 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -25,7 +25,7 @@ import { CustomersService } from './customers.service'; import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; import { RequestUser, User } from 'src/decorators/user.decorator'; import type { StringValue } from 'ms'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { EnforceMfa } from './dtos/enforce-mfa'; @ApiTags('Customers') diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 056c5fb..8b28171 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -59,6 +59,14 @@ export class CustomersService implements OnModuleInit { ); } + async getCustomer(customerId: string) { + return await lastValueFrom( + this.customerService.CustomerFindOneById({ + id: customerId + }) + ) + } + async getLinks(customerId: string) { try { const result = await lastValueFrom( @@ -141,6 +149,7 @@ export class CustomersService implements OnModuleInit { // const decoded = jwt.decode(jwt_token, { complete: true }); return jwt_token; } + async getMonitoringDashboardUrl(metadata: Metadata) { logger.info('CustomersService - getMonitoringDashboardUrl'); @@ -158,6 +167,7 @@ export class CustomersService implements OnModuleInit { return res; } + async getLogsDashboardUrl(metadata: Metadata) { logger.info('CustomersService - getMixPanelLogsDashboardUrl'); diff --git a/src/modules/network-policy/network-policy.controller.ts b/src/modules/network-policy/network-policy.controller.ts index 8ae8371..f45ffdd 100644 --- a/src/modules/network-policy/network-policy.controller.ts +++ b/src/modules/network-policy/network-policy.controller.ts @@ -8,7 +8,7 @@ import { import { NetworkPoliciesDTO } from './dto/network-policy.dto'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { NetworkPolicyService } from './network-policy.service'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; @Controller('network-policy') export class NetworkPolicyController { diff --git a/src/modules/oauth/oauth.controller.ts b/src/modules/oauth/oauth.controller.ts index 1e87866..e1af4c9 100644 --- a/src/modules/oauth/oauth.controller.ts +++ b/src/modules/oauth/oauth.controller.ts @@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport'; import { ConnectionClientService } from '../connection/client.service'; import jwt from 'jsonwebtoken'; import DadosferaLogger from '@dadosfera/dadosfera-logs/dist'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; @ApiTags('oauth') @Controller('oauth') diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index b8951ed..a2fa72a 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator'; import { OpenDataService } from './open-data.service'; import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; import { Metadata } from '@grpc/grpc-js'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { request } from 'http'; import { Request } from 'express'; diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 512a705..6986f25 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -32,7 +32,7 @@ import { PipelinesService } from './pipelines.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { RequestUser, User } from 'src/decorators/user.decorator'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { diff --git a/src/modules/users/users.controller.ts b/src/modules/users/users.controller.ts index 9e00f7c..5fdc603 100644 --- a/src/modules/users/users.controller.ts +++ b/src/modules/users/users.controller.ts @@ -29,7 +29,7 @@ import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import ErrorBuilder from 'src/utils/ErrorBuilder'; import ErrorCodes from 'src/utils/errorCodes'; import { LanguageEnum } from 'src/utils/languages.enum'; diff --git a/src/utils/ PackTheMetadata.ts b/src/utils/PackTheMetadata.ts similarity index 100% rename from src/utils/ PackTheMetadata.ts rename to src/utils/PackTheMetadata.ts From 62d0e8b041b9926644f3c54f09f5b2d386cb93a1 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 12 May 2025 12:12:58 -0300 Subject: [PATCH 02/31] FEAT: assign pubic key endpoint --- src/modules/assign/assign.controller.ts | 24 +++++++++++++ src/modules/assign/assign.module.ts | 15 +++++++++ src/modules/assign/assign.service.ts | 37 +++++++++++++++++++++ src/modules/assign/dto/create-assign.dto.ts | 3 ++ 4 files changed, 79 insertions(+) create mode 100644 src/modules/assign/assign.controller.ts create mode 100644 src/modules/assign/assign.module.ts create mode 100644 src/modules/assign/assign.service.ts create mode 100644 src/modules/assign/dto/create-assign.dto.ts diff --git a/src/modules/assign/assign.controller.ts b/src/modules/assign/assign.controller.ts new file mode 100644 index 0000000..e83488c --- /dev/null +++ b/src/modules/assign/assign.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Post, Body, Put, Get} from '@nestjs/common'; +import { AssignService } from './assign.service'; +import { CreateAssignDto } from './dto/create-assign.dto'; +import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; + +@Controller('assign') +@Authenticated() +export class AssignController { + constructor(private readonly assignService: AssignService) {} + + @Put('/public-key') + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule('public-assign-embed') + create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) { + const metadata = PackTheMetadata(user); + return this.assignService.create(createAssignDto, metadata); + } + +} diff --git a/src/modules/assign/assign.module.ts b/src/modules/assign/assign.module.ts new file mode 100644 index 0000000..3b0fb09 --- /dev/null +++ b/src/modules/assign/assign.module.ts @@ -0,0 +1,15 @@ +import { Module } from '@nestjs/common'; +import { AssignService } from './assign.service'; +import { AssignController } from './assign.controller'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { ClientsModule } from '@nestjs/microservices'; +import { DucClient } from '../duc/client.config'; + +const client = new DucClient(); + +@Module({ + imports: [ClientsModule.register([client.providerOptions])], + controllers: [AssignController], + providers: [AssignService, DadosferaLogger] +}) +export class AssignModule {} diff --git a/src/modules/assign/assign.service.ts b/src/modules/assign/assign.service.ts new file mode 100644 index 0000000..b48ef80 --- /dev/null +++ b/src/modules/assign/assign.service.ts @@ -0,0 +1,37 @@ +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { CreateAssignDto } from './dto/create-assign.dto'; +import { Metadata } from '@grpc/grpc-js'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { ClientGrpc } from '@nestjs/microservices'; +import { DucClient } from 'src/modules/duc/client.config'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; + +@Injectable() +export class AssignService implements OnModuleInit { + + ducService: AssingProtoService; + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.ducService =this.grpcClient.getService( + ProtoServices.AssingProtoService, + ); + } + + async create(createAssignDto: CreateAssignDto, metadata: Metadata) { + return await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata)) + } + + async get(createAssignDto: CreateAssignDto, metadata: Metadata) { + return await lastValueFrom(this.ducService.GetAssignPublicKey(createAssignDto, metadata)) + } +} diff --git a/src/modules/assign/dto/create-assign.dto.ts b/src/modules/assign/dto/create-assign.dto.ts new file mode 100644 index 0000000..67b784e --- /dev/null +++ b/src/modules/assign/dto/create-assign.dto.ts @@ -0,0 +1,3 @@ +export class CreateAssignDto { + publicKey: string; +} From 10580dfa033fb3598bf49bc5fd379b6102ee6ce6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 12 May 2025 12:13:39 -0300 Subject: [PATCH 03/31] FEAT: share endpoint --- src/modules/catalog/share/share.controller.ts | 87 +++++++ src/modules/catalog/share/share.module.ts | 26 ++ src/modules/catalog/share/share.service.ts | 231 ++++++++++++++++++ .../dto/create-share-metadata.dto.ts | 9 + .../share-metadata.controller.ts | 31 +++ .../share-metadata/share-metadata.module.ts | 16 ++ .../share-metadata/share-metadata.service.ts | 61 +++++ 7 files changed, 461 insertions(+) create mode 100644 src/modules/catalog/share/share.controller.ts create mode 100644 src/modules/catalog/share/share.module.ts create mode 100644 src/modules/catalog/share/share.service.ts create mode 100644 src/modules/share-metadata/dto/create-share-metadata.dto.ts create mode 100644 src/modules/share-metadata/share-metadata.controller.ts create mode 100644 src/modules/share-metadata/share-metadata.module.ts create mode 100644 src/modules/share-metadata/share-metadata.service.ts diff --git a/src/modules/catalog/share/share.controller.ts b/src/modules/catalog/share/share.controller.ts new file mode 100644 index 0000000..a5240bf --- /dev/null +++ b/src/modules/catalog/share/share.controller.ts @@ -0,0 +1,87 @@ +import { + Controller, + Get, + Inject, + Param, + UseFilters, +} from '@nestjs/common'; +import { + ApiHeaders, + ApiTags, +} from '@nestjs/swagger'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { + IColumnsMetadataResponse, + IDocsResponse, + IPreviewResponse, + +} from '../dtos'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { Language } from 'src/decorators/language.decorator'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { ShareService } from './share.service'; + +@ApiTags('Catalog') +@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) +@Controller('catalog/data-asset/share') +@UseFilters(new GrpcToHttpExceptionFilter()) +export class ShareController { + logger: DadosferaLogger; + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private catalogShareService: ShareService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Get('/:id') + async getShareDataAsset( + @Param('id') id: string, + @User() user: RequestUser + ) { + this.logger.info(`GET //:id`); + return await this.catalogShareService.getOneDataAssetPublic(id, user); + } + + @Get('/:id/columns-metadata') + async getShareDataAssetColumnsMetadata( + @Language() language: LanguageEnum, + @Param('id') id: string, + @User() user: RequestUser + ): Promise { + this.logger.info(`GET /:id/columns-metadata`); + + const columns_metadata = + await this.catalogShareService.getDatasetColumnsMetadata(id, user); + + + return { columns_metadata }; + } + + @Get('/:id/preview') + async getShareDataAssetPreview( + @Language() language: LanguageEnum, + @Param('id') id: string, + @User() user: RequestUser + ): Promise { + this.logger.info(`GET /:id/preview`); + const preview = await this.catalogShareService.getDatasetPreview(id, user); + + return { preview }; + } + + @Get('/:id/docs') + async getShareDataAssetDocs( + @Language() language: LanguageEnum, + @Param('id') id: string, + @User() user: RequestUser + ): Promise { + this.logger.info(`GET /:id/docs`); + const docs = await this.catalogShareService.getDataDocs(id, user); + + return { docs }; + } + +} diff --git a/src/modules/catalog/share/share.module.ts b/src/modules/catalog/share/share.module.ts new file mode 100644 index 0000000..1f0c7fb --- /dev/null +++ b/src/modules/catalog/share/share.module.ts @@ -0,0 +1,26 @@ +import { Module } from "@nestjs/common"; +import { CatalogClientConfiguration } from "../catalog-client"; +import { ClientsModule } from "@nestjs/microservices"; +import { RolesModule } from "src/modules/roles/roles.module"; +import { UsersModule } from "src/modules/users/users.module"; +import { CustomersModule } from "src/modules/customers/customers.module"; +import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module"; +import { ShareController } from "./share.controller"; +import DadosferaLogger from "@dadosfera/dadosfera-logs"; +import { ShareService } from "./share.service"; + +const client = new CatalogClientConfiguration(); + +@Module({ + imports: [ + ClientsModule.register([client.providerOptions]), + UsersModule, + RolesModule, + CustomersModule, + ShareMetadataModule + ], + controllers: [ShareController], + providers: [ShareService, DadosferaLogger], + exports: [ShareModule], +}) +export class ShareModule {} \ No newline at end of file diff --git a/src/modules/catalog/share/share.service.ts b/src/modules/catalog/share/share.service.ts new file mode 100644 index 0000000..b81096a --- /dev/null +++ b/src/modules/catalog/share/share.service.ts @@ -0,0 +1,231 @@ +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { + ProtoServices, + ReadService, +} from '@dadosfera/protospack-v2/dist/lib/Catalog'; +import { + ForbiddenException, + Inject, + NotFoundException, + OnModuleInit, +} from '@nestjs/common'; +import { CatalogClientConfiguration } from '../catalog-client'; +import { ClientGrpc } from '@nestjs/microservices'; +import { UsersService } from 'src/modules/users/users.service'; +import { RolesService } from 'src/modules/roles/roles.service'; +import { RequestUser } from 'src/decorators/user.decorator'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { Metadata } from '@grpc/grpc-js'; +import { lastValueFrom } from 'rxjs'; +import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service'; +import { isJWT } from 'class-validator'; + +export class ShareService implements OnModuleInit { + catalogReadService: ReadService.CatalogReadServices; + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + @Inject(CatalogClientConfiguration.name) + private readonly grpcClient: ClientGrpc, + private readonly userService: UsersService, + private readonly roleService: RolesService, + private shareMetadataService: ShareMetadataService, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.catalogReadService = + this.grpcClient.getService( + ProtoServices.CatalogReadServices, + ); + } + + async getDatasetColumnsMetadata(id: string, user: RequestUser) { + const shareMetadata = await this.getShareMetadata(id, user); + const metadata = PackTheMetadata({ + customer_id: shareMetadata.customerId, + customer_name: shareMetadata.customerName, + }); + const { columns_metadata } = await lastValueFrom( + this.catalogReadService.GetDatasetColumnsMetadata( + { id: shareMetadata.assetId, type: undefined }, + metadata, + ), + ); + const result = JSON.parse(columns_metadata); + return result; + } + + async getDatasetPreview(id: string, user: RequestUser) { + const shareMetadata = await this.getShareMetadata(id, user); + const metadata = PackTheMetadata({ + customer_id: shareMetadata.customerId, + customer_name: shareMetadata.customerName, + }); + const { preview } = await lastValueFrom( + this.catalogReadService.GetDatasetPreview( + { id: shareMetadata.assetId, type: undefined }, + metadata, + ), + ); + const result = JSON.parse(preview); + return result; + } + + async getOneDataAssetPublic(id: string, user: RequestUser) { + this.logger.info("getOneDataAssetPublic: " + JSON.stringify({ + id, + userId: user?.user_id, + customerId: user?.customer_id + })) + try { + const shareMetadata = await this.getShareMetadata(id, user); + + this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata)) + const metadata = PackTheMetadata({ + customer_id: shareMetadata.customerId, + customer_name: shareMetadata.customerName, + }); + + const { data_asset } = await this.getOneDataAsset({ + customer_id: shareMetadata.customerId, + id: shareMetadata.assetId, + metadata, + }); + this.logger.info('found asset: ' + JSON.stringify(data_asset)); + delete data_asset.p_roles; + delete data_asset.p_users; + data_asset.share_type = 'public'; + if (data_asset.share_type !== 'public') throw new NotFoundException(); + + return { data_asset }; + } catch (error) { + this.logger.error(error); + throw error; + } + + } + + private async getOneDataAsset(data: { + id: string; + customer_id: string; + metadata: Metadata; + }) { + const { customer_id, id, metadata } = data; + const { data_asset } = await lastValueFrom( + this.catalogReadService.GetOneDataAsset( + { id, type: undefined }, + metadata, + ), + ); + let asset = JSON.parse(data_asset); + asset = { + ...asset, + p_roles: asset.roles, + p_users: asset.users, + }; + asset = await this.getAssetsUsersAndRoles([asset], customer_id); + + return { data_asset: asset[0] }; + } + + async getDataDocs(id: string, user: RequestUser) { + const shareMetadata = await this.getShareMetadata(id, user); + const metadata = PackTheMetadata({ + customer_id: shareMetadata.customerId, + customer_name: shareMetadata.customerName, + }); + + const { documentation } = await lastValueFrom( + this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), + ); + console.log(documentation); + const docs = JSON.parse(documentation); + return docs; + } + + private async getAssetsUsersAndRoles( + data_assets: Array, + customer_id: string, + ) { + const { users: customer_users } = + await this.userService.findAllUsersByCustomerId(customer_id); + const { roles: customer_roles } = await this.roleService.roleSearch( + {}, + { customer_id }, + ); + return data_assets.map((data_asset) => { + const owner = customer_users.find( + (u) => u.id === data_asset.owner, + )?.username; + + const roles = []; + const users = []; + for (const role_id of data_asset.roles) { + const role = customer_roles.find((r) => r.id === role_id); + if (role) roles.push({ id: role.id, name: role.name }); + } + for (const user_id of data_asset.users) { + const user = customer_users.find((r) => r.id === user_id); + if (user) users.push({ id: user.id, username: user.username }); + } + return { + ...data_asset, + roles, + users, + owner, + } as typeof data_asset; + }); + } + + // private async validateShareAssign(token: string) { + // const tokenDecoded = jwt.decode(token, { + // complete: true, + // }); + + // jwt.verify(token, this.pemValue, { + // algorithms: ['RS256'], + // }); + + // const shareId = (tokenDecoded.payload as JwtPayload).sub; + // const metadata = PackTheMetadata({}); + // return await this.shareMetadataService.get(shareId, metadata); + // } + + private async getShareMetadata(id: string, user: RequestUser) { + const metadata = PackTheMetadata({}); + const info = await this.shareMetadataService.get(id, metadata); + if (isJWT(id) && info ){ + return info; + } + + if (info.type === 'private') { + if (!user) { + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + } + + const is_data_manager = user.permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + + const is_get = user.permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, + ); + + if (is_data_manager || is_get) { + return; + } + + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + } + return info; + } +} diff --git a/src/modules/share-metadata/dto/create-share-metadata.dto.ts b/src/modules/share-metadata/dto/create-share-metadata.dto.ts new file mode 100644 index 0000000..db06de6 --- /dev/null +++ b/src/modules/share-metadata/dto/create-share-metadata.dto.ts @@ -0,0 +1,9 @@ +export class CreateShareMetadataDto { + assetId: string; + proposedId: string; +} + +export class UpdateShareMetadataDto { + shareId: string; + type: string; +} diff --git a/src/modules/share-metadata/share-metadata.controller.ts b/src/modules/share-metadata/share-metadata.controller.ts new file mode 100644 index 0000000..bfa6ca5 --- /dev/null +++ b/src/modules/share-metadata/share-metadata.controller.ts @@ -0,0 +1,31 @@ +import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common'; +import { ShareMetadataService } from './share-metadata.service'; +import { Authenticated } from 'src/decorators/authentication.decorator'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto'; + +@Controller('share') +@Authenticated() +export class ShareMetadataController { + constructor(private readonly shareMetadataService: ShareMetadataService) {} + + @Put('/') + findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) { + const metadata = PackTheMetadata(user); + return this.shareMetadataService.create(data, metadata); + } + + @Patch('/') + updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) { + const metadata = PackTheMetadata(user); + return this.shareMetadataService.updateShare(data, metadata); + } + + + @Get('/:id') + get(@Param('id') id: string, @User() user: RequestUser) { + const metadata = PackTheMetadata(user); + return this.shareMetadataService.get(id, metadata); + } +} diff --git a/src/modules/share-metadata/share-metadata.module.ts b/src/modules/share-metadata/share-metadata.module.ts new file mode 100644 index 0000000..f75e78e --- /dev/null +++ b/src/modules/share-metadata/share-metadata.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common'; +import { ShareMetadataService } from './share-metadata.service'; +import { ShareMetadataController } from './share-metadata.controller'; +import { DucClient } from '../duc/client.config'; +import { ClientsModule } from '@nestjs/microservices'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +const client = new DucClient(); + +@Module({ + imports: [ClientsModule.register([client.providerOptions])], + controllers: [ShareMetadataController], + providers: [ShareMetadataService, DadosferaLogger], + exports: [ShareMetadataService] +}) +export class ShareMetadataModule {} diff --git a/src/modules/share-metadata/share-metadata.service.ts b/src/modules/share-metadata/share-metadata.service.ts new file mode 100644 index 0000000..0915faa --- /dev/null +++ b/src/modules/share-metadata/share-metadata.service.ts @@ -0,0 +1,61 @@ +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { Metadata } from '@grpc/grpc-js'; +import { ClientGrpc } from '@nestjs/microservices'; +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { lastValueFrom } from 'rxjs'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto'; + +@Injectable() +export class ShareMetadataService implements OnModuleInit { + ducService: ShareMetadataProtoService; + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.ducService =this.grpcClient.getService( + ProtoServices.ShareMetadataProtoService, + ); + } + + create(createShareDto: CreateShareMetadataDto, metadata: Metadata) { + return lastValueFrom( + this.ducService.FindOrCreateShareMetadata( + { + assetId: createShareDto.assetId, + proposedId: createShareDto.proposedId + }, + metadata, + ), + ); + } + + updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) { + return lastValueFrom( + this.ducService.ChangeShareMetadataType( + updateShare, + metadata, + ), + ); + } + + get(id: string, metadata: Metadata) { + return lastValueFrom( + this.ducService.GetShareMetadata( + { + id, + }, + metadata, + ), + ); + } +} From b312849fffdaf554ab389d55f1dc858c7927a958 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 12 May 2025 12:15:16 -0300 Subject: [PATCH 04/31] FEAT: catalog share endpoint for embeds --- docsfera.json | 492 ++++++++++++++++-- package-lock.json | 8 +- package.json | 2 +- src/app.module.ts | 8 +- .../{controllers => }/catalog.controller.ts | 302 ++++++++--- src/modules/catalog/catalog.module.ts | 11 +- src/modules/catalog/catalog.service.ts | 153 +----- .../controllers/catalog.public.controller.ts | 184 ------- 8 files changed, 719 insertions(+), 441 deletions(-) rename src/modules/catalog/{controllers => }/catalog.controller.ts (80%) delete mode 100644 src/modules/catalog/controllers/catalog.public.controller.ts diff --git a/docsfera.json b/docsfera.json index 2fd799a..762ab8d 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3873,6 +3873,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3903,6 +3906,11 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + } ] } }, @@ -3939,6 +3947,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3970,7 +3981,7 @@ } }, { - "name": "c", + "name": "shared", "required": true, "in": "query", "schema": { @@ -3985,6 +3996,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] }, "put": { @@ -4037,6 +4056,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4082,6 +4104,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4122,6 +4147,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4151,14 +4179,6 @@ "schema": { "type": "string" } - }, - { - "name": "c", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { @@ -4175,6 +4195,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] } }, @@ -4201,14 +4229,6 @@ "schema": { "type": "string" } - }, - { - "name": "c", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { @@ -4225,6 +4245,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] } }, @@ -4251,14 +4279,6 @@ "schema": { "type": "string" } - }, - { - "name": "c", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { @@ -4275,6 +4295,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] }, "post": { @@ -4317,6 +4345,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4355,6 +4386,11 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + } ] } }, @@ -4390,6 +4426,11 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + } ] } }, @@ -4444,6 +4485,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4499,6 +4543,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4548,6 +4595,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4595,6 +4645,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4634,6 +4687,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4674,6 +4730,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4720,6 +4779,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4767,6 +4829,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4812,6 +4877,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4851,6 +4919,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4883,6 +4954,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4926,6 +5000,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5167,6 +5244,263 @@ ] } }, + "/catalog/data-asset/share/{id}": { + "get": { + "operationId": "ShareController_getShareDataAsset", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/catalog/data-asset/share/{id}/columns-metadata": { + "get": { + "operationId": "ShareController_getShareDataAssetColumnsMetadata", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IColumnsMetadataResponse" + } + } + } + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/catalog/data-asset/share/{id}/preview": { + "get": { + "operationId": "ShareController_getShareDataAssetPreview", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IPreviewResponse" + } + } + } + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/catalog/data-asset/share/{id}/docs": { + "get": { + "operationId": "ShareController_getShareDataAssetDocs", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IDocsResponse" + } + } + } + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/share": { + "put": { + "operationId": "ShareMetadataController_findOrcreate", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateShareMetadataDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "ShareMetadataController_updateShareType", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateShareMetadataDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } + }, + "/share/{id}": { + "get": { + "operationId": "ShareMetadataController_get", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } + }, "/terms-of-use/token": { "get": { "operationId": "TermsOfUseController_getToken", @@ -5796,20 +6130,6 @@ ] } }, - "/health": { - "get": { - "operationId": "HealthController_check", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Health" - ] - } - }, "/network-policy": { "get": { "operationId": "NetworkPolicyController_getNetworks", @@ -5899,6 +6219,59 @@ } ] } + }, + "/assign/public-key": { + "put": { + "operationId": "AssignController_create", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAssignDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/health": { + "get": { + "operationId": "HealthController_check", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Health" + ] + } } }, "info": { @@ -8419,6 +8792,36 @@ "links" ] }, + "CreateShareMetadataDto": { + "type": "object", + "properties": { + "assetId": { + "type": "string" + }, + "proposedId": { + "type": "string" + } + }, + "required": [ + "assetId", + "proposedId" + ] + }, + "UpdateShareMetadataDto": { + "type": "object", + "properties": { + "shareId": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "shareId", + "type" + ] + }, "ConnectionTestPingReq": { "type": "object", "properties": { @@ -8779,6 +9182,17 @@ "required": [ "policies" ] + }, + "CreateAssignDto": { + "type": "object", + "properties": { + "publicKey": { + "type": "string" + } + }, + "required": [ + "publicKey" + ] } } } diff --git a/package-lock.json b/package-lock.json index c9d852f..50c8341 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1400,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.23", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", - "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-NfnvIENsIval+liPIPoVkFjixmKCKu9tRqvXQu3qTk67kZicNBvEOd9UezED1h1a5kX70+CbSR4mlXUZZ7ZVGA==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index fd3eb0b..40b4faa 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index fedb388..c9a8b35 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -29,6 +29,8 @@ import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; import { ThemeModule } from './modules/theme/theme.module'; import { NetworkPolicyModule } from './modules/network-policy/network-policy.module'; +import { AssignModule } from './modules/assign/assign.module'; +import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module'; @Module({ providers: [ @@ -62,9 +64,11 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod CustomersModule, OpenDataModule, ThemeModule, + NetworkPolicyModule, + AssignModule, + ShareMetadataModule, //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule, - NetworkPolicyModule + HealthModule ], }) export class AppModule {} diff --git a/src/modules/catalog/controllers/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts similarity index 80% rename from src/modules/catalog/controllers/catalog.controller.ts rename to src/modules/catalog/catalog.controller.ts index ca5d88a..f0727f0 100644 --- a/src/modules/catalog/controllers/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -28,12 +28,9 @@ import { RequireAllPermissions, RequireModule, RequireSomePermission, -} from '../../../decorators/authentication.decorator'; -import { - DADOSFERA_MODULES_KEYS, - PERMISSIONS_GROUPS, -} from '../../../authentication/permissions.enum'; -import { CatalogService } from '../catalog.service'; +} from '../../decorators/authentication.decorator'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { CatalogService } from './catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; @@ -52,7 +49,7 @@ import { IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, -} from '../dtos'; +} from './dtos'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; @@ -198,6 +195,209 @@ export class CatalogController { return res; } + @Get('data-asset/:id') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAsset( + @User() user: RequestUser, + @Param('id') id: string, + @Query('shared') shared?: 'true', + ) { + const { username, user_id, customer_id, customer_name, permissions } = user; + + this.logger.info(`GET /data-asset/${id}`, { + username, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + let has_permission = false; + const metadata = PackTheMetadata({ + username, + user_id, + customer_id, + customer_name, + }); + + const user_roles = await this.catalogService.getUserRolesIds(user_id); + + const { data_asset } = await this.catalogService.getOneDataAsset({ + customer_id, + id, + metadata, + }); + has_permission = + is_data_manager || + data_asset?.owner === username || + (user_roles as Array).some((r) => data_asset.p_roles.includes(r)) || + data_asset.p_users.includes(user_id); + if ( + shared === 'true' && + (data_asset.share_type === undefined || data_asset.share_type === 'none') + ) + throw new NotFoundException(); + if (!has_permission) + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + delete data_asset.p_roles; + delete data_asset.p_users; + + return { data_asset }; + } + + @ApiInternalOnlyEndpoint() + @Get('data-asset/rls/:id') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetRls( + @User() user: RequestUser, + @Headers() headers, + @Param('id') id: string, + ) { + const { username, user_id, customer_id, customer_name, permissions } = user; + + this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { + username, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + let has_permission = false; + const metadata = PackTheMetadata({ + username, + user_id, + customer_id, + customer_name, + }); + + const user_roles = await this.catalogService.getUserRolesIds(user_id); + + const { data_asset } = await this.catalogService.getOneDataAsset({ + customer_id, + id, + metadata, + }); + + if (data_asset?.owner === username) has_permission = true; + + for (const role of user_roles) { + if (data_asset.p_roles.includes(role)) has_permission = true; + } + + if (data_asset.p_users.includes(user_id)) has_permission = true; + + if (is_data_manager || has_permission) { + delete data_asset.p_roles; + delete data_asset.p_users; + return { data_asset }; + } + + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + } + + @Get('data-asset/:id/columns-metadata') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetColumnsMetadata( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + ): Promise { + const { customer_name, customer_id, user_id, username } = user; + + this.logger.info(`/catalog - columns-metadata`, { + user_id, + customer_name, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + }); + + const columns_metadata = + await this.catalogService.getDatasetColumnsMetadata(id, metadata); + + return { columns_metadata }; + } + + @Get('data-asset/:id/preview') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetPreview( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + ): Promise { + const { customer_name, customer_id, user_id, username, customer_modules } = user; + + this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { + user_id, + customer_name, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + is_mask: customer_modules.some(mod => mod === 'pii') + }); + + const preview = await this.catalogService.getDatasetPreview(id, metadata); + + return { preview }; + } + + @Get('data-asset/:id/docs') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetDocs( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + ): Promise { + const { customer_name, customer_id, user_id, username } = user; + + this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { + user_id, + customer_name, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + }); + + const docs = await this.catalogService.getDataDocs(id, metadata); + + return { docs }; + } + @Put('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, @@ -448,62 +648,6 @@ export class CatalogController { return response; } - @ApiInternalOnlyEndpoint() - @Get('data-asset/rls/:id') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAssetRls( - @User() user: RequestUser, - @Headers() headers, - @Param('id') id: string, - ) { - const { username, user_id, customer_id, customer_name, permissions } = user; - - this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { - username, - customer_name, - }); - - const is_data_manager = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - let has_permission = false; - const metadata = PackTheMetadata({ - username, - user_id, - customer_id, - customer_name, - }); - - const user_roles = await this.catalogService.getUserRolesIds(user_id); - - const { data_asset } = await this.catalogService.getOneDataAsset({ - customer_id, - id, - metadata, - }); - - if (data_asset?.owner === username) has_permission = true; - - for (const role of user_roles) { - if (data_asset.p_roles.includes(role)) has_permission = true; - } - - if (data_asset.p_users.includes(user_id)) has_permission = true; - - if (is_data_manager || has_permission) { - delete data_asset.p_roles; - delete data_asset.p_users; - return { data_asset }; - } - - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - } - @Post('rls-rule') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) { @@ -715,32 +859,30 @@ export class CatalogController { } @Get('pii-reporter') - @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) - @RequireModule(DADOSFERA_MODULES_KEYS.PII) - async getPiiReporter( - @User() user: RequestUser, - @Res() res: Response, - @Query('type') contentType: TypeParser = 'pdf', - ) { + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule( + DADOSFERA_MODULES_KEYS.PII + ) + async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") { this.logger.info('GET pii-reporter'); const metadata = PackTheMetadata(user); try { - const { file, filename, type } = await this.catalogService.getPiiReporter( - metadata, - contentType, - ); - res.setHeader( - 'Content-Disposition', - `attachment; filename="${filename}"`, - ); + const { + file, + filename, + type + } = await this.catalogService.getPiiReporter(metadata, contentType); + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); res.setHeader('Content-Type', type); // use res.end to send buffer return res.end(file); } catch (error) { - console.error(error); + console.error(error) this.logger.error(error.message); } } -} +} \ No newline at end of file diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index d990344..8320d37 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -2,15 +2,15 @@ import { Module } from '@nestjs/common'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { CatalogController } from './controllers/catalog.controller'; -import { CatalogService } from './catalog.service'; +import { CatalogController } from './catalog.controller'; import { CatalogClientConfiguration } from './catalog-client'; import { ClientsModule } from '@nestjs/microservices'; import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { UsersModule } from '../users/users.module'; import { RolesModule } from '../roles/roles.module'; import { CustomersModule } from '../customers/customers.module'; -import { CatalogPublicController } from './controllers/catalog.public.controller'; +import { ShareModule } from './share/share.module'; +import { CatalogService } from './catalog.service'; const client = new CatalogClientConfiguration(); @@ -20,9 +20,10 @@ const client = new CatalogClientConfiguration(); OldPipelineModule, UsersModule, RolesModule, - CustomersModule + CustomersModule, + ShareModule, ], - controllers: [CatalogController, CatalogPublicController], + controllers: [CatalogController], providers: [CatalogService, DadosferaLogger], exports: [CatalogService], }) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index f7a2ed0..9bf0047 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -12,11 +12,9 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces'; import { BadRequestException, - ForbiddenException, HttpException, HttpStatus, Inject, - NotFoundException, OnModuleInit, } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; @@ -37,19 +35,8 @@ import { GetRlsRulesRequest, PiiMetadata, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; -import { writeFileSync } from 'fs'; -import path from 'path'; -import { HtmlParser } from 'src/utils/FileParser/html-parser'; -import { PiiDto } from './dtos/pii.dto'; -import { CsvParser } from 'src/utils/FileParser/csv-parser'; -import { PDFParser } from 'src/utils/FileParser/pdf-parser'; import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; -import { RequestUser } from 'src/decorators/user.decorator'; -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; -import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { CustomersService } from '../customers/customers.service'; -import { LanguageEnum } from 'src/utils/languages.enum'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -63,7 +50,6 @@ class CatalogService implements OnModuleInit { private readonly grpcClient: ClientGrpc, private readonly userService: UsersService, private readonly roleService: RolesService, - private readonly customerService: CustomersService ) { this.logger = dadosferaLogger.logger; } @@ -248,77 +234,6 @@ class CatalogService implements OnModuleInit { return { data_assets: response, total }; } - async getOneDataAssetPrivate(user: RequestUser, id: string) { - const { username, user_id, customer_id, customer_name, permissions } = user; - - this.logger.info("getOneDataAssetPrivate: " + user_id + " - " + id) - - this.logger.info(`data: `, { - username, - customer_name, - }); - - const is_data_manager = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - - const is_get_catalog = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, - ); - let has_permission = false; - const metadata = PackTheMetadata({ - username, - user_id, - customer_id, - customer_name, - }); - - const user_roles = await this.getUserRolesIds(user_id); - - const { data_asset } = await this.getOneDataAsset({ - customer_id, - id, - metadata, - }); - - has_permission = - is_data_manager || - is_get_catalog || - data_asset?.owner === username || - (user_roles as Array).some((r) => - data_asset.p_roles.includes(r), - ) || - data_asset.p_users.includes(user_id); - - - if (!has_permission) - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - delete data_asset.p_roles; - delete data_asset.p_users; - - return { data_asset }; - } - - async getOneDataAssetPublic(customerId: string, id: string) { - this.logger.info("getOneDataAssetPublic: " + customerId + " - " + id) - const metadata = await this.createMetadataToPublicAccess(customerId); - const { data_asset } = await this.getOneDataAsset({ - customer_id: customerId, - id, - metadata, - }); - this.logger.info("found asset: " + JSON.stringify(data_asset)) - delete data_asset.p_roles; - delete data_asset.p_users; - data_asset.share_type = 'public'; - if (data_asset.share_type !== 'public') - throw new NotFoundException(); - - return { data_asset }; - } - async getOneDataAsset(data: { id: string; customer_id: string; @@ -395,7 +310,7 @@ class CatalogService implements OnModuleInit { const { documentation } = await lastValueFrom( this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), ); - console.log(documentation); + const docs = JSON.parse(documentation); return docs; } @@ -558,26 +473,27 @@ class CatalogService implements OnModuleInit { async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; - this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, { ...body.logMetadata }); - this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); - this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, { ...body.logMetadata }); + this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); try { - const { data, status } = await axios.post(endpoint, { ...body.table_metadata }); + const { data, status } = await axios.post(endpoint, {...body.table_metadata}); this.logger.info( `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); return data.id; } catch (error) { this.logger.error( - `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${error.response?.status - } because of ${JSON.stringify(error.response?.data) || error.message}`, { ...body.logMetadata }); + `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${ + error.response?.status + } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata}); throw new Error(error.response?.data?.message || error.message); } } @@ -587,49 +503,51 @@ class CatalogService implements OnModuleInit { this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; - + try { - this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, { ...body.logMetadata }); - this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); - this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, { ...body.logMetadata }); + this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); const { data, status } = await axios.post(endpoint, body.column_metadata); this.logger.info( `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); return data.map((column) => column.id); } catch (error) { this.logger.error( - `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${error.response?.status - } because of ${error.response?.data || error.message}`, { ...body.logMetadata }); + `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${ + error.response?.status + } because of ${error.response?.data || error.message}`, {...body.logMetadata}); throw new Error(error.response?.data?.message || error.message); } } async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; - this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, { ...body.logMetadata }); - this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); - this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, { ...body.logMetadata }); + this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); try { const { data, status } = await axios.post(endpoint, body.data_preview); this.logger.info( `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); return data.id; } catch (error) { this.logger.error( - `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${error.response?.status + `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${ + error.response?.status } because of ${error.response?.data || error.message}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); throw new Error(error.response?.data?.message || error.message); } @@ -658,23 +576,6 @@ class CatalogService implements OnModuleInit { ); return res; } - - async createMetadataToPublicAccess(customerId?: string, language?: LanguageEnum) { - if (!customerId) { - throw new NotFoundException(); - } - - const { - customer - } = await this.customerService.getCustomer(customerId); - - return PackTheMetadata({ - customer_id: customer.id, - customer_name: customer.name, - language, - }) - } - } -export { CatalogService }; +export { CatalogService }; \ No newline at end of file diff --git a/src/modules/catalog/controllers/catalog.public.controller.ts b/src/modules/catalog/controllers/catalog.public.controller.ts deleted file mode 100644 index 6d3be81..0000000 --- a/src/modules/catalog/controllers/catalog.public.controller.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { - Controller, - ForbiddenException, - Get, - Headers, - Inject, - Param, - Query, - UseFilters, -} from '@nestjs/common'; -import { - ApiHeaders, - ApiTags, -} from '@nestjs/swagger'; -import { - RequireSomePermission, -} from '../../../decorators/authentication.decorator'; -import { - PERMISSIONS_GROUPS, -} from '../../../authentication/permissions.enum'; -import { CatalogService } from '../catalog.service'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { RequestUser, User } from 'src/decorators/user.decorator'; -import { - IColumnsMetadataResponse, - IDocsResponse, - IPreviewResponse, - -} from '../dtos'; -import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { Language } from 'src/decorators/language.decorator'; -import { LanguageEnum } from 'src/utils/languages.enum'; -import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; - - -@ApiTags('Catalog') -@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) -@Controller('catalog') -@UseFilters(new GrpcToHttpExceptionFilter()) -export class CatalogPublicController { - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private catalogService: CatalogService, - ) { - this.logger = dadosferaLogger.logger; - } - - private async createAccessToPublicAssets( - user?: RequestUser, - customerId?: string, - language?: LanguageEnum, - ) { - if (user) { - const { customer_name, customer_id, user_id, username } = user; - return PackTheMetadata({ - customer_name, - customer_id, - user_id, - username, - language, - }); - } else { - return await this.catalogService.createMetadataToPublicAccess( - customerId, - language, - ); - } - } - - private checkDefaultPermission(user: RequestUser) { - const is_data_manager = user.permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - - const is_get = user.permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, - ); - - if (is_data_manager || is_get) { - return; - } - - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - } - - @Get('data-asset/:id') - async getDataAsset( - @User() user: RequestUser, - @Param('id') id: string, - @Query('c') customerId?: string, - ) { - this.logger.info(`GET /data-asset/${id}`); - try { - if (user) { - return await this.catalogService.getOneDataAssetPrivate(user, id); - } else { - return await this.catalogService.getOneDataAssetPublic(customerId, id); - } - } catch (error) { - this.logger.error(error.message); - throw error; - } - } - - - - @Get('data-asset/:id/columns-metadata') - async getDataAssetColumnsMetadata( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - @Query('c') customerId: string, - ): Promise { - const metadata = await this.createAccessToPublicAssets( - user, - customerId, - language, - ); - this.logger.info(`/catalog - columns-metadata`); - - const columns_metadata = - await this.catalogService.getDatasetColumnsMetadata(id, metadata); - - if (user) { - this.checkDefaultPermission(user); - } - - return { columns_metadata }; - } - - @Get('data-asset/:id/preview') - async getDataAssetPreview( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - @Query('c') customerId: string, - ): Promise { - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`); - - const metadata = await this.createAccessToPublicAssets( - user, - customerId, - language, - ); - - const preview = await this.catalogService.getDatasetPreview(id, metadata); - - if (user) { - this.checkDefaultPermission(user); - } - - return { preview }; - } - - @Get('data-asset/:id/docs') - async getDataAssetDocs( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - @Query('c') customerId: string, - ): Promise { - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`); - - const metadata = await this.createAccessToPublicAssets( - user, - customerId, - language, - ); - - const docs = await this.catalogService.getDataDocs(id, metadata); - - if (user) { - this.checkDefaultPermission(user); - } - - return { docs }; - } - -} From 3c4f45678a3365551b49084d29c4a1955f734eb8 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 12 May 2025 12:25:45 -0300 Subject: [PATCH 05/31] CHORE: update protospack --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 50c8341..a67c2e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.24", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1400,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "0.0.0", - "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", - "integrity": "sha512-NfnvIENsIval+liPIPoVkFjixmKCKu9tRqvXQu3qTk67kZicNBvEOd9UezED1h1a5kX70+CbSR4mlXUZZ7ZVGA==", + "version": "3.37.0-beta.24", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.24.tgz", + "integrity": "sha512-uf6cJkq6LjROWxJEGjSXQNengtR74cXtybJUkZRj2c438yjQ6FChEczJdx5053m0qnPpbV86QRrayB/XxVPmvQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 40b4faa..fc1e8ac 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.24", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From d15cc4642a38a60ac44158fc34e923ef1b857804 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 13 May 2025 09:21:44 -0300 Subject: [PATCH 06/31] FEAT: assign embed and public key changes --- docsfera.json | 33 ++++++++++++++++++- package-lock.json | 8 ++--- package.json | 2 +- src/authentication/permissions.enum.ts | 3 +- src/modules/assign/assign.controller.ts | 13 ++++++-- src/modules/assign/assign.service.ts | 7 ++-- .../dto/create-share-metadata.dto.ts | 1 + .../share-metadata/share-metadata.service.ts | 3 +- 8 files changed, 57 insertions(+), 13 deletions(-) diff --git a/docsfera.json b/docsfera.json index 762ab8d..f6896c9 100644 --- a/docsfera.json +++ b/docsfera.json @@ -6257,6 +6257,33 @@ "access-token": [] } ] + }, + "get": { + "operationId": "AssignController_get", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] } }, "/health": { @@ -8800,11 +8827,15 @@ }, "proposedId": { "type": "string" + }, + "type": { + "type": "string" } }, "required": [ "assetId", - "proposedId" + "proposedId", + "type" ] }, "UpdateShareMetadataDto": { diff --git a/package-lock.json b/package-lock.json index a67c2e4..17d66f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.24", + "@dadosfera/protospack-v2": "3.37.0-beta.25", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1400,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.24", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.24.tgz", - "integrity": "sha512-uf6cJkq6LjROWxJEGjSXQNengtR74cXtybJUkZRj2c438yjQ6FChEczJdx5053m0qnPpbV86QRrayB/XxVPmvQ==", + "version": "3.37.0-beta.25", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.25.tgz", + "integrity": "sha512-UxAlpThepqC/HpEOVHhPdRwQP1QwGP6k5mhlStWzYwr/OsTBv5nzefZ6Y+dwXSoNFbPYt2z9xlDnRnvrUWLjAA==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index fc1e8ac..70d6f75 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.24", + "@dadosfera/protospack-v2": "3.37.0-beta.25", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index ee74e22..839a52b 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -614,7 +614,8 @@ export const DADOSFERA_MODULES_KEYS = { LOG_DASHBOARD: 'logs-dashboard', ACCESS_DASHBOARD: 'access-dashboard', DANGER_ZONE: 'danger-zone', - PII: 'pii' + PII: 'pii', + PUBLIC_ASSIGN_EMBED: 'public-assign-embed', } export const DADOSFERA_MODULES: Array = [ diff --git a/src/modules/assign/assign.controller.ts b/src/modules/assign/assign.controller.ts index e83488c..361524c 100644 --- a/src/modules/assign/assign.controller.ts +++ b/src/modules/assign/assign.controller.ts @@ -3,7 +3,7 @@ import { AssignService } from './assign.service'; import { CreateAssignDto } from './dto/create-assign.dto'; import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator'; import { RequestUser, User } from 'src/decorators/user.decorator'; -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; @Controller('assign') @@ -15,10 +15,19 @@ export class AssignController { @RequireSomePermission( PERMISSIONS_GROUPS.USERS.permissions.ADMIN ) - @RequireModule('public-assign-embed') + @RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED) create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) { const metadata = PackTheMetadata(user); return this.assignService.create(createAssignDto, metadata); } + @Get('/public-key') + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED) + async get(@User() user: RequestUser) { + const metadata = PackTheMetadata(user); + return await this.assignService.get(metadata); + } } diff --git a/src/modules/assign/assign.service.ts b/src/modules/assign/assign.service.ts index b48ef80..01a8c20 100644 --- a/src/modules/assign/assign.service.ts +++ b/src/modules/assign/assign.service.ts @@ -28,10 +28,11 @@ export class AssignService implements OnModuleInit { } async create(createAssignDto: CreateAssignDto, metadata: Metadata) { - return await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata)) + const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata)) + return data; } - async get(createAssignDto: CreateAssignDto, metadata: Metadata) { - return await lastValueFrom(this.ducService.GetAssignPublicKey(createAssignDto, metadata)) + async get(metadata: Metadata) { + return await lastValueFrom(this.ducService.GetAssignPublicKey(metadata)) } } diff --git a/src/modules/share-metadata/dto/create-share-metadata.dto.ts b/src/modules/share-metadata/dto/create-share-metadata.dto.ts index db06de6..6935de1 100644 --- a/src/modules/share-metadata/dto/create-share-metadata.dto.ts +++ b/src/modules/share-metadata/dto/create-share-metadata.dto.ts @@ -1,6 +1,7 @@ export class CreateShareMetadataDto { assetId: string; proposedId: string; + type: string; } export class UpdateShareMetadataDto { diff --git a/src/modules/share-metadata/share-metadata.service.ts b/src/modules/share-metadata/share-metadata.service.ts index 0915faa..e69865b 100644 --- a/src/modules/share-metadata/share-metadata.service.ts +++ b/src/modules/share-metadata/share-metadata.service.ts @@ -32,7 +32,8 @@ export class ShareMetadataService implements OnModuleInit { this.ducService.FindOrCreateShareMetadata( { assetId: createShareDto.assetId, - proposedId: createShareDto.proposedId + proposedId: createShareDto.proposedId, + type: createShareDto.type }, metadata, ), From ced2a975472cbd9a86fc5513765d006de598bdc7 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 19 May 2025 16:36:54 -0300 Subject: [PATCH 07/31] FIX: add logger and return info --- src/modules/catalog/share/share.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/catalog/share/share.service.ts b/src/modules/catalog/share/share.service.ts index b81096a..1729f43 100644 --- a/src/modules/catalog/share/share.service.ts +++ b/src/modules/catalog/share/share.service.ts @@ -198,6 +198,7 @@ export class ShareService implements OnModuleInit { private async getShareMetadata(id: string, user: RequestUser) { const metadata = PackTheMetadata({}); + this.logger.info('GET share metadata') const info = await this.shareMetadataService.get(id, metadata); if (isJWT(id) && info ){ return info; @@ -219,7 +220,7 @@ export class ShareService implements OnModuleInit { ); if (is_data_manager || is_get) { - return; + return info; } throw new ForbiddenException( From 6fc8c0b4a89a405a0200b5b241e2e8851fa548f3 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 20 May 2025 16:50:04 -0300 Subject: [PATCH 08/31] FEAT: add mixpanel service to track share access --- docsfera.json | 27 ++-- src/modules/catalog/catalog.module.ts | 1 + src/modules/catalog/share/share.controller.ts | 7 +- src/modules/catalog/share/share.module.ts | 4 +- src/modules/catalog/share/share.service.ts | 19 ++- src/modules/mixpanel/mixpanel.controller.ts | 80 +----------- src/modules/mixpanel/mixpanel.module.ts | 7 +- src/modules/mixpanel/mixpanel.service.ts | 118 ++++++++++++++++++ 8 files changed, 165 insertions(+), 98 deletions(-) create mode 100644 src/modules/mixpanel/mixpanel.service.ts diff --git a/docsfera.json b/docsfera.json index f6896c9..ebc41e9 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5501,6 +5501,17 @@ ] } }, + "/trackEvent/{id}": { + "post": { + "operationId": "MixpanelController_trackEvent", + "parameters": [], + "responses": { + "201": { + "description": "" + } + } + } + }, "/terms-of-use/token": { "get": { "operationId": "TermsOfUseController_getToken", @@ -5995,22 +6006,6 @@ ] } }, - "/trackEvent/{id}": { - "post": { - "operationId": "MixpanelController_trackEvent", - "parameters": [], - "responses": { - "201": { - "description": "" - } - }, - "security": [ - { - "access-token": [] - } - ] - } - }, "/open-data/sharing-ocean-data": { "post": { "operationId": "OpenDataController_createUser", diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index 8320d37..8ce1579 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -11,6 +11,7 @@ import { RolesModule } from '../roles/roles.module'; import { CustomersModule } from '../customers/customers.module'; import { ShareModule } from './share/share.module'; import { CatalogService } from './catalog.service'; +import { MixpanelModule } from '../mixpanel/mixpanel.module'; const client = new CatalogClientConfiguration(); diff --git a/src/modules/catalog/share/share.controller.ts b/src/modules/catalog/share/share.controller.ts index a5240bf..efc394e 100644 --- a/src/modules/catalog/share/share.controller.ts +++ b/src/modules/catalog/share/share.controller.ts @@ -3,6 +3,7 @@ import { Get, Inject, Param, + Req, UseFilters, } from '@nestjs/common'; import { @@ -21,6 +22,7 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; import { ShareService } from './share.service'; +import { Request } from 'express'; @ApiTags('Catalog') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -39,10 +41,11 @@ export class ShareController { @Get('/:id') async getShareDataAsset( @Param('id') id: string, - @User() user: RequestUser + @User() user: RequestUser, + @Req() request: Request ) { this.logger.info(`GET //:id`); - return await this.catalogShareService.getOneDataAssetPublic(id, user); + return await this.catalogShareService.getOneDataAssetPublic(id, user, request); } @Get('/:id/columns-metadata') diff --git a/src/modules/catalog/share/share.module.ts b/src/modules/catalog/share/share.module.ts index 1f0c7fb..ce7021c 100644 --- a/src/modules/catalog/share/share.module.ts +++ b/src/modules/catalog/share/share.module.ts @@ -8,6 +8,7 @@ import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.m import { ShareController } from "./share.controller"; import DadosferaLogger from "@dadosfera/dadosfera-logs"; import { ShareService } from "./share.service"; +import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module"; const client = new CatalogClientConfiguration(); @@ -17,7 +18,8 @@ const client = new CatalogClientConfiguration(); UsersModule, RolesModule, CustomersModule, - ShareMetadataModule + ShareMetadataModule, + MixpanelModule ], controllers: [ShareController], providers: [ShareService, DadosferaLogger], diff --git a/src/modules/catalog/share/share.service.ts b/src/modules/catalog/share/share.service.ts index 1729f43..da8d3fe 100644 --- a/src/modules/catalog/share/share.service.ts +++ b/src/modules/catalog/share/share.service.ts @@ -20,6 +20,8 @@ import { Metadata } from '@grpc/grpc-js'; import { lastValueFrom } from 'rxjs'; import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service'; import { isJWT } from 'class-validator'; +import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service'; +import { Request } from 'express'; export class ShareService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -32,7 +34,8 @@ export class ShareService implements OnModuleInit { private readonly grpcClient: ClientGrpc, private readonly userService: UsersService, private readonly roleService: RolesService, - private shareMetadataService: ShareMetadataService, + private readonly shareMetadataService: ShareMetadataService, + private readonly mixpanelService: MixpanelService, ) { this.logger = dadosferaLogger.logger; } @@ -76,7 +79,7 @@ export class ShareService implements OnModuleInit { return result; } - async getOneDataAssetPublic(id: string, user: RequestUser) { + async getOneDataAssetPublic(id: string, user: RequestUser, request: Request) { this.logger.info("getOneDataAssetPublic: " + JSON.stringify({ id, userId: user?.user_id, @@ -85,6 +88,18 @@ export class ShareService implements OnModuleInit { try { const shareMetadata = await this.getShareMetadata(id, user); + const mixpanelTracker = { + asset: shareMetadata.assetId, + type: isJWT(id) ? 'assigned' : shareMetadata.type, + customer: shareMetadata.customerName + } + + if (user) { + await this.mixpanelService.track("share_page", user, request, mixpanelTracker); + } else { + await this.mixpanelService.trackShare(request, mixpanelTracker); + } + this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata)) const metadata = PackTheMetadata({ customer_id: shareMetadata.customerId, diff --git a/src/modules/mixpanel/mixpanel.controller.ts b/src/modules/mixpanel/mixpanel.controller.ts index 04e8950..477569b 100644 --- a/src/modules/mixpanel/mixpanel.controller.ts +++ b/src/modules/mixpanel/mixpanel.controller.ts @@ -3,14 +3,13 @@ import { init } from 'mixpanel'; import { Authenticated } from 'src/decorators/authentication.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { RequestUser, User } from 'src/decorators/user.decorator'; +import { MixpanelService } from './mixpanel.service'; @ApiInternalOnlyController() -@Authenticated() @Controller('trackEvent') export class MixpanelController { constructor( - @Inject('MIXPANEL_TOKEN') - private readonly mixpanelToken: string, + private mixpanelService: MixpanelService ) {} @Post(':id') async trackEvent( @@ -20,82 +19,11 @@ export class MixpanelController { @Req() request ) { delete body.info; - const mixpanel = init(this.mixpanelToken); - - const separator = user.username.includes('-') ? '-' : '.'; - const removeValues = [ - '.dadosferatech.dadosfera', - '.demo.dadosfera', - '.dadosferademo', - '.dadosferarh.dadosfera', - '.dadosferatech.dadosfera2', - '.dadosferatech.dadosfera', - '.dadosfera.fin', - '.dadosferafin.dadosfera', - '.praxio.dadosfera', - '.dadosfera.tech', - '.treinamentos@dadosfera.ai', - '.dadosfera2', - '.treinamentosfera', - '.dadosfera', - ]; - - let username = user.username; - - removeValues.forEach((value) => { - username = username.replace(value, ''); - }); - - username = username.split('@')?.[0]; - username = username.split('+')?.[0]; - - let firstName = username - .substring(0, username.indexOf(separator)) - .replace('dadosfera', ''); - let lastName = username - .substring(username.lastIndexOf(separator) + 1) - .replace('dadosfera', ''); - - if (!firstName) { - firstName = lastName; - lastName = ''; - } - - firstName = this.capitalize(firstName); - lastName = this.capitalize(lastName); - - await mixpanel.people.set(user.username, { - $first_name: firstName, - $last_name: lastName, - $name: this.getFullName(firstName, lastName), - $email: user.username.includes('@') - ? user.username - : user.username + '@dadosfera.ai', - customer_name: user.customer_name - }); - - await mixpanel.track(id, { - distinct_id: user.username, - customer: user.customer_name, - env: process.env.ENV, - $ip: request.ip, - $os: request.headers['sec-ch-ua-platform'] || '', - $browser: request.headers['user-agent'], - ...body, - }); + + await this.mixpanelService.track(id, user, request, body) return { id, body, user: user.username }; } - capitalize(sentence: string): string { - if (!sentence) { - return ''; - } - return sentence[0].toUpperCase() + sentence.substring(1); - } - - getFullName(firstName: string, lastName: string) { - return `${firstName}${lastName ? ' ' + lastName : ''}`; - } } diff --git a/src/modules/mixpanel/mixpanel.module.ts b/src/modules/mixpanel/mixpanel.module.ts index 98f130d..f3ed70e 100644 --- a/src/modules/mixpanel/mixpanel.module.ts +++ b/src/modules/mixpanel/mixpanel.module.ts @@ -1,6 +1,8 @@ import { Module } from '@nestjs/common'; import { getSecretFromSecretsManager } from 'src/utils/SecretManager'; import { MixpanelController } from './mixpanel.controller'; +import { MixpanelService } from './mixpanel.service'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; @Module({ controllers: [MixpanelController], @@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller'; { provide: 'MIXPANEL_TOKEN', useValue: getSecretFromSecretsManager( - `${process.env.ENV}/root/mixpanel_token`, + `prd/root/mixpanel_token`, ), }, + MixpanelService, + DadosferaLogger ], + exports: [MixpanelService] }) export class MixpanelModule {} diff --git a/src/modules/mixpanel/mixpanel.service.ts b/src/modules/mixpanel/mixpanel.service.ts new file mode 100644 index 0000000..3dd1879 --- /dev/null +++ b/src/modules/mixpanel/mixpanel.service.ts @@ -0,0 +1,118 @@ +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { Inject } from '@nestjs/common'; +import { Request } from 'express'; +import mixpanel, { init } from 'mixpanel'; +import { RequestUser } from 'src/decorators/user.decorator'; + +export class MixpanelService { + logger: DadosferaLogger; + + constructor( + @Inject('MIXPANEL_TOKEN') + private readonly mixpanelToken: string, + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + async track(eventName: string, user: RequestUser, request: Request, body: any) { + this.logger.info("track: " + JSON.stringify({ + eventName, + ...body + })) + const mixpanel = init(this.mixpanelToken); + await this.setPeople(user, mixpanel); + + await mixpanel.track(eventName, { + distinct_id: user.username, + customer: user.customer_name, + env: process.env.ENV, + $ip: request.ip, + $os: request.headers['sec-ch-ua-platform'] || '', + $browser: request.headers['user-agent'], + ...body, + }); + return; + } + + async trackShare(request: Request, body: any) { + this.logger.info("trackShare: " + JSON.stringify(body)) + const mixpanel = init(this.mixpanelToken); + + await mixpanel.track("share_page", { + env: process.env.ENV, + $ip: request.ip, + $os: request.headers['sec-ch-ua-platform'] || '', + $browser: request.headers['user-agent'], + ...body, + }); + return; + } + + private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) { + const separator = user.username.includes('-') ? '-' : '.'; + const removeValues = [ + '.dadosferatech.dadosfera', + '.demo.dadosfera', + '.dadosferademo', + '.dadosferarh.dadosfera', + '.dadosferatech.dadosfera2', + '.dadosferatech.dadosfera', + '.dadosfera.fin', + '.dadosferafin.dadosfera', + '.praxio.dadosfera', + '.dadosfera.tech', + '.treinamentos@dadosfera.ai', + '.dadosfera2', + '.treinamentosfera', + '.dadosfera', + ]; + + let username = user.username; + + removeValues.forEach((value) => { + username = username.replace(value, ''); + }); + + username = username.split('@')?.[0]; + username = username.split('+')?.[0]; + + let firstName = username + .substring(0, username.indexOf(separator)) + .replace('dadosfera', ''); + let lastName = username + .substring(username.lastIndexOf(separator) + 1) + .replace('dadosfera', ''); + + if (!firstName) { + firstName = lastName; + lastName = ''; + } + + firstName = this.capitalize(firstName); + lastName = this.capitalize(lastName); + + await mixpanel.people.set(user.username, { + $first_name: firstName, + $last_name: lastName, + $name: this.getFullName(firstName, lastName), + $email: user.username.includes('@') + ? user.username + : user.username + '@dadosfera.ai', + customer_name: user.customer_name, + }); + } + + private capitalize(sentence: string): string { + if (!sentence) { + return ''; + } + + return sentence[0].toUpperCase() + sentence.substring(1); + } + + private getFullName(firstName: string, lastName: string) { + return `${firstName}${lastName ? ' ' + lastName : ''}`; + } +} From 5a36762544bed596222da329822a7ab4acf974bc Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 26 May 2025 10:24:48 -0300 Subject: [PATCH 09/31] UPDATE: route /catalog/register-dataset will have an limit of 10mb --- src/main.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.ts b/src/main.ts index 46f1701..7511269 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import helmet from 'helmet'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { json, urlencoded } from 'express'; import { AppModule } from './app.module'; import { writeFileSync } from 'fs'; @@ -25,6 +26,8 @@ async function bootstrap() { }, }); app.use(helmet()); + app.use('/catalog/register-dataset', json({ limit: '10mb' })); + app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); configureSwagger(app); await app.listen(3333); if (process.env.KILL_AFTER_START) await app.close(); From a89f57e6902f1e5c09c876e8d9edf76baa643d46 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 30 May 2025 10:13:37 -0300 Subject: [PATCH 10/31] FIX: return pi_factory_url and tr_factory_url to stg url --- helmfiles/stg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 5782e36..530416b 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,11 +9,11 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.dadosfera.ai + value: pi-factory.stg.dadosfera.ai - name: maestro.in_factory_url value: in-factory.stg.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.dadosfera.ai + value: in-factory.stg.dadosfera.ai - name: maestro.open_customer_id value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id From 54cb3f8e7ae14ba53582f9c11b890174e5fe89cb Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 6 Jun 2025 15:53:12 -0300 Subject: [PATCH 11/31] FEAT: add api key endpoint --- docsfera.json | 235 +++++++++++++++++++++ package-lock.json | 8 +- package.json | 2 +- src/app.module.ts | 4 +- src/authentication/authentication.guard.ts | 30 ++- src/main.ts | 7 +- src/modules/api-key/api-key.controller.ts | 68 ++++++ src/modules/api-key/api-key.module.ts | 18 ++ src/modules/api-key/api-key.service.ts | 50 +++++ src/modules/api-key/dto/api-key.dto.ts | 39 ++++ 10 files changed, 449 insertions(+), 12 deletions(-) create mode 100644 src/modules/api-key/api-key.controller.ts create mode 100644 src/modules/api-key/api-key.module.ts create mode 100644 src/modules/api-key/api-key.service.ts create mode 100644 src/modules/api-key/dto/api-key.dto.ts diff --git a/docsfera.json b/docsfera.json index b6e8be8..b333454 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5968,6 +5968,158 @@ } ] } + }, + "/api-key": { + "post": { + "operationId": "ApiKeyController_create", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyDto" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "ApiKeyController_findAll", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/api-key/{id}": { + "delete": { + "operationId": "ApiKeyController_remove", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } } }, "info": { @@ -8848,6 +9000,89 @@ "required": [ "policies" ] + }, + "CreateApiKeyDto": { + "type": "object", + "properties": { + "permissions": { + "description": "Array of permission IDs", + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "permissions" + ] + }, + "CreateApiKeyResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + }, + "key": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by", + "key" + ] + }, + "ApiKeyBaseResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by" + ] } } } diff --git a/package-lock.json b/package-lock.json index c9d852f..2add2f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "3.38.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1400,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.23", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", - "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", + "version": "3.38.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.1.tgz", + "integrity": "sha512-4+yNZMlhvEkHlxJcaeta+jtJ9owskZcg0yjKdZzBfT8PCclctgertLHmrG0+kDb3Y/na/rpPPHYULRp7AO4zDQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index fd3eb0b..b59aa9c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "3.38.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index fedb388..0555eaf 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -29,6 +29,7 @@ import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; import { ThemeModule } from './modules/theme/theme.module'; import { NetworkPolicyModule } from './modules/network-policy/network-policy.module'; +import { ApiKeyModule } from './modules/api-key/api-key.module'; @Module({ providers: [ @@ -64,7 +65,8 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod ThemeModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, - NetworkPolicyModule + NetworkPolicyModule, + ApiKeyModule ], }) export class AppModule {} diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index bc69f8c..0506581 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -17,6 +17,7 @@ import { import { RequestUser } from '../decorators/user.decorator'; import ErrorBuilder from '../utils/ErrorBuilder'; import ErrorCodes from '../utils/errorCodes'; +import { ApiKeyService } from 'src/modules/api-key/api-key.service'; @Injectable() export class AuthenticationGuard @@ -31,6 +32,7 @@ export class AuthenticationGuard dadosferaLogger: DadosferaLogger, private reflector: Reflector, private authClient: AuthClientService, + private apiKeyService: ApiKeyService ) { this.pems = new Map(); this.logger = dadosferaLogger.logger; @@ -48,20 +50,40 @@ export class AuthenticationGuard }); } - canActivate(ctx: ExecutionContext): boolean { + async canActivate(ctx: ExecutionContext): Promise { const authFunctions = this.reflector.getAllAndMerge< AuthenticationFunction[] >(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]); const mustBeAuthenticated = authFunctions.length > 0; - const request = ctx.switchToHttp().getRequest(); - const accessToken = this.validateToken(request, mustBeAuthenticated); - if (!mustBeAuthenticated) { // no need to be authenticated return true; } + const request = ctx.switchToHttp().getRequest(); + const apiKey = request.get('X-api-key'); + if (apiKey) { + const { + api_key + } = await this.apiKeyService.get(apiKey); + + request.user = { + user_id: api_key.user_id, + username: api_key.username, + permissions: api_key.permissions, + customer_id: api_key.customer_id, + customer_name: api_key.customer_name, + customer_tier: api_key.customer_tier, + customer_modules: api_key.customer_modules, + access_token: apiKey, + }; + + return true; + } + + const accessToken = this.validateToken(request, mustBeAuthenticated); + if (!accessToken) { // couldn't load valid token throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED); diff --git a/src/main.ts b/src/main.ts index 7511269..6fd8445 100644 --- a/src/main.ts +++ b/src/main.ts @@ -26,8 +26,11 @@ async function bootstrap() { }, }); app.use(helmet()); - app.use('/catalog/register-dataset', json({ limit: '10mb' })); - app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); + if (process.env.ENV === 'prd') { + app.use('/catalog/register-dataset', json({ limit: '10mb' })); + app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); + } + configureSwagger(app); await app.listen(3333); if (process.env.KILL_AFTER_START) await app.close(); diff --git a/src/modules/api-key/api-key.controller.ts b/src/modules/api-key/api-key.controller.ts new file mode 100644 index 0000000..bc2c51c --- /dev/null +++ b/src/modules/api-key/api-key.controller.ts @@ -0,0 +1,68 @@ +import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common'; +import { ApiKeyService } from './api-key.service'; +import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto'; +import { Authenticated } from 'src/decorators/authentication.decorator'; +import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +@Controller('api-key') +@Authenticated() +@ApiTags('ApiKey') +@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) +@UseFilters(new GrpcToHttpExceptionFilter()) +export class ApiKeyController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private readonly apiKeyService: ApiKeyService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Post() + @ApiResponse({ type: CreateApiKeyResponseDto }) + async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise { + this.logger.info('POST /api-key', { + permissions: createApiKeyDto.permissions, + method: 'create' + }); + const result = await this.apiKeyService.create(createApiKeyDto, user); + this.logger.info('POST /api-key success', { + id: result.id, + method: 'create' + }); + return result; + } + + @Get() + @ApiResponse({ type: [ApiKeyBaseResponseDto] }) + async findAll(@User() user: RequestUser): Promise { + this.logger.info('GET /api-key', { + method: 'findAll' + }); + const result = await this.apiKeyService.findAll(user); + this.logger.info('GET /api-key success', { + count: result.length, + method: 'findAll' + }); + return result; + } + + @Delete(':id') + async remove(@Param('id') id: string, @User() user: RequestUser): Promise { + this.logger.info('DELETE /api-key/:id', { + id, + method: 'remove' + }); + await this.apiKeyService.remove(id, user); + this.logger.info('DELETE /api-key/:id success', { + id, + method: 'remove' + }); + } +} diff --git a/src/modules/api-key/api-key.module.ts b/src/modules/api-key/api-key.module.ts new file mode 100644 index 0000000..1455425 --- /dev/null +++ b/src/modules/api-key/api-key.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +import { ApiKeyService } from './api-key.service'; +import { ApiKeyController } from './api-key.controller'; +import { ClientsModule } from '@nestjs/microservices'; +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +const ducClient = new DucClient(); + +@Module({ + imports: [ + ClientsModule.register([ducClient.providerOptions]) + ], + controllers: [ApiKeyController], + providers: [ApiKeyService, DadosferaLogger], + exports: [ApiKeyService] +}) +export class ApiKeyModule {} diff --git a/src/modules/api-key/api-key.service.ts b/src/modules/api-key/api-key.service.ts new file mode 100644 index 0000000..ea96801 --- /dev/null +++ b/src/modules/api-key/api-key.service.ts @@ -0,0 +1,50 @@ +import { Injectable, Inject, OnModuleInit } from '@nestjs/common'; +import { ClientGrpc } from '@nestjs/microservices'; +import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto'; +import { RequestUser } from 'src/decorators/user.decorator'; +import { DucClient } from '../duc/client.config'; +import { PackTheMetadata } from '../../utils/ PackTheMetadata'; +import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { lastValueFrom } from 'rxjs'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; + +@Injectable() +export class ApiKeyService implements OnModuleInit { + private apiKeyService: ApiKeyWriteProtoService; + + constructor( + @Inject(DucClient.name) private readonly client: ClientGrpc, + ) {} + + onModuleInit() { + this.apiKeyService = this.client.getService(ProtoServices.ApiKeyWriteProtoService); + } + + create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise { + const metadata = PackTheMetadata(user); + + return lastValueFrom(this.apiKeyService.CreateApiKey({ + permissions: createApiKeyDto.permissions + }, metadata)); + } + + async findAll(user: RequestUser): Promise { + const metadata = PackTheMetadata(user); + console.log(metadata) + + const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata)); + return data.api_keys; + } + + async remove(id: string, user: RequestUser) { + const metadata = PackTheMetadata(user); + + await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata)); + } + + async get(key: string) { + const metadata = PackTheMetadata({}); + + return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata)); + } +} diff --git a/src/modules/api-key/dto/api-key.dto.ts b/src/modules/api-key/dto/api-key.dto.ts new file mode 100644 index 0000000..2bbabf3 --- /dev/null +++ b/src/modules/api-key/dto/api-key.dto.ts @@ -0,0 +1,39 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsArray, IsNumber } from 'class-validator'; + +export class PermissionDto { + @ApiProperty({ type: Number }) + id: number; + + @ApiProperty({ type: String }) + name: string; +} + +export class ApiKeyBaseResponseDto { + @ApiProperty({ type: String, format: 'uuid' }) + id: string; + + @ApiProperty({ type: String }) + key_mask: string; + + @ApiProperty({ type: [PermissionDto] }) + permissions: PermissionDto[]; + + @ApiProperty({ type: String, format: 'date-time' }) + created_at: string; + + @ApiProperty({ type: String }) + created_by: string; +} + +export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto { + @ApiProperty({ type: String }) + key: string; +} + +export class CreateApiKeyDto { + @ApiProperty({ type: [Number], description: 'Array of permission IDs' }) + @IsArray() + @IsNumber({}, { each: true }) + permissions: number[]; +} \ No newline at end of file From 01263017e7612de36accf04239f61d450a0796de Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Jun 2025 14:24:26 -0300 Subject: [PATCH 12/31] FIX: comment block list --- src/modules/open-data/open-data.controller.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index b8951ed..e952838 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -45,11 +45,11 @@ export class OpenDataController { ) { this.logger.info('createUser for open data' + JSON.stringify(request.headers)); - const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"]; - if (!corslist.includes(origin)) { - this.logger.info('block request by cors list: '+ origin); - throw new ForbiddenException(); - } + // const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"]; + // if (!corslist.includes(origin)) { + // this.logger.info('block request by cors list: '+ origin); + // throw new ForbiddenException(); + // } const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID; From fea587ad366e8081eaa328a381d0c85c9922cb2c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 11:51:56 -0300 Subject: [PATCH 13/31] FEAT: control login by dedicated customer id injected --- environment.d.ts | 1 + helmfiles/stg.yaml | 26 +++++++++++++++++++++ src/modules/auth/auth.controller.ts | 4 +--- src/modules/auth/auth.service.ts | 35 +++++++++++++++++++++++++---- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/environment.d.ts b/environment.d.ts index c910516..3558f26 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -14,6 +14,7 @@ declare global { AWS_REGION: string; OPEN_GROUP_ID: string; OPEN_CUSTOMER_ID: string; + DEDICATED_PROXY: string; } } } diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 5782e36..616588e 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -18,5 +18,31 @@ charts: value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 + - name: replicaCount + value: 1 + + # Environment to test Network Policies + - name: private + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: maestro.duc_url + value: duc.stg.dadosfera.ai + - name: hostname + value: private.stg.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory.stg.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d + - name: maestro.open_group_id + value: e3f98a2f-7748-4981-8505-7695c8ca8218 + # Customer id + - name: maestro.dedicated_proxy + value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd - name: replicaCount value: 1 \ No newline at end of file diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 7f2f8f0..66870d5 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -86,11 +86,9 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('origin') origin = '', ): Promise { this.logger.info('/auth - SignIn'); - const frontHost = origin.replace(/^https?:\/\//, ''); - const metadata = PackTheMetadata({ language, custom_host: frontHost }); + const metadata = PackTheMetadata({ language }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 5974954..be71d66 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -1,4 +1,4 @@ -import { OnModuleInit, Inject, Injectable } from '@nestjs/common'; +import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { lastValueFrom } from 'rxjs'; @@ -17,6 +17,7 @@ import { AuthResetPasswordRequest, AuthVerifyResetPasswordCodeRequest, AuthConfirmResetPasswordRequest, + AuthSignInResponse, } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; @@ -53,15 +54,41 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } + checkDedicatedProxy(customerId: string) { + const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; + this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); + if (DEDICATED_PROXY !== '') { + this.logger.info('Customer ID: ' + customerId); + if (DEDICATED_PROXY !== customerId) { + throw new ForbiddenException(); + } + } + } + async signIn( { username, password, totp }: AuthSignInRequest, metadata: Metadata, ) { this.logger.info('SignIn'); - return lastValueFrom( - this.authService.AuthSignIn({ username, password, totp }, metadata), - ); + let result: AuthSignInResponse; + + try { + result = await lastValueFrom( + this.authService.AuthSignIn({ username, password, totp }, metadata), + ); + + } catch (error) { + this.logger.error('SignIn - Error during sign-in'); + this.logger.error(error); + throw error; + } + + if (result.customer) { + this.checkDedicatedProxy(result.customer.id); + } + + return result } async refreshAccessToken( From 7f9755493a9d2a7b8ea1d3074d1c1d1536c6d7a9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 12:29:51 -0300 Subject: [PATCH 14/31] FIX: experimental hostname --- helmfiles/stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 616588e..297c5aa 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -30,7 +30,7 @@ charts: - name: maestro.duc_url value: duc.stg.dadosfera.ai - name: hostname - value: private.stg.dadosfera.ai + value: private-maestro.stg.dadosfera.ai - name: maestro.pi_factory_url value: pi-factory.dadosfera.ai - name: maestro.in_factory_url From 41842f7c6afc2ed5f5e63058d09efb366932a666 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 13:56:57 -0300 Subject: [PATCH 15/31] FIX: chart private maestro --- helmfiles/stg.yaml | 4 +++- maestro/templates/deployment.yaml | 14 +++++++------- maestro/templates/ingress-private.yaml | 4 ++-- maestro/templates/ingress.yaml | 6 +++--- maestro/templates/secret.yaml | 6 +++--- maestro/templates/service.yaml | 8 ++++---- maestro/values.yaml | 2 ++ 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 297c5aa..8039c29 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -22,11 +22,13 @@ charts: value: 1 # Environment to test Network Policies - - name: private + - name: private-maestro chart: ../maestro values: - ../maestro/values.yaml set: + - name: app_name + value: maestro-private - name: maestro.duc_url value: duc.stg.dadosfera.ai - name: hostname diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index c648514..6629db6 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: maestro + name: {{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - app: maestro + app: {{ .Values.app_name }} strategy: rollingUpdate: @@ -20,7 +20,7 @@ spec: template: metadata: labels: - app: maestro + app: {{ .Values.app_name }} spec: imagePullSecrets: @@ -108,15 +108,15 @@ spec: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_SECRET_ACCESS_KEY - name: AWS_DEFAULT_REGION valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_DEFAULT_REGION diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index 97708bc..b729904 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -10,7 +10,7 @@ metadata: generation: 1 labels: - app: maestro + app: {{ .Values.app_name }} name: open-data namespace: applications spec: @@ -21,7 +21,7 @@ spec: paths: - backend: service: - name: maestro + name: {{ .Values.app_name }} port: number: {{ .Values.ingress.port }} path: /open-data/sharing-ocean-data diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index faf1305..78b360c 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -9,8 +9,8 @@ metadata: generation: 1 labels: - app: maestro - name: maestro + app: {{ .Values.app_name }} + name: {{ .Values.app_name }} namespace: applications spec: ingressClassName: nginx @@ -20,7 +20,7 @@ spec: paths: - backend: service: - name: maestro + name: {{ .Values.app_name }} port: number: {{ .Values.ingress.port }} path: / diff --git a/maestro/templates/secret.yaml b/maestro/templates/secret.yaml index 9e2fd37..3bbf5ec 100644 --- a/maestro/templates/secret.yaml +++ b/maestro/templates/secret.yaml @@ -1,17 +1,17 @@ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: prd-maestro + name: prd-{{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: refreshInterval: 1h secretStoreRef: name: secretsmanager-prd kind: SecretStore target: - name: prd-maestro + name: prd-{{ .Values.app_name }} creationPolicy: Owner data: - secretKey: AWS_ACCESS_KEY_ID diff --git a/maestro/templates/service.yaml b/maestro/templates/service.yaml index 75f635a..1f7c988 100644 --- a/maestro/templates/service.yaml +++ b/maestro/templates/service.yaml @@ -1,18 +1,18 @@ apiVersion: v1 kind: Service metadata: - name: maestro + name: {{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: type: ClusterIP ports: - - name: maestro + - name: {{ .Values.app_name }} protocol: TCP port: {{ .Values.service.port }} targetPort: {{ .Values.service.targetPort }} selector: - app: maestro + app: {{ .Values.app_name }} diff --git a/maestro/values.yaml b/maestro/values.yaml index 14dd9e4..e0ec85c 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -9,6 +9,7 @@ image: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: 1.56.0 +app_name: maestro containerPort: 3333 imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret" service: @@ -42,6 +43,7 @@ maestro: upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 + dedicated_proxy: "" autoscaling: enabled: false minReplicas: 1 From 9e597fadba9e4d69d01f02a06dafd02631d7e806 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 14:04:45 -0300 Subject: [PATCH 16/31] FIX: open data ingress route --- maestro/templates/ingress-private.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index b729904..70f8eb6 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -11,7 +11,7 @@ metadata: generation: 1 labels: app: {{ .Values.app_name }} - name: open-data + name: open-data-{{ .Values.app_name }} namespace: applications spec: ingressClassName: nginx From 17de31f1a65aa6c33c8d132844bd7f2bc40c1eef Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 14:45:13 -0300 Subject: [PATCH 17/31] FIX: send env DEDICATED_PROXY --- maestro/templates/deployment.yaml | 2 ++ maestro/templates/ingress-private.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 6629db6..ac9eaae 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -100,6 +100,8 @@ spec: value: {{ .Values.maestro.open_customer_id }} - name: OPEN_GROUP_ID value: {{ .Values.maestro.open_group_id }} + - name: DEDICATED_PROXY + value: {{ .Values.maestro.dedicated_proxy }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index 70f8eb6..1ace950 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -11,7 +11,11 @@ metadata: generation: 1 labels: app: {{ .Values.app_name }} + {{- if .Values.maestro.dedicated_proxy}} name: open-data-{{ .Values.app_name }} + {{- else }} + name: open-data + {{- end }} namespace: applications spec: ingressClassName: nginx From cb98099a913e026efb5ab94518deda110697c1ec Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 13 Jun 2025 15:07:50 -0300 Subject: [PATCH 18/31] FIX: update timeout --- src/utils/FileParser/pdf-parser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/FileParser/pdf-parser.ts b/src/utils/FileParser/pdf-parser.ts index 1294b34..a954708 100644 --- a/src/utils/FileParser/pdf-parser.ts +++ b/src/utils/FileParser/pdf-parser.ts @@ -22,7 +22,7 @@ export class PDFParser implements Parser { const html = await this.htmlParser.parse(data); await page.setContent(html, { waitUntil: 'networkidle0', - timeout: 30000, + timeout: 90000, }); // Configurações adicionais para garantir um PDF válido @@ -43,7 +43,7 @@ export class PDFParser implements Parser { pageRanges: '', tagged: true, outline: false, - timeout: 30000, + timeout: 90000, }); await browser.close(); From 0aa9c065a5398d6443f5626763b94f25a6ada7fe Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 16 Jun 2025 10:34:11 -0300 Subject: [PATCH 19/31] FEAT: block user from login when maestro is dedicated --- src/authentication/authentication.guard.ts | 13 +++++++++++++ src/modules/auth/auth.service.ts | 22 +++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index bc69f8c..a41bc74 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -4,6 +4,7 @@ import { OnApplicationBootstrap, ExecutionContext, Inject, + ForbiddenException, } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import assert from 'assert'; @@ -113,6 +114,18 @@ export class AuthenticationGuard return false; } + // Bloquear outros customer de usar o maestor dedicado + const DEDICATED_PROXY = process.env.DEDICATED_PROXY + if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) { + throw new ForbiddenException(); + } + + // Bloquear o customer de acesso o maestro publico + const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(); + } + request.accessTokenPayload = accessTokenPayload; request.user = { user_id: accessTokenPayload.user_id, diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index be71d66..a8ceedd 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -54,14 +54,22 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } - checkDedicatedProxy(customerId: string) { + checkDedicatedProxy({ + customer + }: AuthSignInResponse) { const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); - if (DEDICATED_PROXY !== '') { - this.logger.info('Customer ID: ' + customerId); - if (DEDICATED_PROXY !== customerId) { - throw new ForbiddenException(); - } + this.logger.info('Customer ID: ' + customer.id); + + if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) { + throw new ForbiddenException(); + } + + // Bloquear o customer de acesso o maestro publico + this.logger.info('Check if customer have network policy: ' + customer.modules); + const hasNetworkPolicyModule = customer.modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(); } } @@ -85,7 +93,7 @@ export class AuthClientService implements OnModuleInit { } if (result.customer) { - this.checkDedicatedProxy(result.customer.id); + this.checkDedicatedProxy(result); } return result From 53059c0a0d66a7e1c2e946e10be5cfdd8f82cb8c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 16 Jun 2025 18:09:25 -0300 Subject: [PATCH 20/31] FIX: import --- docsfera.json | 243 ++++++++++++++++++++++++- src/modules/api-key/api-key.service.ts | 2 +- 2 files changed, 236 insertions(+), 9 deletions(-) diff --git a/docsfera.json b/docsfera.json index ebc41e9..96612ac 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,14 +20,6 @@ ], "type": "string" } - }, - { - "name": "origin", - "required": true, - "in": "header", - "schema": { - "type": "string" - } } ], "requestBody": { @@ -6281,6 +6273,158 @@ ] } }, + "/api-key": { + "post": { + "operationId": "ApiKeyController_create", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyDto" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "ApiKeyController_findAll", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/api-key/{id}": { + "delete": { + "operationId": "ApiKeyController_remove", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/health": { "get": { "operationId": "HealthController_check", @@ -9219,6 +9363,89 @@ "required": [ "publicKey" ] + }, + "CreateApiKeyDto": { + "type": "object", + "properties": { + "permissions": { + "description": "Array of permission IDs", + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "permissions" + ] + }, + "CreateApiKeyResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + }, + "key": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by", + "key" + ] + }, + "ApiKeyBaseResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by" + ] } } } diff --git a/src/modules/api-key/api-key.service.ts b/src/modules/api-key/api-key.service.ts index ea96801..3e92053 100644 --- a/src/modules/api-key/api-key.service.ts +++ b/src/modules/api-key/api-key.service.ts @@ -3,10 +3,10 @@ import { ClientGrpc } from '@nestjs/microservices'; import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto'; import { RequestUser } from 'src/decorators/user.decorator'; import { DucClient } from '../duc/client.config'; -import { PackTheMetadata } from '../../utils/ PackTheMetadata'; import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; import { lastValueFrom } from 'rxjs'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; @Injectable() export class ApiKeyService implements OnModuleInit { From 19d748ae09f0b70f79cb15f06b1d6dfe43196042 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 14:47:39 -0300 Subject: [PATCH 21/31] CHORE: add restricted ip and unimed chart --- helmfiles/prd.yaml | 31 +++++++++++++++++++++++++++++++ helmfiles/stg.yaml | 4 +++- maestro/templates/ingress.yaml | 3 +++ maestro/values.yaml | 1 + 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index e4146e8..c426fa9 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -4,6 +4,8 @@ charts: values: - ../maestro/values.yaml set: + - name: app_name + value: maestro - name: maestro.duc_url value: duc.dadosfera.ai - name: hostname @@ -20,3 +22,32 @@ charts: value: c0afdcce-c5be-40d0-9d1d-2d271121f14a - name: replicaCount value: 2 + + - name: unimed + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: app_name + value: maestro-unimed + - name: maestro.duc_url + value: duc.dadosfera.ai + - name: hostname + value: maestro.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d + - name: maestro.open_group_id + value: c0afdcce-c5be-40d0-9d1d-2d271121f14a + # Customer id + - name: maestro.dedicated_proxy + value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd + - name: replicaCount + value: 1 + - name: restricted-ip + value: "177.52.172.0/24" diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 8039c29..089ed03 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -47,4 +47,6 @@ charts: - name: maestro.dedicated_proxy value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd - name: replicaCount - value: 1 \ No newline at end of file + value: 1 + - name: maestro.restricted_ip + value: "57.151.113.140/30" \ No newline at end of file diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index 78b360c..ccd8360 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -6,6 +6,9 @@ metadata: nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; + {{- if .Values.maestro.restricted_ip}} + nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }} + {{- end }} generation: 1 labels: diff --git a/maestro/values.yaml b/maestro/values.yaml index e0ec85c..f568ed9 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -44,6 +44,7 @@ maestro: open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 dedicated_proxy: "" + restricted_ip: "" autoscaling: enabled: false minReplicas: 1 From bac74c95771a4cab6fc06f3d209c57e7cc6a0f4e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 15:36:26 -0300 Subject: [PATCH 22/31] FIX: default value and throw correct Error Class --- src/authentication/authentication.guard.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index a41bc74..108dce8 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -115,16 +115,16 @@ export class AuthenticationGuard } // Bloquear outros customer de usar o maestor dedicado - const DEDICATED_PROXY = process.env.DEDICATED_PROXY + const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) { - throw new ForbiddenException(); + throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN); } // Bloquear o customer de acesso o maestro publico - const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); - if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { - throw new ForbiddenException(); - } + // const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + // if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + // throw new ForbiddenException(); + // } request.accessTokenPayload = accessTokenPayload; request.user = { From b10459cb15e706446d0c79c83a93182e0394b6f6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 17:51:14 -0300 Subject: [PATCH 23/31] FIX: maestro custom hostname to unimed --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index c426fa9..421bb64 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -33,7 +33,7 @@ charts: - name: maestro.duc_url value: duc.dadosfera.ai - name: hostname - value: maestro.dadosfera.ai + value: maestro-unimed.dadosfera.ai - name: maestro.pi_factory_url value: pi-factory.dadosfera.ai - name: maestro.in_factory_url From 6910a2f04cd7a754f034a7ee21a26bff641e46b9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 18:38:20 -0300 Subject: [PATCH 24/31] FIX: correct helm value --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 421bb64..3c42dde 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -49,5 +49,5 @@ charts: value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd - name: replicaCount value: 1 - - name: restricted-ip + - name: maestro.restricted_ip value: "177.52.172.0/24" From 5d8e908339386b4727303ec534322d7726f871e5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 24 Jun 2025 10:26:50 -0300 Subject: [PATCH 25/31] FIX: add nginx proxy timeout 5 min --- maestro/templates/ingress.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index ccd8360..1594baa 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -3,6 +3,9 @@ kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; From bdd371ffe88b3490eff5e460533fba997028381f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 25 Jun 2025 10:58:18 -0300 Subject: [PATCH 26/31] FIX: add dadosfera vpn ip in unimed chart --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 3c42dde..15a9241 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -50,4 +50,4 @@ charts: - name: replicaCount value: 1 - name: maestro.restricted_ip - value: "177.52.172.0/24" + value: "177.52.172.0/24,57.151.113.140/30" From d638d518f8e6f9e85bc6b35ae220d0e1019cf188 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 25 Jun 2025 11:06:41 -0300 Subject: [PATCH 27/31] FIX: change unimed chart name --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 15a9241..b44aaf3 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -23,7 +23,7 @@ charts: - name: replicaCount value: 2 - - name: unimed + - name: unimed-maestro chart: ../maestro values: - ../maestro/values.yaml From d68e436a8bd4ec495d71f9b3da830f33caf2fa47 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 5 Jul 2025 09:38:16 -0300 Subject: [PATCH 28/31] FEAT: export users in csv --- docsfera.json | 35 +++++++++++++++++++++++++++ src/modules/users/dtos/entities.ts | 11 +++++++++ src/modules/users/users.controller.ts | 23 ++++++++++++++++++ src/modules/users/users.service.ts | 30 +++++++++++++++++++++++ 4 files changed, 99 insertions(+) diff --git a/docsfera.json b/docsfera.json index b6e8be8..77461b3 100644 --- a/docsfera.json +++ b/docsfera.json @@ -969,6 +969,41 @@ ] } }, + "/users/download": { + "get": { + "operationId": "UsersController_downloadUsersInCsv", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Users" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/users/hierarchies": { "get": { "operationId": "UsersController_getAllHierarchies", diff --git a/src/modules/users/dtos/entities.ts b/src/modules/users/dtos/entities.ts index 9272e52..8e8b852 100644 --- a/src/modules/users/dtos/entities.ts +++ b/src/modules/users/dtos/entities.ts @@ -174,3 +174,14 @@ export class GetAllDepartmentsRes { @ApiProperty() departments: string[]; } + + +export interface UserReporter { + name: string; + email: string; + mfaStatus: string; + status: string; + lastLogin: string; + createdAt: string; + updatedAt: string; +} diff --git a/src/modules/users/users.controller.ts b/src/modules/users/users.controller.ts index 9e00f7c..c80317a 100644 --- a/src/modules/users/users.controller.ts +++ b/src/modules/users/users.controller.ts @@ -12,6 +12,7 @@ import { Post, Put, Query, + Res, UseFilters, } from '@nestjs/common'; import { @@ -24,6 +25,7 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { Authenticated, RequireAllPermissions, + RequireSomePermission, } from 'src/decorators/authentication.decorator'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; @@ -52,6 +54,7 @@ import { UpdateUserRes, } from './dtos/entities'; import { UsersService } from './users.service'; +import { Response } from 'express'; @ApiInternalOnlyController() @ApiTags('Users') @@ -80,6 +83,26 @@ export class UsersController { return await this.userService.findAllUsersByCustomerId(user.customer_id); } + @Get('/download') + @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + async downloadUsersInCsv( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Res() res: Response + ) { + this.logger.info('downloadUsersInCsv'); + this.userService.setLanguage(language); + const { + file, + filename + } = await this.userService.downloadUsersInCsv(user.customer_id); + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', 'text/csv'); + + res.end(file); + } + @Get('hierarchies') @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse({ type: GetAllHierarchiesRes }) diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index e6717fa..0f52284 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -21,6 +21,7 @@ import { IUserByCustomer, SetUserRolesReq, UpdateUserReq, + UserReporter, } from './dtos/entities'; import { RolesService } from '../roles/roles.service'; import { HIERARCHIES } from './hierarchies'; @@ -29,6 +30,7 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces import { EnrichErrorCode } from 'src/utils/ErrorBuilder'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; +import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; @Injectable() export class UsersService implements OnModuleInit { @@ -78,6 +80,34 @@ export class UsersService implements OnModuleInit { }; } + async downloadUsersInCsv(customerId: string) { + const { users } = await lastValueFrom( + this.usersClientService.UserFindAllByCustomerId({ customerId }), + ); + + const formatUsers: UserReporter[] = users.map(user => ({ + createdAt: user.createdAt, + email: user.email, + lastLogin: user.lastLogin, + mfaStatus: user.mfaStatus, + name: user.name, + status: user.status, + updatedAt: user.updatedAt + })) + + const parser = ParserBuilder.build('csv'); + + const file = await parser.parse(formatUsers); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `dadosfera-users-${timestamp}.csv`; + + return { + file, + filename + } + } + async findOneById(id: string): Promise<{ user: IUserByCustomer }> { const { user } = await lastValueFrom( this.usersClientService.UserFindOneById({ id }), From c98a5ff896a401a4e74dc56bc1975307e8778c24 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 5 Jul 2025 11:21:05 -0300 Subject: [PATCH 29/31] FEAT: endpoint to dowload assets list --- docsfera.json | 79 +++++++++++++++++++++++ src/modules/catalog/catalog.controller.ts | 46 +++++++++++++ src/modules/catalog/catalog.service.ts | 29 +++++++++ src/modules/catalog/dtos/index.ts | 8 +++ src/modules/users/users.service.ts | 2 +- 5 files changed, 163 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 77461b3..65c1901 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3909,6 +3909,85 @@ ] } }, + "/catalog/download": { + "get": { + "operationId": "CatalogController_dowloadAsserts", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "size", + "required": false, + "in": "query", + "description": "Número de registros a ser retornado", + "schema": { + "minimum": 1, + "maximum": 10000, + "type": "number" + } + }, + { + "name": "page", + "required": false, + "in": "query", + "description": "Usado para paginação, em conjunto com `size`", + "schema": { + "minimum": 1, + "type": "number" + } + }, + { + "name": "sort_by", + "required": false, + "in": "query", + "description": "Campo do data asset para usar na ordenação. Padrão: `created_at` ", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "required": false, + "in": "query", + "description": "Tipo de ordenação - `asc`: crescente; `desc`: decrescente ", + "schema": { + "default": "asc", + "enum": [ + "asc", + "desc" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset": { "get": { "operationId": "CatalogController_findByPipelineAndObject", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 09daa16..fe83be0 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -117,6 +117,52 @@ export class CatalogController { return res; } + @Get('/download') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async dowloadAsserts( + @User() user: RequestUser, + @Query() query: ICatalogAllRequest, + @Res() res: Response + ) { + const { user_id, customer_name, customer_id, username, permissions } = user; + this.logger.info(`/catalog/download - searchCatalog`, { + user_id, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + + const roles = await this.catalogService.getUserRolesIds(user_id); + + const metadata = PackTheMetadata({ + user_id, + customer_id, + customer_name, + username, + roles, + is_data_manager, + }); + + const { + file, + filename + } = await this.catalogService.downloadAssets( + query, + metadata, + customer_id, + ); + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', 'text/csv'); + + res.end(file); + } + @ApiInternalOnlyEndpoint() @Get('data-asset') async findByPipelineAndObject(@User() user: RequestUser, @Query() query) { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index a3db9cc..fcfb9e7 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -25,6 +25,7 @@ import { UsersService } from '../users/users.service'; import { RolesService } from '../roles/roles.service'; import { Metadata } from '@grpc/grpc-js'; import { + AssetReporter, BatchRemoveRlsRulesRequest, IUpdateDataRequest, TriggerCatalogReq, @@ -240,6 +241,34 @@ class CatalogService implements OnModuleInit { return { data_assets: response, total }; } + async downloadAssets( + query: Record, + metadata: Metadata, + customer_id: string, + ) { + const data = await this.searchDataAssets(query, metadata, customer_id); + + const formatData = data.data_assets.map(asset => ({ + id: asset.id, + display_name: asset.display_name, + data_asset_type: asset.data_asset_type, + created_at: asset.created_at, + tags: '[' + asset.tags.join(', ') + ']' + })) + + const parser = ParserBuilder.build('csv'); + + const file = await parser.parse(formatData); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `dadosfera_assets_${timestamp}.csv`; + + return { + file, + filename + } + } + async getOneDataAsset(data: { id: string; customer_id: string; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index b869e5e..ef12e78 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -320,3 +320,11 @@ export class BatchRemoveRlsRulesRequest { @ApiPropertyOptional() id_rls?: string; } + +export type AssetReporter = { + id: string; + display_name: string; + data_asset_type: string; + created_at: string; + tags: string; +} diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index 0f52284..3057010 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -100,7 +100,7 @@ export class UsersService implements OnModuleInit { const file = await parser.parse(formatUsers); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filename = `dadosfera-users-${timestamp}.csv`; + const filename = `dadosfera_users_${timestamp}.csv`; return { file, From bb03b18f4acffcf7c405b6709e00387974f5ceec Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 9 Jul 2025 12:55:49 -0300 Subject: [PATCH 30/31] FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources Oracle Migration Changes: - Create deploy/helm-chart/ directory structure (moved from maestro/) - Create deploy/helmfiles/ directory structure (moved from helmfiles/) - Update chart references in helmfiles from ../maestro to ../helm-chart - Update deploy-manually.yml from Azure AKS to Oracle OKE - Update validate-k8s.yml from Azure AKS to Oracle OKE - Replace Azure CLI with OCI CLI installation and configuration - Replace Azure authentication with OCI authentication using secrets - Replace az aks get-credentials with oci ce cluster create-kubeconfig - Update helmfile paths from helmfiles/ to deploy/helmfiles/ - Remove DockerHub integration (push_to_dockerhub input and related steps) - Change runner from [self-hosted, prd-azure] to [self-hosted, prd-oracle] - Add environment field for proper Oracle deployment environment handling - Add HOME environment variable for OCI CLI Affinity/Resources Standardization: - Add affinity configuration with name=general node selector in values.yaml - Update deployment template to use conditional affinity and resources blocks - Configure staging to disable both affinity and resources (null values) - Enable production to use affinity targeting name=general nodes - Enable production to use standard resource limits (100m-2000m CPU, 1500Mi-2Gi memory) This brings maestro into full compliance with Oracle migration requirements and implements the standardized affinity/resources pattern consistent with in-factory and duc. --- .github/workflows/deploy-manually.yml | 68 ++++++++----------- .github/workflows/validate-k8s.yml | 67 ++++++++++++------ {maestro => deploy/helm-chart}/Chart.yaml | 0 .../helm-chart}/templates/NOTES.txt | 0 .../helm-chart}/templates/deployment.yaml | 20 ++---- .../templates/ingress-private.yaml | 0 .../helm-chart}/templates/ingress.yaml | 1 + .../helm-chart}/templates/secret.yaml | 0 .../helm-chart}/templates/service.yaml | 0 {maestro => deploy/helm-chart}/values.yaml | 10 +++ {helmfiles => deploy/helmfiles}/prd.yaml | 4 +- {helmfiles => deploy/helmfiles}/stg.yaml | 10 ++- maestro/.helmignore | 23 ------- 13 files changed, 101 insertions(+), 102 deletions(-) rename {maestro => deploy/helm-chart}/Chart.yaml (100%) rename {maestro => deploy/helm-chart}/templates/NOTES.txt (100%) rename {maestro => deploy/helm-chart}/templates/deployment.yaml (86%) rename {maestro => deploy/helm-chart}/templates/ingress-private.yaml (100%) rename {maestro => deploy/helm-chart}/templates/ingress.yaml (99%) rename {maestro => deploy/helm-chart}/templates/secret.yaml (100%) rename {maestro => deploy/helm-chart}/templates/service.yaml (100%) rename {maestro => deploy/helm-chart}/values.yaml (86%) rename {helmfiles => deploy/helmfiles}/prd.yaml (91%) rename {helmfiles => deploy/helmfiles}/stg.yaml (79%) delete mode 100644 maestro/.helmignore diff --git a/.github/workflows/deploy-manually.yml b/.github/workflows/deploy-manually.yml index 87e306e..5f95da0 100644 --- a/.github/workflows/deploy-manually.yml +++ b/.github/workflows/deploy-manually.yml @@ -13,11 +13,6 @@ on: options: - stg - prd - push_to_dockerhub: - description: "Push image to Dockerhub?" - required: true - type: boolean - default: false jobs: extract_environment: @@ -118,23 +113,6 @@ jobs: docker compose -f build.docker-compose.yml build docker compose -f build.docker-compose.yml push - - name: Login to Docker Hub - if: ${{inputs.push_to_dockerhub}} - uses: docker/login-action@v2 - with: - username: dadosfera - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build, Tag, and Push Image to Dockerhub - if: ${{inputs.push_to_dockerhub}} - env: - ENV: ${{ needs.extract_environment.outputs.environment }} - IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} - ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} - run: | - docker compose -f build.docker-compose.dockerhub.yml build - docker compose -f build.docker-compose.dockerhub.yml push - # - name: Create ZIP file to Deploy AWS Beanstalk # env: # ENV: ${{ needs.extract_environment.outputs.environment }} @@ -166,7 +144,10 @@ jobs: helmfile-deploy: needs: [extract_environment, semantic_release, build_ecr_image] - runs-on: [self-hosted, "prd-azure"] + env: + HOME: /home/runner + runs-on: [self-hosted, "prd-oracle"] + environment: ${{ needs.extract_environment.outputs.environment }} steps: - name: Checkout code @@ -177,13 +158,17 @@ jobs: with: version: 'v3.9.0' - - name: Install Azure ClI + - name: Install OCI CLI run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | bash + bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults + echo "$HOME/bin" >> $GITHUB_PATH - - uses: azure/login@v2 - with: - creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + - name: Configure OCI CLI + run: | + mkdir -p ~/.oci || true + echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config + echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem + chmod 600 ~/.oci/oci_api_key.pem - name: Set up Python uses: actions/setup-python@v4 @@ -194,24 +179,31 @@ jobs: run: | wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz tar -xzf helmfile_0.148.0_linux_amd64.tar.gz - mv helmfile /usr/local/bin/ + sudo mv helmfile /usr/local/bin/ helmfile --version - name: Install Helm Diff Plugin run: helm plugin install https://github.com/databus23/helm-diff || true - - name: Setup kubectl - uses: azure/setup-kubectl@v1 - with: - version: 'v1.30.1' - - - name: Authenticate with cluster + - name: Authenticate with OKE cluster env: - CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} - run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing + ENV: ${{ needs.extract_environment.outputs.environment }} + STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q" + PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq" + run: | + if [ "$ENV" = "stg" ]; then + CLUSTER_ID=$STG_CLUSTER_ID + elif [ "$ENV" = "prd" ]; then + CLUSTER_ID=$PRD_CLUSTER_ID + else + echo "Unknown environment: $ENV" + exit 1 + fi + + oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT - name: Run Helmfile Apply env: ENV: ${{ needs.extract_environment.outputs.environment }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} - run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG + run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 17ed975..015efc2 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -16,22 +16,18 @@ jobs: run: | if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then echo "environment=prd" >> $GITHUB_OUTPUT - elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then + elif [ "${{ github.event.pull_request.base.ref }}" == "stg" ]; then echo "environment=stg" >> $GITHUB_OUTPUT fi id: extract_environment - helmfile-deploy: + helmfile-check: + env: + HOME: /home/runner needs: [extract_environment] - runs-on: [self-hosted, "prd-azure"] - + environment: ${{ needs.extract_environment.outputs.environment }} + runs-on: [self-hosted, "prd-oracle"] steps: - - name: Summary - env: - ENV: ${{ needs.extract_environment.outputs.environment }} - run: | - echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY - - name: Checkout code uses: actions/checkout@v3 @@ -40,13 +36,28 @@ jobs: with: version: 'v3.9.0' - - name: Install Azure ClI + - name: Determine DNS_HOST based on environment + id: set_dns + env: + ENV: ${{ needs.extract_environment.outputs.environment }} run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | bash + if [ "$ENV" = "prd" ]; then + echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT + elif [ "$ENV" = "stg" ]; then + echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT + fi + + - name: Install OCI CLI + run: | + bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults + echo "$HOME/bin" >> $GITHUB_PATH - - uses: azure/login@v2 - with: - creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + - name: Configure OCI CLI + run: | + mkdir -p ~/.oci || true + echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config + echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem + chmod 600 ~/.oci/oci_api_key.pem - name: Set up Python uses: actions/setup-python@v4 @@ -57,23 +68,35 @@ jobs: run: | wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz tar -xzf helmfile_0.148.0_linux_amd64.tar.gz - mv helmfile /usr/local/bin/ + sudo mv helmfile /usr/local/bin/ helmfile --version - name: Install Helm Diff Plugin run: helm plugin install https://github.com/databus23/helm-diff || true + - name: Authenticate with OKE cluster + env: + ENV: ${{ needs.extract_environment.outputs.environment }} + STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q" + PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq" + run: | + if [ "$ENV" = "stg" ]; then + CLUSTER_ID=$STG_CLUSTER_ID + elif [ "$ENV" = "prd" ]; then + CLUSTER_ID=$PRD_CLUSTER_ID + else + echo "Unknown environment: $ENV" + exit 1 + fi + + oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT + - name: Setup kubectl uses: azure/setup-kubectl@v1 with: version: 'v1.30.1' - - name: Authenticate with cluster - env: - CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} - run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing - - name: Run Helmfile Diff env: ENV: ${{ needs.extract_environment.outputs.environment }} - run: helmfile -f helmfiles/${ENV}.yaml diff + run: helmfile -f deploy/helmfiles/${ENV}.yaml diff diff --git a/maestro/Chart.yaml b/deploy/helm-chart/Chart.yaml similarity index 100% rename from maestro/Chart.yaml rename to deploy/helm-chart/Chart.yaml diff --git a/maestro/templates/NOTES.txt b/deploy/helm-chart/templates/NOTES.txt similarity index 100% rename from maestro/templates/NOTES.txt rename to deploy/helm-chart/templates/NOTES.txt diff --git a/maestro/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml similarity index 86% rename from maestro/templates/deployment.yaml rename to deploy/helm-chart/templates/deployment.yaml index c648514..116ab30 100644 --- a/maestro/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -27,15 +27,10 @@ spec: - name: {{ .Values.imagePullSecrets }} nodeSelector: "beta.kubernetes.io/os": linux + {{- if .Values.affinity }} affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: application - operator: In - values: - - general + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} tolerations: - key: "kubernetes.azure.com/scalesetpriority" @@ -48,13 +43,10 @@ spec: image: {{ .Values.image.repository }}:{{ .Values.image.tag }} ports: - containerPort: {{ .Values.containerPort }} + {{- if .Values.resources }} resources: - requests: - cpu: {{ .Values.resources.requests.cpu }} - memory: {{ .Values.resources.requests.memory }} - limits: - cpu: {{ .Values.resources.limits.cpu }} - memory: {{ .Values.resources.limits.memory }} + {{- toYaml .Values.resources | nindent 12 }} + {{- end }} env: - name: AWS_IDENTITY_POOL_ID value: {{ .Values.maestro.aws_identity_pool_id }} diff --git a/maestro/templates/ingress-private.yaml b/deploy/helm-chart/templates/ingress-private.yaml similarity index 100% rename from maestro/templates/ingress-private.yaml rename to deploy/helm-chart/templates/ingress-private.yaml diff --git a/maestro/templates/ingress.yaml b/deploy/helm-chart/templates/ingress.yaml similarity index 99% rename from maestro/templates/ingress.yaml rename to deploy/helm-chart/templates/ingress.yaml index faf1305..16d3ff6 100644 --- a/maestro/templates/ingress.yaml +++ b/deploy/helm-chart/templates/ingress.yaml @@ -3,6 +3,7 @@ kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; diff --git a/maestro/templates/secret.yaml b/deploy/helm-chart/templates/secret.yaml similarity index 100% rename from maestro/templates/secret.yaml rename to deploy/helm-chart/templates/secret.yaml diff --git a/maestro/templates/service.yaml b/deploy/helm-chart/templates/service.yaml similarity index 100% rename from maestro/templates/service.yaml rename to deploy/helm-chart/templates/service.yaml diff --git a/maestro/values.yaml b/deploy/helm-chart/values.yaml similarity index 86% rename from maestro/values.yaml rename to deploy/helm-chart/values.yaml index 14dd9e4..f85c248 100644 --- a/maestro/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -48,3 +48,13 @@ autoscaling: maxReplicas: 100 targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: name + operator: In + values: + - general diff --git a/helmfiles/prd.yaml b/deploy/helmfiles/prd.yaml similarity index 91% rename from helmfiles/prd.yaml rename to deploy/helmfiles/prd.yaml index e4146e8..f3e9df6 100644 --- a/helmfiles/prd.yaml +++ b/deploy/helmfiles/prd.yaml @@ -1,8 +1,8 @@ charts: - name: maestro - chart: ../maestro + chart: ../helm-chart values: - - ../maestro/values.yaml + - ../helm-chart/values.yaml set: - name: maestro.duc_url value: duc.dadosfera.ai diff --git a/helmfiles/stg.yaml b/deploy/helmfiles/stg.yaml similarity index 79% rename from helmfiles/stg.yaml rename to deploy/helmfiles/stg.yaml index 5782e36..57b32b4 100644 --- a/helmfiles/stg.yaml +++ b/deploy/helmfiles/stg.yaml @@ -1,8 +1,8 @@ charts: - name: maestro - chart: ../maestro + chart: ../helm-chart values: - - ../maestro/values.yaml + - ../helm-chart/values.yaml set: - name: maestro.duc_url value: duc.stg.dadosfera.ai @@ -19,4 +19,8 @@ charts: - name: maestro.open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 - name: replicaCount - value: 1 \ No newline at end of file + value: 1 + - name: affinity + value: null + - name: resources + value: null \ No newline at end of file diff --git a/maestro/.helmignore b/maestro/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/maestro/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ From cac8f93330d331c6108fc3c8449d261a78aaf720 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 9 Jul 2025 12:58:11 -0300 Subject: [PATCH 31/31] CI: update validate-k8s.yml to use beta branch for staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change trigger branches from main/stg to main/beta - Update environment extraction to map beta → stg environment - Align with standard branch naming convention (beta for staging) - Consistent with deploy-manually.yml workflow configuration --- .github/workflows/validate-k8s.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 015efc2..4d23e66 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -4,7 +4,7 @@ on: pull_request: branches: - main - - stg + - beta jobs: extract_environment: @@ -16,7 +16,7 @@ jobs: run: | if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then echo "environment=prd" >> $GITHUB_OUTPUT - elif [ "${{ github.event.pull_request.base.ref }}" == "stg" ]; then + elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then echo "environment=stg" >> $GITHUB_OUTPUT fi id: extract_environment