From 7a6db36554d5d2d2a749f41f9df5d77aa51847f2 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sun, 27 Apr 2025 13:00:49 -0300 Subject: [PATCH] FEAT: generate pii reporter --- docsfera.json | 46 +++++++-- package-lock.json | 58 +++++++++++- package.json | 3 +- src/authentication/permissions.enum.ts | 3 +- src/modules/catalog/catalog.controller.ts | 83 +++++++++++----- src/modules/catalog/catalog.service.ts | 47 ++++++++++ src/template/pii-reporter.html | 109 ++++++++++++++++++++++ 7 files changed, 313 insertions(+), 36 deletions(-) create mode 100644 src/template/pii-reporter.html diff --git a/docsfera.json b/docsfera.json index 1b92192..c2b9b59 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,14 +20,6 @@ ], "type": "string" } - }, - { - "name": "origin", - "required": true, - "in": "header", - "schema": { - "type": "string" - } } ], "requestBody": { @@ -4963,6 +4955,44 @@ ] } }, + "/catalog/pii-reporter": { + "get": { + "operationId": "CatalogController_getPiiReporter", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/terms-of-use/token": { "get": { "operationId": "TermsOfUseController_getToken", diff --git a/package-lock.json b/package-lock.json index 898117c..558985c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.22", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -32,6 +32,7 @@ "cron-parser": "^4.9.0", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", + "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.5", @@ -1397,9 +1398,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.22", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.22.tgz", - "integrity": "sha512-SOu+Z5qc3ms6JjtWpHFYEgYq7Q8Y2GHBewDQj5/Evv4yxpzuANw9Eecd6zzCttV47ceM75A8rnomTj9gGkoTTg==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-3vMGyEolF/xeEeFSRUpOA8YVrbTZfWXrcMxMSCkmx411Tuuv4SkX9nHcYaXhYAkFUV9vR7vfmBfMFz4yq1n7mw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -7373,6 +7374,36 @@ "dev": true, "license": "MIT" }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/handlebars/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -12063,6 +12094,19 @@ "node": ">=4.2.0" } }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/uid": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", @@ -12558,6 +12602,12 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 643821b..ae1463a 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.22", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -50,6 +50,7 @@ "cron-parser": "^4.9.0", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", + "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.5", diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 95a313f..ee74e22 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -613,7 +613,8 @@ export interface DadosferaModule { export const DADOSFERA_MODULES_KEYS = { LOG_DASHBOARD: 'logs-dashboard', ACCESS_DASHBOARD: 'access-dashboard', - DANGER_ZONE: 'danger-zone' + DANGER_ZONE: 'danger-zone', + PII: 'pii' } export const DADOSFERA_MODULES: Array = [ diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 5f2d194..9f80e35 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -15,6 +15,7 @@ import { UseFilters, HttpException, HttpStatus, + Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -25,14 +26,14 @@ import { import { Authenticated, RequireAllPermissions, + RequireModule, RequireSomePermission, } from '../../decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { CatalogService } from './catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; -import { lastValueFrom } from 'rxjs'; import { BatchRemoveRlsRulesRequest, GetDatasetCatalogTaskRes, @@ -59,6 +60,7 @@ import { GetRlsRulesRequest, RegisterDatasetWithMetatadaRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; +import { Response } from 'express'; @ApiTags('Catalog') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -748,25 +750,23 @@ export class CatalogController { return JSON.parse(dashboards); } - @Post('register-dataset') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async registerDatasetWithMetadataRequest( @Body() body: RegisterDatasetWithMetatadaRequest, @User() user: RequestUser, ) { - const { customer_id, customer_name, user_id, username } = user; - const logMetadata = { - customer_name: customer_name, - user_id: user_id, - method: 'POST', - path: '/catalog/register-dataset', - }; + const { customer_id, customer_name, user_id, username } = user; + const logMetadata = { + customer_name: customer_name, + user_id: user_id, + method: 'POST', + path: '/catalog/register-dataset', + }; try { - const metadata = PackTheMetadata({ customer_id, customer_name, @@ -774,18 +774,20 @@ export class CatalogController { username, }); - this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata); + this.logger.log( + `Request from user ${user_id} for customer ${customer_name}`, + logMetadata, + ); // Create table metadata const tableMetadataBody = { table_metadata: body.table_metadata, info: { customer: customer_name, - }, - logMetadata: logMetadata + logMetadata: logMetadata, }; - + const table_metadata_id = await this.catalogService.createTableMetadata( tableMetadataBody, ); @@ -798,8 +800,12 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata); - const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody) + this.logger.info( + `Creating column metadata for table ${table_metadata_id}`, + logMetadata, + ); + const column_metadata_ids = + await this.catalogService.createColumnMetadata(columnMetadataBody); // Create data preview const dataPreviewBody = { @@ -808,15 +814,24 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata); + this.logger.debug( + `Creating data preview for table ${table_metadata_id}`, + logMetadata, + ); const data_preview_id = await this.catalogService.createDataPreview( dataPreviewBody, ); // Catalog dataset item - this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata); + this.logger.info( + `Cataloging dataset item for table ${table_metadata_id}`, + logMetadata, + ); await this.catalogService.catalogDatasetItem(table_metadata_id, metadata); - this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata); + this.logger.info( + `Dataset registration completed successfully for table ${table_metadata_id}`, + logMetadata, + ); return { message: 'Dataset registered successfully', table_metadata_id: table_metadata_id, @@ -824,7 +839,10 @@ export class CatalogController { data_preview_id: data_preview_id, }; } catch (error) { - this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata); + this.logger.error( + `Failed to register dataset. The following error occurred: ${error.response.data}`, + logMetadata, + ); throw new HttpException( { @@ -837,4 +855,25 @@ export class CatalogController { ); } } + + @Get('pii-reporter') + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule( + DADOSFERA_MODULES_KEYS.PII + ) + async getPiiReporter(@User() user: RequestUser, @Res() res: Response) { + this.logger.info('GET pii-reporter'); + const metadata = PackTheMetadata(user); + + try { + const reporterHtml = await this.catalogService.getPiiReporter(metadata); + + res.setHeader('Content-Type', 'text/html'); + return res.send(reporterHtml); + } catch (error) { + this.logger.error(error); + } + } } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index adfa0a2..ac7f1d6 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -34,6 +34,8 @@ import { GetNimbusDashboardsRequest, GetRlsRulesRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; +import { readFileSync } from 'fs'; +import path from 'path'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -80,6 +82,51 @@ class CatalogService implements OnModuleInit { )}.dadosfera.ai`; } + async getPiiReporter(metadata: Metadata) { + const { + data + } = await lastValueFrom( + this.catalogWriteService.GetPiiReporter({}, metadata) + ) + + const result = data.map((props) => ({ + ...props, + pii_rules: this.formatPiiRules(JSON.parse(props.pii_rules)), + })); + + // 3. Ler o template HTML + const templatePath = path.join( + __dirname, + '../../../src/template/pii-reporter.html', + ); + const templateSource = readFileSync(templatePath, 'utf8'); + + // 4. Compilar o template + const template = Handlebars.compile(templateSource); + + // 5. Aplicar os dados ao template + return template({ + dados: result, + dataGeracao: new Date().toLocaleString('pt-BR'), + }); + } + + private formatPiiRules( + data: Record< + string, + { + avg_score: number; + max_score: number; + } + >, + ) { + return Object.keys(data) + .filter((key) => { + return data[key].max_score > 0.7; + }) + .join(', '); + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/template/pii-reporter.html b/src/template/pii-reporter.html new file mode 100644 index 0000000..637332a --- /dev/null +++ b/src/template/pii-reporter.html @@ -0,0 +1,109 @@ + + + + + + Dadosfera Relatório de PII + + + + +
+
+ +

Relatório de PII

+
+ +
+

Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.

+
+ + + + + + + + + + + + + + {{#each dados}} + + + + + + + + + {{/each}} + +
DatabaseSchemaTabelaColunaTipo de DadoRegras PII
{{database_name}}{{table_schema}}{{table_name}}{{column_name}}{{data_type}}{{pii_rules}}
+ +

Gerado em: {{dataGeracao}}

+
+ + \ No newline at end of file