gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-10-13 11:09:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

FEAT: generate pii reporter

Browse Source
This commit is contained in:
marcos-silva-rodrigues
2025-04-27 13:00:49 -03:00
parent 9e0495fc26
commit 7a6db36554
7 changed files with 313 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docsfera.json
+38 -8
View File
@@ -20,14 +20,6 @@
],
"type": "string"
}
},
{
"name": "origin",
"required": true,
"in": "header",
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -4963,6 +4955,44 @@
]
}
},
"/catalog/pii-reporter": {
"get": {
"operationId": "CatalogController_getPiiReporter",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/terms-of-use/token": {
"get": {
"operationId": "TermsOfUseController_getToken",
package-lock.json
Generated
+54 -4
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.22",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -32,6 +32,7 @@
"cron-parser": "^4.9.0",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.50.0",
"handlebars": "^4.7.8",
"helmet": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"jwk-to-pem": "^2.0.5",
@@ -1397,9 +1398,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.37.0-beta.22",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.22.tgz",
"integrity": "sha512-SOu+Z5qc3ms6JjtWpHFYEgYq7Q8Y2GHBewDQj5/Evv4yxpzuANw9Eecd6zzCttV47ceM75A8rnomTj9gGkoTTg==",
"version": "0.0.0",
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz",
"integrity": "sha512-3vMGyEolF/xeEeFSRUpOA8YVrbTZfWXrcMxMSCkmx411Tuuv4SkX9nHcYaXhYAkFUV9vR7vfmBfMFz4yq1n7mw==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -7373,6 +7374,36 @@
"dev": true,
"license": "MIT"
},
"node_modules/handlebars": {
"version": "4.7.8",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz",
"integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==",
"license": "MIT",
"dependencies": {
"minimist": "^1.2.5",
"neo-async": "^2.6.2",
"source-map": "^0.6.1",
"wordwrap": "^1.0.0"
},
"bin": {
"handlebars": "bin/handlebars"
},
"engines": {
"node": ">=0.4.7"
},
"optionalDependencies": {
"uglify-js": "^3.1.4"
}
},
"node_modules/handlebars/node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -12063,6 +12094,19 @@
"node": ">=4.2.0"
}
},
"node_modules/uglify-js": {
"version": "3.19.3",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz",
"integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==",
"license": "BSD-2-Clause",
"optional": true,
"bin": {
"uglifyjs": "bin/uglifyjs"
},
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/uid": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz",
@@ -12558,6 +12602,12 @@
"node": ">=0.10.0"
}
},
"node_modules/wordwrap": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
"integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==",
"license": "MIT"
},
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
package.json
+2 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.22",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -50,6 +50,7 @@
"cron-parser": "^4.9.0",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.50.0",
"handlebars": "^4.7.8",
"helmet": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"jwk-to-pem": "^2.0.5",
src/authentication/permissions.enum.ts
+2 -1
View File
@@ -613,7 +613,8 @@ export interface DadosferaModule {
export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone'
DANGER_ZONE: 'danger-zone',
PII: 'pii'
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
src/modules/catalog/catalog.controller.ts
+61 -22
View File
@@ -15,6 +15,7 @@ import {
UseFilters,
HttpException,
HttpStatus,
Res,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -25,14 +26,14 @@ import {
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from '../../decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { lastValueFrom } from 'rxjs';
import {
BatchRemoveRlsRulesRequest,
GetDatasetCatalogTaskRes,
@@ -59,6 +60,7 @@ import {
GetRlsRulesRequest,
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -748,25 +750,23 @@ export class CatalogController {
return JSON.parse(dashboards);
}
@Post('register-dataset')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async registerDatasetWithMetadataRequest(
@Body() body: RegisterDatasetWithMetatadaRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
try {
const metadata = PackTheMetadata({
customer_id,
customer_name,
@@ -774,18 +774,20 @@ export class CatalogController {
username,
});
this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
this.logger.log(
`Request from user ${user_id} for customer ${customer_name}`,
logMetadata,
);
// Create table metadata
const tableMetadataBody = {
table_metadata: body.table_metadata,
info: {
customer: customer_name,
},
logMetadata: logMetadata
logMetadata: logMetadata,
};
const table_metadata_id = await this.catalogService.createTableMetadata(
tableMetadataBody,
);
@@ -798,8 +800,12 @@ export class CatalogController {
customer: customer_name,
},
};
this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
this.logger.info(
`Creating column metadata for table ${table_metadata_id}`,
logMetadata,
);
const column_metadata_ids =
await this.catalogService.createColumnMetadata(columnMetadataBody);
// Create data preview
const dataPreviewBody = {
@@ -808,15 +814,24 @@ export class CatalogController {
customer: customer_name,
},
};
this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
this.logger.debug(
`Creating data preview for table ${table_metadata_id}`,
logMetadata,
);
const data_preview_id = await this.catalogService.createDataPreview(
dataPreviewBody,
);
// Catalog dataset item
this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
this.logger.info(
`Cataloging dataset item for table ${table_metadata_id}`,
logMetadata,
);
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
this.logger.info(
`Dataset registration completed successfully for table ${table_metadata_id}`,
logMetadata,
);
return {
message: 'Dataset registered successfully',
table_metadata_id: table_metadata_id,
@@ -824,7 +839,10 @@ export class CatalogController {
data_preview_id: data_preview_id,
};
} catch (error) {
this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
this.logger.error(
`Failed to register dataset. The following error occurred: ${error.response.data}`,
logMetadata,
);
throw new HttpException(
{
@@ -837,4 +855,25 @@ export class CatalogController {
);
}
}
@Get('pii-reporter')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response) {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const reporterHtml = await this.catalogService.getPiiReporter(metadata);
res.setHeader('Content-Type', 'text/html');
return res.send(reporterHtml);
} catch (error) {
this.logger.error(error);
}
}
}
src/modules/catalog/catalog.service.ts
+47
View File
@@ -34,6 +34,8 @@ import {
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { readFileSync } from 'fs';
import path from 'path';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -80,6 +82,51 @@ class CatalogService implements OnModuleInit {
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata) {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
const result = data.map((props) => ({
...props,
pii_rules: this.formatPiiRules(JSON.parse(props.pii_rules)),
}));
// 3. Ler o template HTML
const templatePath = path.join(
__dirname,
'../../../src/template/pii-reporter.html',
);
const templateSource = readFileSync(templatePath, 'utf8');
// 4. Compilar o template
const template = Handlebars.compile(templateSource);
// 5. Aplicar os dados ao template
return template({
dados: result,
dataGeracao: new Date().toLocaleString('pt-BR'),
});
}
private formatPiiRules(
data: Record<
string,
{
avg_score: number;
max_score: number;
}
>,
) {
return Object.keys(data)
.filter((key) => {
return data[key].max_score > 0.7;
})
.join(', ');
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
src/template/pii-reporter.html
+109
View File
@@ -0,0 +1,109 @@
<!DOCTYPE html>
<html lang="pt-br">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Dadosfera Relatório de PII</title>
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
<style>
body {
font-family: 'Quicksand', sans-serif;
color: #5c5c5c;
margin: 0;
padding: 20px;
}
.container {
max-width: 1200px;
margin: 0 auto;
}
.header {
display: flex;
align-items: center;
margin-bottom: 30px;
}
.logo {
max-width: 200px;
height: auto;
}
h1 {
color: #0d003b;
font-weight: 700;
margin-left: 20px;
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 20px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border: 1px solid #d0d0d0; /* Borda externa da tabela */
}
th {
background-color: #1700a2;
color: white;
font-weight: bold;
text-align: left;
padding: 12px 15px;
border: 1px solid #3a26b8; /* Borda um pouco mais escura para o header */
}
td {
padding: 10px 15px;
border: 1px solid #d0d0d0; /* Borda clara da cor do texto */
}
tr:nth-child(even) {
background-color: #f9f9f9;
}
tr:nth-child(odd) {
background-color: white;
}
.info-section {
margin-top: 30px;
color: #5c5c5c;
}
.timestamp {
font-style: italic;
text-align: right;
margin-top: 20px;
font-size: 0.9em;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
<h1>Relatório de PII</h1>
</div>
<div class="info-section">
<p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
</div>
<table>
<thead>
<tr>
<th>Database</th>
<th>Schema</th>
<th>Tabela</th>
<th>Coluna</th>
<th>Tipo de Dado</th>
<th>Regras PII</th>
</tr>
</thead>
<tbody>
{{#each dados}}
<tr>
<td>{{database_name}}</td>
<td>{{table_schema}}</td>
<td>{{table_name}}</td>
<td>{{column_name}}</td>
<td>{{data_type}}</td>
<td>{{pii_rules}}</td>
</tr>
{{/each}}
</tbody>
</table>
<p class="timestamp">Gerado em: {{dataGeracao}}</p>
</div>
</body>
</html>