Merge pull request #295 from dadosfera/feature/pii-rules

Feature/pii rules

Dockerfile

@@ -4,9 +4,19 @@ FROM base_image AS build_base
 WORKDIR /app
 RUN apk update
 # needed packages to build dependencies from source
-RUN apk add --no-cache aws-cli
+RUN apk add --no-cache \
+      aws-cli \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont
 COPY package*.json ./
 
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
 
 # run aws cli without mounting secret, because CI already has AWS credentials
 FROM build_base AS ci_image

docsfera.json

@@ -4963,6 +4963,52 @@
           ]
         }
       },
+      "/catalog/pii-reporter": {
+        "get": {
+          "operationId": "CatalogController_getPiiReporter",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": true,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/terms-of-use/token": {
         "get": {
           "operationId": "TermsOfUseController_getToken",

helmfiles/stg.yaml

@@ -9,9 +9,9 @@ charts:
       - name: hostname
         value: maestro.stg.dadosfera.ai
       - name: maestro.pi_factory_url
-        value: pi-factory.dadosfera.ai
+        value: pi-factory.stg.dadosfera.ai
       - name: maestro.in_factory_url
-        value: in-factory.dadosfera.ai
+        value: in-factory.stg.dadosfera.ai
       - name: maestro.tr_factory_url
         value: in-factory.dadosfera.ai
       - name: maestro.open_customer_id

package.json

@@ -30,7 +30,7 @@
     "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.37.0-beta.22",
+    "@dadosfera/protospack-v2": "3.37.0-beta.23",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",
@@ -48,8 +48,10 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
     "cron-parser": "^4.9.0",
+    "csv": "^6.3.11",
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.50.0",
+    "handlebars": "^4.7.8",
     "helmet": "^5.1.1",
     "jsonwebtoken": "^9.0.2",
     "jwk-to-pem": "^2.0.5",
@@ -61,6 +63,7 @@
     "passport-google-oauth20": "^2.0.0",
     "passport-hubspot-oauth2": "^1.0.3",
     "passport-mailchimp": "^1.1.0",
+    "puppeteer": "^24.7.2",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",

src/authentication/permissions.enum.ts

@@ -613,7 +613,8 @@ export interface DadosferaModule {
 export const DADOSFERA_MODULES_KEYS = {
   LOG_DASHBOARD: 'logs-dashboard',
   ACCESS_DASHBOARD: 'access-dashboard',
-  DANGER_ZONE: 'danger-zone'
+  DANGER_ZONE: 'danger-zone',
+  PII: 'pii'
 }
 
 export const DADOSFERA_MODULES: Array<DadosferaModule> = [

src/modules/catalog/catalog.controller.ts

@@ -15,6 +15,7 @@ import {
   UseFilters,
   HttpException,
   HttpStatus,
+  Res,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -25,14 +26,14 @@ import {
 import {
   Authenticated,
   RequireAllPermissions,
+  RequireModule,
   RequireSomePermission,
 } from '../../decorators/authentication.decorator';
-import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
-import { lastValueFrom } from 'rxjs';
 import {
   BatchRemoveRlsRulesRequest,
   GetDatasetCatalogTaskRes,
@@ -59,6 +60,7 @@ import {
   GetRlsRulesRequest,
   RegisterDatasetWithMetatadaRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+import { Response } from 'express';
 
 @ApiTags('Catalog')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -344,7 +346,7 @@ export class CatalogController {
     @Language() language: LanguageEnum,
     @Param('id') id: string,
   ): Promise<IPreviewResponse> {
-    const { customer_name, customer_id, user_id, username } = user;
+    const { customer_name, customer_id, user_id, username, customer_modules } = user;
 
     this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
       user_id,
@@ -357,6 +359,7 @@ export class CatalogController {
       user_id,
       username,
       language,
+      is_mask: customer_modules.some(mod => mod === 'pii')
     });
 
     const preview = await this.catalogService.getDatasetPreview(id, metadata);
@@ -748,25 +751,23 @@ export class CatalogController {
     return JSON.parse(dashboards);
   }
 
-
   @Post('register-dataset')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
-    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
   async registerDatasetWithMetadataRequest(
     @Body() body: RegisterDatasetWithMetatadaRequest,
     @User() user: RequestUser,
   ) {
-      const { customer_id, customer_name, user_id, username } = user;
-      const logMetadata = {
-        customer_name: customer_name,
-        user_id: user_id,
-        method: 'POST',
-        path: '/catalog/register-dataset',
-      };
+    const { customer_id, customer_name, user_id, username } = user;
+    const logMetadata = {
+      customer_name: customer_name,
+      user_id: user_id,
+      method: 'POST',
+      path: '/catalog/register-dataset',
+    };
     try {
-
       const metadata = PackTheMetadata({
         customer_id,
         customer_name,
@@ -774,18 +775,20 @@ export class CatalogController {
         username,
       });
 
-      this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
+      this.logger.log(
+        `Request from user ${user_id} for customer ${customer_name}`,
+        logMetadata,
+      );
 
       // Create table metadata
       const tableMetadataBody = {
         table_metadata: body.table_metadata,
         info: {
           customer: customer_name,
-          
         },
-        logMetadata: logMetadata
+        logMetadata: logMetadata,
       };
-      
+
       const table_metadata_id = await this.catalogService.createTableMetadata(
         tableMetadataBody,
       );
@@ -798,8 +801,12 @@ export class CatalogController {
           customer: customer_name,
         },
       };
-      this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
-      const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
+      this.logger.info(
+        `Creating column metadata for table ${table_metadata_id}`,
+        logMetadata,
+      );
+      const column_metadata_ids =
+        await this.catalogService.createColumnMetadata(columnMetadataBody);
 
       // Create data preview
       const dataPreviewBody = {
@@ -808,15 +815,24 @@ export class CatalogController {
           customer: customer_name,
         },
       };
-      this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
+      this.logger.debug(
+        `Creating data preview for table ${table_metadata_id}`,
+        logMetadata,
+      );
       const data_preview_id = await this.catalogService.createDataPreview(
         dataPreviewBody,
       );
 
       // Catalog dataset item
-      this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
+      this.logger.info(
+        `Cataloging dataset item for table ${table_metadata_id}`,
+        logMetadata,
+      );
       await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
-      this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
+      this.logger.info(
+        `Dataset registration completed successfully for table ${table_metadata_id}`,
+        logMetadata,
+      );
       return {
         message: 'Dataset registered successfully',
         table_metadata_id: table_metadata_id,
@@ -824,7 +840,10 @@ export class CatalogController {
         data_preview_id: data_preview_id,
       };
     } catch (error) {
-      this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
+      this.logger.error(
+        `Failed to register dataset. The following error occurred: ${error.response.data}`,
+        logMetadata,
+      );
 
       throw new HttpException(
         {
@@ -837,4 +856,37 @@ export class CatalogController {
       );
     }
   }
+
+  @Get('pii-reporter')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.USERS.permissions.ADMIN
+  )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.PII
+  )
+  async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') type: "html" | "csv") {
+    this.logger.info('GET pii-reporter');
+    const metadata = PackTheMetadata(user);
+
+    try {
+      const reporter = await this.catalogService.getPiiReporter(metadata, type);
+
+      if (type === "html") {
+        res.setHeader('Content-Type', 'text/html');
+      } else if (type === "csv") {
+        res.setHeader('Content-Type', 'text/csv');
+      } else if (type === "pdf") {
+        res.setHeader('Content-Type', 'application/pdf');
+      }
+
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const filename = `relatorio-pii-${timestamp}.${type}`;
+
+      res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+
+      return res.send(reporter);
+    } catch (error) {
+      this.logger.error(error);
+    }
+  }
 }

src/modules/catalog/catalog.service.ts

@@ -34,6 +34,12 @@ import {
   GetNimbusDashboardsRequest,
   GetRlsRulesRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+import { writeFileSync } from 'fs';
+import path from 'path';
+import { HtmlParser } from 'src/utils/FileParser/html-parser';
+import { PiiDto } from './dtos/pii.dto';
+import { CsvParser } from 'src/utils/FileParser/csv-parser';
+import { PDFParser } from 'src/utils/FileParser/pdf-parser';
 
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
@@ -80,6 +86,49 @@ class CatalogService implements OnModuleInit {
     )}.dadosfera.ai`;
   }
 
+  async getPiiReporter(metadata: Metadata, type: "html"| "csv") {
+    try {
+      const {
+        data
+      } = await lastValueFrom(
+        this.catalogWriteService.GetPiiReporter({}, metadata)
+      )
+      this.logger.info("Finish grpc call")
+  
+      // 3. Ler o template HTML
+      const templatePath = path.join(
+        __dirname,
+        '../../../src/template/pii-reporter.html',
+      );
+
+      this.logger.info("templatePath: " + templatePath);
+
+      if (type === "html") {
+        const htmlParser = new HtmlParser<PiiDto>(templatePath);
+        return htmlParser.parse(data);
+      } else if (type === "csv") {
+        const csvParser = new CsvParser<PiiDto>();
+        return await csvParser.parse(data)
+      } else if (type === "pdf") {
+        const htmlParser = new HtmlParser<PiiDto>(templatePath);
+        const pdfParser = new PDFParser();
+        const pdf =  await pdfParser.parse(htmlParser.parse(data));
+        const pdfPath = path.join(
+          __dirname,
+          '../../../src/template/teste.pdf',
+        );
+        writeFileSync(pdfPath, pdf);
+        return pdf
+      }
+
+      throw new Error("Unsupported Type: " + type)
+   
+    } catch (error) {
+      this.logger.error(error);
+    }
+
+  }
+
   async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
     if (!data.embed) data.embed = undefined;

src/modules/catalog/dtos/pii.dto.ts

@@ -0,0 +1,8 @@
+export class PiiDto {
+  database_name: string;
+  table_schema: string;
+  table_name: string;
+  column_name: string;
+  data_type: string;
+  pii_rules: string;
+}

src/template/pii-reporter.html

@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<html lang="pt-br">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Dadosfera Relatório de PII</title>
+    <link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
+    <style>
+        @page {
+            size: A4;
+            margin: 20mm 15mm 20mm 15mm;
+        }
+        body {
+            font-family: 'Quicksand', sans-serif;
+            color: #5c5c5c;
+            margin: 0;
+            padding: 20px;
+        }
+        .container {
+            margin: 0 24px;
+        }
+        .header {
+            display: flex;
+            align-items: center;
+            margin-bottom: 30px;
+        }
+        .logo {
+            max-width: 200px;
+            height: auto;
+        }
+        h1 {
+            color: #0d003b;
+            font-weight: 700;
+            margin-left: 20px;
+        }
+        table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 20px;
+            box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+            border: 1px solid #d0d0d0; /* Borda externa da tabela */
+        }
+        th {
+            background-color: #1700a2;
+            color: white;
+            font-weight: bold;
+            text-align: left;
+            padding: 12px 15px;
+            border: 1px solid #3a26b8; /* Borda um pouco mais escura para o header */
+        }
+        td {
+            padding: 10px 15px;
+            border: 1px solid #d0d0d0; /* Borda clara da cor do texto */
+        }
+        tr:nth-child(even) {
+            background-color: #f9f9f9;
+        }
+        tr:nth-child(odd) {
+            background-color: white;
+        }
+        .info-section {
+            margin-top: 30px;
+            color: #5c5c5c;
+        }
+        .timestamp {
+            font-style: italic;
+            text-align: right;
+            margin-top: 20px;
+            font-size: 0.9em;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
+            <h1>Relatório de PII</h1>
+        </div>
+
+        <div class="info-section">
+            <p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
+        </div>
+
+        <table>
+            <thead>
+                <tr>
+                    <th>Database</th>
+                    <th>Schema</th>
+                    <th>Tabela</th>
+                    <th>Coluna</th>
+                    <th>Tipo de Dado</th>
+                    <th>Regras PII</th>
+                </tr>
+            </thead>
+            <tbody>
+                {{#each dados}}
+                <tr>
+                    <td>{{database_name}}</td>
+                    <td>{{table_schema}}</td>
+                    <td>{{table_name}}</td>
+                    <td>{{column_name}}</td>
+                    <td>{{data_type}}</td>
+                    <td>{{pii_rules}}</td>
+                </tr>
+                {{/each}}
+            </tbody>
+        </table>
+
+        <p class="timestamp">Gerado em: {{dataGeracao}}</p>
+    </div>
+</body>
+</html>

src/utils/ PackTheMetadata.ts

@@ -14,6 +14,7 @@ interface IMetadata {
   host?: string;
   origin?: string;
   custom_host?: string;
+  is_mask?: boolean;
 }
 
 export function PackTheMetadata(info: IMetadata): Metadata {

src/utils/FileParser/csv-parser.ts

@@ -0,0 +1,14 @@
+import { stringify } from 'csv'
+
+export class CsvParser<T> {
+  async parse(data: T[]) {
+    return new Promise((resolve, reject) => {
+      stringify(data, { header: true }, function (err, output) {
+        if (err) reject(err);
+        console.log(output)
+  
+        resolve(output)
+      });
+    })
+  }
+}

src/utils/FileParser/html-parser.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from 'fs';
+import Handlebars from 'handlebars';
+
+export class HtmlParser<T> {
+  constructor(private readonly templatePath: string) {}
+  parse(data: T[]) {
+    const templateSource = readFileSync(this.templatePath, 'utf8');
+
+    const template = Handlebars.compile(templateSource);
+
+    return template({
+      dados: data,
+      dataGeracao: new Date().toLocaleString('pt-BR'),
+    });
+  }
+}

src/utils/FileParser/pdf-parser.ts

@@ -0,0 +1,48 @@
+import puppeteer from "puppeteer";
+
+export class PDFParser<T extends string> {
+  async parse(data: T) {
+    try {
+      const browser = await puppeteer.launch({
+        executablePath: process.env.PUPPETEER_EXECUTABLE_PATH,
+        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
+      });
+      
+      const page = await browser.newPage();
+      page.setDefaultNavigationTimeout(0);
+  
+      await page.setContent(data, { 
+        waitUntil: 'networkidle0',
+        timeout: 30000
+      });
+
+      // Configurações adicionais para garantir um PDF válido
+      const pdfBuffer = await page.pdf({
+        format: 'A4',
+        printBackground: true,
+        margin: {
+          top: '20mm',
+          right: '15mm',
+          bottom: '20mm',
+          left: '15mm'
+        },
+        preferCSSPageSize: true,
+        displayHeaderFooter: false,
+        headerTemplate: '',
+        footerTemplate: '',
+        omitBackground: false,
+        pageRanges: '',
+        tagged: true,
+        outline: false,
+        timeout: 30000
+      });
+  
+      await browser.close();
+  
+      return pdfBuffer;
+    } catch (error) {
+      console.error('Error generating PDF:', error);
+      throw error;
+    }
+  }
+}