Compare commits

...
Author SHA1 Message Date
marcos-silva-rodrigues 5d8e908339 FIX: add nginx proxy timeout 5 min 2025-06-24 10:26:50 -03:00
Marcos Rodrigues Silva 3b48dd1613 Merge pull request #317 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 17:07:55 -03:00
marcos-silva-rodrigues bac74c9577 FIX: default value and throw correct Error Class 2025-06-23 15:36:26 -03:00
marcos-silva-rodrigues 19d748ae09 CHORE: add restricted ip and unimed chart 2025-06-23 14:47:39 -03:00
marcos-silva-rodrigues 4db865371f Merge branch 'bugfix/update-pupperter-timeout' into release/2025-06 2025-06-23 14:12:25 -03:00
marcos-silva-rodrigues 0aa9c065a5 FEAT: block user from login when maestro is dedicated 2025-06-16 10:34:11 -03:00
marcos-silva-rodrigues cb98099a91 FIX: update timeout 2025-06-13 15:07:50 -03:00
marcos-silva-rodrigues 17de31f1a6 FIX: send env DEDICATED_PROXY 2025-06-12 14:45:13 -03:00
marcos-silva-rodrigues 9e597fadba FIX: open data ingress route 2025-06-12 14:04:45 -03:00
marcos-silva-rodrigues 41842f7c6a FIX: chart private maestro 2025-06-12 13:56:57 -03:00
marcos-silva-rodrigues 7f9755493a FIX: experimental hostname 2025-06-12 12:29:51 -03:00
marcos-silva-rodrigues fea587ad36 FEAT: control login by dedicated customer id injected 2025-06-12 11:51:56 -03:00
Marcos Rodrigues Silva b2b7537fee Merge pull request #308 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 15:25:27 -03:00
marcos-silva-rodrigues 01263017e7 FIX: comment block list 2025-06-10 14:24:26 -03:00
Rafael Santana 8aded98a11 Merge pull request #304 from dadosfera/beta
Beta
2025-05-26 10:51:13 -03:00
Rafael Santana 0b5a0d0d7e Merge pull request #303 from dadosfera/adjusting-nestjs-payload
Adjusting nestjs payload
2025-05-26 10:40:43 -03:00
Rafael 5a36762544 UPDATE: route /catalog/register-dataset will have an limit of 10mb 2025-05-26 10:24:48 -03:00
Marcos Rodrigues Silva f579171024 Merge pull request #302 from dadosfera/beta
Default value to header origin
2025-05-15 11:18:17 -03:00
Marcos Rodrigues Silva 885e5f4568 Merge pull request #301 from dadosfera/fix/remove-default-host
Fix/remove default host
2025-05-15 11:00:45 -03:00
marcos-silva-rodrigues 81d8cd8a07 FIX: default value to header origin 2025-05-15 11:00:07 -03:00
Marcos Rodrigues Silva f153f41357 Merge pull request #300 from dadosfera/beta
Beta
2025-05-14 18:30:54 -03:00
marcos-silva-rodrigues 2b9e3ff3bc Merge branch 'main' into beta 2025-05-13 14:32:34 -03:00
Marcos Rodrigues Silva 17cdaac380 Merge pull request #299 from dadosfera/fix/stg-envs
FIX: stg to prd for sbm environment
2025-05-09 11:07:22 -03:00
marcos-silva-rodrigues e4eb388e18 FIX: stg to prd for sbm environment 2025-05-09 11:06:23 -03:00
Marcos Rodrigues Silva 4bea0816b6 Merge pull request #298 from dadosfera/feature/pii-rules
FIX: install chromiun in base_image
2025-04-29 16:22:22 -03:00
marcos-silva-rodrigues fb491328d6 FIX: install chromiun in base_image 2025-04-29 16:20:37 -03:00
Marcos Rodrigues Silva 4e043d93ce Merge pull request #297 from dadosfera/feature/pii-rules
REFACTOR: move template to assets folder
2025-04-29 15:41:16 -03:00
marcos-silva-rodrigues 0b525b835f REFACTOR: move template to assets folder 2025-04-29 15:40:37 -03:00
Marcos Rodrigues Silva 3c8c39ec26 Merge pull request #296 from dadosfera/feature/pii-rules
REFACTOR: create builder for parser reporter and fix pdf
2025-04-29 15:20:46 -03:00
marcos-silva-rodrigues 36997403e1 REFACTOR: create builder for parser reporterand fix pdf 2025-04-29 15:18:46 -03:00
Marcos Rodrigues Silva 80ebd98f6f Merge pull request #294 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-23 17:46:18 -03:00
Marcos Rodrigues Silva 7044d6606e Merge pull request #278 from dadosfera/beta
Beta
2025-04-08 18:04:41 -03:00
26 changed files with 298 additions and 117 deletions
+13
View File
@@ -50,4 +50,17 @@ WORKDIR /app
COPY --from=prod_build /app/dist ./dist
COPY --from=prod_build /app/node_modules ./node_modules
COPY --from=prod_build /app/package*.json ./
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT npm run start:prod
-8
View File
@@ -20,14 +20,6 @@
],
"type": "string"
}
},
{
"name": "origin",
"required": true,
"in": "header",
"schema": {
"type": "string"
}
}
],
"requestBody": {
+1
View File
@@ -14,6 +14,7 @@ declare global {
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
}
}
}
+31
View File
@@ -4,6 +4,8 @@ charts:
values:
- ../maestro/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
@@ -20,3 +22,32 @@ charts:
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
- name: restricted-ip
value: "177.52.172.0/24"
+32 -2
View File
@@ -9,7 +9,7 @@ charts:
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.stg.dadosfera.ai
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
@@ -19,4 +19,34 @@ charts:
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
value: 1
# Environment to test Network Policies
- name: private-maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: app_name
value: maestro-private
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: private-maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: maestro.restricted_ip
value: "57.151.113.140/30"
+9 -7
View File
@@ -1,16 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: maestro
app: {{ .Values.app_name }}
strategy:
rollingUpdate:
@@ -20,7 +20,7 @@ spec:
template:
metadata:
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
imagePullSecrets:
@@ -100,6 +100,8 @@ spec:
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -108,15 +110,15 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
+6 -2
View File
@@ -10,8 +10,12 @@ metadata:
generation: 1
labels:
app: maestro
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
@@ -21,7 +25,7 @@ spec:
paths:
- backend:
service:
name: maestro
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
+9 -3
View File
@@ -3,14 +3,20 @@ kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: maestro
name: maestro
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
@@ -20,7 +26,7 @@ spec:
paths:
- backend:
service:
name: maestro
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
+3 -3
View File
@@ -1,17 +1,17 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-maestro
name: prd-{{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-maestro
name: prd-{{ .Values.app_name }}
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
+4 -4
View File
@@ -1,18 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
type: ClusterIP
ports:
- name: maestro
- name: {{ .Values.app_name }}
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: maestro
app: {{ .Values.app_name }}
+3
View File
@@ -9,6 +9,7 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
@@ -42,6 +43,8 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
autoscaling:
enabled: false
minReplicas: 1
+1
View File
@@ -4,6 +4,7 @@
"compilerOptions": {
"assets": [
"**/*.proto",
"assets/**/*",
{
"include": "i18n/**/*",
"watchAssets": true
@@ -7,51 +7,69 @@
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
<style>
@page {
size: A4;
margin: 20mm 15mm 20mm 15mm;
size: A4 landscape; /* Alterado para paisagem (landscape) */
margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
}
body {
font-family: 'Quicksand', sans-serif;
color: #5c5c5c;
margin: 0;
padding: 20px;
padding: 10px;
font-size: 12px; /* Reduzindo o tamanho da fonte */
}
.container {
margin: 0 24px;
margin: 0;
width: 100%;
}
.header {
display: flex;
align-items: center;
margin-bottom: 30px;
margin-bottom: 20px;
}
.logo {
max-width: 200px;
max-width: 150px; /* Reduzida para economizar espaço */
height: auto;
}
h1 {
color: #0d003b;
font-weight: 700;
margin-left: 20px;
font-size: 24px; /* Tamanho ajustado */
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 20px;
margin-top: 15px;
table-layout: fixed; /* Importante: define larguras fixas */
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border: 1px solid #d0d0d0; /* Borda externa da tabela */
border: 1px solid #d0d0d0;
}
th {
background-color: #1700a2;
color: white;
font-weight: bold;
text-align: left;
padding: 12px 15px;
border: 1px solid #3a26b8; /* Borda um pouco mais escura para o header */
padding: 8px 10px;
border: 1px solid #3a26b8;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
td {
padding: 10px 15px;
border: 1px solid #d0d0d0; /* Borda clara da cor do texto */
padding: 6px 10px;
border: 1px solid #d0d0d0;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
/* Definindo larguras específicas para cada coluna */
th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
tr:nth-child(even) {
background-color: #f9f9f9;
}
@@ -59,13 +77,13 @@
background-color: white;
}
.info-section {
margin-top: 30px;
margin-top: 20px;
color: #5c5c5c;
}
.timestamp {
font-style: italic;
text-align: right;
margin-top: 20px;
margin-top: 15px;
font-size: 0.9em;
}
</style>
@@ -4,6 +4,7 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -113,6 +114,18 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
// const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
// if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
// throw new ForbiddenException();
// }
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
+3
View File
@@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import helmet from 'helmet';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { json, urlencoded } from 'express';
import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
@@ -25,6 +26,8 @@ async function bootstrap() {
},
});
app.use(helmet());
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
+1 -3
View File
@@ -86,11 +86,9 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ language, custom_host: frontHost });
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
}
+39 -4
View File
@@ -1,4 +1,4 @@
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
@@ -17,6 +17,7 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
@@ -53,15 +54,49 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
}
async refreshAccessToken(
+12 -16
View File
@@ -61,6 +61,7 @@ import {
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
import { TypeParser } from 'src/utils/FileParser/parser-types';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -864,29 +865,24 @@ export class CatalogController {
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') type: "html" | "csv") {
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const reporter = await this.catalogService.getPiiReporter(metadata, type);
if (type === "html") {
res.setHeader('Content-Type', 'text/html');
} else if (type === "csv") {
res.setHeader('Content-Type', 'text/csv');
} else if (type === "pdf") {
res.setHeader('Content-Type', 'application/pdf');
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
return res.send(reporter);
// use res.end to send buffer
return res.end(file);
} catch (error) {
this.logger.error(error);
console.error(error)
this.logger.error(error.message);
}
}
}
+23 -27
View File
@@ -33,6 +33,7 @@ import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
@@ -40,6 +41,8 @@ import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -86,7 +89,8 @@ class CatalogService implements OnModuleInit {
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: "html"| "csv") {
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
try {
const {
data
@@ -94,37 +98,29 @@ class CatalogService implements OnModuleInit {
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
// 3. Ler o template HTML
const templatePath = path.join(
__dirname,
'../../../src/template/pii-reporter.html',
);
this.logger.info("templatePath: " + templatePath);
const parser = ParserBuilder.build<PiiMetadata>(type);
if (type === "html") {
const htmlParser = new HtmlParser<PiiDto>(templatePath);
return htmlParser.parse(data);
} else if (type === "csv") {
const csvParser = new CsvParser<PiiDto>();
return await csvParser.parse(data)
} else if (type === "pdf") {
const htmlParser = new HtmlParser<PiiDto>(templatePath);
const pdfParser = new PDFParser();
const pdf = await pdfParser.parse(htmlParser.parse(data));
const pdfPath = path.join(
__dirname,
'../../../src/template/teste.pdf',
);
writeFileSync(pdfPath, pdf);
return pdf
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
throw new Error("Unsupported Type: " + type)
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
} catch (error) {
this.logger.error(error);
this.logger.error(error.message);
throw error;
}
}
@@ -45,11 +45,11 @@ export class OpenDataController {
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
if (!corslist.includes(origin)) {
this.logger.info('block request by cors list: '+ origin);
throw new ForbiddenException();
}
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
+3 -3
View File
@@ -1,11 +1,11 @@
import { stringify } from 'csv'
import { Parser } from './parser.interface';
export class CsvParser<T> {
export class CsvParser<T> implements Parser<T>{
async parse(data: T[]) {
return new Promise((resolve, reject) => {
return new Promise<string>((resolve, reject) => {
stringify(data, { header: true }, function (err, output) {
if (err) reject(err);
console.log(output)
resolve(output)
});
+3 -2
View File
@@ -1,9 +1,10 @@
import { readFileSync } from 'fs';
import Handlebars from 'handlebars';
import { Parser } from './parser.interface';
export class HtmlParser<T> {
export class HtmlParser<T> implements Parser<T> {
constructor(private readonly templatePath: string) {}
parse(data: T[]) {
async parse(data: T[]) {
const templateSource = readFileSync(this.templatePath, 'utf8');
const template = Handlebars.compile(templateSource);
+1
View File
@@ -0,0 +1 @@
export type TypeParser = "html" | "csv" | "pdf"
+25
View File
@@ -0,0 +1,25 @@
import { TypeParser } from './parser-types';
import { HtmlParser } from './html-parser';
import { CsvParser } from './csv-parser';
import { PDFParser } from './pdf-parser';
import { Parser } from './parser.interface';
export class ParserBuilder {
static build<T>(type: TypeParser): Parser<T> {
const templatePath = '/app/dist/assets/template/pii-reporter.html';
if (type === 'html') {
const htmlParser = new HtmlParser<T>(templatePath);
return htmlParser;
} else if (type === 'csv') {
const csvParser = new CsvParser<T>();
return csvParser;
} else if (type === 'pdf') {
const htmlParser = new HtmlParser<T>(templatePath);
const pdfParser = new PDFParser(htmlParser);
return pdfParser;
}
throw new Error('Unsupported Type: ' + type);
}
}
+3
View File
@@ -0,0 +1,3 @@
export interface Parser<T> {
parse(data: T[]): Promise<string | Buffer>
}
+23 -14
View File
@@ -1,30 +1,39 @@
import puppeteer from "puppeteer";
import puppeteer from 'puppeteer';
import { Parser } from './parser.interface';
import { HtmlParser } from './html-parser';
export class PDFParser<T extends string> {
async parse(data: T) {
export class PDFParser<T> implements Parser<T> {
constructor(private htmlParser: HtmlParser<T>) {}
async parse(data: T[]) {
try {
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
],
});
const page = await browser.newPage();
page.setDefaultNavigationTimeout(0);
await page.setContent(data, {
const html = await this.htmlParser.parse(data);
await page.setContent(html, {
waitUntil: 'networkidle0',
timeout: 30000
timeout: 90000,
});
// Configurações adicionais para garantir um PDF válido
const pdfBuffer = await page.pdf({
const pdf = await page.pdf({
format: 'A4',
printBackground: true,
margin: {
top: '20mm',
right: '15mm',
bottom: '20mm',
left: '15mm'
left: '15mm',
},
preferCSSPageSize: true,
displayHeaderFooter: false,
@@ -34,12 +43,12 @@ export class PDFParser<T extends string> {
pageRanges: '',
tagged: true,
outline: false,
timeout: 30000
timeout: 90000,
});
await browser.close();
return pdfBuffer;
return Buffer.from(pdf);
} catch (error) {
console.error('Error generating PDF:', error);
throw error;