Compare commits

...
Author SHA1 Message Date
marcos-silva-rodrigues f3550255c4 FIX: validate if permission is public 2025-06-11 15:18:55 -03:00
marcos-silva-rodrigues 54cb3f8e7a FEAT: add api key endpoint 2025-06-06 15:53:12 -03:00
Rafael Santana 8aded98a11 Merge pull request #304 from dadosfera/beta
Beta
2025-05-26 10:51:13 -03:00
Rafael Santana 0b5a0d0d7e Merge pull request #303 from dadosfera/adjusting-nestjs-payload
Adjusting nestjs payload
2025-05-26 10:40:43 -03:00
Rafael 5a36762544 UPDATE: route /catalog/register-dataset will have an limit of 10mb 2025-05-26 10:24:48 -03:00
Marcos Rodrigues Silva f579171024 Merge pull request #302 from dadosfera/beta
Default value to header origin
2025-05-15 11:18:17 -03:00
Marcos Rodrigues Silva 885e5f4568 Merge pull request #301 from dadosfera/fix/remove-default-host
Fix/remove default host
2025-05-15 11:00:45 -03:00
marcos-silva-rodrigues 81d8cd8a07 FIX: default value to header origin 2025-05-15 11:00:07 -03:00
Marcos Rodrigues Silva f153f41357 Merge pull request #300 from dadosfera/beta
Beta
2025-05-14 18:30:54 -03:00
marcos-silva-rodrigues 2b9e3ff3bc Merge branch 'main' into beta 2025-05-13 14:32:34 -03:00
Marcos Rodrigues Silva 17cdaac380 Merge pull request #299 from dadosfera/fix/stg-envs
FIX: stg to prd for sbm environment
2025-05-09 11:07:22 -03:00
marcos-silva-rodrigues e4eb388e18 FIX: stg to prd for sbm environment 2025-05-09 11:06:23 -03:00
Marcos Rodrigues Silva 4bea0816b6 Merge pull request #298 from dadosfera/feature/pii-rules
FIX: install chromiun in base_image
2025-04-29 16:22:22 -03:00
marcos-silva-rodrigues fb491328d6 FIX: install chromiun in base_image 2025-04-29 16:20:37 -03:00
Marcos Rodrigues Silva 4e043d93ce Merge pull request #297 from dadosfera/feature/pii-rules
REFACTOR: move template to assets folder
2025-04-29 15:41:16 -03:00
marcos-silva-rodrigues 0b525b835f REFACTOR: move template to assets folder 2025-04-29 15:40:37 -03:00
Marcos Rodrigues Silva 3c8c39ec26 Merge pull request #296 from dadosfera/feature/pii-rules
REFACTOR: create builder for parser reporter and fix pdf
2025-04-29 15:20:46 -03:00
marcos-silva-rodrigues 36997403e1 REFACTOR: create builder for parser reporterand fix pdf 2025-04-29 15:18:46 -03:00
Marcos Rodrigues Silva 80ebd98f6f Merge pull request #294 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-23 17:46:18 -03:00
Marcos Rodrigues Silva 7044d6606e Merge pull request #278 from dadosfera/beta
Beta
2025-04-08 18:04:41 -03:00
25 changed files with 670 additions and 96 deletions
+13
View File
@@ -50,4 +50,17 @@ WORKDIR /app
COPY --from=prod_build /app/dist ./dist
COPY --from=prod_build /app/node_modules ./node_modules
COPY --from=prod_build /app/package*.json ./
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT npm run start:prod
+244 -8
View File
@@ -20,14 +20,6 @@
],
"type": "string"
}
},
{
"name": "origin",
"required": true,
"in": "header",
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -5976,6 +5968,167 @@
}
]
}
},
"/api-key": {
"post": {
"operationId": "ApiKeyController_create",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyDto"
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyResponseDto"
}
}
}
},
"default": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyResponseDto"
}
}
}
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
},
"get": {
"operationId": "ApiKeyController_findAll",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyBaseResponseDto"
}
}
}
}
},
"default": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyBaseResponseDto"
}
}
}
}
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/api-key/{id}": {
"delete": {
"operationId": "ApiKeyController_remove",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
}
},
"info": {
@@ -8856,6 +9009,89 @@
"required": [
"policies"
]
},
"CreateApiKeyDto": {
"type": "object",
"properties": {
"permissions": {
"description": "Array of permission IDs",
"type": "array",
"items": {
"type": "number"
}
}
},
"required": [
"permissions"
]
},
"CreateApiKeyResponseDto": {
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uuid"
},
"key_mask": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PermissionDto"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
},
"key": {
"type": "string"
}
},
"required": [
"id",
"key_mask",
"permissions",
"created_at",
"created_by",
"key"
]
},
"ApiKeyBaseResponseDto": {
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uuid"
},
"key_mask": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PermissionDto"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
}
},
"required": [
"id",
"key_mask",
"permissions",
"created_at",
"created_by"
]
}
}
}
+1 -1
View File
@@ -9,7 +9,7 @@ charts:
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.stg.dadosfera.ai
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
+1
View File
@@ -4,6 +4,7 @@
"compilerOptions": {
"assets": [
"**/*.proto",
"assets/**/*",
{
"include": "i18n/**/*",
"watchAssets": true
+4 -4
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.23",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1400,9 +1400,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.37.0-beta.23",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz",
"integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==",
"version": "3.38.0-beta.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.1.tgz",
"integrity": "sha512-4+yNZMlhvEkHlxJcaeta+jtJ9owskZcg0yjKdZzBfT8PCclctgertLHmrG0+kDb3Y/na/rpPPHYULRp7AO4zDQ==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
+1 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.23",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+3 -1
View File
@@ -29,6 +29,7 @@ import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
providers: [
@@ -64,7 +65,8 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
ThemeModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule
NetworkPolicyModule,
ApiKeyModule
],
})
export class AppModule {}
@@ -7,51 +7,69 @@
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
<style>
@page {
size: A4;
margin: 20mm 15mm 20mm 15mm;
size: A4 landscape; /* Alterado para paisagem (landscape) */
margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
}
body {
font-family: 'Quicksand', sans-serif;
color: #5c5c5c;
margin: 0;
padding: 20px;
padding: 10px;
font-size: 12px; /* Reduzindo o tamanho da fonte */
}
.container {
margin: 0 24px;
margin: 0;
width: 100%;
}
.header {
display: flex;
align-items: center;
margin-bottom: 30px;
margin-bottom: 20px;
}
.logo {
max-width: 200px;
max-width: 150px; /* Reduzida para economizar espaço */
height: auto;
}
h1 {
color: #0d003b;
font-weight: 700;
margin-left: 20px;
font-size: 24px; /* Tamanho ajustado */
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 20px;
margin-top: 15px;
table-layout: fixed; /* Importante: define larguras fixas */
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border: 1px solid #d0d0d0; /* Borda externa da tabela */
border: 1px solid #d0d0d0;
}
th {
background-color: #1700a2;
color: white;
font-weight: bold;
text-align: left;
padding: 12px 15px;
border: 1px solid #3a26b8; /* Borda um pouco mais escura para o header */
padding: 8px 10px;
border: 1px solid #3a26b8;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
td {
padding: 10px 15px;
border: 1px solid #d0d0d0; /* Borda clara da cor do texto */
padding: 6px 10px;
border: 1px solid #d0d0d0;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
/* Definindo larguras específicas para cada coluna */
th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
tr:nth-child(even) {
background-color: #f9f9f9;
}
@@ -59,13 +77,13 @@
background-color: white;
}
.info-section {
margin-top: 30px;
margin-top: 20px;
color: #5c5c5c;
}
.timestamp {
font-style: italic;
text-align: right;
margin-top: 20px;
margin-top: 15px;
font-size: 0.9em;
}
</style>
+26 -4
View File
@@ -17,6 +17,7 @@ import {
import { RequestUser } from '../decorators/user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
@Injectable()
export class AuthenticationGuard
@@ -31,6 +32,7 @@ export class AuthenticationGuard
dadosferaLogger: DadosferaLogger,
private reflector: Reflector,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService
) {
this.pems = new Map();
this.logger = dadosferaLogger.logger;
@@ -48,20 +50,40 @@ export class AuthenticationGuard
});
}
canActivate(ctx: ExecutionContext): boolean {
async canActivate(ctx: ExecutionContext): Promise<boolean> {
const authFunctions = this.reflector.getAllAndMerge<
AuthenticationFunction[]
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
const mustBeAuthenticated = authFunctions.length > 0;
const request = ctx.switchToHttp().getRequest();
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!mustBeAuthenticated) {
// no need to be authenticated
return true;
}
const request = ctx.switchToHttp().getRequest();
const apiKey = request.get('X-api-key');
if (apiKey) {
const {
api_key
} = await this.apiKeyService.get(apiKey);
request.user = {
user_id: api_key.user_id,
username: api_key.username,
permissions: api_key.permissions,
customer_id: api_key.customer_id,
customer_name: api_key.customer_name,
customer_tier: api_key.customer_tier,
customer_modules: api_key.customer_modules,
access_token: apiKey,
};
return true;
}
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!accessToken) {
// couldn't load valid token
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
+16
View File
@@ -603,6 +603,22 @@ export const PERMISSIONS_GROUPS = {
},
},
};
export const PUBLIC_PERMISSIONS_SEQID = [
46, // AUTH.GENERATE_TOKEN
23, 13, 29, 5, // PIPELINE
37, 38, // CONNECTION
35, // NETWORK_CONFIG
7, 19, 25, 1, // CATALOG
14, // SNOWFLAKE
11, // DATAVIZ
31, // INTELLIGENCE
34, // USERS
43, // PROCESS
47 // CUSTOMER
];
export interface DadosferaModule {
name: string;
description: string;
+6
View File
@@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import helmet from 'helmet';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { json, urlencoded } from 'express';
import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
@@ -25,6 +26,11 @@ async function bootstrap() {
},
});
app.use(helmet());
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
}
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
+72
View File
@@ -0,0 +1,72 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated, RequireAllPermissions } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('api-key')
@Authenticated()
@ApiTags('ApiKey')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
export class ApiKeyController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
method: 'create'
});
const result = await this.apiKeyService.create(createApiKeyDto, user);
this.logger.info('POST /api-key success', {
id: result.id,
method: 'create'
});
return result;
}
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
});
const result = await this.apiKeyService.findAll(user);
this.logger.info('GET /api-key success', {
count: result.length,
method: 'findAll'
});
return result;
}
@Delete(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
method: 'remove'
});
await this.apiKeyService.remove(id, user);
this.logger.info('DELETE /api-key/:id success', {
id,
method: 'remove'
});
}
}
+18
View File
@@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { ApiKeyController } from './api-key.controller';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([ducClient.providerOptions])
],
controllers: [ApiKeyController],
providers: [ApiKeyService, DadosferaLogger],
exports: [ApiKeyService]
})
export class ApiKeyModule {}
+85
View File
@@ -0,0 +1,85 @@
import {
Injectable,
Inject,
OnModuleInit,
BadRequestException,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
CreateApiKeyDto,
CreateApiKeyResponseDto,
ApiKeyBaseResponseDto,
} from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PackTheMetadata } from '../../utils/ PackTheMetadata';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PUBLIC_PERMISSIONS_SEQID } from 'src/authentication/permissions.enum';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(
ProtoServices.ApiKeyWriteProtoService,
);
}
create(
createApiKeyDto: CreateApiKeyDto,
user: RequestUser,
): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
const invalidPermissions = [];
for (const permission of createApiKeyDto.permissions) {
if (!PUBLIC_PERMISSIONS_SEQID.includes(permission)) {
invalidPermissions.push(permission);
}
}
if (invalidPermissions.length > 0) {
throw new BadRequestException(
`Invalid permissions: ${invalidPermissions.join(', ')}`,
);
}
return lastValueFrom(
this.apiKeyService.CreateApiKey(
{
permissions: createApiKeyDto.permissions,
},
metadata,
),
);
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata);
const data = await lastValueFrom(
this.apiKeyService.ListApiKeys({}, metadata),
);
return data.api_keys;
}
async remove(id: string, user: RequestUser) {
const metadata = PackTheMetadata(user);
await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
}
async get(key: string) {
const metadata = PackTheMetadata({});
return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
}
}
+39
View File
@@ -0,0 +1,39 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsArray, IsNumber } from 'class-validator';
export class PermissionDto {
@ApiProperty({ type: Number })
id: number;
@ApiProperty({ type: String })
name: string;
}
export class ApiKeyBaseResponseDto {
@ApiProperty({ type: String, format: 'uuid' })
id: string;
@ApiProperty({ type: String })
key_mask: string;
@ApiProperty({ type: [PermissionDto] })
permissions: PermissionDto[];
@ApiProperty({ type: String, format: 'date-time' })
created_at: string;
@ApiProperty({ type: String })
created_by: string;
}
export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
@ApiProperty({ type: String })
key: string;
}
export class CreateApiKeyDto {
@ApiProperty({ type: [Number], description: 'Array of permission IDs' })
@IsArray()
@IsNumber({}, { each: true })
permissions: number[];
}
+1 -1
View File
@@ -86,7 +86,7 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Headers('origin') origin = '',
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
+12 -16
View File
@@ -61,6 +61,7 @@ import {
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
import { TypeParser } from 'src/utils/FileParser/parser-types';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -864,29 +865,24 @@ export class CatalogController {
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') type: "html" | "csv") {
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const reporter = await this.catalogService.getPiiReporter(metadata, type);
if (type === "html") {
res.setHeader('Content-Type', 'text/html');
} else if (type === "csv") {
res.setHeader('Content-Type', 'text/csv');
} else if (type === "pdf") {
res.setHeader('Content-Type', 'application/pdf');
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
return res.send(reporter);
// use res.end to send buffer
return res.end(file);
} catch (error) {
this.logger.error(error);
console.error(error)
this.logger.error(error.message);
}
}
}
+23 -27
View File
@@ -33,6 +33,7 @@ import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
@@ -40,6 +41,8 @@ import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -86,7 +89,8 @@ class CatalogService implements OnModuleInit {
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: "html"| "csv") {
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
try {
const {
data
@@ -94,37 +98,29 @@ class CatalogService implements OnModuleInit {
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
// 3. Ler o template HTML
const templatePath = path.join(
__dirname,
'../../../src/template/pii-reporter.html',
);
this.logger.info("templatePath: " + templatePath);
const parser = ParserBuilder.build<PiiMetadata>(type);
if (type === "html") {
const htmlParser = new HtmlParser<PiiDto>(templatePath);
return htmlParser.parse(data);
} else if (type === "csv") {
const csvParser = new CsvParser<PiiDto>();
return await csvParser.parse(data)
} else if (type === "pdf") {
const htmlParser = new HtmlParser<PiiDto>(templatePath);
const pdfParser = new PDFParser();
const pdf = await pdfParser.parse(htmlParser.parse(data));
const pdfPath = path.join(
__dirname,
'../../../src/template/teste.pdf',
);
writeFileSync(pdfPath, pdf);
return pdf
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
throw new Error("Unsupported Type: " + type)
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
} catch (error) {
this.logger.error(error);
this.logger.error(error.message);
throw error;
}
}
+15
View File
@@ -20,6 +20,7 @@ import { PermissionsService } from '../permissions/permissions.service';
import { LanguageEnum } from 'src/utils/languages.enum';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PUBLIC_PERMISSIONS_SEQID } from 'src/authentication/permissions.enum';
interface GetRolesPermissionsName {
id: string;
@@ -103,6 +104,20 @@ export class RolesService {
const meta = new Metadata();
meta.add('access_token', access_token);
const { description, name, permissionIds, userIds } = data;
const invalidPermissions = [];
for (const permission of permissionIds) {
if (!PUBLIC_PERMISSIONS_SEQID.includes(permission)) {
invalidPermissions.push(permission);
}
}
if (invalidPermissions.length > 0) {
throw new BadRequestException(
`Invalid permissions: ${invalidPermissions.join(', ')}`,
);
}
const role = await lastValueFrom(
this.rolesClientService.RoleCreate(
{
+3 -3
View File
@@ -1,11 +1,11 @@
import { stringify } from 'csv'
import { Parser } from './parser.interface';
export class CsvParser<T> {
export class CsvParser<T> implements Parser<T>{
async parse(data: T[]) {
return new Promise((resolve, reject) => {
return new Promise<string>((resolve, reject) => {
stringify(data, { header: true }, function (err, output) {
if (err) reject(err);
console.log(output)
resolve(output)
});
+3 -2
View File
@@ -1,9 +1,10 @@
import { readFileSync } from 'fs';
import Handlebars from 'handlebars';
import { Parser } from './parser.interface';
export class HtmlParser<T> {
export class HtmlParser<T> implements Parser<T> {
constructor(private readonly templatePath: string) {}
parse(data: T[]) {
async parse(data: T[]) {
const templateSource = readFileSync(this.templatePath, 'utf8');
const template = Handlebars.compile(templateSource);
+1
View File
@@ -0,0 +1 @@
export type TypeParser = "html" | "csv" | "pdf"
+25
View File
@@ -0,0 +1,25 @@
import { TypeParser } from './parser-types';
import { HtmlParser } from './html-parser';
import { CsvParser } from './csv-parser';
import { PDFParser } from './pdf-parser';
import { Parser } from './parser.interface';
export class ParserBuilder {
static build<T>(type: TypeParser): Parser<T> {
const templatePath = '/app/dist/assets/template/pii-reporter.html';
if (type === 'html') {
const htmlParser = new HtmlParser<T>(templatePath);
return htmlParser;
} else if (type === 'csv') {
const csvParser = new CsvParser<T>();
return csvParser;
} else if (type === 'pdf') {
const htmlParser = new HtmlParser<T>(templatePath);
const pdfParser = new PDFParser(htmlParser);
return pdfParser;
}
throw new Error('Unsupported Type: ' + type);
}
}
+3
View File
@@ -0,0 +1,3 @@
export interface Parser<T> {
parse(data: T[]): Promise<string | Buffer>
}
+23 -14
View File
@@ -1,30 +1,39 @@
import puppeteer from "puppeteer";
import puppeteer from 'puppeteer';
import { Parser } from './parser.interface';
import { HtmlParser } from './html-parser';
export class PDFParser<T extends string> {
async parse(data: T) {
export class PDFParser<T> implements Parser<T> {
constructor(private htmlParser: HtmlParser<T>) {}
async parse(data: T[]) {
try {
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
],
});
const page = await browser.newPage();
page.setDefaultNavigationTimeout(0);
await page.setContent(data, {
const html = await this.htmlParser.parse(data);
await page.setContent(html, {
waitUntil: 'networkidle0',
timeout: 30000
timeout: 30000,
});
// Configurações adicionais para garantir um PDF válido
const pdfBuffer = await page.pdf({
const pdf = await page.pdf({
format: 'A4',
printBackground: true,
margin: {
top: '20mm',
right: '15mm',
bottom: '20mm',
left: '15mm'
left: '15mm',
},
preferCSSPageSize: true,
displayHeaderFooter: false,
@@ -34,12 +43,12 @@ export class PDFParser<T extends string> {
pageRanges: '',
tagged: true,
outline: false,
timeout: 30000
timeout: 30000,
});
await browser.close();
return pdfBuffer;
return Buffer.from(pdf);
} catch (error) {
console.error('Error generating PDF:', error);
throw error;