From 9cc538cdcdf72252027bb00b9552489f73ca0989 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 1 Apr 2025 17:57:05 -0300 Subject: [PATCH 01/24] FEAT: add host in metadata --- src/modules/auth/auth.controller.ts | 5 ++++- src/utils/ PackTheMetadata.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 1097d51..e6bc02a 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -85,9 +85,10 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, + @Headers('host') host: string ): Promise { this.logger.info('/auth - SignIn'); - const metadata = PackTheMetadata({ language }); + const metadata = PackTheMetadata({ language, host }); return this.authClient.signIn({ username, password, totp }, metadata); } @@ -97,6 +98,7 @@ export class AuthController { async refreshAccessToken( @Body() body: AuthRefreshAccessTokenReq, @Language() language: LanguageEnum, + @Headers('host') host: string ) { this.logger.info('/auth - RefreshAccessToken'); const { refreshToken, customerName: customer_name } = body; @@ -104,6 +106,7 @@ export class AuthController { const metadata = PackTheMetadata({ customer_name, language, + host }); return this.authClient.refreshAccessToken({ refreshToken }, metadata); diff --git a/src/utils/ PackTheMetadata.ts b/src/utils/ PackTheMetadata.ts index 23556ea..ec528f6 100644 --- a/src/utils/ PackTheMetadata.ts +++ b/src/utils/ PackTheMetadata.ts @@ -11,6 +11,7 @@ interface IMetadata { roles?: string[]; is_data_manager?: boolean; access_token?: string; + host?: string; } export function PackTheMetadata(info: IMetadata): Metadata { From e5d9ac2fbfafec2255cd988370d93fae7f3afe0d Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 2 Apr 2025 16:48:10 -0300 Subject: [PATCH 02/24] FEAT: network policy endpoint --- src/app.module.ts | 4 +- .../network-policy/dto/network-policy.dto.ts | 6 ++ .../network-policy.controller.ts | 52 ++++++++++++++++ .../network-policy/network-policy.module.ts | 19 ++++++ .../network-policy/network-policy.service.ts | 60 +++++++++++++++++++ 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 src/modules/network-policy/dto/network-policy.dto.ts create mode 100644 src/modules/network-policy/network-policy.controller.ts create mode 100644 src/modules/network-policy/network-policy.module.ts create mode 100644 src/modules/network-policy/network-policy.service.ts diff --git a/src/app.module.ts b/src/app.module.ts index 3e54fde..fedb388 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -28,6 +28,7 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module'; import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; import { ThemeModule } from './modules/theme/theme.module'; +import { NetworkPolicyModule } from './modules/network-policy/network-policy.module'; @Module({ providers: [ @@ -62,7 +63,8 @@ import { ThemeModule } from './modules/theme/theme.module'; OpenDataModule, ThemeModule, //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule + HealthModule, + NetworkPolicyModule ], }) export class AppModule {} diff --git a/src/modules/network-policy/dto/network-policy.dto.ts b/src/modules/network-policy/dto/network-policy.dto.ts new file mode 100644 index 0000000..ec4f6c2 --- /dev/null +++ b/src/modules/network-policy/dto/network-policy.dto.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class NetworkPoliciesDTO { + @ApiProperty() + policies: string [] +} \ No newline at end of file diff --git a/src/modules/network-policy/network-policy.controller.ts b/src/modules/network-policy/network-policy.controller.ts new file mode 100644 index 0000000..25f3074 --- /dev/null +++ b/src/modules/network-policy/network-policy.controller.ts @@ -0,0 +1,52 @@ +import { Body, Controller, Delete, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { ApiOkResponse } from '@nestjs/swagger'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { + Authenticated, + RequireAllPermissions, +} from 'src/decorators/authentication.decorator'; +import { NetworkPoliciesDTO } from './dto/network-policy.dto'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { NetworkPolicyService } from './network-policy.service'; +import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; + +@Controller('network-policy') +export class NetworkPolicyController { + constructor(private networkPolicyService: NetworkPolicyService) {} + + @Post() + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse() + @HttpCode(HttpStatus.OK) + async applyNetworkPolicies( + @User() user: RequestUser, + @Body() data: NetworkPoliciesDTO, + ) { + const metadata = PackTheMetadata(user); + + return await this.networkPolicyService.apply( + data.policies, + user.customer_id, + metadata, + ); + } + + @Delete() + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse() + @HttpCode(HttpStatus.OK) + async removeNetworkPolicies( + @User() user: RequestUser, + @Body() data: NetworkPoliciesDTO, + ) { + const metadata = PackTheMetadata(user); + + return await this.networkPolicyService.delete( + data.policies, + user.customer_id, + metadata, + ); + } +} diff --git a/src/modules/network-policy/network-policy.module.ts b/src/modules/network-policy/network-policy.module.ts new file mode 100644 index 0000000..34e2e36 --- /dev/null +++ b/src/modules/network-policy/network-policy.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { NetworkPolicyController } from './network-policy.controller'; +import { NetworkPolicyService } from './network-policy.service'; +import { ClientsModule } from '@nestjs/microservices'; +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +const ducClient = new DucClient(); + +@Module({ + imports: [ + ClientsModule.register([ + ducClient.providerOptions + ]), + ], + controllers: [NetworkPolicyController], + providers: [NetworkPolicyService, DadosferaLogger] +}) +export class NetworkPolicyModule {} diff --git a/src/modules/network-policy/network-policy.service.ts b/src/modules/network-policy/network-policy.service.ts new file mode 100644 index 0000000..a411c77 --- /dev/null +++ b/src/modules/network-policy/network-policy.service.ts @@ -0,0 +1,60 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { DucClient } from '../duc/client.config'; +import { ClientGrpc } from '@nestjs/microservices'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { Metadata } from '@grpc/grpc-js'; +import { lastValueFrom } from 'rxjs'; + +@Injectable() +export class NetworkPolicyService { + private customerService: CustomersProtoService; + private logger: DadosferaLogger; + + constructor( + @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.customerService = this.grpcClient.getService( + ProtoServices.CustomersProtoService, + ); + } + + async apply( + networkPolicies: string[], + customerId: string, + metadata: Metadata, + ) { + return await lastValueFrom( + this.customerService.CustomerCreateNetworkPolicy( + { + customerId, + networkPolicies, + }, + metadata, + ), + ); + } + + async delete( + networkPolicies: string[], + customerId: string, + metadata: Metadata, + ) { + return await lastValueFrom( + this.customerService.CustomerRemoveNetworkPolicy( + { + customerId, + networkPolicies, + }, + metadata, + ), + ); + } +} From 301ad369c09cecca7ba1115df6ac3907bbd4543d Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 4 Apr 2025 18:25:48 -0300 Subject: [PATCH 03/24] FEAT: networks get endpoint --- package-lock.json | 8 ++++---- package.json | 2 +- .../network-policy/network-policy.controller.ts | 16 ++++++++++++++-- .../network-policy/network-policy.service.ts | 14 ++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 31af08d..6b1c84e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.37.0-beta.10 ", + "@dadosfera/protospack-v2": "3.37.0-alpha.20", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1298,9 +1298,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.10", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.10.tgz", - "integrity": "sha512-nKSu9VpOPFSTeX0yN4aEY0HjvGxgbLPYXl217NHgLm+a/+i01USDaxjwIOqP5hhDvhnMNaiCq5awiQTYbhF/7A==", + "version": "3.37.0-alpha.20", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-alpha.20.tgz", + "integrity": "sha512-rs8YUgnwA8dTohzSEAeilU7WAscodf26/yrsqC/lq0Dz2s3Hi09TXItnLJ5ktv5xWaBQ/eZjyc5zPjpcWL3myQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 5a9463a..d2c3b74 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.37.0-beta.10 ", + "@dadosfera/protospack-v2": "3.37.0-alpha.20", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/network-policy/network-policy.controller.ts b/src/modules/network-policy/network-policy.controller.ts index 25f3074..8ae8371 100644 --- a/src/modules/network-policy/network-policy.controller.ts +++ b/src/modules/network-policy/network-policy.controller.ts @@ -1,4 +1,4 @@ -import { Body, Controller, Delete, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Post } from '@nestjs/common'; import { ApiOkResponse } from '@nestjs/swagger'; import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { @@ -14,11 +14,23 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; export class NetworkPolicyController { constructor(private networkPolicyService: NetworkPolicyService) {} + @Get() + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @HttpCode(HttpStatus.OK) + async getNetworks( + @User() user: RequestUser, + ) { + return await this.networkPolicyService.getByCustomer( + user.customer_id + ); + } + @Post() @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse() - @HttpCode(HttpStatus.OK) + @HttpCode(HttpStatus.CREATED) async applyNetworkPolicies( @User() user: RequestUser, @Body() data: NetworkPoliciesDTO, diff --git a/src/modules/network-policy/network-policy.service.ts b/src/modules/network-policy/network-policy.service.ts index a411c77..ed39915 100644 --- a/src/modules/network-policy/network-policy.service.ts +++ b/src/modules/network-policy/network-policy.service.ts @@ -6,6 +6,8 @@ import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/int import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; import { Metadata } from '@grpc/grpc-js'; import { lastValueFrom } from 'rxjs'; +import { NetworkPoliciesDTO } from './dto/network-policy.dto'; + @Injectable() export class NetworkPolicyService { @@ -26,6 +28,18 @@ export class NetworkPolicyService { ); } + async getByCustomer(id: string): Promise { + const { + customer + } = await lastValueFrom(this.customerService.CustomerFindOneById({ + id + })); + + return { + policies: customer.networkPolicies + } + } + async apply( networkPolicies: string[], customerId: string, From 9676b51f01a43a2798513bce4fbefac96775e582 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 10 Apr 2025 11:27:07 -0300 Subject: [PATCH 04/24] FIX: comment host metadata --- docsfera.json | 86 +++++++++++++++++++++++++++++ src/modules/auth/auth.controller.ts | 3 +- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index cbd0471..a39fbf8 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,6 +20,14 @@ ], "type": "string" } + }, + { + "name": "host", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -64,6 +72,14 @@ ], "type": "string" } + }, + { + "name": "host", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -5715,6 +5731,62 @@ "Health" ] } + }, + "/network-policy": { + "post": { + "operationId": "NetworkPolicyController_applyNetworkPolicies", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NetworkPoliciesDTO" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "NetworkPolicyController_removeNetworkPolicies", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NetworkPoliciesDTO" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } } }, "info": { @@ -8566,6 +8638,20 @@ "required": [ "theme" ] + }, + "NetworkPoliciesDTO": { + "type": "object", + "properties": { + "policies": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "policies" + ] } } } diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 0fdba0d..bd4b618 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -91,7 +91,8 @@ export class AuthController { @Headers('host') host: string ): Promise { this.logger.info('/auth - SignIn'); - const metadata = PackTheMetadata({ language, host }); + this.logger.info('host: ' + host); + const metadata = PackTheMetadata({ language }); return this.authClient.signIn({ username, password, totp }, metadata); } From deb32b5c0f3ace2f925dedb8138414f299007579 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 10 Apr 2025 11:44:52 -0300 Subject: [PATCH 05/24] FIX: send host metadata --- src/modules/auth/auth.controller.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index bd4b618..3ffb408 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -92,7 +92,8 @@ export class AuthController { ): Promise { this.logger.info('/auth - SignIn'); this.logger.info('host: ' + host); - const metadata = PackTheMetadata({ language }); + const metadata = PackTheMetadata({ language, host }); + this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } From 3e0dbc740131cca1422a72f11e1c739a55dd9e71 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 10 Apr 2025 11:53:53 -0300 Subject: [PATCH 06/24] FIX: remove host metadata --- src/modules/auth/auth.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 3ffb408..439d8fb 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -92,7 +92,7 @@ export class AuthController { ): Promise { this.logger.info('/auth - SignIn'); this.logger.info('host: ' + host); - const metadata = PackTheMetadata({ language, host }); + const metadata = PackTheMetadata({ language }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } From 16cb00f46f4ba5da0781ce2490a5fe22532f05aa Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Apr 2025 15:06:02 -0300 Subject: [PATCH 07/24] UPDATE: send host --- src/modules/auth/auth.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 439d8fb..3ffb408 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -92,7 +92,7 @@ export class AuthController { ): Promise { this.logger.info('/auth - SignIn'); this.logger.info('host: ' + host); - const metadata = PackTheMetadata({ language }); + const metadata = PackTheMetadata({ language, host }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } From e886bea05c62028d9e348f4558fa8e7ce933726b Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Apr 2025 17:18:45 -0300 Subject: [PATCH 08/24] UPDATE: send origin header --- src/utils/ PackTheMetadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/ PackTheMetadata.ts b/src/utils/ PackTheMetadata.ts index ec528f6..1d17aab 100644 --- a/src/utils/ PackTheMetadata.ts +++ b/src/utils/ PackTheMetadata.ts @@ -11,7 +11,7 @@ interface IMetadata { roles?: string[]; is_data_manager?: boolean; access_token?: string; - host?: string; + origin?: string; } export function PackTheMetadata(info: IMetadata): Metadata { From 2c964253ec0ca52ff0e18de24ad77ea969946b69 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Apr 2025 17:33:34 -0300 Subject: [PATCH 09/24] UPDATE: send header in metadata --- src/modules/auth/auth.controller.ts | 5 +++-- src/utils/ PackTheMetadata.ts | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 3ffb408..9d30323 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -88,11 +88,12 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, + @Headers('origin') origin: string, @Headers('host') host: string ): Promise { this.logger.info('/auth - SignIn'); - this.logger.info('host: ' + host); - const metadata = PackTheMetadata({ language, host }); + this.logger.info('origin: ' + origin); + const metadata = PackTheMetadata({ language, origin, customHost: host }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } diff --git a/src/utils/ PackTheMetadata.ts b/src/utils/ PackTheMetadata.ts index 1d17aab..3b1e289 100644 --- a/src/utils/ PackTheMetadata.ts +++ b/src/utils/ PackTheMetadata.ts @@ -11,7 +11,9 @@ interface IMetadata { roles?: string[]; is_data_manager?: boolean; access_token?: string; + host?: string; origin?: string; + customHost?: string; } export function PackTheMetadata(info: IMetadata): Metadata { From bacbf42a04b40f5652ee4235ce231d24011548cd Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Apr 2025 18:00:51 -0300 Subject: [PATCH 10/24] UPDATE: send host --- src/modules/auth/auth.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 9d30323..e065deb 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -93,7 +93,7 @@ export class AuthController { ): Promise { this.logger.info('/auth - SignIn'); this.logger.info('origin: ' + origin); - const metadata = PackTheMetadata({ language, origin, customHost: host }); + const metadata = PackTheMetadata({ language, origin, customHost: host, host }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } From 0a5f1388297e2f83ee6a1e378a22d60bb84c7846 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 14 Apr 2025 08:30:38 -0300 Subject: [PATCH 11/24] FIX: remove host and send custom host --- src/modules/auth/auth.controller.ts | 4 +--- src/utils/ PackTheMetadata.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index e065deb..163ec64 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -88,12 +88,10 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('origin') origin: string, @Headers('host') host: string ): Promise { this.logger.info('/auth - SignIn'); - this.logger.info('origin: ' + origin); - const metadata = PackTheMetadata({ language, origin, customHost: host, host }); + const metadata = PackTheMetadata({ language, custom_host: host }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } diff --git a/src/utils/ PackTheMetadata.ts b/src/utils/ PackTheMetadata.ts index 3b1e289..82ab453 100644 --- a/src/utils/ PackTheMetadata.ts +++ b/src/utils/ PackTheMetadata.ts @@ -13,7 +13,7 @@ interface IMetadata { access_token?: string; host?: string; origin?: string; - customHost?: string; + custom_host?: string; } export function PackTheMetadata(info: IMetadata): Metadata { From f4b8addda4121fef2c4c02167c35983c2c62a0ec Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 14 Apr 2025 08:45:59 -0300 Subject: [PATCH 12/24] FIX: send origin removing http prefix --- src/modules/auth/auth.controller.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 163ec64..b31ada8 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -88,10 +88,11 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('host') host: string + @Headers('origin') origin: string ): Promise { this.logger.info('/auth - SignIn'); - const metadata = PackTheMetadata({ language, custom_host: host }); + const frontHost = origin.replace(/^https?:\/\//, ''); + const metadata = PackTheMetadata({ language, custom_host: frontHost }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } From c8a3ace0525ed2d9574f94f701ba2f3be02d17dd Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 14 Apr 2025 14:14:46 -0300 Subject: [PATCH 13/24] UPDATE: change in-factory, pi-factory, tr-factory to prod --- helmfiles/stg.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 530416b..263570f 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,11 +9,11 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.stg.dadosfera.ai + value: pi-factory.dadosfera.ai - name: maestro.in_factory_url - value: in-factory.stg.dadosfera.ai + value: in-factory.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.stg.dadosfera.ai + value: in-factory.dadosfera.ai - name: maestro.open_customer_id value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id From df8520afd3ec2c139a4b1a6e2ef4044950fdefe1 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 15 Apr 2025 14:24:41 -0300 Subject: [PATCH 14/24] FIX: send origin header --- docsfera.json | 15 ++++++-- src/modules/auth/auth.controller.ts | 49 ++++++++++++++------------ src/modules/auth/auth.service.ts | 53 +++++++++++++++++------------ 3 files changed, 72 insertions(+), 45 deletions(-) diff --git a/docsfera.json b/docsfera.json index 8765bd2..73ab232 100644 --- a/docsfera.json +++ b/docsfera.json @@ -22,7 +22,7 @@ } }, { - "name": "host", + "name": "origin", "required": true, "in": "header", "schema": { @@ -74,7 +74,7 @@ } }, { - "name": "host", + "name": "origin", "required": true, "in": "header", "schema": { @@ -265,6 +265,14 @@ ], "type": "string" } + }, + { + "name": "origin", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "responses": { @@ -5771,6 +5779,9 @@ } }, "responses": { + "200": { + "description": "" + }, "201": { "description": "", "content": { diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index b31ada8..226036a 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -54,7 +54,6 @@ import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; - @ApiTags('Auth') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @UseFilters(new GrpcToHttpExceptionFilter()) @@ -63,7 +62,6 @@ export class AuthController { logger: DadosferaLogger; redirectUrl: string; - constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, @@ -88,7 +86,7 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('origin') origin: string + @Headers('origin') origin: string, ): Promise { this.logger.info('/auth - SignIn'); const frontHost = origin.replace(/^https?:\/\//, ''); @@ -103,15 +101,15 @@ export class AuthController { async refreshAccessToken( @Body() body: AuthRefreshAccessTokenReq, @Language() language: LanguageEnum, - @Headers('host') host: string + @Headers('origin') origin: string, ) { this.logger.info('/auth - RefreshAccessToken'); const { refreshToken, customerName: customer_name } = body; - + const frontHost = origin.replace(/^https?:\/\//, ''); const metadata = PackTheMetadata({ - customer_name, language, - host + customer_name, + custom_host: frontHost, }); return this.authClient.refreshAccessToken({ refreshToken }, metadata); @@ -185,16 +183,23 @@ export class AuthController { @ApiInternalOnlyEndpoint() @Post('confirm-reset-password') @HttpCode(HttpStatus.OK) - async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) { + async confirmResetPassword( + @Body() body: AuthConfirmResetPasswordRequest, + @Headers('origin') origin: string, + ) { this.logger.info('/auth - confirm-reset-password'); - + const frontHost = origin.replace(/^https?:\/\//, ''); + const metadata = PackTheMetadata({ custom_host: frontHost }); const { username, code, newPassword } = body; - return this.authClient.confirmResetPassword({ - username, - code, - newPassword, - }); + return this.authClient.confirmResetPassword( + { + username, + code, + newPassword, + }, + metadata, + ); } @ApiInternalOnlyEndpoint() @@ -360,18 +365,18 @@ export class AuthController { async blockUsers( @Language() language: LanguageEnum, @User() user: RequestUser, - @Body() body: BulkEditRequest + @Body() body: BulkEditRequest, ) { this.logger.info('blockUsers - Starting request'); - + try { const metadata = PackTheMetadata(user); this.logger.debug('Calling blockUsers service', { metadata: { access_token: metadata.get('access_token'), - language: metadata.get('language') - } + language: metadata.get('language'), + }, }); const result = await this.authClient.blockUsers(body.users, metadata); @@ -380,7 +385,7 @@ export class AuthController { } catch (error) { this.logger.error('blockUsers - Error', { error: error.message, - stack: error.stack + stack: error.stack, }); throw error; } @@ -393,7 +398,7 @@ export class AuthController { async unblockUsers( @Language() language: LanguageEnum, @User() user: RequestUser, - @Body() body: BulkEditRequest + @Body() body: BulkEditRequest, ) { this.logger.info('unblockUsers'); const metadata = PackTheMetadata(user); @@ -408,11 +413,11 @@ export class AuthController { async resetUsers( @Language() language: LanguageEnum, @User() user: RequestUser, - @Body() body: BulkEditRequest + @Body() body: BulkEditRequest, ) { this.logger.info('resetUsers'); const metadata = PackTheMetadata(user); - return this.authClient.resetUsers(body.users, metadata); + return this.authClient.resetUsers(body.users, metadata); } } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index f99ff3a..5974954 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -113,19 +113,21 @@ export class AuthClientService implements OnModuleInit { ); } - async confirmResetPassword({ - username, - code, - newPassword, - }: AuthConfirmResetPasswordRequest) { + async confirmResetPassword( + { username, code, newPassword }: AuthConfirmResetPasswordRequest, + metadata: Metadata, + ) { this.logger.info('confirmResetPassword'); return lastValueFrom( - this.authService.AuthConfirmResetPassword({ - username, - code, - newPassword, - }), + this.authService.AuthConfirmResetPassword( + { + username, + code, + newPassword, + }, + metadata, + ), ); } @@ -170,15 +172,18 @@ export class AuthClientService implements OnModuleInit { ); } - async blockUsers(users: string[],metadata: Metadata): Promise { + async blockUsers( + users: string[], + metadata: Metadata, + ): Promise { this.logger.info('blockUsers - Service starting'); - + try { this.logger.debug('Calling BlockUser gRPC method', { metadata: { access_token: metadata.get('access_token'), - language: metadata.get('language') - } + language: metadata.get('language'), + }, }); const response = await lastValueFrom( @@ -196,26 +201,32 @@ export class AuthClientService implements OnModuleInit { } } - async unblockUsers(users: string[],metadata: Metadata): Promise { + async unblockUsers( + users: string[], + metadata: Metadata, + ): Promise { this.logger.info('unblockUsers'); return await lastValueFrom( this.authService.UnblockUser({ users }, metadata), ); } - async resetUsers(users: string[], metadata: Metadata): Promise { + async resetUsers( + users: string[], + metadata: Metadata, + ): Promise { this.logger.info('resetUsers'); - + try { this.logger.debug('Calling ResetUser gRPC method', { metadata: { access_token: metadata.get('access_token'), - language: metadata.get('language') - } + language: metadata.get('language'), + }, }); const response = await lastValueFrom( - this.authService.ResetUser({ users }, metadata) + this.authService.ResetUser({ users }, metadata), ); this.logger.info('resetUsers - Success', { response }); @@ -223,7 +234,7 @@ export class AuthClientService implements OnModuleInit { } catch (error) { this.logger.error('resetUsers - Error', { error: error.message, - stack: error.stack + stack: error.stack, }); throw error; } From 8f5cb13cc31bdad63c6c09de9730fb4fde559175 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 21 Apr 2025 18:53:34 -0300 Subject: [PATCH 15/24] FEAT: enforce mfa --- docsfera.json | 93 ++++++++++++++++++- package-lock.json | 8 +- package.json | 2 +- src/modules/auth/dtos/login.ts | 2 + src/modules/customers/customers.controller.ts | 11 +++ src/modules/customers/customers.service.ts | 10 ++ src/modules/customers/dtos/enforce-mfa.ts | 6 ++ 7 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 src/modules/customers/dtos/enforce-mfa.ts diff --git a/docsfera.json b/docsfera.json index b256567..fc85ccf 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5473,6 +5473,54 @@ ] } }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/customers/{id}/links": { "get": { "operationId": "CustomersController_getCustomerLinks", @@ -5620,6 +5668,34 @@ "Customers" ], "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, { "access-token": [] }, @@ -5921,6 +5997,9 @@ }, "themeEnabled": { "type": "boolean" + }, + "enforceMfa": { + "type": "boolean" } }, "required": [ @@ -5931,7 +6010,8 @@ "tier", "scheduleLimit", "links", - "themeEnabled" + "themeEnabled", + "enforceMfa" ] }, "AuthUser": { @@ -8600,6 +8680,17 @@ "tags" ] }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, "CustomerLink": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index 3776d14..0bf3b62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-alpha.20", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1397,9 +1397,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-alpha.20", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-alpha.20.tgz", - "integrity": "sha512-rs8YUgnwA8dTohzSEAeilU7WAscodf26/yrsqC/lq0Dz2s3Hi09TXItnLJ5ktv5xWaBQ/eZjyc5zPjpcWL3myQ==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-reoXI3vbFXUwzZ/o1tQSH22auXair53bJoozz8nIyYjYyAHtJ9+5IW3SgKyPTFrkiewziFEOwRQ3MJE0B3Haag==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index d2c3b74..c824a10 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-alpha.20", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index b3890fe..0cd90ae 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -87,6 +87,8 @@ export class AuthCustomer { @ApiProperty() themeEnabled: boolean; + @ApiProperty() + enforceMfa: boolean; } export class AuthSignInReq implements AuthSignInRequest { diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 280dce3..f9f89dc 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -7,6 +7,7 @@ import { HttpStatus, Inject, Param, + Post, Put, Query, UseFilters, @@ -17,6 +18,7 @@ import { Authenticated, RequireAllPermissions, RequireModule, + RequireSomePermission, } from 'src/decorators/authentication.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { CustomersService } from './customers.service'; @@ -24,6 +26,7 @@ import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; import { RequestUser, User } from 'src/decorators/user.decorator'; import type { StringValue } from 'ms'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { EnforceMfa } from './dtos/enforce-mfa'; @ApiTags('Customers') @Controller('customers') @@ -39,6 +42,14 @@ export class CustomersController { this.logger = dadosferaLogger.logger; } + @Post(':id/mfa') + @Authenticated() + @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) { + this.logger.info('enableMfaEnforce', { id }); + return await this.customersService.enableEnforceMfa(id, data.enabled); + } + @Get(':id/links') @Authenticated() @ApiOkResponse({ type: CustomerLinksResponse }) diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index ed302fd..056c5fb 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -34,6 +34,7 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; // This function will accept any string, which may result in a bug. @Injectable() export class CustomersService implements OnModuleInit { + private customerService: CustomersProtoService; private logger: DadosferaLogger; private pipelineReadService: ReadService.PipelineV2ReadService; @@ -203,4 +204,13 @@ export class CustomersService implements OnModuleInit { ) ); } + + async enableEnforceMfa(id: string, enabled: boolean) { + return await lastValueFrom( + this.customerService.CustomerUpdateEnforceMfa({ + customerId: id, + enforceMfa: enabled + }) + ) + } } diff --git a/src/modules/customers/dtos/enforce-mfa.ts b/src/modules/customers/dtos/enforce-mfa.ts new file mode 100644 index 0000000..4709579 --- /dev/null +++ b/src/modules/customers/dtos/enforce-mfa.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class EnforceMfa { + @ApiProperty() + enabled: boolean +} \ No newline at end of file From 9e87562e23e4dfd78dff25a53055f2846346de2e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 22 Apr 2025 09:43:34 -0300 Subject: [PATCH 16/24] FEAT: update protospack --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0bf3b62..898117c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.22", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1397,9 +1397,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "0.0.0", - "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", - "integrity": "sha512-reoXI3vbFXUwzZ/o1tQSH22auXair53bJoozz8nIyYjYyAHtJ9+5IW3SgKyPTFrkiewziFEOwRQ3MJE0B3Haag==", + "version": "3.37.0-beta.22", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.22.tgz", + "integrity": "sha512-SOu+Z5qc3ms6JjtWpHFYEgYq7Q8Y2GHBewDQj5/Evv4yxpzuANw9Eecd6zzCttV47ceM75A8rnomTj9gGkoTTg==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index c824a10..643821b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.22", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From f2437a8b3cc88d84d69b204d308e22a9d5836ebc Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 22 Apr 2025 14:46:39 -0300 Subject: [PATCH 17/24] FIX: add decorator to require danger zone module --- docsfera.json | 3 +++ src/authentication/permissions.enum.ts | 3 ++- src/modules/customers/customers.controller.ts | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index fc85ccf..1b92192 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5515,6 +5515,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 1e01531..95a313f 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -612,7 +612,8 @@ export interface DadosferaModule { export const DADOSFERA_MODULES_KEYS = { LOG_DASHBOARD: 'logs-dashboard', - ACCESS_DASHBOARD: 'access-dashboard' + ACCESS_DASHBOARD: 'access-dashboard', + DANGER_ZONE: 'danger-zone' } export const DADOSFERA_MODULES: Array = [ diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index f9f89dc..bcab557 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -45,6 +45,7 @@ export class CustomersController { @Post(':id/mfa') @Authenticated() @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @RequireModule(DADOSFERA_MODULES_KEYS.DANGER_ZONE) async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) { this.logger.info('enableMfaEnforce', { id }); return await this.customersService.enableEnforceMfa(id, data.enabled); From 7a6db36554d5d2d2a749f41f9df5d77aa51847f2 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sun, 27 Apr 2025 13:00:49 -0300 Subject: [PATCH 18/24] FEAT: generate pii reporter --- docsfera.json | 46 +++++++-- package-lock.json | 58 +++++++++++- package.json | 3 +- src/authentication/permissions.enum.ts | 3 +- src/modules/catalog/catalog.controller.ts | 83 +++++++++++----- src/modules/catalog/catalog.service.ts | 47 ++++++++++ src/template/pii-reporter.html | 109 ++++++++++++++++++++++ 7 files changed, 313 insertions(+), 36 deletions(-) create mode 100644 src/template/pii-reporter.html diff --git a/docsfera.json b/docsfera.json index 1b92192..c2b9b59 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,14 +20,6 @@ ], "type": "string" } - }, - { - "name": "origin", - "required": true, - "in": "header", - "schema": { - "type": "string" - } } ], "requestBody": { @@ -4963,6 +4955,44 @@ ] } }, + "/catalog/pii-reporter": { + "get": { + "operationId": "CatalogController_getPiiReporter", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/terms-of-use/token": { "get": { "operationId": "TermsOfUseController_getToken", diff --git a/package-lock.json b/package-lock.json index 898117c..558985c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.22", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -32,6 +32,7 @@ "cron-parser": "^4.9.0", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", + "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.5", @@ -1397,9 +1398,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.22", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.22.tgz", - "integrity": "sha512-SOu+Z5qc3ms6JjtWpHFYEgYq7Q8Y2GHBewDQj5/Evv4yxpzuANw9Eecd6zzCttV47ceM75A8rnomTj9gGkoTTg==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-3vMGyEolF/xeEeFSRUpOA8YVrbTZfWXrcMxMSCkmx411Tuuv4SkX9nHcYaXhYAkFUV9vR7vfmBfMFz4yq1n7mw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -7373,6 +7374,36 @@ "dev": true, "license": "MIT" }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/handlebars/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -12063,6 +12094,19 @@ "node": ">=4.2.0" } }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/uid": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", @@ -12558,6 +12602,12 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 643821b..ae1463a 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.22", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -50,6 +50,7 @@ "cron-parser": "^4.9.0", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", + "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.5", diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 95a313f..ee74e22 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -613,7 +613,8 @@ export interface DadosferaModule { export const DADOSFERA_MODULES_KEYS = { LOG_DASHBOARD: 'logs-dashboard', ACCESS_DASHBOARD: 'access-dashboard', - DANGER_ZONE: 'danger-zone' + DANGER_ZONE: 'danger-zone', + PII: 'pii' } export const DADOSFERA_MODULES: Array = [ diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 5f2d194..9f80e35 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -15,6 +15,7 @@ import { UseFilters, HttpException, HttpStatus, + Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -25,14 +26,14 @@ import { import { Authenticated, RequireAllPermissions, + RequireModule, RequireSomePermission, } from '../../decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { CatalogService } from './catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; -import { lastValueFrom } from 'rxjs'; import { BatchRemoveRlsRulesRequest, GetDatasetCatalogTaskRes, @@ -59,6 +60,7 @@ import { GetRlsRulesRequest, RegisterDatasetWithMetatadaRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; +import { Response } from 'express'; @ApiTags('Catalog') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -748,25 +750,23 @@ export class CatalogController { return JSON.parse(dashboards); } - @Post('register-dataset') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async registerDatasetWithMetadataRequest( @Body() body: RegisterDatasetWithMetatadaRequest, @User() user: RequestUser, ) { - const { customer_id, customer_name, user_id, username } = user; - const logMetadata = { - customer_name: customer_name, - user_id: user_id, - method: 'POST', - path: '/catalog/register-dataset', - }; + const { customer_id, customer_name, user_id, username } = user; + const logMetadata = { + customer_name: customer_name, + user_id: user_id, + method: 'POST', + path: '/catalog/register-dataset', + }; try { - const metadata = PackTheMetadata({ customer_id, customer_name, @@ -774,18 +774,20 @@ export class CatalogController { username, }); - this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata); + this.logger.log( + `Request from user ${user_id} for customer ${customer_name}`, + logMetadata, + ); // Create table metadata const tableMetadataBody = { table_metadata: body.table_metadata, info: { customer: customer_name, - }, - logMetadata: logMetadata + logMetadata: logMetadata, }; - + const table_metadata_id = await this.catalogService.createTableMetadata( tableMetadataBody, ); @@ -798,8 +800,12 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata); - const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody) + this.logger.info( + `Creating column metadata for table ${table_metadata_id}`, + logMetadata, + ); + const column_metadata_ids = + await this.catalogService.createColumnMetadata(columnMetadataBody); // Create data preview const dataPreviewBody = { @@ -808,15 +814,24 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata); + this.logger.debug( + `Creating data preview for table ${table_metadata_id}`, + logMetadata, + ); const data_preview_id = await this.catalogService.createDataPreview( dataPreviewBody, ); // Catalog dataset item - this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata); + this.logger.info( + `Cataloging dataset item for table ${table_metadata_id}`, + logMetadata, + ); await this.catalogService.catalogDatasetItem(table_metadata_id, metadata); - this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata); + this.logger.info( + `Dataset registration completed successfully for table ${table_metadata_id}`, + logMetadata, + ); return { message: 'Dataset registered successfully', table_metadata_id: table_metadata_id, @@ -824,7 +839,10 @@ export class CatalogController { data_preview_id: data_preview_id, }; } catch (error) { - this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata); + this.logger.error( + `Failed to register dataset. The following error occurred: ${error.response.data}`, + logMetadata, + ); throw new HttpException( { @@ -837,4 +855,25 @@ export class CatalogController { ); } } + + @Get('pii-reporter') + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule( + DADOSFERA_MODULES_KEYS.PII + ) + async getPiiReporter(@User() user: RequestUser, @Res() res: Response) { + this.logger.info('GET pii-reporter'); + const metadata = PackTheMetadata(user); + + try { + const reporterHtml = await this.catalogService.getPiiReporter(metadata); + + res.setHeader('Content-Type', 'text/html'); + return res.send(reporterHtml); + } catch (error) { + this.logger.error(error); + } + } } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index adfa0a2..ac7f1d6 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -34,6 +34,8 @@ import { GetNimbusDashboardsRequest, GetRlsRulesRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; +import { readFileSync } from 'fs'; +import path from 'path'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -80,6 +82,51 @@ class CatalogService implements OnModuleInit { )}.dadosfera.ai`; } + async getPiiReporter(metadata: Metadata) { + const { + data + } = await lastValueFrom( + this.catalogWriteService.GetPiiReporter({}, metadata) + ) + + const result = data.map((props) => ({ + ...props, + pii_rules: this.formatPiiRules(JSON.parse(props.pii_rules)), + })); + + // 3. Ler o template HTML + const templatePath = path.join( + __dirname, + '../../../src/template/pii-reporter.html', + ); + const templateSource = readFileSync(templatePath, 'utf8'); + + // 4. Compilar o template + const template = Handlebars.compile(templateSource); + + // 5. Aplicar os dados ao template + return template({ + dados: result, + dataGeracao: new Date().toLocaleString('pt-BR'), + }); + } + + private formatPiiRules( + data: Record< + string, + { + avg_score: number; + max_score: number; + } + >, + ) { + return Object.keys(data) + .filter((key) => { + return data[key].max_score > 0.7; + }) + .join(', '); + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/template/pii-reporter.html b/src/template/pii-reporter.html new file mode 100644 index 0000000..637332a --- /dev/null +++ b/src/template/pii-reporter.html @@ -0,0 +1,109 @@ + + + + + + Dadosfera Relatório de PII + + + + +
+
+ +

Relatório de PII

+
+ +
+

Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.

+
+ + + + + + + + + + + + + + {{#each dados}} + + + + + + + + + {{/each}} + +
DatabaseSchemaTabelaColunaTipo de DadoRegras PII
{{database_name}}{{table_schema}}{{table_name}}{{column_name}}{{data_type}}{{pii_rules}}
+ +

Gerado em: {{dataGeracao}}

+
+ + \ No newline at end of file From 58e3c7be94f9521a4bb2b6ced71f1bbe28645e55 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 29 Apr 2025 14:00:23 -0300 Subject: [PATCH 19/24] FEAT: pii --- Dockerfile | 12 +- docsfera.json | 16 + package-lock.json | 709 +++++++++++++++++++++- package.json | 4 +- src/modules/catalog/catalog.controller.ts | 23 +- src/modules/catalog/catalog.service.ts | 84 +-- src/modules/catalog/dtos/pii.dto.ts | 8 + src/template/pii-reporter.html | 7 +- src/utils/ PackTheMetadata.ts | 1 + src/utils/FileParser/csv-parser.ts | 14 + src/utils/FileParser/html-parser.ts | 16 + src/utils/FileParser/pdf-parser.ts | 48 ++ 12 files changed, 883 insertions(+), 59 deletions(-) create mode 100644 src/modules/catalog/dtos/pii.dto.ts create mode 100644 src/utils/FileParser/csv-parser.ts create mode 100644 src/utils/FileParser/html-parser.ts create mode 100644 src/utils/FileParser/pdf-parser.ts diff --git a/Dockerfile b/Dockerfile index 79bae4a..3186c71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,19 @@ FROM base_image AS build_base WORKDIR /app RUN apk update # needed packages to build dependencies from source -RUN apk add --no-cache aws-cli +RUN apk add --no-cache \ + aws-cli \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont COPY package*.json ./ +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + # run aws cli without mounting secret, because CI already has AWS credentials FROM build_base AS ci_image diff --git a/docsfera.json b/docsfera.json index c2b9b59..b8d9671 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,6 +20,14 @@ ], "type": "string" } + }, + { + "name": "origin", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -4970,6 +4978,14 @@ ], "type": "string" } + }, + { + "name": "type", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { diff --git a/package-lock.json b/package-lock.json index 558985c..c9d852f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -30,6 +30,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cron-parser": "^4.9.0", + "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", "handlebars": "^4.7.8", @@ -44,6 +45,7 @@ "passport-google-oauth20": "^2.0.0", "passport-hubspot-oauth2": "^1.0.3", "passport-mailchimp": "^1.1.0", + "puppeteer": "^24.7.2", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.5.5", @@ -1398,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "0.0.0", - "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", - "integrity": "sha512-3vMGyEolF/xeEeFSRUpOA8YVrbTZfWXrcMxMSCkmx411Tuuv4SkX9nHcYaXhYAkFUV9vR7vfmBfMFz4yq1n7mw==", + "version": "3.37.0-beta.23", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", + "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -2946,6 +2948,27 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.2.tgz", + "integrity": "sha512-i4Ez+s9oRWQbNjtI/3+jxr7OH508mjAKvza0ekPJem0ZtmsYHP3B5dq62+IaBHKaGCOuqJxXzvFLUhJvQ6jtsQ==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.0", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.1", + "tar-fs": "^3.0.8", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.6", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz", @@ -3541,6 +3564,12 @@ "node": ">= 6" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -3986,6 +4015,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", @@ -4636,6 +4675,18 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4708,6 +4759,12 @@ "form-data": "^4.0.0" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, "node_modules/babel-jest": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.5.1.tgz", @@ -4814,6 +4871,78 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.1.3.tgz", + "integrity": "sha512-OeEZYIg+2qepaWLyphaOXHAHKo3xkM8y3BeGAvHdMN8GNWvEAU1Yw6rYpGzu/wDDbKxgEjVeVDpgGhDzaeMpjg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.1.tgz", + "integrity": "sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -4855,6 +4984,15 @@ "node": ">= 0.8" } }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -5058,6 +5196,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -5244,6 +5391,19 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-4.1.1.tgz", + "integrity": "sha512-biR7t4vF3YluE6RlMSk9IWk+b9U+WWyzHp+N2pL9vRTk+UXHYRTVp7jTK58ZNzMLBgoLMHY4QyJMbeuw3eKxqg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -5688,6 +5848,48 @@ "dev": true, "license": "MIT" }, + "node_modules/csv": { + "version": "6.3.11", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.11.tgz", + "integrity": "sha512-a8bhT76Q546jOElHcTrkzWY7Py925mfLO/jqquseH61ThOebYwOjLbWHBqdRB4K1VpU36sTyIei6Jwj7QdEZ7g==", + "license": "MIT", + "dependencies": { + "csv-generate": "^4.4.2", + "csv-parse": "^5.6.0", + "csv-stringify": "^6.5.2", + "stream-transform": "^3.3.3" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.2.tgz", + "integrity": "sha512-W6nVsf+rz0J3yo9FOjeer7tmzBJKaTTxf7K0uw6GZgRocZYPVpuSWWa5/aoWWrjQZj4/oNIKTYapOM7hiNjVMA==", + "license": "MIT" + }, + "node_modules/csv-parse": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", + "license": "MIT" + }, + "node_modules/csv-stringify": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", + "integrity": "sha512-RFPahj0sXcmUyjrObAK+DOWtMvMIFV328n4qZJhgX3x2RqkQgOTU2mCUmiFR0CzM6AzChlRSUErjiJeEt8BaQA==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -5802,6 +6004,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -5840,6 +6056,12 @@ "node": ">=8" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1425554", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz", + "integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==", + "license": "BSD-3-Clause" + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -6142,6 +6364,15 @@ "node": ">=10.13.0" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/error-callsites": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/error-callsites/-/error-callsites-2.0.4.tgz", @@ -6251,7 +6482,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "esprima": "^4.0.1", @@ -6273,7 +6503,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=4.0" @@ -6283,7 +6512,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, "license": "BSD-3-Clause", "optional": true, "engines": { @@ -6494,7 +6722,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -6561,7 +6788,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" @@ -6791,6 +7017,41 @@ "node": ">=4" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -6804,6 +7065,12 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -6900,6 +7167,15 @@ "bser": "2.1.1" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -7273,6 +7549,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-uri": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", + "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -7755,6 +8045,25 @@ "node": ">= 0.10" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "license": "MIT", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8753,6 +9062,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "license": "MIT" + }, "node_modules/jsdom": { "version": "16.7.0", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", @@ -9442,6 +9757,12 @@ "node": ">=8" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mixpanel": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz", @@ -9553,6 +9874,15 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/next-line": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", @@ -9906,6 +10236,73 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -10186,6 +10583,12 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -10403,6 +10806,15 @@ "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==", "license": "MIT" }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -10464,6 +10876,75 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -10502,6 +10983,91 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "24.7.2", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.7.2.tgz", + "integrity": "sha512-ifYqoY6wGs0yZeFuFPn8BE9FhuveXkarF+eO18I2e/axdoCh4Qh1AE+qXdJBhdaeoPt6eRNTY4Dih29Jbq8wow==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.2", + "chromium-bidi": "4.1.1", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1425554", + "puppeteer-core": "24.7.2", + "typed-query-selector": "^2.12.0" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "24.7.2", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.7.2.tgz", + "integrity": "sha512-P9pZyTmJqKODFCnkZgemCpoFA4LbAa8+NumHVQKyP5X9IgdNS1ZnAnIh1sMAwhF8/xEUGf7jt+qmNLlKieFw1Q==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.2", + "chromium-bidi": "4.1.1", + "debug": "^4.4.0", + "devtools-protocol": "0.0.1425554", + "typed-query-selector": "^2.12.0", + "ws": "^8.18.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", @@ -11179,6 +11745,53 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.4.tgz", + "integrity": "sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ==", + "license": "MIT", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/sonic-boom": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", @@ -11286,6 +11899,12 @@ "readable-stream": "^3.0.6" } }, + "node_modules/stream-transform": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.3.tgz", + "integrity": "sha512-dALXrXe+uq4aO5oStdHKlfCM/b3NBdouigvxVPxCdrMRAU6oHh3KNss20VbTPQNQmjAHzZGKGe66vgwegFEIog==", + "license": "MIT" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -11294,6 +11913,19 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -11541,6 +12173,31 @@ "node": ">=6" } }, + "node_modules/tar-fs": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", + "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", @@ -11650,6 +12307,15 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -12065,6 +12731,12 @@ "node": ">= 0.6" } }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -12740,6 +13412,16 @@ "node": ">=12" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -12762,6 +13444,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.3.tgz", + "integrity": "sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index ae1463a..fd3eb0b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -48,6 +48,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cron-parser": "^4.9.0", + "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", "handlebars": "^4.7.8", @@ -62,6 +63,7 @@ "passport-google-oauth20": "^2.0.0", "passport-hubspot-oauth2": "^1.0.3", "passport-mailchimp": "^1.1.0", + "puppeteer": "^24.7.2", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.5.5", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 9f80e35..ab6290a 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -346,7 +346,7 @@ export class CatalogController { @Language() language: LanguageEnum, @Param('id') id: string, ): Promise { - const { customer_name, customer_id, user_id, username } = user; + const { customer_name, customer_id, user_id, username, customer_modules } = user; this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, @@ -359,6 +359,7 @@ export class CatalogController { user_id, username, language, + is_mask: customer_modules.some(mod => mod === 'pii') }); const preview = await this.catalogService.getDatasetPreview(id, metadata); @@ -863,15 +864,27 @@ export class CatalogController { @RequireModule( DADOSFERA_MODULES_KEYS.PII ) - async getPiiReporter(@User() user: RequestUser, @Res() res: Response) { + async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') type: "html" | "csv") { this.logger.info('GET pii-reporter'); const metadata = PackTheMetadata(user); try { - const reporterHtml = await this.catalogService.getPiiReporter(metadata); + const reporter = await this.catalogService.getPiiReporter(metadata, type); - res.setHeader('Content-Type', 'text/html'); - return res.send(reporterHtml); + if (type === "html") { + res.setHeader('Content-Type', 'text/html'); + } else if (type === "csv") { + res.setHeader('Content-Type', 'text/csv'); + } else if (type === "pdf") { + res.setHeader('Content-Type', 'application/pdf'); + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `relatorio-pii-${timestamp}.${type}`; + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + + return res.send(reporter); } catch (error) { this.logger.error(error); } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index ac7f1d6..68036c3 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -34,8 +34,12 @@ import { GetNimbusDashboardsRequest, GetRlsRulesRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; -import { readFileSync } from 'fs'; +import { writeFileSync } from 'fs'; import path from 'path'; +import { HtmlParser } from 'src/utils/FileParser/html-parser'; +import { PiiDto } from './dtos/pii.dto'; +import { CsvParser } from 'src/utils/FileParser/csv-parser'; +import { PDFParser } from 'src/utils/FileParser/pdf-parser'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -82,49 +86,47 @@ class CatalogService implements OnModuleInit { )}.dadosfera.ai`; } - async getPiiReporter(metadata: Metadata) { - const { - data - } = await lastValueFrom( - this.catalogWriteService.GetPiiReporter({}, metadata) - ) + async getPiiReporter(metadata: Metadata, type: "html"| "csv") { + try { + const { + data + } = await lastValueFrom( + this.catalogWriteService.GetPiiReporter({}, metadata) + ) + this.logger.info("Finish grpc call") + + // 3. Ler o template HTML + const templatePath = path.join( + __dirname, + '../../../src/template/pii-reporter.html', + ); - const result = data.map((props) => ({ - ...props, - pii_rules: this.formatPiiRules(JSON.parse(props.pii_rules)), - })); + this.logger.info("templatePath: " + templatePath); - // 3. Ler o template HTML - const templatePath = path.join( - __dirname, - '../../../src/template/pii-reporter.html', - ); - const templateSource = readFileSync(templatePath, 'utf8'); - - // 4. Compilar o template - const template = Handlebars.compile(templateSource); - - // 5. Aplicar os dados ao template - return template({ - dados: result, - dataGeracao: new Date().toLocaleString('pt-BR'), - }); - } - - private formatPiiRules( - data: Record< - string, - { - avg_score: number; - max_score: number; + if (type === "html") { + const htmlParser = new HtmlParser(templatePath); + return htmlParser.parse(data); + } else if (type === "csv") { + const csvParser = new CsvParser(); + return await csvParser.parse(data) + } else if (type === "pdf") { + const htmlParser = new HtmlParser(templatePath); + const pdfParser = new PDFParser(); + const pdf = await pdfParser.parse(htmlParser.parse(data)); + const pdfPath = path.join( + __dirname, + '../../../src/template/teste.pdf', + ); + writeFileSync(pdfPath, pdf); + return pdf } - >, - ) { - return Object.keys(data) - .filter((key) => { - return data[key].max_score > 0.7; - }) - .join(', '); + + throw new Error("Unsupported Type: " + type) + + } catch (error) { + this.logger.error(error); + } + } async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { diff --git a/src/modules/catalog/dtos/pii.dto.ts b/src/modules/catalog/dtos/pii.dto.ts new file mode 100644 index 0000000..c456b2b --- /dev/null +++ b/src/modules/catalog/dtos/pii.dto.ts @@ -0,0 +1,8 @@ +export class PiiDto { + database_name: string; + table_schema: string; + table_name: string; + column_name: string; + data_type: string; + pii_rules: string; +} diff --git a/src/template/pii-reporter.html b/src/template/pii-reporter.html index 637332a..d4b4505 100644 --- a/src/template/pii-reporter.html +++ b/src/template/pii-reporter.html @@ -6,6 +6,10 @@ Dadosfera Relatório de PII diff --git a/src/utils/FileParser/csv-parser.ts b/src/utils/FileParser/csv-parser.ts index 61e1ae1..c481710 100644 --- a/src/utils/FileParser/csv-parser.ts +++ b/src/utils/FileParser/csv-parser.ts @@ -1,11 +1,11 @@ import { stringify } from 'csv' +import { Parser } from './parser.interface'; -export class CsvParser { +export class CsvParser implements Parser{ async parse(data: T[]) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { stringify(data, { header: true }, function (err, output) { if (err) reject(err); - console.log(output) resolve(output) }); diff --git a/src/utils/FileParser/html-parser.ts b/src/utils/FileParser/html-parser.ts index 7e0b23c..0e7f155 100644 --- a/src/utils/FileParser/html-parser.ts +++ b/src/utils/FileParser/html-parser.ts @@ -1,9 +1,10 @@ import { readFileSync } from 'fs'; import Handlebars from 'handlebars'; +import { Parser } from './parser.interface'; -export class HtmlParser { +export class HtmlParser implements Parser { constructor(private readonly templatePath: string) {} - parse(data: T[]) { + async parse(data: T[]) { const templateSource = readFileSync(this.templatePath, 'utf8'); const template = Handlebars.compile(templateSource); diff --git a/src/utils/FileParser/parser-types.ts b/src/utils/FileParser/parser-types.ts new file mode 100644 index 0000000..021ac28 --- /dev/null +++ b/src/utils/FileParser/parser-types.ts @@ -0,0 +1 @@ +export type TypeParser = "html" | "csv" | "pdf" \ No newline at end of file diff --git a/src/utils/FileParser/parser.builder.ts b/src/utils/FileParser/parser.builder.ts new file mode 100644 index 0000000..3d70c2e --- /dev/null +++ b/src/utils/FileParser/parser.builder.ts @@ -0,0 +1,29 @@ +import path from 'path'; +import { TypeParser } from './parser-types'; +import { HtmlParser } from './html-parser'; +import { CsvParser } from './csv-parser'; +import { PDFParser } from './pdf-parser'; +import { Parser } from './parser.interface'; + +export class ParserBuilder { + static build(type: TypeParser): Parser { + const templatePath = path.join( + __dirname, + '../../../src/template/pii-reporter.html', + ); + + if (type === 'html') { + const htmlParser = new HtmlParser(templatePath); + return htmlParser; + } else if (type === 'csv') { + const csvParser = new CsvParser(); + return csvParser; + } else if (type === 'pdf') { + const htmlParser = new HtmlParser(templatePath); + const pdfParser = new PDFParser(htmlParser); + return pdfParser; + } + + throw new Error('Unsupported Type: ' + type); + } +} diff --git a/src/utils/FileParser/parser.interface.ts b/src/utils/FileParser/parser.interface.ts new file mode 100644 index 0000000..26af780 --- /dev/null +++ b/src/utils/FileParser/parser.interface.ts @@ -0,0 +1,3 @@ +export interface Parser { + parse(data: T[]): Promise +} \ No newline at end of file diff --git a/src/utils/FileParser/pdf-parser.ts b/src/utils/FileParser/pdf-parser.ts index bf0433a..1294b34 100644 --- a/src/utils/FileParser/pdf-parser.ts +++ b/src/utils/FileParser/pdf-parser.ts @@ -1,30 +1,39 @@ -import puppeteer from "puppeteer"; +import puppeteer from 'puppeteer'; +import { Parser } from './parser.interface'; +import { HtmlParser } from './html-parser'; -export class PDFParser { - async parse(data: T) { +export class PDFParser implements Parser { + constructor(private htmlParser: HtmlParser) {} + + async parse(data: T[]) { try { const browser = await puppeteer.launch({ executablePath: process.env.PUPPETEER_EXECUTABLE_PATH, - args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'] + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + ], }); - + const page = await browser.newPage(); page.setDefaultNavigationTimeout(0); - - await page.setContent(data, { + + const html = await this.htmlParser.parse(data); + await page.setContent(html, { waitUntil: 'networkidle0', - timeout: 30000 + timeout: 30000, }); // Configurações adicionais para garantir um PDF válido - const pdfBuffer = await page.pdf({ + const pdf = await page.pdf({ format: 'A4', printBackground: true, margin: { top: '20mm', right: '15mm', bottom: '20mm', - left: '15mm' + left: '15mm', }, preferCSSPageSize: true, displayHeaderFooter: false, @@ -34,12 +43,12 @@ export class PDFParser { pageRanges: '', tagged: true, outline: false, - timeout: 30000 + timeout: 30000, }); - + await browser.close(); - - return pdfBuffer; + + return Buffer.from(pdf); } catch (error) { console.error('Error generating PDF:', error); throw error; From 0b525b835fa9643e4a5b88f0370cc28bb541efdc Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 29 Apr 2025 15:40:37 -0300 Subject: [PATCH 22/24] REFACTOR: move template to assets folder --- nest-cli.json | 1 + src/{ => assets}/template/pii-reporter.html | 0 src/utils/FileParser/parser.builder.ts | 6 +----- 3 files changed, 2 insertions(+), 5 deletions(-) rename src/{ => assets}/template/pii-reporter.html (100%) diff --git a/nest-cli.json b/nest-cli.json index 69ccf04..5085879 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -4,6 +4,7 @@ "compilerOptions": { "assets": [ "**/*.proto", + "assets/**/*", { "include": "i18n/**/*", "watchAssets": true diff --git a/src/template/pii-reporter.html b/src/assets/template/pii-reporter.html similarity index 100% rename from src/template/pii-reporter.html rename to src/assets/template/pii-reporter.html diff --git a/src/utils/FileParser/parser.builder.ts b/src/utils/FileParser/parser.builder.ts index 3d70c2e..7e2d5d9 100644 --- a/src/utils/FileParser/parser.builder.ts +++ b/src/utils/FileParser/parser.builder.ts @@ -1,4 +1,3 @@ -import path from 'path'; import { TypeParser } from './parser-types'; import { HtmlParser } from './html-parser'; import { CsvParser } from './csv-parser'; @@ -7,10 +6,7 @@ import { Parser } from './parser.interface'; export class ParserBuilder { static build(type: TypeParser): Parser { - const templatePath = path.join( - __dirname, - '../../../src/template/pii-reporter.html', - ); + const templatePath = '/app/dist/assets/template/pii-reporter.html'; if (type === 'html') { const htmlParser = new HtmlParser(templatePath); From fb491328d65179fecee20fb61d6cce93214d83b4 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 29 Apr 2025 16:20:37 -0300 Subject: [PATCH 23/24] FIX: install chromiun in base_image --- Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3186c71..eac9682 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,4 +50,17 @@ WORKDIR /app COPY --from=prod_build /app/dist ./dist COPY --from=prod_build /app/node_modules ./node_modules COPY --from=prod_build /app/package*.json ./ +RUN apk update +# needed packages to build dependencies from source +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + ENTRYPOINT npm run start:prod From e4eb388e1871d10f41edfe0ebef6e9497b5fdc07 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 9 May 2025 11:06:23 -0300 Subject: [PATCH 24/24] FIX: stg to prd for sbm environment --- helmfiles/stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 0aac3e7..5782e36 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,7 +9,7 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.stg.dadosfera.ai + value: pi-factory.dadosfera.ai - name: maestro.in_factory_url value: in-factory.stg.dadosfera.ai - name: maestro.tr_factory_url