From 5a36762544bed596222da329822a7ab4acf974bc Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 26 May 2025 10:24:48 -0300 Subject: [PATCH 01/21] UPDATE: route /catalog/register-dataset will have an limit of 10mb --- src/main.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.ts b/src/main.ts index 46f1701..7511269 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import helmet from 'helmet'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { json, urlencoded } from 'express'; import { AppModule } from './app.module'; import { writeFileSync } from 'fs'; @@ -25,6 +26,8 @@ async function bootstrap() { }, }); app.use(helmet()); + app.use('/catalog/register-dataset', json({ limit: '10mb' })); + app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); configureSwagger(app); await app.listen(3333); if (process.env.KILL_AFTER_START) await app.close(); From 01263017e7612de36accf04239f61d450a0796de Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Jun 2025 14:24:26 -0300 Subject: [PATCH 02/21] FIX: comment block list --- src/modules/open-data/open-data.controller.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index b8951ed..e952838 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -45,11 +45,11 @@ export class OpenDataController { ) { this.logger.info('createUser for open data' + JSON.stringify(request.headers)); - const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"]; - if (!corslist.includes(origin)) { - this.logger.info('block request by cors list: '+ origin); - throw new ForbiddenException(); - } + // const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"]; + // if (!corslist.includes(origin)) { + // this.logger.info('block request by cors list: '+ origin); + // throw new ForbiddenException(); + // } const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID; From fea587ad366e8081eaa328a381d0c85c9922cb2c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 11:51:56 -0300 Subject: [PATCH 03/21] FEAT: control login by dedicated customer id injected --- environment.d.ts | 1 + helmfiles/stg.yaml | 26 +++++++++++++++++++++ src/modules/auth/auth.controller.ts | 4 +--- src/modules/auth/auth.service.ts | 35 +++++++++++++++++++++++++---- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/environment.d.ts b/environment.d.ts index c910516..3558f26 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -14,6 +14,7 @@ declare global { AWS_REGION: string; OPEN_GROUP_ID: string; OPEN_CUSTOMER_ID: string; + DEDICATED_PROXY: string; } } } diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 5782e36..616588e 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -18,5 +18,31 @@ charts: value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 + - name: replicaCount + value: 1 + + # Environment to test Network Policies + - name: private + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: maestro.duc_url + value: duc.stg.dadosfera.ai + - name: hostname + value: private.stg.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory.stg.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d + - name: maestro.open_group_id + value: e3f98a2f-7748-4981-8505-7695c8ca8218 + # Customer id + - name: maestro.dedicated_proxy + value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd - name: replicaCount value: 1 \ No newline at end of file diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 7f2f8f0..66870d5 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -86,11 +86,9 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('origin') origin = '', ): Promise { this.logger.info('/auth - SignIn'); - const frontHost = origin.replace(/^https?:\/\//, ''); - const metadata = PackTheMetadata({ language, custom_host: frontHost }); + const metadata = PackTheMetadata({ language }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 5974954..be71d66 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -1,4 +1,4 @@ -import { OnModuleInit, Inject, Injectable } from '@nestjs/common'; +import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { lastValueFrom } from 'rxjs'; @@ -17,6 +17,7 @@ import { AuthResetPasswordRequest, AuthVerifyResetPasswordCodeRequest, AuthConfirmResetPasswordRequest, + AuthSignInResponse, } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; @@ -53,15 +54,41 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } + checkDedicatedProxy(customerId: string) { + const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; + this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); + if (DEDICATED_PROXY !== '') { + this.logger.info('Customer ID: ' + customerId); + if (DEDICATED_PROXY !== customerId) { + throw new ForbiddenException(); + } + } + } + async signIn( { username, password, totp }: AuthSignInRequest, metadata: Metadata, ) { this.logger.info('SignIn'); - return lastValueFrom( - this.authService.AuthSignIn({ username, password, totp }, metadata), - ); + let result: AuthSignInResponse; + + try { + result = await lastValueFrom( + this.authService.AuthSignIn({ username, password, totp }, metadata), + ); + + } catch (error) { + this.logger.error('SignIn - Error during sign-in'); + this.logger.error(error); + throw error; + } + + if (result.customer) { + this.checkDedicatedProxy(result.customer.id); + } + + return result } async refreshAccessToken( From 7f9755493a9d2a7b8ea1d3074d1c1d1536c6d7a9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 12:29:51 -0300 Subject: [PATCH 04/21] FIX: experimental hostname --- helmfiles/stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 616588e..297c5aa 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -30,7 +30,7 @@ charts: - name: maestro.duc_url value: duc.stg.dadosfera.ai - name: hostname - value: private.stg.dadosfera.ai + value: private-maestro.stg.dadosfera.ai - name: maestro.pi_factory_url value: pi-factory.dadosfera.ai - name: maestro.in_factory_url From 41842f7c6afc2ed5f5e63058d09efb366932a666 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 13:56:57 -0300 Subject: [PATCH 05/21] FIX: chart private maestro --- helmfiles/stg.yaml | 4 +++- maestro/templates/deployment.yaml | 14 +++++++------- maestro/templates/ingress-private.yaml | 4 ++-- maestro/templates/ingress.yaml | 6 +++--- maestro/templates/secret.yaml | 6 +++--- maestro/templates/service.yaml | 8 ++++---- maestro/values.yaml | 2 ++ 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 297c5aa..8039c29 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -22,11 +22,13 @@ charts: value: 1 # Environment to test Network Policies - - name: private + - name: private-maestro chart: ../maestro values: - ../maestro/values.yaml set: + - name: app_name + value: maestro-private - name: maestro.duc_url value: duc.stg.dadosfera.ai - name: hostname diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index c648514..6629db6 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: maestro + name: {{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - app: maestro + app: {{ .Values.app_name }} strategy: rollingUpdate: @@ -20,7 +20,7 @@ spec: template: metadata: labels: - app: maestro + app: {{ .Values.app_name }} spec: imagePullSecrets: @@ -108,15 +108,15 @@ spec: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_SECRET_ACCESS_KEY - name: AWS_DEFAULT_REGION valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_DEFAULT_REGION diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index 97708bc..b729904 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -10,7 +10,7 @@ metadata: generation: 1 labels: - app: maestro + app: {{ .Values.app_name }} name: open-data namespace: applications spec: @@ -21,7 +21,7 @@ spec: paths: - backend: service: - name: maestro + name: {{ .Values.app_name }} port: number: {{ .Values.ingress.port }} path: /open-data/sharing-ocean-data diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index faf1305..78b360c 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -9,8 +9,8 @@ metadata: generation: 1 labels: - app: maestro - name: maestro + app: {{ .Values.app_name }} + name: {{ .Values.app_name }} namespace: applications spec: ingressClassName: nginx @@ -20,7 +20,7 @@ spec: paths: - backend: service: - name: maestro + name: {{ .Values.app_name }} port: number: {{ .Values.ingress.port }} path: / diff --git a/maestro/templates/secret.yaml b/maestro/templates/secret.yaml index 9e2fd37..3bbf5ec 100644 --- a/maestro/templates/secret.yaml +++ b/maestro/templates/secret.yaml @@ -1,17 +1,17 @@ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: prd-maestro + name: prd-{{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: refreshInterval: 1h secretStoreRef: name: secretsmanager-prd kind: SecretStore target: - name: prd-maestro + name: prd-{{ .Values.app_name }} creationPolicy: Owner data: - secretKey: AWS_ACCESS_KEY_ID diff --git a/maestro/templates/service.yaml b/maestro/templates/service.yaml index 75f635a..1f7c988 100644 --- a/maestro/templates/service.yaml +++ b/maestro/templates/service.yaml @@ -1,18 +1,18 @@ apiVersion: v1 kind: Service metadata: - name: maestro + name: {{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: type: ClusterIP ports: - - name: maestro + - name: {{ .Values.app_name }} protocol: TCP port: {{ .Values.service.port }} targetPort: {{ .Values.service.targetPort }} selector: - app: maestro + app: {{ .Values.app_name }} diff --git a/maestro/values.yaml b/maestro/values.yaml index 14dd9e4..e0ec85c 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -9,6 +9,7 @@ image: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: 1.56.0 +app_name: maestro containerPort: 3333 imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret" service: @@ -42,6 +43,7 @@ maestro: upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 + dedicated_proxy: "" autoscaling: enabled: false minReplicas: 1 From 9e597fadba9e4d69d01f02a06dafd02631d7e806 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 14:04:45 -0300 Subject: [PATCH 06/21] FIX: open data ingress route --- maestro/templates/ingress-private.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index b729904..70f8eb6 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -11,7 +11,7 @@ metadata: generation: 1 labels: app: {{ .Values.app_name }} - name: open-data + name: open-data-{{ .Values.app_name }} namespace: applications spec: ingressClassName: nginx From 17de31f1a65aa6c33c8d132844bd7f2bc40c1eef Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 14:45:13 -0300 Subject: [PATCH 07/21] FIX: send env DEDICATED_PROXY --- maestro/templates/deployment.yaml | 2 ++ maestro/templates/ingress-private.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 6629db6..ac9eaae 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -100,6 +100,8 @@ spec: value: {{ .Values.maestro.open_customer_id }} - name: OPEN_GROUP_ID value: {{ .Values.maestro.open_group_id }} + - name: DEDICATED_PROXY + value: {{ .Values.maestro.dedicated_proxy }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index 70f8eb6..1ace950 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -11,7 +11,11 @@ metadata: generation: 1 labels: app: {{ .Values.app_name }} + {{- if .Values.maestro.dedicated_proxy}} name: open-data-{{ .Values.app_name }} + {{- else }} + name: open-data + {{- end }} namespace: applications spec: ingressClassName: nginx From cb98099a913e026efb5ab94518deda110697c1ec Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 13 Jun 2025 15:07:50 -0300 Subject: [PATCH 08/21] FIX: update timeout --- src/utils/FileParser/pdf-parser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/FileParser/pdf-parser.ts b/src/utils/FileParser/pdf-parser.ts index 1294b34..a954708 100644 --- a/src/utils/FileParser/pdf-parser.ts +++ b/src/utils/FileParser/pdf-parser.ts @@ -22,7 +22,7 @@ export class PDFParser implements Parser { const html = await this.htmlParser.parse(data); await page.setContent(html, { waitUntil: 'networkidle0', - timeout: 30000, + timeout: 90000, }); // Configurações adicionais para garantir um PDF válido @@ -43,7 +43,7 @@ export class PDFParser implements Parser { pageRanges: '', tagged: true, outline: false, - timeout: 30000, + timeout: 90000, }); await browser.close(); From 0aa9c065a5398d6443f5626763b94f25a6ada7fe Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 16 Jun 2025 10:34:11 -0300 Subject: [PATCH 09/21] FEAT: block user from login when maestro is dedicated --- src/authentication/authentication.guard.ts | 13 +++++++++++++ src/modules/auth/auth.service.ts | 22 +++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index bc69f8c..a41bc74 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -4,6 +4,7 @@ import { OnApplicationBootstrap, ExecutionContext, Inject, + ForbiddenException, } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import assert from 'assert'; @@ -113,6 +114,18 @@ export class AuthenticationGuard return false; } + // Bloquear outros customer de usar o maestor dedicado + const DEDICATED_PROXY = process.env.DEDICATED_PROXY + if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) { + throw new ForbiddenException(); + } + + // Bloquear o customer de acesso o maestro publico + const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(); + } + request.accessTokenPayload = accessTokenPayload; request.user = { user_id: accessTokenPayload.user_id, diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index be71d66..a8ceedd 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -54,14 +54,22 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } - checkDedicatedProxy(customerId: string) { + checkDedicatedProxy({ + customer + }: AuthSignInResponse) { const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); - if (DEDICATED_PROXY !== '') { - this.logger.info('Customer ID: ' + customerId); - if (DEDICATED_PROXY !== customerId) { - throw new ForbiddenException(); - } + this.logger.info('Customer ID: ' + customer.id); + + if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) { + throw new ForbiddenException(); + } + + // Bloquear o customer de acesso o maestro publico + this.logger.info('Check if customer have network policy: ' + customer.modules); + const hasNetworkPolicyModule = customer.modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(); } } @@ -85,7 +93,7 @@ export class AuthClientService implements OnModuleInit { } if (result.customer) { - this.checkDedicatedProxy(result.customer.id); + this.checkDedicatedProxy(result); } return result From 19d748ae09f0b70f79cb15f06b1d6dfe43196042 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 14:47:39 -0300 Subject: [PATCH 10/21] CHORE: add restricted ip and unimed chart --- helmfiles/prd.yaml | 31 +++++++++++++++++++++++++++++++ helmfiles/stg.yaml | 4 +++- maestro/templates/ingress.yaml | 3 +++ maestro/values.yaml | 1 + 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index e4146e8..c426fa9 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -4,6 +4,8 @@ charts: values: - ../maestro/values.yaml set: + - name: app_name + value: maestro - name: maestro.duc_url value: duc.dadosfera.ai - name: hostname @@ -20,3 +22,32 @@ charts: value: c0afdcce-c5be-40d0-9d1d-2d271121f14a - name: replicaCount value: 2 + + - name: unimed + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: app_name + value: maestro-unimed + - name: maestro.duc_url + value: duc.dadosfera.ai + - name: hostname + value: maestro.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d + - name: maestro.open_group_id + value: c0afdcce-c5be-40d0-9d1d-2d271121f14a + # Customer id + - name: maestro.dedicated_proxy + value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd + - name: replicaCount + value: 1 + - name: restricted-ip + value: "177.52.172.0/24" diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 8039c29..089ed03 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -47,4 +47,6 @@ charts: - name: maestro.dedicated_proxy value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd - name: replicaCount - value: 1 \ No newline at end of file + value: 1 + - name: maestro.restricted_ip + value: "57.151.113.140/30" \ No newline at end of file diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index 78b360c..ccd8360 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -6,6 +6,9 @@ metadata: nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; + {{- if .Values.maestro.restricted_ip}} + nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }} + {{- end }} generation: 1 labels: diff --git a/maestro/values.yaml b/maestro/values.yaml index e0ec85c..f568ed9 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -44,6 +44,7 @@ maestro: open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 dedicated_proxy: "" + restricted_ip: "" autoscaling: enabled: false minReplicas: 1 From bac74c95771a4cab6fc06f3d209c57e7cc6a0f4e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 15:36:26 -0300 Subject: [PATCH 11/21] FIX: default value and throw correct Error Class --- src/authentication/authentication.guard.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index a41bc74..108dce8 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -115,16 +115,16 @@ export class AuthenticationGuard } // Bloquear outros customer de usar o maestor dedicado - const DEDICATED_PROXY = process.env.DEDICATED_PROXY + const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) { - throw new ForbiddenException(); + throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN); } // Bloquear o customer de acesso o maestro publico - const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); - if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { - throw new ForbiddenException(); - } + // const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + // if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + // throw new ForbiddenException(); + // } request.accessTokenPayload = accessTokenPayload; request.user = { From b10459cb15e706446d0c79c83a93182e0394b6f6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 17:51:14 -0300 Subject: [PATCH 12/21] FIX: maestro custom hostname to unimed --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index c426fa9..421bb64 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -33,7 +33,7 @@ charts: - name: maestro.duc_url value: duc.dadosfera.ai - name: hostname - value: maestro.dadosfera.ai + value: maestro-unimed.dadosfera.ai - name: maestro.pi_factory_url value: pi-factory.dadosfera.ai - name: maestro.in_factory_url From 6910a2f04cd7a754f034a7ee21a26bff641e46b9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 23 Jun 2025 18:38:20 -0300 Subject: [PATCH 13/21] FIX: correct helm value --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 421bb64..3c42dde 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -49,5 +49,5 @@ charts: value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd - name: replicaCount value: 1 - - name: restricted-ip + - name: maestro.restricted_ip value: "177.52.172.0/24" From 5d8e908339386b4727303ec534322d7726f871e5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 24 Jun 2025 10:26:50 -0300 Subject: [PATCH 14/21] FIX: add nginx proxy timeout 5 min --- maestro/templates/ingress.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index ccd8360..1594baa 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -3,6 +3,9 @@ kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; From bdd371ffe88b3490eff5e460533fba997028381f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 25 Jun 2025 10:58:18 -0300 Subject: [PATCH 15/21] FIX: add dadosfera vpn ip in unimed chart --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 3c42dde..15a9241 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -50,4 +50,4 @@ charts: - name: replicaCount value: 1 - name: maestro.restricted_ip - value: "177.52.172.0/24" + value: "177.52.172.0/24,57.151.113.140/30" From d638d518f8e6f9e85bc6b35ae220d0e1019cf188 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 25 Jun 2025 11:06:41 -0300 Subject: [PATCH 16/21] FIX: change unimed chart name --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 15a9241..b44aaf3 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -23,7 +23,7 @@ charts: - name: replicaCount value: 2 - - name: unimed + - name: unimed-maestro chart: ../maestro values: - ../maestro/values.yaml From d68e436a8bd4ec495d71f9b3da830f33caf2fa47 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 5 Jul 2025 09:38:16 -0300 Subject: [PATCH 17/21] FEAT: export users in csv --- docsfera.json | 35 +++++++++++++++++++++++++++ src/modules/users/dtos/entities.ts | 11 +++++++++ src/modules/users/users.controller.ts | 23 ++++++++++++++++++ src/modules/users/users.service.ts | 30 +++++++++++++++++++++++ 4 files changed, 99 insertions(+) diff --git a/docsfera.json b/docsfera.json index b6e8be8..77461b3 100644 --- a/docsfera.json +++ b/docsfera.json @@ -969,6 +969,41 @@ ] } }, + "/users/download": { + "get": { + "operationId": "UsersController_downloadUsersInCsv", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Users" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/users/hierarchies": { "get": { "operationId": "UsersController_getAllHierarchies", diff --git a/src/modules/users/dtos/entities.ts b/src/modules/users/dtos/entities.ts index 9272e52..8e8b852 100644 --- a/src/modules/users/dtos/entities.ts +++ b/src/modules/users/dtos/entities.ts @@ -174,3 +174,14 @@ export class GetAllDepartmentsRes { @ApiProperty() departments: string[]; } + + +export interface UserReporter { + name: string; + email: string; + mfaStatus: string; + status: string; + lastLogin: string; + createdAt: string; + updatedAt: string; +} diff --git a/src/modules/users/users.controller.ts b/src/modules/users/users.controller.ts index 9e00f7c..c80317a 100644 --- a/src/modules/users/users.controller.ts +++ b/src/modules/users/users.controller.ts @@ -12,6 +12,7 @@ import { Post, Put, Query, + Res, UseFilters, } from '@nestjs/common'; import { @@ -24,6 +25,7 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { Authenticated, RequireAllPermissions, + RequireSomePermission, } from 'src/decorators/authentication.decorator'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; @@ -52,6 +54,7 @@ import { UpdateUserRes, } from './dtos/entities'; import { UsersService } from './users.service'; +import { Response } from 'express'; @ApiInternalOnlyController() @ApiTags('Users') @@ -80,6 +83,26 @@ export class UsersController { return await this.userService.findAllUsersByCustomerId(user.customer_id); } + @Get('/download') + @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + async downloadUsersInCsv( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Res() res: Response + ) { + this.logger.info('downloadUsersInCsv'); + this.userService.setLanguage(language); + const { + file, + filename + } = await this.userService.downloadUsersInCsv(user.customer_id); + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', 'text/csv'); + + res.end(file); + } + @Get('hierarchies') @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse({ type: GetAllHierarchiesRes }) diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index e6717fa..0f52284 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -21,6 +21,7 @@ import { IUserByCustomer, SetUserRolesReq, UpdateUserReq, + UserReporter, } from './dtos/entities'; import { RolesService } from '../roles/roles.service'; import { HIERARCHIES } from './hierarchies'; @@ -29,6 +30,7 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces import { EnrichErrorCode } from 'src/utils/ErrorBuilder'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; +import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; @Injectable() export class UsersService implements OnModuleInit { @@ -78,6 +80,34 @@ export class UsersService implements OnModuleInit { }; } + async downloadUsersInCsv(customerId: string) { + const { users } = await lastValueFrom( + this.usersClientService.UserFindAllByCustomerId({ customerId }), + ); + + const formatUsers: UserReporter[] = users.map(user => ({ + createdAt: user.createdAt, + email: user.email, + lastLogin: user.lastLogin, + mfaStatus: user.mfaStatus, + name: user.name, + status: user.status, + updatedAt: user.updatedAt + })) + + const parser = ParserBuilder.build('csv'); + + const file = await parser.parse(formatUsers); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `dadosfera-users-${timestamp}.csv`; + + return { + file, + filename + } + } + async findOneById(id: string): Promise<{ user: IUserByCustomer }> { const { user } = await lastValueFrom( this.usersClientService.UserFindOneById({ id }), From c98a5ff896a401a4e74dc56bc1975307e8778c24 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 5 Jul 2025 11:21:05 -0300 Subject: [PATCH 18/21] FEAT: endpoint to dowload assets list --- docsfera.json | 79 +++++++++++++++++++++++ src/modules/catalog/catalog.controller.ts | 46 +++++++++++++ src/modules/catalog/catalog.service.ts | 29 +++++++++ src/modules/catalog/dtos/index.ts | 8 +++ src/modules/users/users.service.ts | 2 +- 5 files changed, 163 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 77461b3..65c1901 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3909,6 +3909,85 @@ ] } }, + "/catalog/download": { + "get": { + "operationId": "CatalogController_dowloadAsserts", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "size", + "required": false, + "in": "query", + "description": "Número de registros a ser retornado", + "schema": { + "minimum": 1, + "maximum": 10000, + "type": "number" + } + }, + { + "name": "page", + "required": false, + "in": "query", + "description": "Usado para paginação, em conjunto com `size`", + "schema": { + "minimum": 1, + "type": "number" + } + }, + { + "name": "sort_by", + "required": false, + "in": "query", + "description": "Campo do data asset para usar na ordenação. Padrão: `created_at` ", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "required": false, + "in": "query", + "description": "Tipo de ordenação - `asc`: crescente; `desc`: decrescente ", + "schema": { + "default": "asc", + "enum": [ + "asc", + "desc" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset": { "get": { "operationId": "CatalogController_findByPipelineAndObject", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 09daa16..fe83be0 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -117,6 +117,52 @@ export class CatalogController { return res; } + @Get('/download') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async dowloadAsserts( + @User() user: RequestUser, + @Query() query: ICatalogAllRequest, + @Res() res: Response + ) { + const { user_id, customer_name, customer_id, username, permissions } = user; + this.logger.info(`/catalog/download - searchCatalog`, { + user_id, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + + const roles = await this.catalogService.getUserRolesIds(user_id); + + const metadata = PackTheMetadata({ + user_id, + customer_id, + customer_name, + username, + roles, + is_data_manager, + }); + + const { + file, + filename + } = await this.catalogService.downloadAssets( + query, + metadata, + customer_id, + ); + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', 'text/csv'); + + res.end(file); + } + @ApiInternalOnlyEndpoint() @Get('data-asset') async findByPipelineAndObject(@User() user: RequestUser, @Query() query) { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index a3db9cc..fcfb9e7 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -25,6 +25,7 @@ import { UsersService } from '../users/users.service'; import { RolesService } from '../roles/roles.service'; import { Metadata } from '@grpc/grpc-js'; import { + AssetReporter, BatchRemoveRlsRulesRequest, IUpdateDataRequest, TriggerCatalogReq, @@ -240,6 +241,34 @@ class CatalogService implements OnModuleInit { return { data_assets: response, total }; } + async downloadAssets( + query: Record, + metadata: Metadata, + customer_id: string, + ) { + const data = await this.searchDataAssets(query, metadata, customer_id); + + const formatData = data.data_assets.map(asset => ({ + id: asset.id, + display_name: asset.display_name, + data_asset_type: asset.data_asset_type, + created_at: asset.created_at, + tags: '[' + asset.tags.join(', ') + ']' + })) + + const parser = ParserBuilder.build('csv'); + + const file = await parser.parse(formatData); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `dadosfera_assets_${timestamp}.csv`; + + return { + file, + filename + } + } + async getOneDataAsset(data: { id: string; customer_id: string; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index b869e5e..ef12e78 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -320,3 +320,11 @@ export class BatchRemoveRlsRulesRequest { @ApiPropertyOptional() id_rls?: string; } + +export type AssetReporter = { + id: string; + display_name: string; + data_asset_type: string; + created_at: string; + tags: string; +} diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index 0f52284..3057010 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -100,7 +100,7 @@ export class UsersService implements OnModuleInit { const file = await parser.parse(formatUsers); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filename = `dadosfera-users-${timestamp}.csv`; + const filename = `dadosfera_users_${timestamp}.csv`; return { file, From 16c81371606cf22dc6bb15e1bdcf48de370501a6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 7 Jul 2025 17:26:26 -0300 Subject: [PATCH 19/21] CHORE: Update unimed ips --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index b44aaf3..14d4689 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -50,4 +50,4 @@ charts: - name: replicaCount value: 1 - name: maestro.restricted_ip - value: "177.52.172.0/24,57.151.113.140/30" + value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30" From 66fea0722d27f10461bd0dcfc6dc8f2eadfd84aa Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Jul 2025 11:12:35 -0300 Subject: [PATCH 20/21] FEAT: enabled network check in app.dadosfera --- src/authentication/authentication.guard.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index 108dce8..09f4845 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -121,10 +121,10 @@ export class AuthenticationGuard } // Bloquear o customer de acesso o maestro publico - // const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); - // if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { - // throw new ForbiddenException(); - // } + const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN); + } request.accessTokenPayload = accessTokenPayload; request.user = { From eb80967608c64e055e5ca66f2383bdf2f8d2e551 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Jul 2025 11:36:23 -0300 Subject: [PATCH 21/21] FIX: tests --- src/authentication/authentication.guard.spec.ts | 1 + src/decorators/user.decorator.spec.ts | 1 + src/modules/users/users.controller.spec.ts | 11 +++++++++++ src/modules/users/users.service.spec.ts | 11 +++++++++++ 4 files changed, 24 insertions(+) diff --git a/src/authentication/authentication.guard.spec.ts b/src/authentication/authentication.guard.spec.ts index 6ee4f36..e34d41c 100644 --- a/src/authentication/authentication.guard.spec.ts +++ b/src/authentication/authentication.guard.spec.ts @@ -99,6 +99,7 @@ describe('authentication.guard', () => { customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11', customer_name: 'dadosfera', customer_tier: 'BASIC', + customer_modules: [] }; beforeAll(async () => { diff --git a/src/decorators/user.decorator.spec.ts b/src/decorators/user.decorator.spec.ts index 2f9b0eb..9daee29 100644 --- a/src/decorators/user.decorator.spec.ts +++ b/src/decorators/user.decorator.spec.ts @@ -52,6 +52,7 @@ describe('user.decorator', () => { customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11', customer_name: 'dadosfera', customer_tier: 'BASIC', + customer_modules: [], access_token: '', }; diff --git a/src/modules/users/users.controller.spec.ts b/src/modules/users/users.controller.spec.ts index 3bba70b..7b43898 100644 --- a/src/modules/users/users.controller.spec.ts +++ b/src/modules/users/users.controller.spec.ts @@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module'; import { PermissionsModule } from '../permissions/permissions.module'; // const client = new DucClient(); +jest.mock('puppeteer', () => ({ + launch: jest.fn().mockResolvedValue({ + newPage: jest.fn().mockResolvedValue({ + goto: jest.fn(), + evaluate: jest.fn(), + close: jest.fn() + }), + close: jest.fn() + }) +})); + const logger = { info: (...args) => args, diff --git a/src/modules/users/users.service.spec.ts b/src/modules/users/users.service.spec.ts index 788c2e9..b7cfc60 100644 --- a/src/modules/users/users.service.spec.ts +++ b/src/modules/users/users.service.spec.ts @@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module'; // const client = new DucClient(); +jest.mock('puppeteer', () => ({ + launch: jest.fn().mockResolvedValue({ + newPage: jest.fn().mockResolvedValue({ + goto: jest.fn(), + evaluate: jest.fn(), + close: jest.fn() + }), + close: jest.fn() + }) +})); + const logger = { info: (...args) => args, error: (...args) => args,