FIX: add nginx proxy timeout 5 min

Release/2025 06

environment.d.ts

@@ -14,6 +14,7 @@ declare global {
       AWS_REGION: string;
       OPEN_GROUP_ID: string;
       OPEN_CUSTOMER_ID: string;
+      DEDICATED_PROXY: string;
     }
   }
 }

helmfiles/prd.yaml

@@ -4,6 +4,8 @@ charts:
     values:
     - ../maestro/values.yaml
     set:
+      - name: app_name
+        value: maestro
       - name: maestro.duc_url
         value: duc.dadosfera.ai
       - name: hostname
@@ -20,3 +22,32 @@ charts:
         value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
       - name: replicaCount
         value: 2
+  
+  - name: unimed
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: app_name
+        value: maestro-unimed
+      - name: maestro.duc_url
+        value: duc.dadosfera.ai
+      - name: hostname
+        value: maestro.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
+      # Customer id
+      - name: maestro.dedicated_proxy
+        value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
+      - name: replicaCount
+        value: 1
+      - name: restricted-ip
+        value: "177.52.172.0/24"

helmfiles/stg.yaml

@@ -19,4 +19,34 @@ charts:
       - name: maestro.open_group_id
         value: e3f98a2f-7748-4981-8505-7695c8ca8218
       - name: replicaCount
-        value: 1
+        value: 1
+  
+    # Environment to test Network Policies
+  - name: private-maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: app_name
+        value: maestro-private
+      - name: maestro.duc_url
+        value: duc.stg.dadosfera.ai
+      - name: hostname
+        value: private-maestro.stg.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.stg.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: e3f98a2f-7748-4981-8505-7695c8ca8218
+      # Customer id
+      - name: maestro.dedicated_proxy
+        value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
+      - name: replicaCount
+        value: 1
+      - name: maestro.restricted_ip
+        value: "57.151.113.140/30"

maestro/templates/deployment.yaml

@@ -1,16 +1,16 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: maestro
+  name: {{ .Values.app_name }}
   namespace: applications
   labels:
-    app: maestro
+    app: {{ .Values.app_name }}
 
 spec:
   replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
-      app: maestro
+      app: {{ .Values.app_name }}
   
   strategy:
     rollingUpdate:
@@ -20,7 +20,7 @@ spec:
   template:
     metadata:
       labels:
-        app: maestro
+        app: {{ .Values.app_name }}
     
     spec:
       imagePullSecrets:
@@ -100,6 +100,8 @@ spec:
               value: {{ .Values.maestro.open_customer_id }}
             - name: OPEN_GROUP_ID
               value: {{ .Values.maestro.open_group_id }}
+            - name: DEDICATED_PROXY
+              value: {{ .Values.maestro.dedicated_proxy }}
             - name: JWT_PRIVATE_KEY
               valueFrom:
                 secretKeyRef:
@@ -108,15 +110,15 @@ spec:
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
-                  name: prd-maestro
+                  name: prd-{{ .Values.app_name }}
                   key: AWS_ACCESS_KEY_ID
             - name: AWS_SECRET_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
-                  name: prd-maestro
+                  name: prd-{{ .Values.app_name }}
                   key: AWS_SECRET_ACCESS_KEY
             - name: AWS_DEFAULT_REGION
               valueFrom:
                 secretKeyRef:
-                  name: prd-maestro
+                  name: prd-{{ .Values.app_name }}
                   key: AWS_DEFAULT_REGION

maestro/templates/ingress-private.yaml

@@ -10,8 +10,12 @@ metadata:
       
   generation: 1
   labels:
-    app: maestro
+    app: {{ .Values.app_name }}
+  {{- if .Values.maestro.dedicated_proxy}}
+  name: open-data-{{ .Values.app_name }}
+  {{- else }}
   name: open-data
+  {{- end }}
   namespace: applications
 spec:
   ingressClassName: nginx
@@ -21,7 +25,7 @@ spec:
       paths:
       - backend:
           service:
-            name: maestro
+            name: {{ .Values.app_name }}
             port:
               number: {{ .Values.ingress.port }}
         path: /open-data/sharing-ocean-data

maestro/templates/ingress.yaml

@@ -3,14 +3,20 @@ kind: Ingress
 metadata:
   annotations:
     nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
     nginx.ingress.kubernetes.io/server-snippet: |
       underscores_in_headers on;
       ignore_invalid_headers on;
+    {{- if .Values.maestro.restricted_ip}}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
+    {{- end }}
       
   generation: 1
   labels:
-    app: maestro
+    app: {{ .Values.app_name }}
-  name: maestro
+  name: {{ .Values.app_name }}
   namespace: applications
 spec:
   ingressClassName: nginx
@@ -20,7 +26,7 @@ spec:
       paths:
       - backend:
           service:
-            name: maestro
+            name: {{ .Values.app_name }}
             port:
               number: {{ .Values.ingress.port }}
         path: /

maestro/templates/secret.yaml

@@ -1,17 +1,17 @@
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
-  name: prd-maestro
+  name: prd-{{ .Values.app_name }}
   namespace: applications
   labels:
-    app: maestro
+    app: {{ .Values.app_name }}
 spec:
   refreshInterval: 1h
   secretStoreRef:
     name: secretsmanager-prd
     kind: SecretStore
   target:
-    name: prd-maestro
+    name: prd-{{ .Values.app_name }}
     creationPolicy: Owner
   data:
   - secretKey: AWS_ACCESS_KEY_ID

maestro/templates/service.yaml

@@ -1,18 +1,18 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: maestro
+  name: {{ .Values.app_name }}
   namespace: applications
   labels:
-    app: maestro
+    app: {{ .Values.app_name }}
 
 spec:
   type: ClusterIP
   ports:
-  - name: maestro
+  - name: {{ .Values.app_name }}
     protocol: TCP
     port: {{ .Values.service.port }}
     targetPort: {{ .Values.service.targetPort }}
   selector:
-    app: maestro
+    app: {{ .Values.app_name }}
 

maestro/values.yaml

@@ -9,6 +9,7 @@ image:
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
   tag: 1.56.0
+app_name: maestro
 containerPort: 3333
 imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
 service:
@@ -42,6 +43,8 @@ maestro:
   upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
   open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
   open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
+  dedicated_proxy: ""
+  restricted_ip: ""
 autoscaling:
   enabled: false
   minReplicas: 1

src/authentication/authentication.guard.ts

@@ -4,6 +4,7 @@ import {
   OnApplicationBootstrap,
   ExecutionContext,
   Inject,
+  ForbiddenException,
 } from '@nestjs/common';
 import { Reflector } from '@nestjs/core';
 import assert from 'assert';
@@ -113,6 +114,18 @@ export class AuthenticationGuard
       return false;
     }
 
+    // Bloquear outros customer de usar o maestor dedicado
+    const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
+    if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
+      throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
+    }
+
+    // Bloquear o customer de acesso o maestro publico
+    // const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
+    // if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
+    //   throw new ForbiddenException();
+    // }
+
     request.accessTokenPayload = accessTokenPayload;
     request.user = {
       user_id: accessTokenPayload.user_id,

src/main.ts

@@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import helmet from 'helmet';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { json, urlencoded } from 'express';
 
 import { AppModule } from './app.module';
 import { writeFileSync } from 'fs';
@@ -25,6 +26,8 @@ async function bootstrap() {
     },
   });
   app.use(helmet());
+  app.use('/catalog/register-dataset', json({ limit: '10mb' }));
+  app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
   configureSwagger(app);
   await app.listen(3333);
   if (process.env.KILL_AFTER_START) await app.close();

src/modules/auth/auth.controller.ts

@@ -86,11 +86,9 @@ export class AuthController {
   async signIn(
     @Body() { username, password, totp }: AuthSignInReq,
     @Language() language: LanguageEnum,
-    @Headers('origin') origin = '',
   ): Promise<AuthSignInRes> {
     this.logger.info('/auth - SignIn');
-    const frontHost = origin.replace(/^https?:\/\//, '');
+    const metadata = PackTheMetadata({ language });
-    const metadata = PackTheMetadata({ language, custom_host: frontHost });
     this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
     return this.authClient.signIn({ username, password, totp }, metadata);
   }

src/modules/auth/auth.service.ts

@@ -1,4 +1,4 @@
-import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
+import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { lastValueFrom } from 'rxjs';
@@ -17,6 +17,7 @@ import {
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
   AuthConfirmResetPasswordRequest,
+  AuthSignInResponse,
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { DucClient } from '../duc/client.config';
 import { Metadata } from '@grpc/grpc-js';
@@ -53,15 +54,49 @@ export class AuthClientService implements OnModuleInit {
     return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
   }
 
+  checkDedicatedProxy({
+    customer
+  }: AuthSignInResponse) {
+    const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
+    this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
+    this.logger.info('Customer ID: ' + customer.id);
+
+    if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
+      throw new ForbiddenException();
+    }
+
+    // Bloquear o customer de acesso o maestro publico
+    this.logger.info('Check if customer have network policy: ' + customer.modules);
+    const hasNetworkPolicyModule = customer.modules.includes('network-policy');
+    if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
+      throw new ForbiddenException();
+    }
+  }
+
   async signIn(
     { username, password, totp }: AuthSignInRequest,
     metadata: Metadata,
   ) {
     this.logger.info('SignIn');
 
-    return lastValueFrom(
+    let result: AuthSignInResponse;
-      this.authService.AuthSignIn({ username, password, totp }, metadata),
+
-    );
+    try {
+      result = await lastValueFrom(
+        this.authService.AuthSignIn({ username, password, totp }, metadata),
+      );
+
+    } catch (error) {
+      this.logger.error('SignIn - Error during sign-in');
+      this.logger.error(error);
+      throw error;
+    }
+
+    if (result.customer) {
+      this.checkDedicatedProxy(result);
+    }
+
+    return result
   }
 
   async refreshAccessToken(

src/modules/open-data/open-data.controller.ts

@@ -45,11 +45,11 @@ export class OpenDataController {
   ) {
     this.logger.info('createUser for open data' + JSON.stringify(request.headers));
 
-    const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
+    // const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
-    if (!corslist.includes(origin)) {
+    // if (!corslist.includes(origin)) {
-      this.logger.info('block request by cors list: '+ origin);
+    //   this.logger.info('block request by cors list: '+ origin);
-      throw new ForbiddenException();
+    //   throw new ForbiddenException();
-    }
+    // }
 
     const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
     const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;

src/utils/FileParser/pdf-parser.ts

@@ -22,7 +22,7 @@ export class PDFParser<T> implements Parser<T> {
       const html = await this.htmlParser.parse(data);
       await page.setContent(html, {
         waitUntil: 'networkidle0',
-        timeout: 30000,
+        timeout: 90000,
       });
 
       // Configurações adicionais para garantir um PDF válido
@@ -43,7 +43,7 @@ export class PDFParser<T> implements Parser<T> {
         pageRanges: '',
         tagged: true,
         outline: false,
-        timeout: 30000,
+        timeout: 90000,
       });
 
       await browser.close();