From 16c81371606cf22dc6bb15e1bdcf48de370501a6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 7 Jul 2025 17:26:26 -0300 Subject: [PATCH 01/10] CHORE: Update unimed ips --- helmfiles/prd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index b44aaf3..14d4689 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -50,4 +50,4 @@ charts: - name: replicaCount value: 1 - name: maestro.restricted_ip - value: "177.52.172.0/24,57.151.113.140/30" + value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30" From 66fea0722d27f10461bd0dcfc6dc8f2eadfd84aa Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Jul 2025 11:12:35 -0300 Subject: [PATCH 02/10] FEAT: enabled network check in app.dadosfera --- src/authentication/authentication.guard.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index 108dce8..09f4845 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -121,10 +121,10 @@ export class AuthenticationGuard } // Bloquear o customer de acesso o maestro publico - // const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); - // if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { - // throw new ForbiddenException(); - // } + const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN); + } request.accessTokenPayload = accessTokenPayload; request.user = { From eb80967608c64e055e5ca66f2383bdf2f8d2e551 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 11 Jul 2025 11:36:23 -0300 Subject: [PATCH 03/10] FIX: tests --- src/authentication/authentication.guard.spec.ts | 1 + src/decorators/user.decorator.spec.ts | 1 + src/modules/users/users.controller.spec.ts | 11 +++++++++++ src/modules/users/users.service.spec.ts | 11 +++++++++++ 4 files changed, 24 insertions(+) diff --git a/src/authentication/authentication.guard.spec.ts b/src/authentication/authentication.guard.spec.ts index 6ee4f36..e34d41c 100644 --- a/src/authentication/authentication.guard.spec.ts +++ b/src/authentication/authentication.guard.spec.ts @@ -99,6 +99,7 @@ describe('authentication.guard', () => { customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11', customer_name: 'dadosfera', customer_tier: 'BASIC', + customer_modules: [] }; beforeAll(async () => { diff --git a/src/decorators/user.decorator.spec.ts b/src/decorators/user.decorator.spec.ts index 2f9b0eb..9daee29 100644 --- a/src/decorators/user.decorator.spec.ts +++ b/src/decorators/user.decorator.spec.ts @@ -52,6 +52,7 @@ describe('user.decorator', () => { customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11', customer_name: 'dadosfera', customer_tier: 'BASIC', + customer_modules: [], access_token: '', }; diff --git a/src/modules/users/users.controller.spec.ts b/src/modules/users/users.controller.spec.ts index 3bba70b..7b43898 100644 --- a/src/modules/users/users.controller.spec.ts +++ b/src/modules/users/users.controller.spec.ts @@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module'; import { PermissionsModule } from '../permissions/permissions.module'; // const client = new DucClient(); +jest.mock('puppeteer', () => ({ + launch: jest.fn().mockResolvedValue({ + newPage: jest.fn().mockResolvedValue({ + goto: jest.fn(), + evaluate: jest.fn(), + close: jest.fn() + }), + close: jest.fn() + }) +})); + const logger = { info: (...args) => args, diff --git a/src/modules/users/users.service.spec.ts b/src/modules/users/users.service.spec.ts index 788c2e9..b7cfc60 100644 --- a/src/modules/users/users.service.spec.ts +++ b/src/modules/users/users.service.spec.ts @@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module'; // const client = new DucClient(); +jest.mock('puppeteer', () => ({ + launch: jest.fn().mockResolvedValue({ + newPage: jest.fn().mockResolvedValue({ + goto: jest.fn(), + evaluate: jest.fn(), + close: jest.fn() + }), + close: jest.fn() + }) +})); + const logger = { info: (...args) => args, error: (...args) => args, From c1e4f0b18abf19c314af473faa608e97e00b63c7 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 11:48:15 -0300 Subject: [PATCH 04/10] FIX: remove express session --- package-lock.json | 100 -------------------------------- package.json | 1 - src/@types/express-session.d.ts | 8 --- src/main.ts | 15 ++--- 4 files changed, 8 insertions(+), 116 deletions(-) delete mode 100644 src/@types/express-session.d.ts diff --git a/package-lock.json b/package-lock.json index 986889b..4092f52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,6 @@ "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", - "express-session": "^1.18.1", "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", @@ -7055,75 +7054,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/express-session": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.18.1.tgz", - "integrity": "sha512-a5mtTqEaZvBCL9A9aqkrtfz+3SMDhOVUnjafjo+s7A9Txkq+SVX2DLvSp1Zrv4uCXa3lMSK3viWnh9Gg07PBUA==", - "license": "MIT", - "dependencies": { - "cookie": "0.7.2", - "cookie-signature": "1.0.7", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-headers": "~1.0.2", - "parseurl": "~1.3.3", - "safe-buffer": "5.2.1", - "uid-safe": "~2.1.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/express-session/node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/express-session/node_modules/cookie-signature": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", - "license": "MIT" - }, - "node_modules/express-session/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/express-session/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/express-session/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/express/node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -10375,15 +10305,6 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -11445,15 +11366,6 @@ "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", "license": "MIT" }, - "node_modules/random-bytes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -13172,18 +13084,6 @@ "node": ">=8" } }, - "node_modules/uid-safe": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", - "license": "MIT", - "dependencies": { - "random-bytes": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/uid2": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/uid2/-/uid2-0.0.4.tgz", diff --git a/package.json b/package.json index 605dc93..ac37a76 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", - "express-session": "^1.18.1", "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", diff --git a/src/@types/express-session.d.ts b/src/@types/express-session.d.ts deleted file mode 100644 index 6531c85..0000000 --- a/src/@types/express-session.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -import 'express-session'; - -declare module 'express-session' { - interface SessionData { - state: string | undefined; - code_verifier: string | undefined; - } -} \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index eff6350..e6421fc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,12 +3,12 @@ import { NestFactory } from '@nestjs/core'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import helmet from 'helmet'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; - +import { json, urlencoded } from 'express'; import { AppModule } from './app.module'; import { writeFileSync } from 'fs'; import { execSync } from 'child_process'; import { INestApplication } from '@nestjs/common'; -import session from 'express-session'; + async function bootstrap() { DadosferaLogger.setupLogger({ serviceName: 'maestro', @@ -26,11 +26,12 @@ async function bootstrap() { }, }); app.use(helmet()); - app.use(session({ - secret: process.env.SESSION_SECRET, - resave: false, - saveUninitialized: false, - })); + + if (process.env.ENV === 'prd') { + app.use('/catalog/register-dataset', json({ limit: '10mb' })); + app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); + } + configureSwagger(app); await app.listen(3333); if (process.env.KILL_AFTER_START) await app.close(); From dda3e8baafd4a2249bb6bcb4be90c1a4d1c135d9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 17:39:15 -0300 Subject: [PATCH 05/10] FIX: add logger --- src/modules/identity-provider/identity-provider.controller.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/modules/identity-provider/identity-provider.controller.ts b/src/modules/identity-provider/identity-provider.controller.ts index 813d925..8c9010d 100644 --- a/src/modules/identity-provider/identity-provider.controller.ts +++ b/src/modules/identity-provider/identity-provider.controller.ts @@ -189,6 +189,8 @@ export class IdentityProviderController { const redirectUrl = await this.identityProviderService.loginIdentityProvider(id); + + this.logger.info(`Redirecting to: ${redirectUrl}`); return { url: redirectUrl, }; From fbeb8dfd913a30ec33c8d4ee940b8b09f73a47dc Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 17:56:43 -0300 Subject: [PATCH 06/10] FIX: redis env --- maestro/templates/deployment.yaml | 8 ++++++++ maestro/values.yaml | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index c648514..bcc7da2 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -100,6 +100,14 @@ spec: value: {{ .Values.maestro.open_customer_id }} - name: OPEN_GROUP_ID value: {{ .Values.maestro.open_group_id }} + - name: DEDICATED_PROXY + value: {{ .Values.maestro.dedicated_proxy }} + - name: REDIS_DATABASE + value: "{{ .Values.maestro.redis_database }}" + - name: REDIS_HOST + value: {{ .Values.maestro.redis_host }} + - name: REDIS_PORT + value: "{{ .Values.maestro.redis_port }}" - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/maestro/values.yaml b/maestro/values.yaml index 14dd9e4..ee59bc5 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -42,6 +42,11 @@ maestro: upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 + dedicated_proxy: "" + restricted_ip: "" + redis_host: "product-redis-prd.z4xvqj.0001.use1.cache.amazonaws.com" + redis_port: "6379" + redis_database: "0" autoscaling: enabled: false minReplicas: 1 From 3eba3f414f6f3181d26f043a716f4ecc6c244503 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 18:15:47 -0300 Subject: [PATCH 07/10] FIX: send redirect uri --- .../identity-provider.controller.ts | 15 ++++++++++----- .../identity-provider.service.ts | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/modules/identity-provider/identity-provider.controller.ts b/src/modules/identity-provider/identity-provider.controller.ts index 8c9010d..6d8aed6 100644 --- a/src/modules/identity-provider/identity-provider.controller.ts +++ b/src/modules/identity-provider/identity-provider.controller.ts @@ -8,7 +8,6 @@ import { HttpStatus, Inject, Param, - Patch, Post, Put, Redirect, @@ -155,9 +154,8 @@ export class IdentityProviderController { } const origin = req.headers['origin'] as string; - const langHeader = req.headers['dadosfera-lang'] as string; - const lang = langHeader.substring(0, 2) + langHeader.substring(2).toUpperCase(); + const lang = language.substring(0, 2) + language.substring(2).toUpperCase(); const callbackUrl = process.env.ENV !== "prd" ? `${origin}/auth/callback` : `${origin}/${lang}/auth/callback`; return await this.identityProviderService.getTokenByIdp(code, state, callbackUrl); @@ -184,11 +182,18 @@ export class IdentityProviderController { @Get(':id') @HttpCode(HttpStatus.OK) @Redirect() - async loginIdp(@Param('id') id: string) { + async loginIdp(@Param('id') id: string, @Req() req: Request, @Language() language: LanguageEnum) { this.logger.info('GET /identity-providers/:id'); + + const origin = req.headers['origin'] as string || req.headers['referer'] as string; + console.log('Origin:', origin); + + const lang = language.substring(0, 2) + language.substring(2).toUpperCase(); + const callbackUrl = process.env.ENV !== "prd" ? `${origin}/auth/callback` : `${origin}/${lang}/auth/callback`; + const redirectUrl = - await this.identityProviderService.loginIdentityProvider(id); + await this.identityProviderService.loginIdentityProvider(id, callbackUrl); this.logger.info(`Redirecting to: ${redirectUrl}`); return { diff --git a/src/modules/identity-provider/identity-provider.service.ts b/src/modules/identity-provider/identity-provider.service.ts index 456343c..4cca0fe 100644 --- a/src/modules/identity-provider/identity-provider.service.ts +++ b/src/modules/identity-provider/identity-provider.service.ts @@ -44,7 +44,7 @@ export class IdentityProviderService implements OnModuleInit { ); } - async loginIdentityProvider(id: string) { + async loginIdentityProvider(id: string, callbackUrl: string) { const idp = await lastValueFrom( this.identityProviderService.FindIdentityProvider({ id }), ); @@ -83,6 +83,7 @@ export class IdentityProviderService implements OnModuleInit { code_challenge_method: 'S256', state, nonce, + redirect_uri: callbackUrl, }); const idpUrl = url + '&identity_provider=' + idp.name; return idpUrl; From e81cfdcce10775a66782aa68102e3f4764051bc5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 18:36:36 -0300 Subject: [PATCH 08/10] FIX: using only header origin --- src/modules/identity-provider/identity-provider.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/identity-provider/identity-provider.controller.ts b/src/modules/identity-provider/identity-provider.controller.ts index 6d8aed6..b01ed30 100644 --- a/src/modules/identity-provider/identity-provider.controller.ts +++ b/src/modules/identity-provider/identity-provider.controller.ts @@ -186,7 +186,7 @@ export class IdentityProviderController { this.logger.info('GET /identity-providers/:id'); - const origin = req.headers['origin'] as string || req.headers['referer'] as string; + const origin = req.headers['origin'] as string; console.log('Origin:', origin); const lang = language.substring(0, 2) + language.substring(2).toUpperCase(); From 4c7111dcb6d97e9512859884c3ed1f764add93e0 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 18:45:47 -0300 Subject: [PATCH 09/10] FIX: using referer headaer --- .../identity-provider/identity-provider.controller.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/modules/identity-provider/identity-provider.controller.ts b/src/modules/identity-provider/identity-provider.controller.ts index b01ed30..26197b6 100644 --- a/src/modules/identity-provider/identity-provider.controller.ts +++ b/src/modules/identity-provider/identity-provider.controller.ts @@ -186,11 +186,14 @@ export class IdentityProviderController { this.logger.info('GET /identity-providers/:id'); - const origin = req.headers['origin'] as string; + const frontDomain = req.headers['origin'] as string || req.headers['referer'] as string; console.log('Origin:', origin); + const host = frontDomain.lastIndexOf('/') !== -1 + ? frontDomain.substring(0, frontDomain.lastIndexOf('/')) + : frontDomain; const lang = language.substring(0, 2) + language.substring(2).toUpperCase(); - const callbackUrl = process.env.ENV !== "prd" ? `${origin}/auth/callback` : `${origin}/${lang}/auth/callback`; + const callbackUrl = process.env.ENV !== "prd" ? `${host}/auth/callback` : `${host}/${lang}/auth/callback`; const redirectUrl = await this.identityProviderService.loginIdentityProvider(id, callbackUrl); From e9880bcf68c74fef1128bdf54860f3a4419d1ca7 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 16 Jul 2025 18:49:27 -0300 Subject: [PATCH 10/10] FIX: add logger in header --- src/modules/identity-provider/identity-provider.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/identity-provider/identity-provider.controller.ts b/src/modules/identity-provider/identity-provider.controller.ts index 26197b6..c25b2b4 100644 --- a/src/modules/identity-provider/identity-provider.controller.ts +++ b/src/modules/identity-provider/identity-provider.controller.ts @@ -187,7 +187,7 @@ export class IdentityProviderController { const frontDomain = req.headers['origin'] as string || req.headers['referer'] as string; - console.log('Origin:', origin); + this.logger.info(`Front domain: ${frontDomain}`); const host = frontDomain.lastIndexOf('/') !== -1 ? frontDomain.substring(0, frontDomain.lastIndexOf('/')) : frontDomain;