From e4eb388e1871d10f41edfe0ebef6e9497b5fdc07 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 9 May 2025 11:06:23 -0300 Subject: [PATCH 01/13] FIX: stg to prd for sbm environment --- helmfiles/stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 0aac3e7..5782e36 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,7 +9,7 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.stg.dadosfera.ai + value: pi-factory.dadosfera.ai - name: maestro.in_factory_url value: in-factory.stg.dadosfera.ai - name: maestro.tr_factory_url From 81d8cd8a07d948e6228c5e260ff4426ce61c7d4c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 15 May 2025 11:00:07 -0300 Subject: [PATCH 02/13] FIX: default value to header origin --- docsfera.json | 8 -------- src/modules/auth/auth.controller.ts | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/docsfera.json b/docsfera.json index b8d9671..b6e8be8 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,14 +20,6 @@ ], "type": "string" } - }, - { - "name": "origin", - "required": true, - "in": "header", - "schema": { - "type": "string" - } } ], "requestBody": { diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 226036a..7f2f8f0 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -86,7 +86,7 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('origin') origin: string, + @Headers('origin') origin = '', ): Promise { this.logger.info('/auth - SignIn'); const frontHost = origin.replace(/^https?:\/\//, ''); From 5a36762544bed596222da329822a7ab4acf974bc Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 26 May 2025 10:24:48 -0300 Subject: [PATCH 03/13] UPDATE: route /catalog/register-dataset will have an limit of 10mb --- src/main.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.ts b/src/main.ts index 46f1701..7511269 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import helmet from 'helmet'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { json, urlencoded } from 'express'; import { AppModule } from './app.module'; import { writeFileSync } from 'fs'; @@ -25,6 +26,8 @@ async function bootstrap() { }, }); app.use(helmet()); + app.use('/catalog/register-dataset', json({ limit: '10mb' })); + app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); configureSwagger(app); await app.listen(3333); if (process.env.KILL_AFTER_START) await app.close(); From a89f57e6902f1e5c09c876e8d9edf76baa643d46 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 30 May 2025 10:13:37 -0300 Subject: [PATCH 04/13] FIX: return pi_factory_url and tr_factory_url to stg url --- helmfiles/stg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 5782e36..530416b 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,11 +9,11 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.dadosfera.ai + value: pi-factory.stg.dadosfera.ai - name: maestro.in_factory_url value: in-factory.stg.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.dadosfera.ai + value: in-factory.stg.dadosfera.ai - name: maestro.open_customer_id value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id From 54cb3f8e7ae14ba53582f9c11b890174e5fe89cb Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 6 Jun 2025 15:53:12 -0300 Subject: [PATCH 05/13] FEAT: add api key endpoint --- docsfera.json | 235 +++++++++++++++++++++ package-lock.json | 8 +- package.json | 2 +- src/app.module.ts | 4 +- src/authentication/authentication.guard.ts | 30 ++- src/main.ts | 7 +- src/modules/api-key/api-key.controller.ts | 68 ++++++ src/modules/api-key/api-key.module.ts | 18 ++ src/modules/api-key/api-key.service.ts | 50 +++++ src/modules/api-key/dto/api-key.dto.ts | 39 ++++ 10 files changed, 449 insertions(+), 12 deletions(-) create mode 100644 src/modules/api-key/api-key.controller.ts create mode 100644 src/modules/api-key/api-key.module.ts create mode 100644 src/modules/api-key/api-key.service.ts create mode 100644 src/modules/api-key/dto/api-key.dto.ts diff --git a/docsfera.json b/docsfera.json index b6e8be8..b333454 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5968,6 +5968,158 @@ } ] } + }, + "/api-key": { + "post": { + "operationId": "ApiKeyController_create", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyDto" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "ApiKeyController_findAll", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/api-key/{id}": { + "delete": { + "operationId": "ApiKeyController_remove", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } } }, "info": { @@ -8848,6 +9000,89 @@ "required": [ "policies" ] + }, + "CreateApiKeyDto": { + "type": "object", + "properties": { + "permissions": { + "description": "Array of permission IDs", + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "permissions" + ] + }, + "CreateApiKeyResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + }, + "key": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by", + "key" + ] + }, + "ApiKeyBaseResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by" + ] } } } diff --git a/package-lock.json b/package-lock.json index c9d852f..2add2f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "3.38.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1400,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.23", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", - "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", + "version": "3.38.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.1.tgz", + "integrity": "sha512-4+yNZMlhvEkHlxJcaeta+jtJ9owskZcg0yjKdZzBfT8PCclctgertLHmrG0+kDb3Y/na/rpPPHYULRp7AO4zDQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index fd3eb0b..b59aa9c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "3.38.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index fedb388..0555eaf 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -29,6 +29,7 @@ import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; import { ThemeModule } from './modules/theme/theme.module'; import { NetworkPolicyModule } from './modules/network-policy/network-policy.module'; +import { ApiKeyModule } from './modules/api-key/api-key.module'; @Module({ providers: [ @@ -64,7 +65,8 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod ThemeModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, - NetworkPolicyModule + NetworkPolicyModule, + ApiKeyModule ], }) export class AppModule {} diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index bc69f8c..0506581 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -17,6 +17,7 @@ import { import { RequestUser } from '../decorators/user.decorator'; import ErrorBuilder from '../utils/ErrorBuilder'; import ErrorCodes from '../utils/errorCodes'; +import { ApiKeyService } from 'src/modules/api-key/api-key.service'; @Injectable() export class AuthenticationGuard @@ -31,6 +32,7 @@ export class AuthenticationGuard dadosferaLogger: DadosferaLogger, private reflector: Reflector, private authClient: AuthClientService, + private apiKeyService: ApiKeyService ) { this.pems = new Map(); this.logger = dadosferaLogger.logger; @@ -48,20 +50,40 @@ export class AuthenticationGuard }); } - canActivate(ctx: ExecutionContext): boolean { + async canActivate(ctx: ExecutionContext): Promise { const authFunctions = this.reflector.getAllAndMerge< AuthenticationFunction[] >(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]); const mustBeAuthenticated = authFunctions.length > 0; - const request = ctx.switchToHttp().getRequest(); - const accessToken = this.validateToken(request, mustBeAuthenticated); - if (!mustBeAuthenticated) { // no need to be authenticated return true; } + const request = ctx.switchToHttp().getRequest(); + const apiKey = request.get('X-api-key'); + if (apiKey) { + const { + api_key + } = await this.apiKeyService.get(apiKey); + + request.user = { + user_id: api_key.user_id, + username: api_key.username, + permissions: api_key.permissions, + customer_id: api_key.customer_id, + customer_name: api_key.customer_name, + customer_tier: api_key.customer_tier, + customer_modules: api_key.customer_modules, + access_token: apiKey, + }; + + return true; + } + + const accessToken = this.validateToken(request, mustBeAuthenticated); + if (!accessToken) { // couldn't load valid token throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED); diff --git a/src/main.ts b/src/main.ts index 7511269..6fd8445 100644 --- a/src/main.ts +++ b/src/main.ts @@ -26,8 +26,11 @@ async function bootstrap() { }, }); app.use(helmet()); - app.use('/catalog/register-dataset', json({ limit: '10mb' })); - app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); + if (process.env.ENV === 'prd') { + app.use('/catalog/register-dataset', json({ limit: '10mb' })); + app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' })); + } + configureSwagger(app); await app.listen(3333); if (process.env.KILL_AFTER_START) await app.close(); diff --git a/src/modules/api-key/api-key.controller.ts b/src/modules/api-key/api-key.controller.ts new file mode 100644 index 0000000..bc2c51c --- /dev/null +++ b/src/modules/api-key/api-key.controller.ts @@ -0,0 +1,68 @@ +import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common'; +import { ApiKeyService } from './api-key.service'; +import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto'; +import { Authenticated } from 'src/decorators/authentication.decorator'; +import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +@Controller('api-key') +@Authenticated() +@ApiTags('ApiKey') +@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) +@UseFilters(new GrpcToHttpExceptionFilter()) +export class ApiKeyController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private readonly apiKeyService: ApiKeyService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Post() + @ApiResponse({ type: CreateApiKeyResponseDto }) + async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise { + this.logger.info('POST /api-key', { + permissions: createApiKeyDto.permissions, + method: 'create' + }); + const result = await this.apiKeyService.create(createApiKeyDto, user); + this.logger.info('POST /api-key success', { + id: result.id, + method: 'create' + }); + return result; + } + + @Get() + @ApiResponse({ type: [ApiKeyBaseResponseDto] }) + async findAll(@User() user: RequestUser): Promise { + this.logger.info('GET /api-key', { + method: 'findAll' + }); + const result = await this.apiKeyService.findAll(user); + this.logger.info('GET /api-key success', { + count: result.length, + method: 'findAll' + }); + return result; + } + + @Delete(':id') + async remove(@Param('id') id: string, @User() user: RequestUser): Promise { + this.logger.info('DELETE /api-key/:id', { + id, + method: 'remove' + }); + await this.apiKeyService.remove(id, user); + this.logger.info('DELETE /api-key/:id success', { + id, + method: 'remove' + }); + } +} diff --git a/src/modules/api-key/api-key.module.ts b/src/modules/api-key/api-key.module.ts new file mode 100644 index 0000000..1455425 --- /dev/null +++ b/src/modules/api-key/api-key.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +import { ApiKeyService } from './api-key.service'; +import { ApiKeyController } from './api-key.controller'; +import { ClientsModule } from '@nestjs/microservices'; +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +const ducClient = new DucClient(); + +@Module({ + imports: [ + ClientsModule.register([ducClient.providerOptions]) + ], + controllers: [ApiKeyController], + providers: [ApiKeyService, DadosferaLogger], + exports: [ApiKeyService] +}) +export class ApiKeyModule {} diff --git a/src/modules/api-key/api-key.service.ts b/src/modules/api-key/api-key.service.ts new file mode 100644 index 0000000..ea96801 --- /dev/null +++ b/src/modules/api-key/api-key.service.ts @@ -0,0 +1,50 @@ +import { Injectable, Inject, OnModuleInit } from '@nestjs/common'; +import { ClientGrpc } from '@nestjs/microservices'; +import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto'; +import { RequestUser } from 'src/decorators/user.decorator'; +import { DucClient } from '../duc/client.config'; +import { PackTheMetadata } from '../../utils/ PackTheMetadata'; +import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { lastValueFrom } from 'rxjs'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; + +@Injectable() +export class ApiKeyService implements OnModuleInit { + private apiKeyService: ApiKeyWriteProtoService; + + constructor( + @Inject(DucClient.name) private readonly client: ClientGrpc, + ) {} + + onModuleInit() { + this.apiKeyService = this.client.getService(ProtoServices.ApiKeyWriteProtoService); + } + + create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise { + const metadata = PackTheMetadata(user); + + return lastValueFrom(this.apiKeyService.CreateApiKey({ + permissions: createApiKeyDto.permissions + }, metadata)); + } + + async findAll(user: RequestUser): Promise { + const metadata = PackTheMetadata(user); + console.log(metadata) + + const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata)); + return data.api_keys; + } + + async remove(id: string, user: RequestUser) { + const metadata = PackTheMetadata(user); + + await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata)); + } + + async get(key: string) { + const metadata = PackTheMetadata({}); + + return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata)); + } +} diff --git a/src/modules/api-key/dto/api-key.dto.ts b/src/modules/api-key/dto/api-key.dto.ts new file mode 100644 index 0000000..2bbabf3 --- /dev/null +++ b/src/modules/api-key/dto/api-key.dto.ts @@ -0,0 +1,39 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsArray, IsNumber } from 'class-validator'; + +export class PermissionDto { + @ApiProperty({ type: Number }) + id: number; + + @ApiProperty({ type: String }) + name: string; +} + +export class ApiKeyBaseResponseDto { + @ApiProperty({ type: String, format: 'uuid' }) + id: string; + + @ApiProperty({ type: String }) + key_mask: string; + + @ApiProperty({ type: [PermissionDto] }) + permissions: PermissionDto[]; + + @ApiProperty({ type: String, format: 'date-time' }) + created_at: string; + + @ApiProperty({ type: String }) + created_by: string; +} + +export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto { + @ApiProperty({ type: String }) + key: string; +} + +export class CreateApiKeyDto { + @ApiProperty({ type: [Number], description: 'Array of permission IDs' }) + @IsArray() + @IsNumber({}, { each: true }) + permissions: number[]; +} \ No newline at end of file From 01263017e7612de36accf04239f61d450a0796de Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 10 Jun 2025 14:24:26 -0300 Subject: [PATCH 06/13] FIX: comment block list --- src/modules/open-data/open-data.controller.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index b8951ed..e952838 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -45,11 +45,11 @@ export class OpenDataController { ) { this.logger.info('createUser for open data' + JSON.stringify(request.headers)); - const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"]; - if (!corslist.includes(origin)) { - this.logger.info('block request by cors list: '+ origin); - throw new ForbiddenException(); - } + // const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"]; + // if (!corslist.includes(origin)) { + // this.logger.info('block request by cors list: '+ origin); + // throw new ForbiddenException(); + // } const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID; From fea587ad366e8081eaa328a381d0c85c9922cb2c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 11:51:56 -0300 Subject: [PATCH 07/13] FEAT: control login by dedicated customer id injected --- environment.d.ts | 1 + helmfiles/stg.yaml | 26 +++++++++++++++++++++ src/modules/auth/auth.controller.ts | 4 +--- src/modules/auth/auth.service.ts | 35 +++++++++++++++++++++++++---- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/environment.d.ts b/environment.d.ts index c910516..3558f26 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -14,6 +14,7 @@ declare global { AWS_REGION: string; OPEN_GROUP_ID: string; OPEN_CUSTOMER_ID: string; + DEDICATED_PROXY: string; } } } diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 5782e36..616588e 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -18,5 +18,31 @@ charts: value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 + - name: replicaCount + value: 1 + + # Environment to test Network Policies + - name: private + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: maestro.duc_url + value: duc.stg.dadosfera.ai + - name: hostname + value: private.stg.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory.stg.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d + - name: maestro.open_group_id + value: e3f98a2f-7748-4981-8505-7695c8ca8218 + # Customer id + - name: maestro.dedicated_proxy + value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd - name: replicaCount value: 1 \ No newline at end of file diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 7f2f8f0..66870d5 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -86,11 +86,9 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, - @Headers('origin') origin = '', ): Promise { this.logger.info('/auth - SignIn'); - const frontHost = origin.replace(/^https?:\/\//, ''); - const metadata = PackTheMetadata({ language, custom_host: frontHost }); + const metadata = PackTheMetadata({ language }); this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 5974954..be71d66 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -1,4 +1,4 @@ -import { OnModuleInit, Inject, Injectable } from '@nestjs/common'; +import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { lastValueFrom } from 'rxjs'; @@ -17,6 +17,7 @@ import { AuthResetPasswordRequest, AuthVerifyResetPasswordCodeRequest, AuthConfirmResetPasswordRequest, + AuthSignInResponse, } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; @@ -53,15 +54,41 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } + checkDedicatedProxy(customerId: string) { + const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; + this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); + if (DEDICATED_PROXY !== '') { + this.logger.info('Customer ID: ' + customerId); + if (DEDICATED_PROXY !== customerId) { + throw new ForbiddenException(); + } + } + } + async signIn( { username, password, totp }: AuthSignInRequest, metadata: Metadata, ) { this.logger.info('SignIn'); - return lastValueFrom( - this.authService.AuthSignIn({ username, password, totp }, metadata), - ); + let result: AuthSignInResponse; + + try { + result = await lastValueFrom( + this.authService.AuthSignIn({ username, password, totp }, metadata), + ); + + } catch (error) { + this.logger.error('SignIn - Error during sign-in'); + this.logger.error(error); + throw error; + } + + if (result.customer) { + this.checkDedicatedProxy(result.customer.id); + } + + return result } async refreshAccessToken( From 7f9755493a9d2a7b8ea1d3074d1c1d1536c6d7a9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 12:29:51 -0300 Subject: [PATCH 08/13] FIX: experimental hostname --- helmfiles/stg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 616588e..297c5aa 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -30,7 +30,7 @@ charts: - name: maestro.duc_url value: duc.stg.dadosfera.ai - name: hostname - value: private.stg.dadosfera.ai + value: private-maestro.stg.dadosfera.ai - name: maestro.pi_factory_url value: pi-factory.dadosfera.ai - name: maestro.in_factory_url From 41842f7c6afc2ed5f5e63058d09efb366932a666 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 13:56:57 -0300 Subject: [PATCH 09/13] FIX: chart private maestro --- helmfiles/stg.yaml | 4 +++- maestro/templates/deployment.yaml | 14 +++++++------- maestro/templates/ingress-private.yaml | 4 ++-- maestro/templates/ingress.yaml | 6 +++--- maestro/templates/secret.yaml | 6 +++--- maestro/templates/service.yaml | 8 ++++---- maestro/values.yaml | 2 ++ 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 297c5aa..8039c29 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -22,11 +22,13 @@ charts: value: 1 # Environment to test Network Policies - - name: private + - name: private-maestro chart: ../maestro values: - ../maestro/values.yaml set: + - name: app_name + value: maestro-private - name: maestro.duc_url value: duc.stg.dadosfera.ai - name: hostname diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index c648514..6629db6 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: maestro + name: {{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - app: maestro + app: {{ .Values.app_name }} strategy: rollingUpdate: @@ -20,7 +20,7 @@ spec: template: metadata: labels: - app: maestro + app: {{ .Values.app_name }} spec: imagePullSecrets: @@ -108,15 +108,15 @@ spec: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_SECRET_ACCESS_KEY - name: AWS_DEFAULT_REGION valueFrom: secretKeyRef: - name: prd-maestro + name: prd-{{ .Values.app_name }} key: AWS_DEFAULT_REGION diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index 97708bc..b729904 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -10,7 +10,7 @@ metadata: generation: 1 labels: - app: maestro + app: {{ .Values.app_name }} name: open-data namespace: applications spec: @@ -21,7 +21,7 @@ spec: paths: - backend: service: - name: maestro + name: {{ .Values.app_name }} port: number: {{ .Values.ingress.port }} path: /open-data/sharing-ocean-data diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml index faf1305..78b360c 100644 --- a/maestro/templates/ingress.yaml +++ b/maestro/templates/ingress.yaml @@ -9,8 +9,8 @@ metadata: generation: 1 labels: - app: maestro - name: maestro + app: {{ .Values.app_name }} + name: {{ .Values.app_name }} namespace: applications spec: ingressClassName: nginx @@ -20,7 +20,7 @@ spec: paths: - backend: service: - name: maestro + name: {{ .Values.app_name }} port: number: {{ .Values.ingress.port }} path: / diff --git a/maestro/templates/secret.yaml b/maestro/templates/secret.yaml index 9e2fd37..3bbf5ec 100644 --- a/maestro/templates/secret.yaml +++ b/maestro/templates/secret.yaml @@ -1,17 +1,17 @@ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: prd-maestro + name: prd-{{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: refreshInterval: 1h secretStoreRef: name: secretsmanager-prd kind: SecretStore target: - name: prd-maestro + name: prd-{{ .Values.app_name }} creationPolicy: Owner data: - secretKey: AWS_ACCESS_KEY_ID diff --git a/maestro/templates/service.yaml b/maestro/templates/service.yaml index 75f635a..1f7c988 100644 --- a/maestro/templates/service.yaml +++ b/maestro/templates/service.yaml @@ -1,18 +1,18 @@ apiVersion: v1 kind: Service metadata: - name: maestro + name: {{ .Values.app_name }} namespace: applications labels: - app: maestro + app: {{ .Values.app_name }} spec: type: ClusterIP ports: - - name: maestro + - name: {{ .Values.app_name }} protocol: TCP port: {{ .Values.service.port }} targetPort: {{ .Values.service.targetPort }} selector: - app: maestro + app: {{ .Values.app_name }} diff --git a/maestro/values.yaml b/maestro/values.yaml index 14dd9e4..e0ec85c 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -9,6 +9,7 @@ image: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: 1.56.0 +app_name: maestro containerPort: 3333 imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret" service: @@ -42,6 +43,7 @@ maestro: upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 + dedicated_proxy: "" autoscaling: enabled: false minReplicas: 1 From 9e597fadba9e4d69d01f02a06dafd02631d7e806 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 14:04:45 -0300 Subject: [PATCH 10/13] FIX: open data ingress route --- maestro/templates/ingress-private.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index b729904..70f8eb6 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -11,7 +11,7 @@ metadata: generation: 1 labels: app: {{ .Values.app_name }} - name: open-data + name: open-data-{{ .Values.app_name }} namespace: applications spec: ingressClassName: nginx From 17de31f1a65aa6c33c8d132844bd7f2bc40c1eef Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Jun 2025 14:45:13 -0300 Subject: [PATCH 11/13] FIX: send env DEDICATED_PROXY --- maestro/templates/deployment.yaml | 2 ++ maestro/templates/ingress-private.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 6629db6..ac9eaae 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -100,6 +100,8 @@ spec: value: {{ .Values.maestro.open_customer_id }} - name: OPEN_GROUP_ID value: {{ .Values.maestro.open_group_id }} + - name: DEDICATED_PROXY + value: {{ .Values.maestro.dedicated_proxy }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/maestro/templates/ingress-private.yaml b/maestro/templates/ingress-private.yaml index 70f8eb6..1ace950 100644 --- a/maestro/templates/ingress-private.yaml +++ b/maestro/templates/ingress-private.yaml @@ -11,7 +11,11 @@ metadata: generation: 1 labels: app: {{ .Values.app_name }} + {{- if .Values.maestro.dedicated_proxy}} name: open-data-{{ .Values.app_name }} + {{- else }} + name: open-data + {{- end }} namespace: applications spec: ingressClassName: nginx From cb98099a913e026efb5ab94518deda110697c1ec Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 13 Jun 2025 15:07:50 -0300 Subject: [PATCH 12/13] FIX: update timeout --- src/utils/FileParser/pdf-parser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/FileParser/pdf-parser.ts b/src/utils/FileParser/pdf-parser.ts index 1294b34..a954708 100644 --- a/src/utils/FileParser/pdf-parser.ts +++ b/src/utils/FileParser/pdf-parser.ts @@ -22,7 +22,7 @@ export class PDFParser implements Parser { const html = await this.htmlParser.parse(data); await page.setContent(html, { waitUntil: 'networkidle0', - timeout: 30000, + timeout: 90000, }); // Configurações adicionais para garantir um PDF válido @@ -43,7 +43,7 @@ export class PDFParser implements Parser { pageRanges: '', tagged: true, outline: false, - timeout: 30000, + timeout: 90000, }); await browser.close(); From 0aa9c065a5398d6443f5626763b94f25a6ada7fe Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 16 Jun 2025 10:34:11 -0300 Subject: [PATCH 13/13] FEAT: block user from login when maestro is dedicated --- src/authentication/authentication.guard.ts | 13 +++++++++++++ src/modules/auth/auth.service.ts | 22 +++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/authentication/authentication.guard.ts b/src/authentication/authentication.guard.ts index bc69f8c..a41bc74 100644 --- a/src/authentication/authentication.guard.ts +++ b/src/authentication/authentication.guard.ts @@ -4,6 +4,7 @@ import { OnApplicationBootstrap, ExecutionContext, Inject, + ForbiddenException, } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import assert from 'assert'; @@ -113,6 +114,18 @@ export class AuthenticationGuard return false; } + // Bloquear outros customer de usar o maestor dedicado + const DEDICATED_PROXY = process.env.DEDICATED_PROXY + if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) { + throw new ForbiddenException(); + } + + // Bloquear o customer de acesso o maestro publico + const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(); + } + request.accessTokenPayload = accessTokenPayload; request.user = { user_id: accessTokenPayload.user_id, diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index be71d66..a8ceedd 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -54,14 +54,22 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } - checkDedicatedProxy(customerId: string) { + checkDedicatedProxy({ + customer + }: AuthSignInResponse) { const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); - if (DEDICATED_PROXY !== '') { - this.logger.info('Customer ID: ' + customerId); - if (DEDICATED_PROXY !== customerId) { - throw new ForbiddenException(); - } + this.logger.info('Customer ID: ' + customer.id); + + if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) { + throw new ForbiddenException(); + } + + // Bloquear o customer de acesso o maestro publico + this.logger.info('Check if customer have network policy: ' + customer.modules); + const hasNetworkPolicyModule = customer.modules.includes('network-policy'); + if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { + throw new ForbiddenException(); } } @@ -85,7 +93,7 @@ export class AuthClientService implements OnModuleInit { } if (result.customer) { - this.checkDedicatedProxy(result.customer.id); + this.checkDedicatedProxy(result); } return result