Merge pull request #309 from dadosfera/feature/dedicated-maestro

Feature/dedicated maestro

docsfera.json

@@ -5968,6 +5968,158 @@
             }
           ]
         }
+      },
+      "/api-key": {
+        "post": {
+          "operationId": "ApiKeyController_create",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/CreateApiKeyDto"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/CreateApiKeyResponseDto"
+                  }
+                }
+              }
+            },
+            "default": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/CreateApiKeyResponseDto"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "ApiKey"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        },
+        "get": {
+          "operationId": "ApiKeyController_findAll",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/components/schemas/ApiKeyBaseResponseDto"
+                    }
+                  }
+                }
+              }
+            },
+            "default": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/components/schemas/ApiKeyBaseResponseDto"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "ApiKey"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/api-key/{id}": {
+        "delete": {
+          "operationId": "ApiKeyController_remove",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "ApiKey"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
       }
     },
     "info": {
@@ -8848,6 +9000,89 @@
           "required": [
             "policies"
           ]
+        },
+        "CreateApiKeyDto": {
+          "type": "object",
+          "properties": {
+            "permissions": {
+              "description": "Array of permission IDs",
+              "type": "array",
+              "items": {
+                "type": "number"
+              }
+            }
+          },
+          "required": [
+            "permissions"
+          ]
+        },
+        "CreateApiKeyResponseDto": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "string",
+              "format": "uuid"
+            },
+            "key_mask": {
+              "type": "string"
+            },
+            "permissions": {
+              "type": "array",
+              "items": {
+                "$ref": "#/components/schemas/PermissionDto"
+              }
+            },
+            "created_at": {
+              "type": "string",
+              "format": "date-time"
+            },
+            "created_by": {
+              "type": "string"
+            },
+            "key": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "id",
+            "key_mask",
+            "permissions",
+            "created_at",
+            "created_by",
+            "key"
+          ]
+        },
+        "ApiKeyBaseResponseDto": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "string",
+              "format": "uuid"
+            },
+            "key_mask": {
+              "type": "string"
+            },
+            "permissions": {
+              "type": "array",
+              "items": {
+                "$ref": "#/components/schemas/PermissionDto"
+              }
+            },
+            "created_at": {
+              "type": "string",
+              "format": "date-time"
+            },
+            "created_by": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "id",
+            "key_mask",
+            "permissions",
+            "created_at",
+            "created_by"
+          ]
         }
       }
     }

environment.d.ts

@@ -14,6 +14,7 @@ declare global {
       AWS_REGION: string;
       OPEN_GROUP_ID: string;
       OPEN_CUSTOMER_ID: string;
+      DEDICATED_PROXY: string;
     }
   }
 }

helmfiles/stg.yaml

@@ -8,6 +8,29 @@ charts:
         value: duc.stg.dadosfera.ai
       - name: hostname
         value: maestro.stg.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.stg.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.stg.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.stg.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: e3f98a2f-7748-4981-8505-7695c8ca8218
+      - name: replicaCount
+        value: 1
+  
+    # Environment to test Network Policies
+  - name: private
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: maestro.duc_url
+        value: duc.stg.dadosfera.ai
+      - name: hostname
+        value: private-maestro.stg.dadosfera.ai
       - name: maestro.pi_factory_url
         value: pi-factory.dadosfera.ai
       - name: maestro.in_factory_url
@@ -18,5 +41,8 @@ charts:
         value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
       - name: maestro.open_group_id
         value: e3f98a2f-7748-4981-8505-7695c8ca8218
+      # Customer id
+      - name: maestro.dedicated_proxy
+        value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
       - name: replicaCount
         value: 1

package-lock.json

@@ -12,7 +12,7 @@
         "@aws-sdk/client-secrets-manager": "^3.414.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "3.37.0-beta.23",
+        "@dadosfera/protospack-v2": "3.38.0-beta.1",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1400,9 +1400,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.37.0-beta.23",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz",
-      "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==",
+      "version": "3.38.0-beta.1",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.1.tgz",
+      "integrity": "sha512-4+yNZMlhvEkHlxJcaeta+jtJ9owskZcg0yjKdZzBfT8PCclctgertLHmrG0+kDb3Y/na/rpPPHYULRp7AO4zDQ==",
       "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",

package.json

@@ -30,7 +30,7 @@
     "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.37.0-beta.23",
+    "@dadosfera/protospack-v2": "3.38.0-beta.1",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/app.module.ts

@@ -29,6 +29,7 @@ import { CustomersModule } from './modules/customers/customers.module';
 import { OpenDataModule } from './modules/open-data/open-data.module';
 import { ThemeModule } from './modules/theme/theme.module';
 import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
+import { ApiKeyModule } from './modules/api-key/api-key.module';
 
 @Module({
   providers: [
@@ -64,7 +65,8 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
     ThemeModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
-    NetworkPolicyModule
+    NetworkPolicyModule,
+    ApiKeyModule
   ],
 })
 export class AppModule {}

src/authentication/authentication.guard.ts

@@ -17,6 +17,7 @@ import {
 import { RequestUser } from '../decorators/user.decorator';
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';
+import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 @Injectable()
 export class AuthenticationGuard
@@ -31,6 +32,7 @@ export class AuthenticationGuard
     dadosferaLogger: DadosferaLogger,
     private reflector: Reflector,
     private authClient: AuthClientService,
+    private apiKeyService: ApiKeyService
   ) {
     this.pems = new Map();
     this.logger = dadosferaLogger.logger;
@@ -48,20 +50,40 @@ export class AuthenticationGuard
     });
   }
 
-  canActivate(ctx: ExecutionContext): boolean {
+  async canActivate(ctx: ExecutionContext): Promise<boolean> {
     const authFunctions = this.reflector.getAllAndMerge<
       AuthenticationFunction[]
     >(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
     const mustBeAuthenticated = authFunctions.length > 0;
 
-    const request = ctx.switchToHttp().getRequest();
-    const accessToken = this.validateToken(request, mustBeAuthenticated);
-
     if (!mustBeAuthenticated) {
       // no need to be authenticated
       return true;
     }
 
+    const request = ctx.switchToHttp().getRequest();
+    const apiKey = request.get('X-api-key');
+    if (apiKey) {
+      const {
+        api_key
+      } = await this.apiKeyService.get(apiKey);
+
+      request.user = {
+        user_id: api_key.user_id,
+        username: api_key.username,
+        permissions: api_key.permissions,
+        customer_id: api_key.customer_id,
+        customer_name: api_key.customer_name,
+        customer_tier: api_key.customer_tier,
+        customer_modules: api_key.customer_modules,
+        access_token: apiKey,
+      };
+
+      return true;
+    }
+
+    const accessToken = this.validateToken(request, mustBeAuthenticated);
+
     if (!accessToken) {
       // couldn't load valid token
       throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);

src/main.ts

@@ -3,6 +3,7 @@ import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import helmet from 'helmet';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { json, urlencoded } from 'express';
 
 import { AppModule } from './app.module';
 import { writeFileSync } from 'fs';
@@ -25,6 +26,11 @@ async function bootstrap() {
     },
   });
   app.use(helmet());
+  if (process.env.ENV === 'prd') {
+    app.use('/catalog/register-dataset', json({ limit: '10mb' }));
+    app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
+  }
+
   configureSwagger(app);
   await app.listen(3333);
   if (process.env.KILL_AFTER_START) await app.close();

src/modules/api-key/api-key.controller.ts

@@ -0,0 +1,68 @@
+import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
+import { ApiKeyService } from './api-key.service';
+import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+@Controller('api-key')
+@Authenticated()
+@ApiTags('ApiKey')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
+@UseFilters(new GrpcToHttpExceptionFilter())
+export class ApiKeyController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private readonly apiKeyService: ApiKeyService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post()
+  @ApiResponse({ type: CreateApiKeyResponseDto })
+  async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
+    this.logger.info('POST /api-key', {
+      permissions: createApiKeyDto.permissions,
+      method: 'create'
+    });
+    const result = await this.apiKeyService.create(createApiKeyDto, user);
+    this.logger.info('POST /api-key success', { 
+      id: result.id,
+      method: 'create'
+    });
+    return result;
+  }
+
+  @Get()
+  @ApiResponse({ type: [ApiKeyBaseResponseDto] })
+  async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
+    this.logger.info('GET /api-key', {
+      method: 'findAll'
+    });
+    const result = await this.apiKeyService.findAll(user);
+    this.logger.info('GET /api-key success', { 
+      count: result.length,
+      method: 'findAll'
+    });
+    return result;
+  }
+
+  @Delete(':id')
+  async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
+    this.logger.info('DELETE /api-key/:id', { 
+      id,
+      method: 'remove'
+    });
+    await this.apiKeyService.remove(id, user);
+    this.logger.info('DELETE /api-key/:id success', { 
+      id,
+      method: 'remove'
+    });
+  }
+}

src/modules/api-key/api-key.module.ts

@@ -0,0 +1,18 @@
+import { Module } from '@nestjs/common';
+import { ApiKeyService } from './api-key.service';
+import { ApiKeyController } from './api-key.controller';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+const ducClient = new DucClient();
+
+@Module({
+  imports: [
+    ClientsModule.register([ducClient.providerOptions])
+  ],
+  controllers: [ApiKeyController],
+  providers: [ApiKeyService, DadosferaLogger],
+  exports: [ApiKeyService]
+})
+export class ApiKeyModule {}

src/modules/api-key/api-key.service.ts

@@ -0,0 +1,50 @@
+import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
+import { RequestUser } from 'src/decorators/user.decorator';
+import { DucClient } from '../duc/client.config';
+import { PackTheMetadata } from '../../utils/ PackTheMetadata';
+import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { lastValueFrom } from 'rxjs';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+
+@Injectable()
+export class ApiKeyService implements OnModuleInit {
+  private apiKeyService: ApiKeyWriteProtoService;
+
+  constructor(
+    @Inject(DucClient.name) private readonly client: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
+  }
+
+  create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
+    const metadata = PackTheMetadata(user);
+
+    return lastValueFrom(this.apiKeyService.CreateApiKey({
+      permissions: createApiKeyDto.permissions
+    }, metadata));
+  }
+
+  async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
+    const metadata = PackTheMetadata(user);
+    console.log(metadata)
+
+    const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
+    return data.api_keys;
+  }
+
+  async remove(id: string, user: RequestUser) {
+    const metadata = PackTheMetadata(user);
+
+    await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
+  }
+
+  async get(key: string) {
+    const metadata = PackTheMetadata({});
+
+    return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
+  }
+}

src/modules/api-key/dto/api-key.dto.ts

@@ -0,0 +1,39 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsArray, IsNumber } from 'class-validator';
+
+export class PermissionDto {
+  @ApiProperty({ type: Number })
+  id: number;
+
+  @ApiProperty({ type: String })
+  name: string;
+}
+
+export class ApiKeyBaseResponseDto {
+  @ApiProperty({ type: String, format: 'uuid' })
+  id: string;
+
+  @ApiProperty({ type: String })
+  key_mask: string;
+
+  @ApiProperty({ type: [PermissionDto] })
+  permissions: PermissionDto[];
+
+  @ApiProperty({ type: String, format: 'date-time' })
+  created_at: string;
+
+  @ApiProperty({ type: String })
+  created_by: string;
+}
+
+export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
+  @ApiProperty({ type: String })
+  key: string;
+}
+
+export class CreateApiKeyDto {
+  @ApiProperty({ type: [Number], description: 'Array of permission IDs' })
+  @IsArray()
+  @IsNumber({}, { each: true })
+  permissions: number[];
+} 

src/modules/auth/auth.controller.ts

@@ -86,11 +86,9 @@ export class AuthController {
   async signIn(
     @Body() { username, password, totp }: AuthSignInReq,
     @Language() language: LanguageEnum,
-    @Headers('origin') origin = '',
   ): Promise<AuthSignInRes> {
     this.logger.info('/auth - SignIn');
-    const frontHost = origin.replace(/^https?:\/\//, '');
-    const metadata = PackTheMetadata({ language, custom_host: frontHost });
+    const metadata = PackTheMetadata({ language });
     this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
     return this.authClient.signIn({ username, password, totp }, metadata);
   }

src/modules/auth/auth.service.ts

@@ -1,4 +1,4 @@
-import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
+import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { lastValueFrom } from 'rxjs';
@@ -17,6 +17,7 @@ import {
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
   AuthConfirmResetPasswordRequest,
+  AuthSignInResponse,
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { DucClient } from '../duc/client.config';
 import { Metadata } from '@grpc/grpc-js';
@@ -53,15 +54,41 @@ export class AuthClientService implements OnModuleInit {
     return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
   }
 
+  checkDedicatedProxy(customerId: string) {
+    const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
+    this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
+    if (DEDICATED_PROXY !== '') {
+      this.logger.info('Customer ID: ' + customerId);
+      if (DEDICATED_PROXY !== customerId)  {
+        throw new ForbiddenException();
+      }
+    }
+  }
+
   async signIn(
     { username, password, totp }: AuthSignInRequest,
     metadata: Metadata,
   ) {
     this.logger.info('SignIn');
 
-    return lastValueFrom(
-      this.authService.AuthSignIn({ username, password, totp }, metadata),
-    );
+    let result: AuthSignInResponse;
+
+    try {
+      result = await lastValueFrom(
+        this.authService.AuthSignIn({ username, password, totp }, metadata),
+      );
+
+    } catch (error) {
+      this.logger.error('SignIn - Error during sign-in');
+      this.logger.error(error);
+      throw error;
+    }
+
+    if (result.customer) {
+      this.checkDedicatedProxy(result.customer.id);
+    }
+
+    return result
   }
 
   async refreshAccessToken(

src/modules/open-data/open-data.controller.ts

@@ -45,11 +45,11 @@ export class OpenDataController {
   ) {
     this.logger.info('createUser for open data' + JSON.stringify(request.headers));
 
-    const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
-    if (!corslist.includes(origin)) {
-      this.logger.info('block request by cors list: '+ origin);
-      throw new ForbiddenException();
-    }
+    // const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
+    // if (!corslist.includes(origin)) {
+    //   this.logger.info('block request by cors list: '+ origin);
+    //   throw new ForbiddenException();
+    // }
 
     const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
     const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;