Merge pull request #274 from dadosfera/UPDATE/helmfile

UPDATE: return variable to stg.

docsfera.json

@@ -542,6 +542,123 @@
           ]
         }
       },
+      "/auth/users/block": {
+        "post": {
+          "operationId": "AuthController_blockUsers",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Auth"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/auth/users/unblock": {
+        "post": {
+          "operationId": "AuthController_unblockUsers",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Auth"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/auth/users/reset": {
+        "post": {
+          "operationId": "AuthController_resetUsers",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Auth"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/connections": {
         "post": {
           "operationId": "ConnectionController_createConnection",
@@ -4787,6 +4904,41 @@
           ]
         }
       },
+      "/catalog/register-dataset": {
+        "post": {
+          "operationId": "CatalogController_registerDatasetWithMetadataRequest",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "201": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/terms-of-use/token": {
         "get": {
           "operationId": "TermsOfUseController_getToken",
@@ -5454,6 +5606,14 @@
               "schema": {
                 "type": "string"
               }
+            },
+            {
+              "name": "origin",
+              "required": true,
+              "in": "header",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -5608,6 +5768,9 @@
             "name": {
               "type": "string"
             },
+            "displayName": {
+              "type": "string"
+            },
             "tier": {
               "type": "string"
             },
@@ -5619,15 +5782,20 @@
               "items": {
                 "type": "string"
               }
+            },
+            "themeEnabled": {
+              "type": "boolean"
             }
           },
           "required": [
             "modules",
             "id",
             "name",
+            "displayName",
             "tier",
             "scheduleLimit",
-            "links"
+            "links",
+            "themeEnabled"
           ]
         },
         "AuthUser": {
@@ -8374,18 +8542,18 @@
             "textColor": {
               "type": "string"
             },
-            "logoWhite": {
+            "logo": {
               "type": "string"
             },
-            "logoBlack": {
+            "logoLogin": {
               "type": "string"
             }
           },
           "required": [
             "backgroundColor",
             "textColor",
-            "logoWhite",
-            "logoBlack"
+            "logo",
+            "logoLogin"
           ]
         },
         "CustomerThemeResponse": {

helmfiles/prd.yaml

@@ -18,3 +18,5 @@ charts:
         value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
       - name: maestro.open_group_id
         value: 401573bb-334f-44b2-b30e-88d4cea31ae9
+      - name: replicaCount
+        value: 2

helmfiles/stg.yaml

@@ -9,11 +9,11 @@ charts:
       - name: hostname
         value: maestro.stg.dadosfera.ai
       - name: maestro.pi_factory_url
-        value: pi-factory.dadosfera.ai
+        value: pi-factory.stg.dadosfera.ai
       - name: maestro.in_factory_url
-        value: in-factory.dadosfera.ai
+        value: in-factory.stg.dadosfera.ai
       - name: maestro.tr_factory_url
-        value: in-factory.dadosfera.ai
+        value: in-factory.stg.dadosfera.ai
       - name: maestro.open_customer_id
         value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
       - name: maestro.open_group_id

maestro/templates/deployment.yaml

@@ -35,7 +35,8 @@ spec:
                   - key: application
                     operator: In
                     values:
-                      - backend
+                      - general
+
       tolerations:
       - key: "kubernetes.azure.com/scalesetpriority"
         operator: "Equal"
@@ -95,6 +96,10 @@ spec:
               value: {{ .Values.maestro.tr_factory_url }}
             - name: UPLOAD_FILE_AGENT_CONNECTION
               value: {{ .Values.maestro.upload_file_agent_connection }}
+            - name: OPEN_CUSTOMER_ID
+              value: {{ .Values.maestro.open_customer_id }}
+            - name: OPEN_GROUP_ID
+              value: {{ .Values.maestro.open_group_id }}
             - name: JWT_PRIVATE_KEY
               valueFrom:
                 secretKeyRef:

maestro/templates/ingress-private.yaml

@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip 
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/server-snippet: |
+      underscores_in_headers on;
+      ignore_invalid_headers on;
+      
+  generation: 1
+  labels:
+    app: maestro
+  name: open-data
+  namespace: applications
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{ .Values.hostname }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: maestro
+            port:
+              number: {{ .Values.ingress.port }}
+        path: /open-data/sharing-ocean-data
+        pathType: Prefix

maestro/values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 
 replicaCount: 3
-hostname: maestro-temp.dadosfera.ai
+hostname: maestro.dadosfera.ai
 image:
   repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
   pullPolicy: IfNotPresent

package.json

@@ -30,7 +30,7 @@
     "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.37.0-beta.3",
+    "@dadosfera/protospack-v2": "3.37.0-alpha.19",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/decorators/set-origin.decorator.ts

@@ -1,32 +0,0 @@
-import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common';
-import { Reflector } from '@nestjs/core';
-import { SetMetadata } from '@nestjs/common';
-
-export const SetOrigin = (origin: string) => SetMetadata('allowedOrigin', origin);
-
-@Injectable()
-export class CORSGuard implements CanActivate {
-  constructor(private reflector: Reflector) {}
-
-  canActivate(context: ExecutionContext): boolean {
-    const response = context.switchToHttp().getResponse();
-    const request = context.switchToHttp().getRequest();
-    const origin = request.headers.origin;
-
-    // Obter a origem permitida através do decorador
-    const allowedOrigin = this.reflector.get<string>('allowedOrigin', context.getHandler());
-    if (process.env.ENV === "local") {
-        return true;
-    }
-
-    // Verifica se a origem da requisição é permitida   
-    if (allowedOrigin && origin !== allowedOrigin) {
-      throw new ForbiddenException('Acesso não permitido pela política CORS');
-    }
-
-    response.setHeader('Access-Control-Allow-Origin', allowedOrigin);
-
-    return true;
-  }
-}
-

src/modules/auth/auth.controller.ts

@@ -43,6 +43,7 @@ import {
   AuthRefreshAccessTokenRes,
   AuthSignInReq,
   AuthSignInRes,
+  BulkEditRequest,
 } from './dtos/login';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { AuthGuard } from '@nestjs/passport';
@@ -53,6 +54,7 @@ import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
+
 @ApiTags('Auth')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
@@ -61,6 +63,7 @@ export class AuthController {
   logger: DadosferaLogger;
   redirectUrl: string;
 
+
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
@@ -344,4 +347,77 @@ export class AuthController {
 
     return { token, email, url, language };
   }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('users/block')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async blockUsers(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Body() body: BulkEditRequest
+  ) {
+    this.logger.info('blockUsers - Starting request');
+  
+    try {
+      const metadata = PackTheMetadata({
+        customer_id: user.customer_id,
+        language,
+      });
+
+      this.logger.debug('Calling blockUsers service', {
+        metadata: {
+          access_token: metadata.get('access_token'),
+          language: metadata.get('language')
+        }
+      });
+
+      const result = await this.authClient.blockUsers(body.users, metadata);
+      this.logger.info('blockUsers - Success', { result });
+      return result;
+    } catch (error) {
+      this.logger.error('blockUsers - Error', {
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
+    }
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('users/unblock')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async unblockUsers(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Body() body: BulkEditRequest
+  ) {
+    this.logger.info('unblockUsers');
+    const metadata = PackTheMetadata({
+      customer_id: user.customer_id,
+      language,
+    });
+
+    return this.authClient.unblockUsers(body.users, metadata);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('users/reset')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async resetUsers(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Body() body: BulkEditRequest
+  ) {
+    this.logger.info('resetUsers');
+    const metadata = PackTheMetadata({
+      access_token: user.access_token,
+      customer_id: user.customer_id,
+      language,
+    });
+
+    return this.authClient.resetUsers(body.users,  metadata);
+  }
 }

src/modules/auth/auth.service.ts

@@ -20,6 +20,7 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { DucClient } from '../duc/client.config';
 import { Metadata } from '@grpc/grpc-js';
+import { BulkEditResponse } from './dtos/login';
 
 @Injectable()
 export class AuthClientService implements OnModuleInit {
@@ -168,4 +169,63 @@ export class AuthClientService implements OnModuleInit {
       this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
     );
   }
+
+  async blockUsers(users: string[],metadata: Metadata): Promise<BulkEditResponse> {
+    this.logger.info('blockUsers - Service starting');
+    
+    try {
+      this.logger.debug('Calling BlockUser gRPC method', {
+        metadata: {
+          access_token: metadata.get('access_token'),
+          language: metadata.get('language')
+        }
+      });
+
+      const response = await lastValueFrom<BulkEditResponse>(
+        this.authService.BlockUser({ users }, metadata),
+      );
+
+      this.logger.info('blockUsers - Service success', { response });
+      return response;
+    } catch (error) {
+      this.logger.error('blockUsers - Service error', {
+        error: error.message,
+        stack: error.stack,
+      });
+      throw error;
+    }
+  }
+
+  async unblockUsers(users: string[],metadata: Metadata): Promise<BulkEditResponse> {
+    this.logger.info('unblockUsers');
+    return await lastValueFrom(
+      this.authService.UnblockUser({ users }, metadata),
+    );
+  }
+
+  async resetUsers(users: string[], metadata: Metadata): Promise<BulkEditResponse> {
+    this.logger.info('resetUsers');
+    
+    try {
+      this.logger.debug('Calling ResetUser gRPC method', {
+        metadata: {
+          access_token: metadata.get('access_token'),
+          language: metadata.get('language')
+        }
+      });
+
+      const response = await lastValueFrom<BulkEditResponse>(
+        this.authService.ResetUser({ users }, metadata)
+      );
+
+      this.logger.info('resetUsers - Success', { response });
+      return response;
+    } catch (error) {
+      this.logger.error('resetUsers - Error', {
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
+    }
+  }
 }

src/modules/auth/dtos/login.ts

@@ -67,16 +67,26 @@ export class AuthUser {
 export class AuthCustomer {
   @ApiProperty()
   modules: string[];
+
   @ApiProperty()
   id: string;
+
   @ApiProperty()
   name: string;
+
+  @ApiProperty()
+  displayName: string;
   @ApiProperty()
   tier: string;
+
   @ApiProperty()
   scheduleLimit: string;
+
   @ApiProperty()
   links: Link[];
+
+  @ApiProperty()
+  themeEnabled: boolean;
 }
 
 export class AuthSignInReq implements AuthSignInRequest {
@@ -118,3 +128,13 @@ export class AuthRefreshAccessTokenRes {
   @ApiProperty()
   accessToken: string;
 }
+
+export interface BulkEditRequest {
+  users: string[];
+}
+
+export interface BulkEditResponse {
+  message: string;
+  successfulUsers: string[];
+  failedUsers: string[];
+}

src/modules/catalog/catalog-client.ts

@@ -5,6 +5,7 @@ import {
 } from '@nestjs/microservices';
 import { credentials } from '@grpc/grpc-js';
 import { Catalog } from '@dadosfera/protospack-v2';
+import { PlatformInterfaces } from '@dadosfera/protospack-v2';
 
 const isLocalConnection =
   process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
@@ -19,11 +20,13 @@ export class CatalogClientConfiguration {
       package: [
         Catalog.ProtoPackages.ReadPackage,
         Catalog.ProtoPackages.WritePackage,
+        PlatformInterfaces.ProtoPackages.WritePackage
       ],
       credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         Catalog.ProtoPaths.ReadFilePath,
         Catalog.ProtoPaths.WriteFilePath,
+        PlatformInterfaces.ProtoPaths.WriteFilePath
       ],
       loader: {
         keepCase: true,

src/modules/catalog/catalog.controller.ts

@@ -13,6 +13,8 @@ import {
   Put,
   Query,
   UseFilters,
+  HttpException,
+  HttpStatus,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
+import { lastValueFrom } from 'rxjs';
 import {
   BatchRemoveRlsRulesRequest,
   GetDatasetCatalogTaskRes,
@@ -54,6 +57,7 @@ import {
   AddRlsRuleRequest,
   GetNimbusDashboardsRequest,
   GetRlsRulesRequest,
+  RegisterDatasetWithMetatadaRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 @ApiTags('Catalog')
@@ -743,4 +747,94 @@ export class CatalogController {
 
     return JSON.parse(dashboards);
   }
+
+
+  @Post('register-dataset')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
+  )
+  async registerDatasetWithMetadataRequest(
+    @Body() body: RegisterDatasetWithMetatadaRequest,
+    @User() user: RequestUser,
+  ) {
+      const { customer_id, customer_name, user_id, username } = user;
+      const logMetadata = {
+        customer_name: customer_name,
+        user_id: user_id,
+        method: 'POST',
+        path: '/catalog/register-dataset',
+      };
+    try {
+
+      const metadata = PackTheMetadata({
+        customer_id,
+        customer_name,
+        user_id,
+        username,
+      });
+
+      this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
+
+      // Create table metadata
+      const tableMetadataBody = {
+        table_metadata: body.table_metadata,
+        info: {
+          customer: customer_name,
+          
+        },
+        logMetadata: logMetadata
+      };
+      
+      const table_metadata_id = await this.catalogService.createTableMetadata(
+        tableMetadataBody,
+      );
+      this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
+
+      // Create column metadata
+      const columnMetadataBody = {
+        column_metadata: body.column_metadata,
+        info: {
+          customer: customer_name,
+        },
+      };
+      this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
+      const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
+
+      // Create data preview
+      const dataPreviewBody = {
+        data_preview: body.data_preview,
+        info: {
+          customer: customer_name,
+        },
+      };
+      this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
+      const data_preview_id = await this.catalogService.createDataPreview(
+        dataPreviewBody,
+      );
+
+      // Catalog dataset item
+      this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
+      await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
+      this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
+      return {
+        message: 'Dataset registered successfully',
+        table_metadata_id: table_metadata_id,
+        column_metadata_ids: column_metadata_ids,
+        data_preview_id: data_preview_id,
+      };
+    } catch (error) {
+      this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
+
+      throw new HttpException(
+        {
+          message: 'Ocorreu um erro ao registrar o dataset',
+          error: error.message,
+          code: 'REGISTRATION_FAILED',
+          details: error.message,
+        },
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
+  }
 }

src/modules/catalog/catalog.service.ts

@@ -5,6 +5,11 @@ import {
   ProtoServices,
   Messages,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog';
+import {
+  Messages as PlatformInterfaceMessages,
+  WriteService as PlatformInterfaceWriteService,
+  ProtoServices as PlatformInterfacesProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
 import {
   BadRequestException,
   HttpException,
@@ -33,6 +38,7 @@ import {
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
+  platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
   logger: any;
   constructor(
     @Inject(DadosferaLogger)
@@ -54,9 +60,14 @@ class CatalogService implements OnModuleInit {
       this.grpcClient.getService<WriteService.CatalogWriteServices>(
         ProtoServices.CatalogWriteServices,
       );
+    this.platformWriteService =
+      this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
+        PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
+      );
   }
 
   _getNimbusUrl(body) {
+    this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
     if (process.env.ENV === 'prd') {
@@ -420,6 +431,112 @@ class CatalogService implements OnModuleInit {
     );
     return res.dashboards;
   }
+
+  async createTableMetadata(body: any): Promise<number> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+
+    const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
+
+    this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
+
+    try {
+      const { data, status } = await axios.post(endpoint, {...body.table_metadata});
+
+      this.logger.info(
+        `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.id;
+    } catch (error) {
+      this.logger.error(
+        `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
+          error.response?.status
+        } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async createColumnMetadata(body: any): Promise<number[]> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
+    const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
+
+ 
+
+    try {
+      this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
+      this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+      this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
+      const { data, status } = await axios.post(endpoint, body.column_metadata);
+
+      this.logger.info(
+        `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.map((column) => column.id);
+    } catch (error) {
+      this.logger.error(
+        `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
+          error.response?.status
+        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async createDataPreview(body: any): Promise<number> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+    const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
+
+    this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
+
+    try {
+      const { data, status } = await axios.post(endpoint, body.data_preview);
+
+      this.logger.info(
+        `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.id;
+    } catch (error) {
+      this.logger.error(
+        `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
+          error.response?.status
+        } because of ${error.response?.data || error.message}`,
+        {...body.logMetadata},
+      );
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
+    const customer_name_raw = metadata.get('customer_name');
+
+    const customer_name = customer_name_raw?.[0]?.toString();
+    if (!customer_name) {
+      throw new BadRequestException('Customer name not found in metadata');
+    }
+    const res = await lastValueFrom(
+      this.platformWriteService.CatalogDataAssets(
+        {
+          data_assets: [
+            {
+              data_asset_id: table_metadata_id.toString(),
+              customer_name: customer_name,
+              data_asset_type: 'dataset',
+            },
+          ],
+        },
+        metadata,
+      ),
+    );
+    return res;
+  }
 }
 
 export { CatalogService };

src/modules/customers/customers.service.ts

@@ -145,7 +145,7 @@ export class CustomersService implements OnModuleInit {
     const res = await lastValueFrom(
       this.pipelineReadService.PipelineV2GetDashboardUrl(
         {
-          dashboard_id: '45',
+          dashboard_id: '98',
           exp: '15m',
           metabase_customer_name: 'dadosferatech',
         },

src/modules/mixpanel/mixpanel.controller.ts

@@ -1,4 +1,4 @@
-import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
+import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
 import { init } from 'mixpanel';
 import { Authenticated } from 'src/decorators/authentication.decorator';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -13,7 +13,12 @@ export class MixpanelController {
     private readonly mixpanelToken: string,
   ) {}
   @Post(':id')
-  async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
+  async trackEvent(
+    @Param('id') id,
+    @Body() body,
+    @User() user: RequestUser,
+    @Req() request
+  ) {
     delete body.info;
     const mixpanel = init(this.mixpanelToken);
 
@@ -66,13 +71,16 @@ export class MixpanelController {
       $email: user.username.includes('@')
         ? user.username
         : user.username + '@dadosfera.ai',
-      customer_name: user.customer_name,
+      customer_name: user.customer_name
     });
 
     await mixpanel.track(id, {
       distinct_id: user.username,
       customer: user.customer_name,
       env: process.env.ENV,
+      $ip: request.ip,
+      $os: request.headers['sec-ch-ua-platform'] || '',
+      $browser: request.headers['user-agent'],
       ...body,
     });
 

src/modules/open-data/open-data.controller.ts

@@ -1,5 +1,5 @@
 import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { Body, Controller, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards } from '@nestjs/common';
+import { Body, Controller, ForbiddenException, Header, Headers, HttpCode, HttpException, Inject, Param, Post, Query, Req, Res, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
 import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@@ -8,7 +8,6 @@ import { UsersService } from '../users/users.service';
 import { Language } from 'src/decorators/language.decorator';
 import { OpenDataService } from './open-data.service';
 import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
-import { CORSGuard, SetOrigin } from 'src/decorators/set-origin.decorator';
 import { Metadata } from '@grpc/grpc-js';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { request } from 'http';
@@ -31,8 +30,6 @@ export class OpenDataController {
   }
 
   @Post("/sharing-ocean-data")
-  // @SetOrigin('devsbm.dadosfera.io')
-  // @UseGuards(CORSGuard)
   @HttpCode(200)
   @Header('content-type', 'application/json')
   @ApiOkResponse()
@@ -43,23 +40,20 @@ export class OpenDataController {
     language: string,
     @Req()
     request: Request,
+    @Headers('origin')
+    origin: string
   ) {
-    this.logger.info('createUser for open data'
-        + JSON.stringify({
-          origin: request.headers.origin,
-          language,
-          body
-    }));
+    this.logger.info('createUser for open data' + JSON.stringify(request.headers));
 
-    // "401573bb-334f-44b2-b30e-88d4cea31ae9"
-    // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID;
-    // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e""
-    // const OPENDATA_CUSTOMER_ID =  process.env.OPEN_CUSTOMER_ID;
-    const OPENDATA_CUSTOMER_ID = "b3e3dfe5-b992-4586-a73c-c0b0c00f615d";
+    const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
+    if (!corslist.includes(origin)) {
+      this.logger.info('block request by cors list: '+ origin);
+      throw new ForbiddenException();
+    }
 
-    this.logger.info("OPENDATA_CUSTOMER_ID: " + process.env.OPEN_CUSTOMER_ID)
-    this.logger.info("OPEN_GROUP_ID: " + process.env.OPEN_GROUP_ID)
-    const roles = ["e3f98a2f-7748-4981-8505-7695c8ca8218"];
+    const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
+    const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
+    const roles = [process.env.OPEN_GROUP_ID];
     const metadata = PackTheMetadata({
       language: language || 'en-us'
     }); 
@@ -77,7 +71,6 @@ export class OpenDataController {
       this.logger.error('user data ' + e.message);
     }
 
-
     const user: CreateUserOpenDataDTO = {
       email: data["email"],
       enquiryType: data["enquiry_type"],
@@ -85,11 +78,11 @@ export class OpenDataController {
       lastName: data["last_name"],
       organization: data["organization"]
     }
-    this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID }));
+    this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`);
 
     try {
-      await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata);
-      this.logger.info('user created with sucessfull data');
+      const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata);
+      this.logger.info('user created with id: '+ id);
       return {
         success: true,
         status: 'success',

src/modules/open-data/open-data.service.ts

@@ -40,10 +40,10 @@ export class OpenDataService implements OnModuleInit {
     }
 
     try {
-      await lastValueFrom(
+      const { user } = await lastValueFrom(
         this.usersClientService.SimpleUserCreate(body, metadata),
       );
-      return "User created";
+      return user.id;
     } catch(err) {
       return err;
     }

src/modules/pipelinesV2/pipelines.service.ts

@@ -328,7 +328,7 @@ export class PipelinesService implements OnModuleInit {
     const res = await lastValueFrom(
       this.pipelineReadService.PipelineV2GetDashboardUrl(
         {
-          dashboard_id: '83',
+          dashboard_id: '95',
           exp: '15m',
           metabase_customer_name: 'dadosferatech',
         },

src/modules/theme/dtos/customers.ts

@@ -27,9 +27,9 @@ export class CustomerTheme implements Theme {
   @ApiProperty()
   textColor: string;
   @ApiProperty()
-  logoWhite: string;
+  logo: string;
   @ApiProperty()
-  logoBlack: string;
+  logoLogin: string;
 }
 
 export class CustomerThemeResponse {

src/modules/theme/theme.controller.ts

@@ -59,22 +59,22 @@ export class ThemeController {
       id, 
     }));
 
-    const logoWhite = files.find(file => file.fieldname === 'logoWhite');
-    const logoBlack = files.find(file => file.fieldname === 'logoBlack');
+    const logo = files.find(file => file.fieldname === 'logo');
+    const logoLogin = files.find(file => file.fieldname === 'logoLogin');
 
-    this.validFileSize(logoWhite);
-    this.validFileSize(logoBlack);
-    this.validMimeType(logoWhite);
-    this.validMimeType(logoBlack);
+    this.validFileSize(logo);
+    this.validFileSize(logoLogin);
+    this.validMimeType(logo);
+    this.validMimeType(logoLogin);
 
     try {
       const theme = await this.themeService.createThemeByCustomer(id, {
         ...data,
-        logoWhite,
-        logoBlack
+        logo,
+        logoLogin
       });
       this.logger.info('saveCustomertheme' + JSON.stringify(theme));
-      return { theme: theme };
+      return theme;
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
         this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND');
@@ -98,7 +98,7 @@ export class ThemeController {
     const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png'];
 
     if (file && !mimeTypesValid.includes(file.mimetype)) {
-      throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg ou png`, HttpStatus.BAD_REQUEST);
+      throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, ou png`, HttpStatus.BAD_REQUEST);
     }
   }
 
@@ -109,8 +109,10 @@ export class ThemeController {
 
     try {
       const data = await this.themeService.getThemeByCustomer(id);
-      this.logger.info('Success - getCustomerTheme with id'+ id);
-      return data;
+      this.logger.info('Success - getCustomerTheme'+ JSON.stringify(data));
+      if (data?.theme) return data;
+
+      return { theme: null };
     }catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
         this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');

src/modules/theme/theme.service.ts

@@ -21,11 +21,10 @@ import { resolve } from 'path';
 import { Readable } from 'stream';
 
 type Files = {
-  logoWhite: Express.Multer.File,
-  logoBlack: Express.Multer.File,
+  logo: Express.Multer.File,
+  logoLogin: Express.Multer.File,
 }
 
-// This function will accept any string, which may result in a bug.
 @Injectable()
 export class ThemeService implements OnModuleInit {
   private themeService: ThemeProtoService;
@@ -63,12 +62,12 @@ export class ThemeService implements OnModuleInit {
       chunk: Buffer.alloc(0)
     })
 
-    if(theme.logoBlack) {
-      await this.sendFile(theme.logoBlack, customerThemeRequest$);
+    if(theme.logo) {
+      await this.sendFile(theme.logo, customerThemeRequest$);
     }
 
-    if(theme.logoWhite) {
-      await this.sendFile(theme.logoWhite, customerThemeRequest$);
+    if(theme.logoLogin) {
+      await this.sendFile(theme.logoLogin, customerThemeRequest$);
     }
     customerThemeRequest$.complete();
 
@@ -95,18 +94,20 @@ export class ThemeService implements OnModuleInit {
   }
 
    private async sendFile(file: Express.Multer.File, stream$: ReplaySubject<ThemeRequest>) {
-    const bufferStream = new Readable({
-      highWaterMark: 1024 * 1024, // 1 MB por chunk
-      read() {}
-    });
-    bufferStream.push(file.buffer);
-    bufferStream.push(null);
+    const chunkSize = 4 * 1024 * 1024;
+    const bufferStream = new CustomBufferStream(file.buffer, chunkSize);
+    const parseMimitypeForExtension = {
+      'image/jpeg': '.jpeg',
+      'image/jpg': '.jpg',
+      'image/png': '.png',
+      'image/svg+xml': '.svg',
+    }
+
+    const extension = parseMimitypeForExtension[file.mimetype];
 
     return new Promise((resolve, reject) => {
       bufferStream.on('data', (chunk) => {
-        console.log('enviando chunk', file.fieldname)
-        const mimetype = file.mimetype.split('/')[1]; // example image/jpeg
-        const filename = file.fieldname.concat(".", mimetype);
+        const filename = file.fieldname.concat(extension);
         stream$.next({
           customerId: '',
           displayName: '',
@@ -119,15 +120,39 @@ export class ThemeService implements OnModuleInit {
       });
 
       bufferStream.on('end', () => {
-        console.log('terminou de enviar')
         resolve(file.filename)
       });
 
       bufferStream.on('error', (err) => {
-        console.error('Erro no stream:', err);
         reject(err);
       });
     });
   }
 
 }
+
+class CustomBufferStream extends Readable {
+  buffer: Buffer;
+  offset: number;
+  chunkSize: number;
+
+  constructor(buffer: Buffer, chunkSize: number) {
+    super({ highWaterMark: chunkSize }); // Configura o tamanho do chunk
+    this.buffer = buffer;
+    this.offset = 0;
+    this.chunkSize = chunkSize;
+  }
+
+  _read() {
+    if (this.offset < this.buffer.length) {
+      const end = Math.min(this.offset + this.chunkSize, this.buffer.length);
+
+      const copiedBuf = Uint8Array.prototype.slice.call(this.buffer);
+      const chunk = copiedBuf.slice(this.offset, end);
+      this.offset = end;
+      this.push(chunk);
+    } else {
+      this.push(null);
+    }
+  }
+}

src/utils/ErrorBuilder.ts

@@ -77,6 +77,13 @@ export function EnrichErrorCode(code: string) {
         message: 'Token para recuperar senha inválido',
         code,
       };
+    case ErrorCodes.AUTH.REQUIRED_PASSWORD_RESET:
+      return {
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Não permitido',
+        message: 'Seu usuário foi resetado por um administrador, resete a senha pelo link "Esqueci a senha", para prosseguir',
+        code,
+      };
 
     case ErrorCodes.AUTH.WEAK_NEW_PASSWORD:
       return {

src/utils/errorCodes.ts

@@ -13,6 +13,7 @@ export const AUTH = {
   RESET_PASSWORD_CODE_EXPIRED: 'AUTH.RESET_PASSWORD_CODE_EXPIRED',
   RESET_PASSWORD_CODE_INVALID: 'AUTH.RESET_PASSWORD_CODE_INVALID',
   WEAK_NEW_PASSWORD: 'AUTH.WEAK_NEW_PASSWORD',
+  REQUIRED_PASSWORD_RESET: 'AUTH.REQUIRED_PASSWORD_RESET'
 };
 export const ROLE = {
   INVALID_ID: 'ROLE.INVALID_ID',