Merge pull request #259 from dadosfera/feat/update-customer-dash

UPDATE: customer dash

docsfera.json

@@ -4787,41 +4787,6 @@
           ]
         }
       },
-      "/catalog/register-dataset": {
-        "post": {
-          "operationId": "CatalogController_registerDatasetWithMetadataRequest",
-          "parameters": [
-            {
-              "name": "dadosfera-lang",
-              "in": "header",
-              "required": false,
-              "schema": {
-                "enum": [
-                  "pt-br",
-                  "en-us"
-                ],
-                "type": "string"
-              }
-            }
-          ],
-          "responses": {
-            "201": {
-              "description": ""
-            }
-          },
-          "tags": [
-            "Catalog"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
       "/terms-of-use/token": {
         "get": {
           "operationId": "TermsOfUseController_getToken",

helmfiles/prd.yaml

@@ -17,6 +17,4 @@ charts:
       - name: maestro.open_customer_id
         value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
       - name: maestro.open_group_id
-        value: 401573bb-334f-44b2-b30e-88d4cea31ae9
-      - name: replicaCount
-        value: 2
+        value: 401573bb-334f-44b2-b30e-88d4cea31ae9

maestro/templates/deployment.yaml

@@ -35,8 +35,7 @@ spec:
                   - key: application
                     operator: In
                     values:
-                      - general
-
+                      - backend
       tolerations:
       - key: "kubernetes.azure.com/scalesetpriority"
         operator: "Equal"

maestro/values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 
 replicaCount: 3
-hostname: maestro.dadosfera.ai
+hostname: maestro-temp.dadosfera.ai
 image:
   repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
   pullPolicy: IfNotPresent

package.json

@@ -30,7 +30,7 @@
     "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.37.0-beta.7",
+    "@dadosfera/protospack-v2": "3.37.0-beta.5",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/catalog/catalog-client.ts

@@ -5,7 +5,6 @@ import {
 } from '@nestjs/microservices';
 import { credentials } from '@grpc/grpc-js';
 import { Catalog } from '@dadosfera/protospack-v2';
-import { PlatformInterfaces } from '@dadosfera/protospack-v2';
 
 const isLocalConnection =
   process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
@@ -20,13 +19,11 @@ export class CatalogClientConfiguration {
       package: [
         Catalog.ProtoPackages.ReadPackage,
         Catalog.ProtoPackages.WritePackage,
-        PlatformInterfaces.ProtoPackages.WritePackage
       ],
       credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         Catalog.ProtoPaths.ReadFilePath,
         Catalog.ProtoPaths.WriteFilePath,
-        PlatformInterfaces.ProtoPaths.WriteFilePath
       ],
       loader: {
         keepCase: true,

src/modules/catalog/catalog.controller.ts

@@ -13,8 +13,6 @@ import {
   Put,
   Query,
   UseFilters,
-  HttpException,
-  HttpStatus,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -32,7 +30,6 @@ import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
-import { lastValueFrom } from 'rxjs';
 import {
   BatchRemoveRlsRulesRequest,
   GetDatasetCatalogTaskRes,
@@ -57,7 +54,6 @@ import {
   AddRlsRuleRequest,
   GetNimbusDashboardsRequest,
   GetRlsRulesRequest,
-  RegisterDatasetWithMetatadaRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 @ApiTags('Catalog')
@@ -747,94 +743,4 @@ export class CatalogController {
 
     return JSON.parse(dashboards);
   }
-
-
-  @Post('register-dataset')
-  @RequireSomePermission(
-    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
-    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
-  )
-  async registerDatasetWithMetadataRequest(
-    @Body() body: RegisterDatasetWithMetatadaRequest,
-    @User() user: RequestUser,
-  ) {
-      const { customer_id, customer_name, user_id, username } = user;
-      const logMetadata = {
-        customer_name: customer_name,
-        user_id: user_id,
-        method: 'POST',
-        path: '/catalog/register-dataset',
-      };
-    try {
-
-      const metadata = PackTheMetadata({
-        customer_id,
-        customer_name,
-        user_id,
-        username,
-      });
-
-      this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
-
-      // Create table metadata
-      const tableMetadataBody = {
-        table_metadata: body.table_metadata,
-        info: {
-          customer: customer_name,
-          
-        },
-        logMetadata: logMetadata
-      };
-      
-      const table_metadata_id = await this.catalogService.createTableMetadata(
-        tableMetadataBody,
-      );
-      this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
-
-      // Create column metadata
-      const columnMetadataBody = {
-        column_metadata: body.column_metadata,
-        info: {
-          customer: customer_name,
-        },
-      };
-      this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
-      const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
-
-      // Create data preview
-      const dataPreviewBody = {
-        data_preview: body.data_preview,
-        info: {
-          customer: customer_name,
-        },
-      };
-      this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
-      const data_preview_id = await this.catalogService.createDataPreview(
-        dataPreviewBody,
-      );
-
-      // Catalog dataset item
-      this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
-      await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
-      this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
-      return {
-        message: 'Dataset registered successfully',
-        table_metadata_id: table_metadata_id,
-        column_metadata_ids: column_metadata_ids,
-        data_preview_id: data_preview_id,
-      };
-    } catch (error) {
-      this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
-
-      throw new HttpException(
-        {
-          message: 'Ocorreu um erro ao registrar o dataset',
-          error: error.message,
-          code: 'REGISTRATION_FAILED',
-          details: error.message,
-        },
-        HttpStatus.INTERNAL_SERVER_ERROR,
-      );
-    }
-  }
 }

src/modules/catalog/catalog.service.ts

@@ -5,11 +5,6 @@ import {
   ProtoServices,
   Messages,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog';
-import {
-  Messages as PlatformInterfaceMessages,
-  WriteService as PlatformInterfaceWriteService,
-  ProtoServices as PlatformInterfacesProtoServices,
-} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
 import {
   BadRequestException,
   HttpException,
@@ -38,7 +33,6 @@ import {
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
-  platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
   logger: any;
   constructor(
     @Inject(DadosferaLogger)
@@ -60,14 +54,9 @@ class CatalogService implements OnModuleInit {
       this.grpcClient.getService<WriteService.CatalogWriteServices>(
         ProtoServices.CatalogWriteServices,
       );
-    this.platformWriteService =
-      this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
-        PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
-      );
   }
 
   _getNimbusUrl(body) {
-    this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
     if (process.env.ENV === 'prd') {
@@ -431,112 +420,6 @@ class CatalogService implements OnModuleInit {
     );
     return res.dashboards;
   }
-
-  async createTableMetadata(body: any): Promise<number> {
-    const nimbusUrl = this._getNimbusUrl(body);
-    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
-
-    const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
-
-    this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
-    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
-    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
-
-    try {
-      const { data, status } = await axios.post(endpoint, {...body.table_metadata});
-
-      this.logger.info(
-        `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
-        {...body.logMetadata},
-      );
-      return data.id;
-    } catch (error) {
-      this.logger.error(
-        `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
-          error.response?.status
-        } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
-      throw new Error(error.response?.data?.message || error.message);
-    }
-  }
-
-  async createColumnMetadata(body: any): Promise<number[]> {
-    const nimbusUrl = this._getNimbusUrl(body);
-    this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
-    const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
-
- 
-
-    try {
-      this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
-      this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
-      this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
-      const { data, status } = await axios.post(endpoint, body.column_metadata);
-
-      this.logger.info(
-        `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
-        {...body.logMetadata},
-      );
-      return data.map((column) => column.id);
-    } catch (error) {
-      this.logger.error(
-        `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
-          error.response?.status
-        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
-      throw new Error(error.response?.data?.message || error.message);
-    }
-  }
-
-  async createDataPreview(body: any): Promise<number> {
-    const nimbusUrl = this._getNimbusUrl(body);
-    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
-    const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
-
-    this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
-    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
-    this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
-
-    try {
-      const { data, status } = await axios.post(endpoint, body.data_preview);
-
-      this.logger.info(
-        `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
-        {...body.logMetadata},
-      );
-      return data.id;
-    } catch (error) {
-      this.logger.error(
-        `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
-          error.response?.status
-        } because of ${error.response?.data || error.message}`,
-        {...body.logMetadata},
-      );
-      throw new Error(error.response?.data?.message || error.message);
-    }
-  }
-
-  async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
-    const customer_name_raw = metadata.get('customer_name');
-
-    const customer_name = customer_name_raw?.[0]?.toString();
-    if (!customer_name) {
-      throw new BadRequestException('Customer name not found in metadata');
-    }
-    const res = await lastValueFrom(
-      this.platformWriteService.CatalogDataAssets(
-        {
-          data_assets: [
-            {
-              data_asset_id: table_metadata_id.toString(),
-              customer_name: customer_name,
-              data_asset_type: 'dataset',
-            },
-          ],
-        },
-        metadata,
-      ),
-    );
-    return res;
-  }
 }
 
 export { CatalogService };

src/modules/customers/customers.service.ts

@@ -145,7 +145,7 @@ export class CustomersService implements OnModuleInit {
     const res = await lastValueFrom(
       this.pipelineReadService.PipelineV2GetDashboardUrl(
         {
-          dashboard_id: '45',
+          dashboard_id: '98',
           exp: '15m',
           metabase_customer_name: 'dadosferatech',
         },

src/modules/open-data/open-data.controller.ts

@@ -41,6 +41,9 @@ export class OpenDataController {
     @Req()
     request: Request,
   ) {
+    if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) {
+      throw new ForbiddenException();
+    }
     this.logger.info('createUser for open data');
 
     this.logger.info('user-agent', request.headers['user-agent']);