Merge pull request #409 from dadosfera/feature/catalog-update

FIX: Simplify searchDataAssets by owner logging and remove log code

docsfera.json

@@ -3887,6 +3887,26 @@
               "schema": {
                 "type": "string"
               }
+            },
+            {
+              "name": "catalog_date_from",
+              "required": false,
+              "in": "query",
+              "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-01-01",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "catalog_date_to",
+              "required": false,
+              "in": "query",
+              "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-12-31",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -4026,6 +4046,26 @@
               "schema": {
                 "type": "string"
               }
+            },
+            {
+              "name": "catalog_date_from",
+              "required": false,
+              "in": "query",
+              "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-01-01",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "catalog_date_to",
+              "required": false,
+              "in": "query",
+              "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-12-31",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {

src/modules/catalog/catalog.service.ts

@@ -41,6 +41,7 @@ import {
 import { TypeParser } from 'src/utils/FileParser/parser-types';
 import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
 
+
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
@@ -57,6 +58,7 @@ class CatalogService implements OnModuleInit {
     this.logger = dadosferaLogger.logger;
   }
 
+
   onModuleInit() {
     this.catalogReadService =
       this.grpcClient.getService<ReadService.CatalogReadServices>(
@@ -72,20 +74,24 @@ class CatalogService implements OnModuleInit {
       );
   }
 
+
   _getNimbusUrl(body) {
     this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
+
     if (process.env.ENV === 'prd') {
       return `https://nimbus-${customer}.dadosfera.ai`;
     }
 
+
     return `https://nimbus-${customer}.${process.env.ENV.replace(
       'local',
       'stg',
     )}.dadosfera.ai`;
   }
 
+
   async getPiiReporter(metadata: Metadata, type: TypeParser) {
     this.logger.info('getPiiReporter: ' + type)
     try {
@@ -96,8 +102,10 @@ class CatalogService implements OnModuleInit {
       )
       this.logger.info("Finish grpc call")
 
+
       const parser = ParserBuilder.build<PiiMetadata>(type);
 
+
       this.logger.info('parser file to: ' + type)
       const file = await parser.parse(data)
       this.logger.info('finish parser')
@@ -107,9 +115,11 @@ class CatalogService implements OnModuleInit {
         'pdf': 'application/pdf'
       }
 
+
       const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
       const filename = `relatorio-pii-${timestamp}.${type}`;
 
+
       return {
         file,
         filename: filename,
@@ -120,20 +130,25 @@ class CatalogService implements OnModuleInit {
       throw error;
     }
 
+
   }
 
+
   async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
     if (!data.embed) data.embed = undefined;
 
+
     return lastValueFrom(
       this.catalogWriteService.CreateDataAsset(data, metadata),
     );
   }
 
+
   async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.ManagePermission(data, metadata),
     ).catch((err) => {
@@ -144,9 +159,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.RevokePermission(data, metadata),
     ).catch((err) => {
@@ -157,9 +174,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.MakeAComment(data, metadata),
     ).catch((err) => {
@@ -170,9 +189,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.UpdateAComment(data, metadata),
     ).catch((err) => {
@@ -183,9 +204,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.DeleteDataAsset(data, metadata),
     ).catch((err) => {
@@ -196,73 +219,30 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async getUserRolesIds(userId: string) {
     const result = await this.userService.findOneById(userId).catch(() => null);
 
+
     if (result) {
       return result.user.roles.map((role) => role.id);
     }
 
+
     return [];
   }
 
+
   async searchDataAssets(
     query: Record<string, any>,
     metadata: Metadata,
     customer_id: string,
   ) {
-    this.logger.info('CatalogService - searchDataAssets', { query });
+    this.logger.info('CatalogService - searchDataAssets');
+
 
     const { search, page, size, sort_by, order, ...filters } = query;
 
-    this.logger.debug('Extracted filters:', { filters });
-
-    if (filters.owner) {
-      const { users: customer_users } =
-        await this.userService.findAllUsersByCustomerId(customer_id);
-      
-      this.logger.info('Available users in database count:', { count: customer_users.length });
-      this.logger.info('First 5 users:', {
-        users: customer_users.slice(0, 5).map(u => ({ id: u.id, email: u.email, name: u.name }))
-      });
-
-      const ownerValues = Array.isArray(filters.owner) 
-        ? filters.owner 
-        : (typeof filters.owner === 'string' && filters.owner.includes(',') 
-            ? filters.owner.split(',').map((o: string) => o.trim()) 
-            : [filters.owner]);
-      
-      this.logger.info('Owner values to convert:', { ownerValues, ownerFiltersOriginal: filters.owner });
-
-      const ownerIds = ownerValues
-        .map((ownerValue: string) => {
-          const normalizedOwner = ownerValue.replace(/\s/g, '+');
-          const user = customer_users.find(u => {
-            const isIdMatch = u.id === ownerValue;
-            const isEmailMatch = u.email === ownerValue || u.email === normalizedOwner;
-            const isNameMatch = u.name === ownerValue || u.name === normalizedOwner;
-            this.logger.info('Comparing:', {
-              userId: u.id,
-              userEmail: u.email,
-              userName: u.name,
-              filterValue: ownerValue,
-              normalizedFilter: normalizedOwner,
-              idMatch: isIdMatch,
-              emailMatch: isEmailMatch,
-              nameMatch: isNameMatch,
-            });
-            return isIdMatch || isEmailMatch || isNameMatch;
-          });
-          this.logger.info('Looking for owner result:', { ownerValue, found: !!user, userId: user?.id });
-          return user?.id || ownerValue;
-        })
-        .filter((id: string) => id);
-
-      if (ownerIds.length > 0) {
-        filters.owner = ownerIds;
-      }
-    }
-
     const { data_assets, total } = await lastValueFrom(
       this.catalogReadService.GetAllDataAssets(
         {
@@ -277,16 +257,20 @@ class CatalogService implements OnModuleInit {
       ),
     );
 
+
     const result = JSON.parse(data_assets);
 
+
     const response = await this.getAssetsUsersAndRoles(
       result.data_assets,
       customer_id,
     );
 
+
     return { data_assets: response, total };
   }
 
+
   async downloadAssets(
     query: Record<string, any>,
     metadata: Metadata,
@@ -294,6 +278,7 @@ class CatalogService implements OnModuleInit {
   ) {
     const data = await this.searchDataAssets(query, metadata, customer_id);
 
+
     const formatData = data.data_assets.map(asset => ({
       id: asset.id,
       display_name: asset.display_name,
@@ -302,19 +287,24 @@ class CatalogService implements OnModuleInit {
       tags: '[' + asset.tags.join(', ') + ']'
     }))
 
+
     const parser = ParserBuilder.build<AssetReporter>('csv');
 
+
     const file = await parser.parse(formatData);
 
+
     const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
     const filename = `dadosfera_assets_${timestamp}.csv`;
 
+
     return {
       file,
       filename
     }
   }
 
+
   async getOneDataAsset(data: {
     id: string;
     customer_id: string;
@@ -335,9 +325,11 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
+
     return { data_asset: asset[0] };
   }
 
+
   async getOneDataAssetByPipelineAndObject(data: {
     customer_id: string;
     pipeline: string;
@@ -359,9 +351,11 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
+
     return { data_asset: asset[0] };
   }
 
+
   async updateOneDataAsset(data: {
     data_asset_id: string;
     customer_id: string;
@@ -370,6 +364,7 @@ class CatalogService implements OnModuleInit {
   }) {
     const { body, customer_id, data_asset_id, metadata } = data;
 
+
     const { data_asset } = await lastValueFrom(
       this.catalogWriteService.UpdateDataAsset(
         { id: data_asset_id, changes: JSON.stringify(body) },
@@ -384,6 +379,7 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
+
     return { data_asset: asset[0] };
   }
 
@@ -396,6 +392,7 @@ class CatalogService implements OnModuleInit {
     return docs;
   }
 
+
   async getDatasetPreview(id: string, metadata: Metadata) {
     const { preview } = await lastValueFrom(
       this.catalogReadService.GetDatasetPreview(
@@ -407,6 +404,7 @@ class CatalogService implements OnModuleInit {
     return result;
   }
 
+
   async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
     const { columns_metadata } = await lastValueFrom(
       this.catalogReadService.GetDatasetColumnsMetadata(
@@ -436,6 +434,7 @@ class CatalogService implements OnModuleInit {
     return data;
   }
 
+
   private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
     this.logger.info('Creating data docs via gRPC for other asset types');
     try {
@@ -463,6 +462,7 @@ class CatalogService implements OnModuleInit {
   async findAllTags(data, metadata) {
     this.logger.info('CatalogService - findAllCustomerTags');
 
+
     const response = await lastValueFrom(
       this.catalogReadService.GetCustomerTags(data, metadata),
     )
@@ -475,6 +475,7 @@ class CatalogService implements OnModuleInit {
         throw new Error(err);
       });
 
+
     return response;
   }
   async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
@@ -489,6 +490,7 @@ class CatalogService implements OnModuleInit {
         (u) => u.id === data_asset.owner,
       )?.email;
 
+
       const roles = [];
       const users = [];
       const data_asset_roles = data_asset?.roles || []
@@ -497,6 +499,7 @@ class CatalogService implements OnModuleInit {
         if (role) roles.push({ id: role.id, name: role.name });
       }
 
+
       const data_asset_users = data_asset?.users || []
       for (const user_id of data_asset_users) {
         const user = customer_users.find((r) => r.id === user_id);
@@ -511,6 +514,7 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
     const { session } = await lastValueFrom(
       this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -524,6 +528,7 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
+
   async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogWriteService.AddRlsRule(data, metadata),
@@ -531,6 +536,7 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
+
   async removeRlsRule(id: number, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogWriteService.RemoveRlsRule({ id }, metadata),
@@ -538,12 +544,14 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
+
   async batchRemoveRlsRule(
     query: BatchRemoveRlsRulesRequest,
     metadata: Metadata,
   ) {
     const { id_rls, nimbus_dashboard_id } = query;
 
+
     if (id_rls && nimbus_dashboard_id) {
       throw new BadRequestException(
         "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
@@ -564,6 +572,7 @@ class CatalogService implements OnModuleInit {
     return 'OK';
   }
 
+
   async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogReadService.GetRlsRules(data, metadata),
@@ -571,6 +580,7 @@ class CatalogService implements OnModuleInit {
     return res.rls_rules;
   }
 
+
   async getOneRlsRule(id: number, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogReadService.GetOneRlsRule({ id }, metadata),
@@ -578,6 +588,7 @@ class CatalogService implements OnModuleInit {
     return res.rls_rule;
   }
 
+
   async getNimbusDashboards(
     data: GetNimbusDashboardsRequest,
     metadata: Metadata,
@@ -588,19 +599,24 @@ class CatalogService implements OnModuleInit {
     return res.dashboards;
   }
 
+
   async createTableMetadata(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
 
+
     const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
 
+
     this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
     this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
     this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
 
+
     try {
       const { data, status } = await axios.post(endpoint, {...body.table_metadata});
 
+
       this.logger.info(
         `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
         {...body.logMetadata},
@@ -615,19 +631,23 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+
   async createColumnMetadata(body: any): Promise<number[]> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
     const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
 
+
  
 
+
     try {
       this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
       this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
       this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
       const { data, status } = await axios.post(endpoint, body.column_metadata);
 
+
       this.logger.info(
         `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
         {...body.logMetadata},
@@ -642,18 +662,22 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+
   async createDataPreview(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
     const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
 
+
     this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
     this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
     this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
 
+
     try {
       const { data, status } = await axios.post(endpoint, body.data_preview);
 
+
       this.logger.info(
         `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
         {...body.logMetadata},
@@ -670,9 +694,11 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+
   async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
     const customer_name_raw = metadata.get('customer_name');
 
+
     const customer_name = customer_name_raw?.[0]?.toString();
     if (!customer_name) {
       throw new BadRequestException('Customer name not found in metadata');
@@ -695,4 +721,5 @@ class CatalogService implements OnModuleInit {
   }
 }
 
-export { CatalogService };
+
+export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -153,6 +153,18 @@ export class ICatalogAllRequest {
     example: 'user-id-1,user-id-2',
   })
   owner?: string;
+
+  @ApiPropertyOptional({
+    description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
+    example: '2025-01-01',
+  })
+  catalog_date_from?: string;
+
+  @ApiPropertyOptional({
+    description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
+    example: '2025-12-31',
+  })
+  catalog_date_to?: string;
 }
 
 export class ICatalogAllResponse {