Merge pull request #409 from dadosfera/feature/catalog-update

FIX: Simplify searchDataAssets by owner logging and remove log code

.github/workflows/validate-k8s.yml

@@ -66,13 +66,16 @@ jobs:
 
       - name: Install Helmfile
         run: |
-          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
           tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
           sudo mv helmfile /usr/local/bin/
           helmfile --version
 
-      - name: Install Helm Diff Plugin
-        run: helm plugin install https://github.com/databus23/helm-diff || true
+      - name: Debug Helm env
+        run: |
+          helm env
+          echo "HOME=$HOME"
+          ls -R $HOME/.local/share/helm || true
 
       - name: Authenticate with OKE cluster
         env:

docsfera.json

@@ -3877,6 +3877,36 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "owner",
+              "required": false,
+              "in": "query",
+              "description": "ID do usuário owner para filtrar data assets",
+              "example": "user-id-1,user-id-2",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "catalog_date_from",
+              "required": false,
+              "in": "query",
+              "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-01-01",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "catalog_date_to",
+              "required": false,
+              "in": "query",
+              "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-12-31",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -4006,6 +4036,36 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "owner",
+              "required": false,
+              "in": "query",
+              "description": "ID do usuário owner para filtrar data assets",
+              "example": "user-id-1,user-id-2",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "catalog_date_from",
+              "required": false,
+              "in": "query",
+              "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-01-01",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "catalog_date_to",
+              "required": false,
+              "in": "query",
+              "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
+              "example": "2025-12-31",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {

src/modules/catalog/catalog.service.ts

@@ -41,6 +41,7 @@ import {
 import { TypeParser } from 'src/utils/FileParser/parser-types';
 import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
 
+
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
@@ -57,6 +58,7 @@ class CatalogService implements OnModuleInit {
     this.logger = dadosferaLogger.logger;
   }
 
+
   onModuleInit() {
     this.catalogReadService =
       this.grpcClient.getService<ReadService.CatalogReadServices>(
@@ -72,20 +74,24 @@ class CatalogService implements OnModuleInit {
       );
   }
 
+
   _getNimbusUrl(body) {
     this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
+
     if (process.env.ENV === 'prd') {
       return `https://nimbus-${customer}.dadosfera.ai`;
     }
 
+
     return `https://nimbus-${customer}.${process.env.ENV.replace(
       'local',
       'stg',
     )}.dadosfera.ai`;
   }
 
+
   async getPiiReporter(metadata: Metadata, type: TypeParser) {
     this.logger.info('getPiiReporter: ' + type)
     try {
@@ -96,8 +102,10 @@ class CatalogService implements OnModuleInit {
       )
       this.logger.info("Finish grpc call")
 
+
       const parser = ParserBuilder.build<PiiMetadata>(type);
 
+
       this.logger.info('parser file to: ' + type)
       const file = await parser.parse(data)
       this.logger.info('finish parser')
@@ -107,9 +115,11 @@ class CatalogService implements OnModuleInit {
         'pdf': 'application/pdf'
       }
 
+
       const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
       const filename = `relatorio-pii-${timestamp}.${type}`;
 
+
       return {
         file,
         filename: filename,
@@ -120,20 +130,25 @@ class CatalogService implements OnModuleInit {
       throw error;
     }
 
+
   }
 
+
   async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
     if (!data.embed) data.embed = undefined;
 
+
     return lastValueFrom(
       this.catalogWriteService.CreateDataAsset(data, metadata),
     );
   }
 
+
   async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.ManagePermission(data, metadata),
     ).catch((err) => {
@@ -144,9 +159,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.RevokePermission(data, metadata),
     ).catch((err) => {
@@ -157,9 +174,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.MakeAComment(data, metadata),
     ).catch((err) => {
@@ -170,9 +189,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.UpdateAComment(data, metadata),
     ).catch((err) => {
@@ -183,9 +204,11 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
+
     return lastValueFrom(
       this.catalogWriteService.DeleteDataAsset(data, metadata),
     ).catch((err) => {
@@ -196,16 +219,20 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async getUserRolesIds(userId: string) {
     const result = await this.userService.findOneById(userId).catch(() => null);
 
+
     if (result) {
       return result.user.roles.map((role) => role.id);
     }
 
+
     return [];
   }
 
+
   async searchDataAssets(
     query: Record<string, any>,
     metadata: Metadata,
@@ -213,6 +240,7 @@ class CatalogService implements OnModuleInit {
   ) {
     this.logger.info('CatalogService - searchDataAssets');
 
+
     const { search, page, size, sort_by, order, ...filters } = query;
 
     const { data_assets, total } = await lastValueFrom(
@@ -229,16 +257,20 @@ class CatalogService implements OnModuleInit {
       ),
     );
 
+
     const result = JSON.parse(data_assets);
 
+
     const response = await this.getAssetsUsersAndRoles(
       result.data_assets,
       customer_id,
     );
 
+
     return { data_assets: response, total };
   }
 
+
   async downloadAssets(
     query: Record<string, any>,
     metadata: Metadata,
@@ -246,6 +278,7 @@ class CatalogService implements OnModuleInit {
   ) {
     const data = await this.searchDataAssets(query, metadata, customer_id);
 
+
     const formatData = data.data_assets.map(asset => ({
       id: asset.id,
       display_name: asset.display_name,
@@ -254,19 +287,24 @@ class CatalogService implements OnModuleInit {
       tags: '[' + asset.tags.join(', ') + ']'
     }))
 
+
     const parser = ParserBuilder.build<AssetReporter>('csv');
 
+
     const file = await parser.parse(formatData);
 
+
     const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
     const filename = `dadosfera_assets_${timestamp}.csv`;
 
+
     return {
       file,
       filename
     }
   }
 
+
   async getOneDataAsset(data: {
     id: string;
     customer_id: string;
@@ -287,9 +325,11 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
+
     return { data_asset: asset[0] };
   }
 
+
   async getOneDataAssetByPipelineAndObject(data: {
     customer_id: string;
     pipeline: string;
@@ -311,9 +351,11 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
+
     return { data_asset: asset[0] };
   }
 
+
   async updateOneDataAsset(data: {
     data_asset_id: string;
     customer_id: string;
@@ -322,6 +364,7 @@ class CatalogService implements OnModuleInit {
   }) {
     const { body, customer_id, data_asset_id, metadata } = data;
 
+
     const { data_asset } = await lastValueFrom(
       this.catalogWriteService.UpdateDataAsset(
         { id: data_asset_id, changes: JSON.stringify(body) },
@@ -336,6 +379,7 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
+
     return { data_asset: asset[0] };
   }
 
@@ -348,6 +392,7 @@ class CatalogService implements OnModuleInit {
     return docs;
   }
 
+
   async getDatasetPreview(id: string, metadata: Metadata) {
     const { preview } = await lastValueFrom(
       this.catalogReadService.GetDatasetPreview(
@@ -359,6 +404,7 @@ class CatalogService implements OnModuleInit {
     return result;
   }
 
+
   async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
     const { columns_metadata } = await lastValueFrom(
       this.catalogReadService.GetDatasetColumnsMetadata(
@@ -388,6 +434,7 @@ class CatalogService implements OnModuleInit {
     return data;
   }
 
+
   private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
     this.logger.info('Creating data docs via gRPC for other asset types');
     try {
@@ -415,6 +462,7 @@ class CatalogService implements OnModuleInit {
   async findAllTags(data, metadata) {
     this.logger.info('CatalogService - findAllCustomerTags');
 
+
     const response = await lastValueFrom(
       this.catalogReadService.GetCustomerTags(data, metadata),
     )
@@ -427,6 +475,7 @@ class CatalogService implements OnModuleInit {
         throw new Error(err);
       });
 
+
     return response;
   }
   async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
@@ -439,7 +488,8 @@ class CatalogService implements OnModuleInit {
     return data_assets.map((data_asset) => {
       const owner = customer_users.find(
         (u) => u.id === data_asset.owner,
-      )?.username;
+      )?.email;
+
 
       const roles = [];
       const users = [];
@@ -449,10 +499,11 @@ class CatalogService implements OnModuleInit {
         if (role) roles.push({ id: role.id, name: role.name });
       }
 
+
       const data_asset_users = data_asset?.users || []
       for (const user_id of data_asset_users) {
         const user = customer_users.find((r) => r.id === user_id);
-        if (user) users.push({ id: user.id, username: user.username });
+        if (user) users.push({ id: user.id, email: user.email });
       }
       return {
         ...data_asset,
@@ -463,6 +514,7 @@ class CatalogService implements OnModuleInit {
     });
   }
 
+
   async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
     const { session } = await lastValueFrom(
       this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -476,6 +528,7 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
+
   async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogWriteService.AddRlsRule(data, metadata),
@@ -483,6 +536,7 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
+
   async removeRlsRule(id: number, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogWriteService.RemoveRlsRule({ id }, metadata),
@@ -490,12 +544,14 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
+
   async batchRemoveRlsRule(
     query: BatchRemoveRlsRulesRequest,
     metadata: Metadata,
   ) {
     const { id_rls, nimbus_dashboard_id } = query;
 
+
     if (id_rls && nimbus_dashboard_id) {
       throw new BadRequestException(
         "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
@@ -516,6 +572,7 @@ class CatalogService implements OnModuleInit {
     return 'OK';
   }
 
+
   async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogReadService.GetRlsRules(data, metadata),
@@ -523,6 +580,7 @@ class CatalogService implements OnModuleInit {
     return res.rls_rules;
   }
 
+
   async getOneRlsRule(id: number, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogReadService.GetOneRlsRule({ id }, metadata),
@@ -530,6 +588,7 @@ class CatalogService implements OnModuleInit {
     return res.rls_rule;
   }
 
+
   async getNimbusDashboards(
     data: GetNimbusDashboardsRequest,
     metadata: Metadata,
@@ -540,19 +599,24 @@ class CatalogService implements OnModuleInit {
     return res.dashboards;
   }
 
+
   async createTableMetadata(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
 
+
     const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
 
+
     this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
     this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
     this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
 
+
     try {
       const { data, status } = await axios.post(endpoint, {...body.table_metadata});
 
+
       this.logger.info(
         `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
         {...body.logMetadata},
@@ -567,19 +631,23 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+
   async createColumnMetadata(body: any): Promise<number[]> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
     const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
 
+
  
 
+
     try {
       this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
       this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
       this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
       const { data, status } = await axios.post(endpoint, body.column_metadata);
 
+
       this.logger.info(
         `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
         {...body.logMetadata},
@@ -594,18 +662,22 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+
   async createDataPreview(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
     const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
 
+
     this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
     this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
     this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
 
+
     try {
       const { data, status } = await axios.post(endpoint, body.data_preview);
 
+
       this.logger.info(
         `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
         {...body.logMetadata},
@@ -622,9 +694,11 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+
   async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
     const customer_name_raw = metadata.get('customer_name');
 
+
     const customer_name = customer_name_raw?.[0]?.toString();
     if (!customer_name) {
       throw new BadRequestException('Customer name not found in metadata');
@@ -647,4 +721,5 @@ class CatalogService implements OnModuleInit {
   }
 }
 
-export { CatalogService };
+
+export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -147,6 +147,24 @@ export class ICatalogAllRequest {
     description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
   })
   order?: OrderEnum;
+
+  @ApiPropertyOptional({
+    description: 'ID do usuário owner para filtrar data assets',
+    example: 'user-id-1,user-id-2',
+  })
+  owner?: string;
+
+  @ApiPropertyOptional({
+    description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
+    example: '2025-01-01',
+  })
+  catalog_date_from?: string;
+
+  @ApiPropertyOptional({
+    description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
+    example: '2025-12-31',
+  })
+  catalog_date_to?: string;
 }
 
 export class ICatalogAllResponse {

src/modules/catalog/share/share.service.ts

@@ -180,7 +180,7 @@ export class ShareService implements OnModuleInit {
     return data_assets.map((data_asset) => {
       const owner = customer_users.find(
         (u) => u.id === data_asset.owner,
-      )?.username;
+      )?.email;
 
       const roles = [];
       const users = [];
@@ -190,7 +190,7 @@ export class ShareService implements OnModuleInit {
       }
       for (const user_id of data_asset.users) {
         const user = customer_users.find((r) => r.id === user_id);
-        if (user) users.push({ id: user.id, username: user.username });
+        if (user) users.push({ id: user.id, email: user.email });
       }
       return {
         ...data_asset,