From df3f2489f986823fe2dcd1b80b9f80e22470d612 Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Thu, 11 Dec 2025 22:10:32 +0000 Subject: [PATCH] FIX: Calendar Fix --- docsfera.json | 40 +++++++++++ src/modules/catalog/catalog.service.ts | 96 +++++++++++++++++++++++++- src/modules/catalog/dtos/index.ts | 12 ++++ 3 files changed, 147 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 9cb09aa..d27ab0a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3887,6 +3887,26 @@ "schema": { "type": "string" } + }, + { + "name": "catalog_date_from", + "required": false, + "in": "query", + "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-01-01", + "schema": { + "type": "string" + } + }, + { + "name": "catalog_date_to", + "required": false, + "in": "query", + "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-12-31", + "schema": { + "type": "string" + } } ], "responses": { @@ -4026,6 +4046,26 @@ "schema": { "type": "string" } + }, + { + "name": "catalog_date_from", + "required": false, + "in": "query", + "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-01-01", + "schema": { + "type": "string" + } + }, + { + "name": "catalog_date_to", + "required": false, + "in": "query", + "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-12-31", + "schema": { + "type": "string" + } } ], "responses": { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 4fd9619..8d87d9c 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -39,6 +39,7 @@ import { import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; + class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; catalogWriteService: WriteService.CatalogWriteServices; @@ -55,6 +56,7 @@ class CatalogService implements OnModuleInit { this.logger = dadosferaLogger.logger; } + onModuleInit() { this.catalogReadService = this.grpcClient.getService( @@ -70,20 +72,24 @@ class CatalogService implements OnModuleInit { ); } + _getNimbusUrl(body) { this.logger.debug(`Body: ${JSON.stringify(body)}`); const customer = body.info.customer.toLowerCase(); + if (process.env.ENV === 'prd') { return `https://nimbus-${customer}.dadosfera.ai`; } + return `https://nimbus-${customer}.${process.env.ENV.replace( 'local', 'stg', )}.dadosfera.ai`; } + async getPiiReporter(metadata: Metadata, type: TypeParser) { this.logger.info('getPiiReporter: ' + type) try { @@ -94,8 +100,10 @@ class CatalogService implements OnModuleInit { ) this.logger.info("Finish grpc call") + const parser = ParserBuilder.build(type); + this.logger.info('parser file to: ' + type) const file = await parser.parse(data) this.logger.info('finish parser') @@ -105,9 +113,11 @@ class CatalogService implements OnModuleInit { 'pdf': 'application/pdf' } + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `relatorio-pii-${timestamp}.${type}`; + return { file, filename: filename, @@ -118,20 +128,25 @@ class CatalogService implements OnModuleInit { throw error; } + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; + return lastValueFrom( this.catalogWriteService.CreateDataAsset(data, metadata), ); } + async managePermissions(data: Messages.ManagePermissionRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.ManagePermission(data, metadata), ).catch((err) => { @@ -142,9 +157,11 @@ class CatalogService implements OnModuleInit { }); } + async revokePermissions(data: Messages.RevokePermissionRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.RevokePermission(data, metadata), ).catch((err) => { @@ -155,9 +172,11 @@ class CatalogService implements OnModuleInit { }); } + async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.MakeAComment(data, metadata), ).catch((err) => { @@ -168,9 +187,11 @@ class CatalogService implements OnModuleInit { }); } + async deleteComment(data: Messages.UpdateACommentRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.UpdateAComment(data, metadata), ).catch((err) => { @@ -181,9 +202,11 @@ class CatalogService implements OnModuleInit { }); } + async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.DeleteDataAsset(data, metadata), ).catch((err) => { @@ -194,16 +217,20 @@ class CatalogService implements OnModuleInit { }); } + async getUserRolesIds(userId: string) { const result = await this.userService.findOneById(userId).catch(() => null); + if (result) { return result.user.roles.map((role) => role.id); } + return []; } + async searchDataAssets( query: Record, metadata: Metadata, @@ -211,10 +238,19 @@ class CatalogService implements OnModuleInit { ) { this.logger.info('CatalogService - searchDataAssets', { query }); + const { search, page, size, sort_by, order, ...filters } = query; + this.logger.debug('Extracted filters:', { filters }); + + console.log('MAESTRO VAI CHAMAR PI-FACTORY COM:', { + search, page, size, sort_by, order, + filters: filters + }); + + if (filters.owner) { const { users: customer_users } = await this.userService.findAllUsersByCustomerId(customer_id); @@ -224,6 +260,7 @@ class CatalogService implements OnModuleInit { users: customer_users.slice(0, 5).map(u => ({ id: u.id, email: u.email, name: u.name })) }); + const ownerValues = Array.isArray(filters.owner) ? filters.owner : (typeof filters.owner === 'string' && filters.owner.includes(',') @@ -232,6 +269,7 @@ class CatalogService implements OnModuleInit { this.logger.info('Owner values to convert:', { ownerValues, ownerFiltersOriginal: filters.owner }); + const ownerIds = ownerValues .map((ownerValue: string) => { const normalizedOwner = ownerValue.replace(/\s/g, '+'); @@ -256,11 +294,13 @@ class CatalogService implements OnModuleInit { }) .filter((id: string) => id); + if (ownerIds.length > 0) { filters.owner = ownerIds; } } + const { data_assets, total } = await lastValueFrom( this.catalogReadService.GetAllDataAssets( { @@ -275,16 +315,25 @@ class CatalogService implements OnModuleInit { ), ); + + + console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY'); + + + const result = JSON.parse(data_assets); + const response = await this.getAssetsUsersAndRoles( result.data_assets, customer_id, ); + return { data_assets: response, total }; } + async downloadAssets( query: Record, metadata: Metadata, @@ -292,6 +341,7 @@ class CatalogService implements OnModuleInit { ) { const data = await this.searchDataAssets(query, metadata, customer_id); + const formatData = data.data_assets.map(asset => ({ id: asset.id, display_name: asset.display_name, @@ -300,19 +350,24 @@ class CatalogService implements OnModuleInit { tags: '[' + asset.tags.join(', ') + ']' })) + const parser = ParserBuilder.build('csv'); + const file = await parser.parse(formatData); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `dadosfera_assets_${timestamp}.csv`; + return { file, filename } } + async getOneDataAsset(data: { id: string; customer_id: string; @@ -333,9 +388,11 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); + return { data_asset: asset[0] }; } + async getOneDataAssetByPipelineAndObject(data: { customer_id: string; pipeline: string; @@ -357,9 +414,11 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); + return { data_asset: asset[0] }; } + async updateOneDataAsset(data: { data_asset_id: string; customer_id: string; @@ -368,6 +427,7 @@ class CatalogService implements OnModuleInit { }) { const { body, customer_id, data_asset_id, metadata } = data; + const { data_asset } = await lastValueFrom( this.catalogWriteService.UpdateDataAsset( { id: data_asset_id, changes: JSON.stringify(body) }, @@ -382,9 +442,11 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); + return { data_asset: asset[0] }; } + async getDataDocs(id: string, metadata: Metadata) { const { documentation } = await lastValueFrom( this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), @@ -394,6 +456,7 @@ class CatalogService implements OnModuleInit { return docs; } + async getDatasetPreview(id: string, metadata: Metadata) { const { preview } = await lastValueFrom( this.catalogReadService.GetDatasetPreview( @@ -405,6 +468,7 @@ class CatalogService implements OnModuleInit { return result; } + async getDatasetColumnsMetadata(id: string, metadata: Metadata) { const { columns_metadata } = await lastValueFrom( this.catalogReadService.GetDatasetColumnsMetadata( @@ -416,6 +480,7 @@ class CatalogService implements OnModuleInit { return result; } + async createDataDocs(body) { const nimbusUrl = this._getNimbusUrl(body); const { data } = await axios.post( @@ -425,9 +490,11 @@ class CatalogService implements OnModuleInit { return data; } + async findAllTags(data, metadata) { this.logger.info('CatalogService - findAllCustomerTags'); + const response = await lastValueFrom( this.catalogReadService.GetCustomerTags(data, metadata), ) @@ -440,6 +507,7 @@ class CatalogService implements OnModuleInit { throw new Error(err); }); + return response; } async getAssetsUsersAndRoles(data_assets: Array, customer_id: string) { @@ -454,6 +522,7 @@ class CatalogService implements OnModuleInit { (u) => u.id === data_asset.owner, )?.email; + const roles = []; const users = []; const data_asset_roles = data_asset?.roles || [] @@ -462,6 +531,7 @@ class CatalogService implements OnModuleInit { if (role) roles.push({ id: role.id, name: role.name }); } + const data_asset_users = data_asset?.users || [] for (const user_id of data_asset_users) { const user = customer_users.find((r) => r.id === user_id); @@ -476,6 +546,7 @@ class CatalogService implements OnModuleInit { }); } + async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) { const { session } = await lastValueFrom( this.catalogWriteService.TriggerDatasetCataloging(data, metadata), @@ -489,6 +560,7 @@ class CatalogService implements OnModuleInit { return res; } + async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) { const res = await lastValueFrom( this.catalogWriteService.AddRlsRule(data, metadata), @@ -496,6 +568,7 @@ class CatalogService implements OnModuleInit { return res; } + async removeRlsRule(id: number, metadata: Metadata) { const res = await lastValueFrom( this.catalogWriteService.RemoveRlsRule({ id }, metadata), @@ -503,12 +576,14 @@ class CatalogService implements OnModuleInit { return res; } + async batchRemoveRlsRule( query: BatchRemoveRlsRulesRequest, metadata: Metadata, ) { const { id_rls, nimbus_dashboard_id } = query; + if (id_rls && nimbus_dashboard_id) { throw new BadRequestException( "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'", @@ -529,6 +604,7 @@ class CatalogService implements OnModuleInit { return 'OK'; } + async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) { const res = await lastValueFrom( this.catalogReadService.GetRlsRules(data, metadata), @@ -536,6 +612,7 @@ class CatalogService implements OnModuleInit { return res.rls_rules; } + async getOneRlsRule(id: number, metadata: Metadata) { const res = await lastValueFrom( this.catalogReadService.GetOneRlsRule({ id }, metadata), @@ -543,6 +620,7 @@ class CatalogService implements OnModuleInit { return res.rls_rule; } + async getNimbusDashboards( data: GetNimbusDashboardsRequest, metadata: Metadata, @@ -553,19 +631,24 @@ class CatalogService implements OnModuleInit { return res.dashboards; } + async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); + const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; + this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); + try { const { data, status } = await axios.post(endpoint, {...body.table_metadata}); + this.logger.info( `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, {...body.logMetadata}, @@ -580,19 +663,23 @@ class CatalogService implements OnModuleInit { } } + async createColumnMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; + + try { this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); const { data, status } = await axios.post(endpoint, body.column_metadata); + this.logger.info( `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, {...body.logMetadata}, @@ -607,18 +694,22 @@ class CatalogService implements OnModuleInit { } } + async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; + this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); + try { const { data, status } = await axios.post(endpoint, body.data_preview); + this.logger.info( `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, {...body.logMetadata}, @@ -635,9 +726,11 @@ class CatalogService implements OnModuleInit { } } + async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); + const customer_name = customer_name_raw?.[0]?.toString(); if (!customer_name) { throw new BadRequestException('Customer name not found in metadata'); @@ -660,4 +753,5 @@ class CatalogService implements OnModuleInit { } } -export { CatalogService }; \ No newline at end of file + +export { CatalogService }; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 59de404..39d2708 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -153,6 +153,18 @@ export class ICatalogAllRequest { example: 'user-id-1,user-id-2', }) owner?: string; + + @ApiPropertyOptional({ + description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)', + example: '2025-01-01', + }) + catalog_date_from?: string; + + @ApiPropertyOptional({ + description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)', + example: '2025-12-31', + }) + catalog_date_to?: string; } export class ICatalogAllResponse {