diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index c56313e..7660cd6 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -41,7 +41,6 @@ import { import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; - class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; catalogWriteService: WriteService.CatalogWriteServices; @@ -58,7 +57,6 @@ class CatalogService implements OnModuleInit { this.logger = dadosferaLogger.logger; } - onModuleInit() { this.catalogReadService = this.grpcClient.getService( @@ -74,81 +72,65 @@ class CatalogService implements OnModuleInit { ); } - _getNimbusUrl(body) { this.logger.debug(`Body: ${JSON.stringify(body)}`); const customer = body.info.customer.toLowerCase(); - if (process.env.ENV === 'prd') { return `https://nimbus-${customer}.dadosfera.ai`; } - return `https://nimbus-${customer}.${process.env.ENV.replace( 'local', 'stg', )}.dadosfera.ai`; } - async getPiiReporter(metadata: Metadata, type: TypeParser) { - this.logger.info('getPiiReporter: ' + type) + this.logger.info('getPiiReporter: ' + type); try { - const { - data - } = await lastValueFrom( - this.catalogWriteService.GetPiiReporter({}, metadata) - ) - this.logger.info("Finish grpc call") - + const { data } = await lastValueFrom( + this.catalogWriteService.GetPiiReporter({}, metadata), + ); + this.logger.info('Finish grpc call'); const parser = ParserBuilder.build(type); - - this.logger.info('parser file to: ' + type) - const file = await parser.parse(data) - this.logger.info('finish parser') + this.logger.info('parser file to: ' + type); + const file = await parser.parse(data); + this.logger.info('finish parser'); const mimeTypes: Record = { - 'csv': 'text/csv', - 'html': 'text/html', - 'pdf': 'application/pdf' - } - + csv: 'text/csv', + html: 'text/html', + pdf: 'application/pdf', + }; const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `relatorio-pii-${timestamp}.${type}`; - return { file, filename: filename, - type: mimeTypes[type] - } + type: mimeTypes[type], + }; } catch (error) { this.logger.error(error.message); throw error; } - - } - async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; - return lastValueFrom( this.catalogWriteService.CreateDataAsset(data, metadata), ); } - async managePermissions(data: Messages.ManagePermissionRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); - return lastValueFrom( this.catalogWriteService.ManagePermission(data, metadata), ).catch((err) => { @@ -159,11 +141,9 @@ class CatalogService implements OnModuleInit { }); } - async revokePermissions(data: Messages.RevokePermissionRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); - return lastValueFrom( this.catalogWriteService.RevokePermission(data, metadata), ).catch((err) => { @@ -174,11 +154,9 @@ class CatalogService implements OnModuleInit { }); } - async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); - return lastValueFrom( this.catalogWriteService.MakeAComment(data, metadata), ).catch((err) => { @@ -189,11 +167,9 @@ class CatalogService implements OnModuleInit { }); } - async deleteComment(data: Messages.UpdateACommentRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); - return lastValueFrom( this.catalogWriteService.UpdateAComment(data, metadata), ).catch((err) => { @@ -204,11 +180,9 @@ class CatalogService implements OnModuleInit { }); } - async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); - return lastValueFrom( this.catalogWriteService.DeleteDataAsset(data, metadata), ).catch((err) => { @@ -219,61 +193,110 @@ class CatalogService implements OnModuleInit { }); } - async getUserRolesIds(userId: string) { const result = await this.userService.findOneById(userId).catch(() => null); - if (result) { return result.user.roles.map((role) => role.id); } - return []; } - async searchDataAssets( query: Record, metadata: Metadata, customer_id: string, ) { - this.logger.info('CatalogService - searchDataAssets'); + this.logger.info('CatalogService - searchDataAssets', { query }); - const { search, page, size, sort_by, order, owner, ...filters } = query; - - // Verificar se filters.owner existe - if (owner !== undefined && owner !== null) { + const { search, page, size, sort_by, order, ...filters } = query; + + this.logger.debug('Extracted filters:', { filters }); + + console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', { + search, + page, + size, + sort_by, + order, + filters, + }); + + // 🔥 AJUSTE DO manually AQUI: converte '1'/'0' string em número ou remove + if ( + filters.manually !== undefined && + filters.manually !== null && + filters.manually !== '' + ) { + filters.manually = Number(filters.manually); // 1 ou 0 + } else { + delete filters.manually; + } + + console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', { + search, + page, + size, + sort_by, + order, + filters, + }); + + if (filters.owner) { const { users: customer_users } = await this.userService.findAllUsersByCustomerId(customer_id); - let ownerValues: string[]; - if (Array.isArray(owner)) { - ownerValues = owner; - } else if (typeof owner === 'string' && owner.includes(',')) { - ownerValues = owner.split(',').map((o: string) => o.trim()); - } else { - ownerValues = [owner]; - } - - // Separar "sem owner" (marcador especial null) dos owners normais - const hasEmptyOwner = ownerValues.some(v => v === 'null'); - const normalOwners = ownerValues.filter(v => v !== 'null'); - - // Converter nomes de owners para IDs - const ownerIds = normalOwners.map((ownerValue: string) => { - const user = customer_users.find(u => - u.name === ownerValue || u.email === ownerValue || u.id === ownerValue - ); - return user ? user.id : ownerValue; + this.logger.info('Available users in database count:', { + count: customer_users.length, + }); + this.logger.info('First 5 users:', { + users: customer_users + .slice(0, 5) + .map((u) => ({ id: u.id, email: u.email, name: u.name })), }); - // Se tiver "sem owner", adicionar marcador especial - if (hasEmptyOwner) { - ownerIds.push('null'); - } + const ownerValues = Array.isArray(filters.owner) + ? filters.owner + : typeof filters.owner === 'string' && filters.owner.includes(',') + ? filters.owner.split(',').map((o: string) => o.trim()) + : [filters.owner]; + + this.logger.info('Owner values to convert:', { + ownerValues, + ownerFiltersOriginal: filters.owner, + }); + + const ownerIds = ownerValues + .map((ownerValue: string) => { + const normalizedOwner = ownerValue.replace(/\s/g, '+'); + const user = customer_users.find((u) => { + const isIdMatch = u.id === ownerValue; + const isEmailMatch = + u.email === ownerValue || u.email === normalizedOwner; + const isNameMatch = + u.name === ownerValue || u.name === normalizedOwner; + this.logger.info('Comparing:', { + userId: u.id, + userEmail: u.email, + userName: u.name, + filterValue: ownerValue, + normalizedFilter: normalizedOwner, + idMatch: isIdMatch, + emailMatch: isEmailMatch, + nameMatch: isNameMatch, + }); + return isIdMatch || isEmailMatch || isNameMatch; + }); + this.logger.info('Looking for owner result:', { + ownerValue, + found: !!user, + userId: user?.id, + }); + return user?.id || ownerValue; + }) + .filter((id: string) => id); - // Adicionar ao filters apenas se houver valores if (ownerIds.length > 0) { filters.owner = ownerIds; } @@ -293,20 +316,18 @@ class CatalogService implements OnModuleInit { ), ); + console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY'); const result = JSON.parse(data_assets); - const response = await this.getAssetsUsersAndRoles( result.data_assets, customer_id, ); - return { data_assets: response, total }; } - async downloadAssets( query: Record, metadata: Metadata, @@ -314,33 +335,27 @@ class CatalogService implements OnModuleInit { ) { const data = await this.searchDataAssets(query, metadata, customer_id); - - const formatData = data.data_assets.map(asset => ({ + const formatData = data.data_assets.map((asset) => ({ id: asset.id, display_name: asset.display_name, data_asset_type: asset.data_asset_type, created_at: asset.created_at, - tags: '[' + asset.tags.join(', ') + ']' - })) - + tags: '[' + asset.tags.join(', ') + ']', + })); const parser = ParserBuilder.build('csv'); - const file = await parser.parse(formatData); - const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `dadosfera_assets_${timestamp}.csv`; - return { file, - filename - } + filename, + }; } - async getOneDataAsset(data: { id: string; customer_id: string; @@ -361,11 +376,9 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); - return { data_asset: asset[0] }; } - async getOneDataAssetByPipelineAndObject(data: { customer_id: string; pipeline: string; @@ -387,11 +400,9 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); - return { data_asset: asset[0] }; } - async updateOneDataAsset(data: { data_asset_id: string; customer_id: string; @@ -400,7 +411,6 @@ class CatalogService implements OnModuleInit { }) { const { body, customer_id, data_asset_id, metadata } = data; - const { data_asset } = await lastValueFrom( this.catalogWriteService.UpdateDataAsset( { id: data_asset_id, changes: JSON.stringify(body) }, @@ -415,7 +425,6 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); - return { data_asset: asset[0] }; } @@ -423,12 +432,11 @@ class CatalogService implements OnModuleInit { const { documentation } = await lastValueFrom( this.catalogReadService.GetDatasetDoc({ id }, metadata), ); - + const docs = JSON.parse(documentation); return docs; } - async getDatasetPreview(id: string, metadata: Metadata) { const { preview } = await lastValueFrom( this.catalogReadService.GetDatasetPreview( @@ -440,7 +448,6 @@ class CatalogService implements OnModuleInit { return result; } - async getDatasetColumnsMetadata(id: string, metadata: Metadata) { const { columns_metadata } = await lastValueFrom( this.catalogReadService.GetDatasetColumnsMetadata( @@ -456,7 +463,7 @@ class CatalogService implements OnModuleInit { if (body.asset_type === 'table' || body.asset_type === 'view') { return this.createDataDocsViaNimbus(body); } - + return this.createDataDocsViaGrpc(body, metadata); } @@ -470,7 +477,6 @@ class CatalogService implements OnModuleInit { return data; } - private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) { this.logger.info('Creating data docs via gRPC for other asset types'); try { @@ -483,9 +489,8 @@ class CatalogService implements OnModuleInit { metadata, ), ); - + return response; - } catch (error) { this.logger.error('Error creating data asset docs:', error); throw new HttpException( @@ -498,7 +503,6 @@ class CatalogService implements OnModuleInit { async findAllTags(data, metadata) { this.logger.info('CatalogService - findAllCustomerTags'); - const response = await lastValueFrom( this.catalogReadService.GetCustomerTags(data, metadata), ) @@ -511,9 +515,9 @@ class CatalogService implements OnModuleInit { throw new Error(err); }); - return response; } + async getAssetsUsersAndRoles(data_assets: Array, customer_id: string) { const { users: customer_users } = await this.userService.findAllUsersByCustomerId(customer_id); @@ -526,17 +530,15 @@ class CatalogService implements OnModuleInit { (u) => u.id === data_asset.owner, )?.email; - const roles = []; const users = []; - const data_asset_roles = data_asset?.roles || [] + const data_asset_roles = data_asset?.roles || []; for (const role_id of data_asset_roles) { const role = customer_roles.find((r) => r.id === role_id); if (role) roles.push({ id: role.id, name: role.name }); } - - const data_asset_users = data_asset?.users || [] + const data_asset_users = data_asset?.users || []; for (const user_id of data_asset_users) { const user = customer_users.find((r) => r.id === user_id); if (user) users.push({ id: user.id, email: user.email }); @@ -550,7 +552,6 @@ class CatalogService implements OnModuleInit { }); } - async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) { const { session } = await lastValueFrom( this.catalogWriteService.TriggerDatasetCataloging(data, metadata), @@ -564,7 +565,6 @@ class CatalogService implements OnModuleInit { return res; } - async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) { const res = await lastValueFrom( this.catalogWriteService.AddRlsRule(data, metadata), @@ -572,7 +572,6 @@ class CatalogService implements OnModuleInit { return res; } - async removeRlsRule(id: number, metadata: Metadata) { const res = await lastValueFrom( this.catalogWriteService.RemoveRlsRule({ id }, metadata), @@ -580,14 +579,12 @@ class CatalogService implements OnModuleInit { return res; } - async batchRemoveRlsRule( query: BatchRemoveRlsRulesRequest, metadata: Metadata, ) { const { id_rls, nimbus_dashboard_id } = query; - if (id_rls && nimbus_dashboard_id) { throw new BadRequestException( "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'", @@ -608,7 +605,6 @@ class CatalogService implements OnModuleInit { return 'OK'; } - async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) { const res = await lastValueFrom( this.catalogReadService.GetRlsRules(data, metadata), @@ -616,7 +612,6 @@ class CatalogService implements OnModuleInit { return res.rls_rules; } - async getOneRlsRule(id: number, metadata: Metadata) { const res = await lastValueFrom( this.catalogReadService.GetOneRlsRule({ id }, metadata), @@ -624,7 +619,6 @@ class CatalogService implements OnModuleInit { return res.rls_rule; } - async getNimbusDashboards( data: GetNimbusDashboardsRequest, metadata: Metadata, @@ -635,88 +629,99 @@ class CatalogService implements OnModuleInit { return res.dashboards; } - async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); - + this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; - - this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); - this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); - this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); - + this.logger.info( + `Creating table metadata for table ${body.table_metadata.table_name}`, + { ...body.logMetadata }, + ); + this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); + this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, { + ...body.logMetadata, + }); try { - const { data, status } = await axios.post(endpoint, {...body.table_metadata}); - + const { data, status } = await axios.post(endpoint, { + ...body.table_metadata, + }); this.logger.info( `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); return data.id; } catch (error) { this.logger.error( `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${ error.response?.status - } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata}); + } because of ${JSON.stringify(error.response?.data) || error.message}`, + { ...body.logMetadata }, + ); throw new Error(error.response?.data?.message || error.message); } } - async createColumnMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; - - - - try { - this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); - this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); - this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); - const { data, status } = await axios.post(endpoint, body.column_metadata); - + this.logger.info( + `Creating column metadata for table ${body.column_metadata.table_name}`, + { ...body.logMetadata }, + ); + this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); + this.logger.debug( + `Payload: ${JSON.stringify(body.column_metadata)}`, + { ...body.logMetadata }, + ); + const { data, status } = await axios.post( + endpoint, + body.column_metadata, + ); this.logger.info( `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); return data.map((column) => column.id); } catch (error) { this.logger.error( `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${ error.response?.status - } because of ${error.response?.data || error.message}`, {...body.logMetadata}); + } because of ${error.response?.data || error.message}`, + { ...body.logMetadata }, + ); throw new Error(error.response?.data?.message || error.message); } } - async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; - - this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); - this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); - this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); - + this.logger.info( + `Creating data preview for table ${body.data_preview.table_name}`, + { ...body.logMetadata }, + ); + this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); + this.logger.debug( + `Payload: ${JSON.stringify(body.data_preview)}`, + { ...body.logMetadata }, + ); try { const { data, status } = await axios.post(endpoint, body.data_preview); - this.logger.info( `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); return data.id; } catch (error) { @@ -724,17 +729,15 @@ class CatalogService implements OnModuleInit { `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${ error.response?.status } because of ${error.response?.data || error.message}`, - {...body.logMetadata}, + { ...body.logMetadata }, ); throw new Error(error.response?.data?.message || error.message); } } - async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); - const customer_name = customer_name_raw?.[0]?.toString(); if (!customer_name) { throw new BadRequestException('Customer name not found in metadata'); @@ -757,5 +760,4 @@ class CatalogService implements OnModuleInit { } } - export { CatalogService };