import { BadRequestException, Body, Controller, Delete, ForbiddenException, Get, Headers, Inject, NotFoundException, Param, Post, Put, Query, UseFilters, HttpException, HttpStatus, Res, } from '@nestjs/common'; import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags, } from '@nestjs/swagger'; import { Authenticated, RequireAllPermissions, RequireModule, RequireSomePermission, } from '../../decorators/authentication.decorator'; import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { CatalogService } from './catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; import { BatchRemoveRlsRulesRequest, GetDatasetCatalogTaskRes, ICatalogAllRequest, ICatalogAllResponse, IColumnsMetadataResponse, ICreateDataAsset, IDeleteComment, IDocsResponse, IMakeAComment, IOneDataAsset, IPreviewResponse, IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, } from './dtos'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { AddRlsRuleRequest, GetNimbusDashboardsRequest, GetRlsRulesRequest, RegisterDatasetWithMetatadaRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; import { Response } from 'express'; import { TypeParser } from 'src/utils/FileParser/parser-types'; @ApiTags('Catalog') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @Controller('catalog') @UseFilters(new GrpcToHttpExceptionFilter()) @Authenticated() export class CatalogController { logger: DadosferaLogger; constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, private catalogService: CatalogService, ) { this.logger = dadosferaLogger.logger; } @Get() @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async searchCatalog( @User() user: RequestUser, @Query() query: ICatalogAllRequest, ): Promise { const { user_id, customer_name, customer_id, username, permissions } = user; this.logger.info(`/catalog - searchCatalog`, { user_id, customer_name, }); const is_data_manager = permissions.includes( PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, ); const roles = await this.catalogService.getUserRolesIds(user_id); const metadata = PackTheMetadata({ user_id, customer_id, customer_name, username, roles, is_data_manager, }); const res = await this.catalogService.searchDataAssets( query, metadata, customer_id, ); return res; } @Get('/download') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async dowloadAsserts( @User() user: RequestUser, @Query() query: ICatalogAllRequest, @Res() res: Response ) { const { user_id, customer_name, customer_id, username, permissions } = user; this.logger.info(`/catalog/download - searchCatalog`, { user_id, customer_name, }); const is_data_manager = permissions.includes( PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, ); const roles = await this.catalogService.getUserRolesIds(user_id); const metadata = PackTheMetadata({ user_id, customer_id, customer_name, username, roles, is_data_manager, }); const { file, filename } = await this.catalogService.downloadAssets( query, metadata, customer_id, ); res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); res.setHeader('Content-Type', 'text/csv'); res.end(file); } @ApiInternalOnlyEndpoint() @Get('data-asset') async findByPipelineAndObject(@User() user: RequestUser, @Query() query) { const { username, user_id, customer_id, customer_name, permissions } = user; const { pipeline, object } = query; this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, { username, customer_name, }); if (!pipeline || !object) { throw new BadRequestException('Query params not provided'); } const is_data_manager = permissions.includes( PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, ); let has_permission = false; const metadata = PackTheMetadata({ username, user_id: undefined, customer_id, customer_name, }); const user_roles = await this.catalogService.getUserRolesIds(user_id); const { data_asset } = await this.catalogService.getOneDataAssetByPipelineAndObject({ pipeline: query.pipeline, object: query.object, customer_id, metadata, }); if (data_asset?.owner === username) has_permission = true; for (const role of user_roles) { if (data_asset.p_roles.includes(role)) has_permission = true; } if (data_asset.p_users.includes(user_id)) has_permission = true; if (is_data_manager || has_permission) { delete data_asset.p_roles; delete data_asset.p_users; return { data_asset }; } throw new ForbiddenException( 'You do not have permission to access this data asset.', ); } @ApiInternalOnlyEndpoint() @Get('tags') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async findAllTags(@Body() body) { this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, { user: body.info.user_id, customer: body.info.customer, }); const { user_id, customer, customer_id } = body.info; const metadata = PackTheMetadata({ user_id, customer_id, customer_name: customer, }); const res = await this.catalogService.findAllTags(body, metadata); return res; } @Get('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async getDataAsset( @User() user: RequestUser, @Param('id') id: string, @Query('shared') shared?: 'true', ) { const { username, user_id, customer_id, customer_name, permissions } = user; this.logger.info(`GET /data-asset/${id}`, { username, customer_name, }); const is_data_manager = permissions.includes( PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, ); let has_permission = false; const metadata = PackTheMetadata({ username, user_id, customer_id, customer_name, }); const user_roles = await this.catalogService.getUserRolesIds(user_id); const { data_asset } = await this.catalogService.getOneDataAsset({ customer_id, id, metadata, }); has_permission = is_data_manager || data_asset?.owner === username || (user_roles as Array).some((r) => data_asset.p_roles.includes(r)) || data_asset.p_users.includes(user_id); if ( shared === 'true' && (data_asset.share_type === undefined || data_asset.share_type === 'none') ) throw new NotFoundException(); if (!has_permission) throw new ForbiddenException( 'You do not have permission to access this data asset.', ); delete data_asset.p_roles; delete data_asset.p_users; return { data_asset }; } @ApiInternalOnlyEndpoint() @Get('data-asset/rls/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async getDataAssetRls( @User() user: RequestUser, @Headers() headers, @Param('id') id: string, ) { const { username, user_id, customer_id, customer_name, permissions } = user; this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { username, customer_name, }); const is_data_manager = permissions.includes( PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, ); let has_permission = false; const metadata = PackTheMetadata({ username, user_id, customer_id, customer_name, }); const user_roles = await this.catalogService.getUserRolesIds(user_id); const { data_asset } = await this.catalogService.getOneDataAsset({ customer_id, id, metadata, }); if (data_asset?.owner === username) has_permission = true; for (const role of user_roles) { if (data_asset.p_roles.includes(role)) has_permission = true; } if (data_asset.p_users.includes(user_id)) has_permission = true; if (is_data_manager || has_permission) { delete data_asset.p_roles; delete data_asset.p_users; return { data_asset }; } throw new ForbiddenException( 'You do not have permission to access this data asset.', ); } @Get('data-asset/:id/columns-metadata') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async getDataAssetColumnsMetadata( @User() user: RequestUser, @Language() language: LanguageEnum, @Param('id') id: string, ): Promise { const { customer_name, customer_id, user_id, username } = user; this.logger.info(`/catalog - columns-metadata`, { user_id, customer_name, }); const metadata = PackTheMetadata({ customer_name, customer_id, user_id, username, language, }); const columns_metadata = await this.catalogService.getDatasetColumnsMetadata(id, metadata); return { columns_metadata }; } @Get('data-asset/:id/preview') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async getDataAssetPreview( @User() user: RequestUser, @Language() language: LanguageEnum, @Param('id') id: string, ): Promise { const { customer_name, customer_id, user_id, username, customer_modules } = user; this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, customer_name, }); const metadata = PackTheMetadata({ customer_name, customer_id, user_id, username, language, is_mask: customer_modules.some(mod => mod === 'pii') }); const preview = await this.catalogService.getDatasetPreview(id, metadata); return { preview }; } @Get('data-asset/:id/docs') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async getDataAssetDocs( @User() user: RequestUser, @Language() language: LanguageEnum, @Param('id') id: string, @Query('asset_type') asset_type: string, ): Promise { const { customer_name, customer_id, user_id, username } = user; this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, customer_name, }); const metadata = PackTheMetadata({ customer_name, customer_id, user_id, username, language, }); const docs = await this.catalogService.getDataDocs(id, asset_type, metadata); return { docs }; } @Put('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async updateDataAsset( @User() user: RequestUser, @Language() language: LanguageEnum, @Param('id') data_asset_id: string, @Body() body: IUpdateDataRequest, ): Promise { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, language, }); const result = await this.catalogService.updateOneDataAsset({ body, data_asset_id, customer_id, metadata, }); delete result.data_asset.p_roles; delete result.data_asset.p_users; return result; } @Post('data-asset/:id/docs') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async manageDataAssetDocs( @User() user: RequestUser, @Headers() headers, @Param('id') table_id: string, @Body('docs') docs: string, @Query('asset_type') asset_type: string, ) { const { user_id, customer_name, customer_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, { user_id, customer_name, }); const body = { table_id, docs, asset_type, info: { customer: customer_name, }, } const res = await this.catalogService.createDataDocs(body, metadata); return res; } @ApiInternalOnlyEndpoint() @Put('data-asset/:id/manage-permissions') async manageDataAssetPermissions( @Param('id') id: string, @User() user: RequestUser, @Body() body, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.managePermissions( { ...body, id }, metadata, ); return { data_asset: JSON.parse(response.data_asset) }; } @ApiInternalOnlyEndpoint() @Put('data-asset/:id/revoke-permissions') async revokeDataAssetPermissions( @Param('id') id: string, @User() user: RequestUser, @Body() body, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.revokePermissions( { ...body, id }, metadata, ); return { message: response.message }; } @Post() @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async createDataAsset( @User() user: RequestUser, @Body() body: ICreateDataAsset, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const { data_asset } = await this.catalogService.createDataAsset( body, metadata, ); return { data_asset: JSON.parse(data_asset) }; } @Post('data-asset/:id/comment') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async commentOnDataAsset( @Param('id') id: string, @User() user: RequestUser, @Body() body: IMakeAComment, ) { const { message } = body; const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.commentOnDataAsset( { data_asset_id: id, username, message }, metadata, ); return response; } @Delete('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.DELETE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.deleteDataAsset( { id, type: undefined }, metadata, ); return response; } @Delete('data-asset/:id/comment') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async deleteComment( @Param('id') id: string, @User() user: RequestUser, @Body() body: IDeleteComment, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.deleteComment( { data_asset_id: id, comment: { ...body.comment, created_at: undefined, message: undefined }, }, metadata, ); return response; } @Post('dataset-catalog-task') @RequireAllPermissions( PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK, ) @ApiCreatedResponse({ type: TriggerCatalogRes }) async triggerCatalog( @User() user: RequestUser, @Body() body: TriggerCatalogReq, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const session = await this.catalogService.triggerCatalog(body, metadata); return { session }; } @Get('dataset-catalog-task/:session') @RequireAllPermissions( PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK, ) @ApiOkResponse({ type: GetDatasetCatalogTaskRes }) async getCatalogTask( @User() user: RequestUser, @Param('session') session: string, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.getDatasetCatalogTask( session, metadata, ); return response; } @Post('rls-rule') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.addRlsRule(body, metadata); return response; } @Get('rls-rule/:id') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async getOneRlsRule(@Param('id') id: string, @User() user: RequestUser) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const idInt = parseInt(id); const response = await this.catalogService.getOneRlsRule(idInt, metadata); return response; } @Get('rls-rule') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async getRlsRules( @User() user: RequestUser, @Query() query: GetRlsRulesRequest, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const response = await this.catalogService.getRlsRules(query, metadata); return response; } @Delete('rls-rule/:id') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async removeRlsRule(@Param('id') id: string, @User() user: RequestUser) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const idInt = parseInt(id); const response = await this.catalogService.removeRlsRule(idInt, metadata); return response; } @Delete('rls-rule') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async batchRemoveRlsRules( @Query() query: BatchRemoveRlsRulesRequest, @User() user: RequestUser, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); await this.catalogService.batchRemoveRlsRule(query, metadata); return { message: 'OK' }; } @Get('nimbus-dashboards') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async getNimbusDashboards( @User() user: RequestUser, @Body() body: GetNimbusDashboardsRequest, ) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); const dashboards = await this.catalogService.getNimbusDashboards( body, metadata, ); return JSON.parse(dashboards); } @Post('register-dataset') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async registerDatasetWithMetadataRequest( @Body() body: RegisterDatasetWithMetatadaRequest, @User() user: RequestUser, ) { const { customer_id, customer_name, user_id, username } = user; const logMetadata = { customer_name: customer_name, user_id: user_id, method: 'POST', path: '/catalog/register-dataset', }; try { const metadata = PackTheMetadata({ customer_id, customer_name, user_id, username, }); this.logger.log( `Request from user ${user_id} for customer ${customer_name}`, logMetadata, ); // Create table metadata const tableMetadataBody = { table_metadata: body.table_metadata, info: { customer: customer_name, }, logMetadata: logMetadata, }; const table_metadata_id = await this.catalogService.createTableMetadata( tableMetadataBody, ); this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata); // Create column metadata const columnMetadataBody = { column_metadata: body.column_metadata, info: { customer: customer_name, }, }; this.logger.info( `Creating column metadata for table ${table_metadata_id}`, logMetadata, ); const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody); // Create data preview const dataPreviewBody = { data_preview: body.data_preview, info: { customer: customer_name, }, }; this.logger.debug( `Creating data preview for table ${table_metadata_id}`, logMetadata, ); const data_preview_id = await this.catalogService.createDataPreview( dataPreviewBody, ); // Catalog dataset item this.logger.info( `Cataloging dataset item for table ${table_metadata_id}`, logMetadata, ); await this.catalogService.catalogDatasetItem(table_metadata_id, metadata); this.logger.info( `Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata, ); return { message: 'Dataset registered successfully', table_metadata_id: table_metadata_id, column_metadata_ids: column_metadata_ids, data_preview_id: data_preview_id, }; } catch (error) { this.logger.error( `Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata, ); throw new HttpException( { message: 'Ocorreu um erro ao registrar o dataset', error: error.message, code: 'REGISTRATION_FAILED', details: error.message, }, HttpStatus.INTERNAL_SERVER_ERROR, ); } } @Get('pii-reporter') @RequireSomePermission( PERMISSIONS_GROUPS.USERS.permissions.ADMIN ) @RequireModule( DADOSFERA_MODULES_KEYS.PII ) async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") { this.logger.info('GET pii-reporter'); const metadata = PackTheMetadata(user); try { const { file, filename, type } = await this.catalogService.getPiiReporter(metadata, contentType); res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); res.setHeader('Content-Type', type); // use res.end to send buffer return res.end(file); } catch (error) { console.error(error) this.logger.error(error.message); } } }