From b312849fffdaf554ab389d55f1dc858c7927a958 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 12 May 2025 12:15:16 -0300 Subject: [PATCH] FEAT: catalog share endpoint for embeds --- docsfera.json | 492 ++++++++++++++++-- package-lock.json | 8 +- package.json | 2 +- src/app.module.ts | 8 +- .../{controllers => }/catalog.controller.ts | 302 ++++++++--- src/modules/catalog/catalog.module.ts | 11 +- src/modules/catalog/catalog.service.ts | 153 +----- .../controllers/catalog.public.controller.ts | 184 ------- 8 files changed, 719 insertions(+), 441 deletions(-) rename src/modules/catalog/{controllers => }/catalog.controller.ts (80%) delete mode 100644 src/modules/catalog/controllers/catalog.public.controller.ts diff --git a/docsfera.json b/docsfera.json index 2fd799a..762ab8d 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3873,6 +3873,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3903,6 +3906,11 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + } ] } }, @@ -3939,6 +3947,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -3970,7 +3981,7 @@ } }, { - "name": "c", + "name": "shared", "required": true, "in": "query", "schema": { @@ -3985,6 +3996,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] }, "put": { @@ -4037,6 +4056,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4082,6 +4104,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4122,6 +4147,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4151,14 +4179,6 @@ "schema": { "type": "string" } - }, - { - "name": "c", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { @@ -4175,6 +4195,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] } }, @@ -4201,14 +4229,6 @@ "schema": { "type": "string" } - }, - { - "name": "c", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { @@ -4225,6 +4245,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] } }, @@ -4251,14 +4279,6 @@ "schema": { "type": "string" } - }, - { - "name": "c", - "required": true, - "in": "query", - "schema": { - "type": "string" - } } ], "responses": { @@ -4275,6 +4295,14 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } ] }, "post": { @@ -4317,6 +4345,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4355,6 +4386,11 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + } ] } }, @@ -4390,6 +4426,11 @@ }, "tags": [ "Catalog" + ], + "security": [ + { + "access-token": [] + } ] } }, @@ -4444,6 +4485,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4499,6 +4543,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4548,6 +4595,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4595,6 +4645,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4634,6 +4687,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4674,6 +4730,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4720,6 +4779,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4767,6 +4829,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4812,6 +4877,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4851,6 +4919,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4883,6 +4954,9 @@ "Catalog" ], "security": [ + { + "access-token": [] + }, { "access-token": [] } @@ -4926,6 +5000,9 @@ { "access-token": [] }, + { + "access-token": [] + }, { "access-token": [] } @@ -5167,6 +5244,263 @@ ] } }, + "/catalog/data-asset/share/{id}": { + "get": { + "operationId": "ShareController_getShareDataAsset", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/catalog/data-asset/share/{id}/columns-metadata": { + "get": { + "operationId": "ShareController_getShareDataAssetColumnsMetadata", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IColumnsMetadataResponse" + } + } + } + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/catalog/data-asset/share/{id}/preview": { + "get": { + "operationId": "ShareController_getShareDataAssetPreview", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IPreviewResponse" + } + } + } + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/catalog/data-asset/share/{id}/docs": { + "get": { + "operationId": "ShareController_getShareDataAssetDocs", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IDocsResponse" + } + } + } + } + }, + "tags": [ + "Catalog" + ] + } + }, + "/share": { + "put": { + "operationId": "ShareMetadataController_findOrcreate", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateShareMetadataDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "ShareMetadataController_updateShareType", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateShareMetadataDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } + }, + "/share/{id}": { + "get": { + "operationId": "ShareMetadataController_get", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + } + ] + } + }, "/terms-of-use/token": { "get": { "operationId": "TermsOfUseController_getToken", @@ -5796,20 +6130,6 @@ ] } }, - "/health": { - "get": { - "operationId": "HealthController_check", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Health" - ] - } - }, "/network-policy": { "get": { "operationId": "NetworkPolicyController_getNetworks", @@ -5899,6 +6219,59 @@ } ] } + }, + "/assign/public-key": { + "put": { + "operationId": "AssignController_create", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAssignDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/health": { + "get": { + "operationId": "HealthController_check", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Health" + ] + } } }, "info": { @@ -8419,6 +8792,36 @@ "links" ] }, + "CreateShareMetadataDto": { + "type": "object", + "properties": { + "assetId": { + "type": "string" + }, + "proposedId": { + "type": "string" + } + }, + "required": [ + "assetId", + "proposedId" + ] + }, + "UpdateShareMetadataDto": { + "type": "object", + "properties": { + "shareId": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "shareId", + "type" + ] + }, "ConnectionTestPingReq": { "type": "object", "properties": { @@ -8779,6 +9182,17 @@ "required": [ "policies" ] + }, + "CreateAssignDto": { + "type": "object", + "properties": { + "publicKey": { + "type": "string" + } + }, + "required": [ + "publicKey" + ] } } } diff --git a/package-lock.json b/package-lock.json index c9d852f..50c8341 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1400,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.23", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", - "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-NfnvIENsIval+liPIPoVkFjixmKCKu9tRqvXQu3qTk67kZicNBvEOd9UezED1h1a5kX70+CbSR4mlXUZZ7ZVGA==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index fd3eb0b..40b4faa 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.23", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index fedb388..c9a8b35 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -29,6 +29,8 @@ import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; import { ThemeModule } from './modules/theme/theme.module'; import { NetworkPolicyModule } from './modules/network-policy/network-policy.module'; +import { AssignModule } from './modules/assign/assign.module'; +import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module'; @Module({ providers: [ @@ -62,9 +64,11 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod CustomersModule, OpenDataModule, ThemeModule, + NetworkPolicyModule, + AssignModule, + ShareMetadataModule, //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule, - NetworkPolicyModule + HealthModule ], }) export class AppModule {} diff --git a/src/modules/catalog/controllers/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts similarity index 80% rename from src/modules/catalog/controllers/catalog.controller.ts rename to src/modules/catalog/catalog.controller.ts index ca5d88a..f0727f0 100644 --- a/src/modules/catalog/controllers/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -28,12 +28,9 @@ import { RequireAllPermissions, RequireModule, RequireSomePermission, -} from '../../../decorators/authentication.decorator'; -import { - DADOSFERA_MODULES_KEYS, - PERMISSIONS_GROUPS, -} from '../../../authentication/permissions.enum'; -import { CatalogService } from '../catalog.service'; +} from '../../decorators/authentication.decorator'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { CatalogService } from './catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; @@ -52,7 +49,7 @@ import { IUpdateDataRequest, TriggerCatalogReq, TriggerCatalogRes, -} from '../dtos'; +} from './dtos'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { Language } from 'src/decorators/language.decorator'; import { LanguageEnum } from 'src/utils/languages.enum'; @@ -198,6 +195,209 @@ export class CatalogController { return res; } + @Get('data-asset/:id') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAsset( + @User() user: RequestUser, + @Param('id') id: string, + @Query('shared') shared?: 'true', + ) { + const { username, user_id, customer_id, customer_name, permissions } = user; + + this.logger.info(`GET /data-asset/${id}`, { + username, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + let has_permission = false; + const metadata = PackTheMetadata({ + username, + user_id, + customer_id, + customer_name, + }); + + const user_roles = await this.catalogService.getUserRolesIds(user_id); + + const { data_asset } = await this.catalogService.getOneDataAsset({ + customer_id, + id, + metadata, + }); + has_permission = + is_data_manager || + data_asset?.owner === username || + (user_roles as Array).some((r) => data_asset.p_roles.includes(r)) || + data_asset.p_users.includes(user_id); + if ( + shared === 'true' && + (data_asset.share_type === undefined || data_asset.share_type === 'none') + ) + throw new NotFoundException(); + if (!has_permission) + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + delete data_asset.p_roles; + delete data_asset.p_users; + + return { data_asset }; + } + + @ApiInternalOnlyEndpoint() + @Get('data-asset/rls/:id') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetRls( + @User() user: RequestUser, + @Headers() headers, + @Param('id') id: string, + ) { + const { username, user_id, customer_id, customer_name, permissions } = user; + + this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { + username, + customer_name, + }); + + const is_data_manager = permissions.includes( + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, + ); + let has_permission = false; + const metadata = PackTheMetadata({ + username, + user_id, + customer_id, + customer_name, + }); + + const user_roles = await this.catalogService.getUserRolesIds(user_id); + + const { data_asset } = await this.catalogService.getOneDataAsset({ + customer_id, + id, + metadata, + }); + + if (data_asset?.owner === username) has_permission = true; + + for (const role of user_roles) { + if (data_asset.p_roles.includes(role)) has_permission = true; + } + + if (data_asset.p_users.includes(user_id)) has_permission = true; + + if (is_data_manager || has_permission) { + delete data_asset.p_roles; + delete data_asset.p_users; + return { data_asset }; + } + + throw new ForbiddenException( + 'You do not have permission to access this data asset.', + ); + } + + @Get('data-asset/:id/columns-metadata') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetColumnsMetadata( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + ): Promise { + const { customer_name, customer_id, user_id, username } = user; + + this.logger.info(`/catalog - columns-metadata`, { + user_id, + customer_name, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + }); + + const columns_metadata = + await this.catalogService.getDatasetColumnsMetadata(id, metadata); + + return { columns_metadata }; + } + + @Get('data-asset/:id/preview') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetPreview( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + ): Promise { + const { customer_name, customer_id, user_id, username, customer_modules } = user; + + this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { + user_id, + customer_name, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + is_mask: customer_modules.some(mod => mod === 'pii') + }); + + const preview = await this.catalogService.getDatasetPreview(id, metadata); + + return { preview }; + } + + @Get('data-asset/:id/docs') + @RequireSomePermission( + PERMISSIONS_GROUPS.CATALOG.permissions.GET, + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, + ) + async getDataAssetDocs( + @User() user: RequestUser, + @Language() language: LanguageEnum, + @Param('id') id: string, + ): Promise { + const { customer_name, customer_id, user_id, username } = user; + + this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { + user_id, + customer_name, + }); + + const metadata = PackTheMetadata({ + customer_name, + customer_id, + user_id, + username, + language, + }); + + const docs = await this.catalogService.getDataDocs(id, metadata); + + return { docs }; + } + @Put('data-asset/:id') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, @@ -448,62 +648,6 @@ export class CatalogController { return response; } - @ApiInternalOnlyEndpoint() - @Get('data-asset/rls/:id') - @RequireSomePermission( - PERMISSIONS_GROUPS.CATALOG.permissions.GET, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, - ) - async getDataAssetRls( - @User() user: RequestUser, - @Headers() headers, - @Param('id') id: string, - ) { - const { username, user_id, customer_id, customer_name, permissions } = user; - - this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { - username, - customer_name, - }); - - const is_data_manager = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - let has_permission = false; - const metadata = PackTheMetadata({ - username, - user_id, - customer_id, - customer_name, - }); - - const user_roles = await this.catalogService.getUserRolesIds(user_id); - - const { data_asset } = await this.catalogService.getOneDataAsset({ - customer_id, - id, - metadata, - }); - - if (data_asset?.owner === username) has_permission = true; - - for (const role of user_roles) { - if (data_asset.p_roles.includes(role)) has_permission = true; - } - - if (data_asset.p_users.includes(user_id)) has_permission = true; - - if (is_data_manager || has_permission) { - delete data_asset.p_roles; - delete data_asset.p_users; - return { data_asset }; - } - - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - } - @Post('rls-rule') @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) { @@ -715,32 +859,30 @@ export class CatalogController { } @Get('pii-reporter') - @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) - @RequireModule(DADOSFERA_MODULES_KEYS.PII) - async getPiiReporter( - @User() user: RequestUser, - @Res() res: Response, - @Query('type') contentType: TypeParser = 'pdf', - ) { + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule( + DADOSFERA_MODULES_KEYS.PII + ) + async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") { this.logger.info('GET pii-reporter'); const metadata = PackTheMetadata(user); try { - const { file, filename, type } = await this.catalogService.getPiiReporter( - metadata, - contentType, - ); - res.setHeader( - 'Content-Disposition', - `attachment; filename="${filename}"`, - ); + const { + file, + filename, + type + } = await this.catalogService.getPiiReporter(metadata, contentType); + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); res.setHeader('Content-Type', type); // use res.end to send buffer return res.end(file); } catch (error) { - console.error(error); + console.error(error) this.logger.error(error.message); } } -} +} \ No newline at end of file diff --git a/src/modules/catalog/catalog.module.ts b/src/modules/catalog/catalog.module.ts index d990344..8320d37 100644 --- a/src/modules/catalog/catalog.module.ts +++ b/src/modules/catalog/catalog.module.ts @@ -2,15 +2,15 @@ import { Module } from '@nestjs/common'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { CatalogController } from './controllers/catalog.controller'; -import { CatalogService } from './catalog.service'; +import { CatalogController } from './catalog.controller'; import { CatalogClientConfiguration } from './catalog-client'; import { ClientsModule } from '@nestjs/microservices'; import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module'; import { UsersModule } from '../users/users.module'; import { RolesModule } from '../roles/roles.module'; import { CustomersModule } from '../customers/customers.module'; -import { CatalogPublicController } from './controllers/catalog.public.controller'; +import { ShareModule } from './share/share.module'; +import { CatalogService } from './catalog.service'; const client = new CatalogClientConfiguration(); @@ -20,9 +20,10 @@ const client = new CatalogClientConfiguration(); OldPipelineModule, UsersModule, RolesModule, - CustomersModule + CustomersModule, + ShareModule, ], - controllers: [CatalogController, CatalogPublicController], + controllers: [CatalogController], providers: [CatalogService, DadosferaLogger], exports: [CatalogService], }) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index f7a2ed0..9bf0047 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -12,11 +12,9 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces'; import { BadRequestException, - ForbiddenException, HttpException, HttpStatus, Inject, - NotFoundException, OnModuleInit, } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; @@ -37,19 +35,8 @@ import { GetRlsRulesRequest, PiiMetadata, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; -import { writeFileSync } from 'fs'; -import path from 'path'; -import { HtmlParser } from 'src/utils/FileParser/html-parser'; -import { PiiDto } from './dtos/pii.dto'; -import { CsvParser } from 'src/utils/FileParser/csv-parser'; -import { PDFParser } from 'src/utils/FileParser/pdf-parser'; import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; -import { RequestUser } from 'src/decorators/user.decorator'; -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; -import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { CustomersService } from '../customers/customers.service'; -import { LanguageEnum } from 'src/utils/languages.enum'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -63,7 +50,6 @@ class CatalogService implements OnModuleInit { private readonly grpcClient: ClientGrpc, private readonly userService: UsersService, private readonly roleService: RolesService, - private readonly customerService: CustomersService ) { this.logger = dadosferaLogger.logger; } @@ -248,77 +234,6 @@ class CatalogService implements OnModuleInit { return { data_assets: response, total }; } - async getOneDataAssetPrivate(user: RequestUser, id: string) { - const { username, user_id, customer_id, customer_name, permissions } = user; - - this.logger.info("getOneDataAssetPrivate: " + user_id + " - " + id) - - this.logger.info(`data: `, { - username, - customer_name, - }); - - const is_data_manager = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - - const is_get_catalog = permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, - ); - let has_permission = false; - const metadata = PackTheMetadata({ - username, - user_id, - customer_id, - customer_name, - }); - - const user_roles = await this.getUserRolesIds(user_id); - - const { data_asset } = await this.getOneDataAsset({ - customer_id, - id, - metadata, - }); - - has_permission = - is_data_manager || - is_get_catalog || - data_asset?.owner === username || - (user_roles as Array).some((r) => - data_asset.p_roles.includes(r), - ) || - data_asset.p_users.includes(user_id); - - - if (!has_permission) - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - delete data_asset.p_roles; - delete data_asset.p_users; - - return { data_asset }; - } - - async getOneDataAssetPublic(customerId: string, id: string) { - this.logger.info("getOneDataAssetPublic: " + customerId + " - " + id) - const metadata = await this.createMetadataToPublicAccess(customerId); - const { data_asset } = await this.getOneDataAsset({ - customer_id: customerId, - id, - metadata, - }); - this.logger.info("found asset: " + JSON.stringify(data_asset)) - delete data_asset.p_roles; - delete data_asset.p_users; - data_asset.share_type = 'public'; - if (data_asset.share_type !== 'public') - throw new NotFoundException(); - - return { data_asset }; - } - async getOneDataAsset(data: { id: string; customer_id: string; @@ -395,7 +310,7 @@ class CatalogService implements OnModuleInit { const { documentation } = await lastValueFrom( this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), ); - console.log(documentation); + const docs = JSON.parse(documentation); return docs; } @@ -558,26 +473,27 @@ class CatalogService implements OnModuleInit { async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; - this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, { ...body.logMetadata }); - this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); - this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, { ...body.logMetadata }); + this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); try { - const { data, status } = await axios.post(endpoint, { ...body.table_metadata }); + const { data, status } = await axios.post(endpoint, {...body.table_metadata}); this.logger.info( `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); return data.id; } catch (error) { this.logger.error( - `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${error.response?.status - } because of ${JSON.stringify(error.response?.data) || error.message}`, { ...body.logMetadata }); + `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${ + error.response?.status + } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata}); throw new Error(error.response?.data?.message || error.message); } } @@ -587,49 +503,51 @@ class CatalogService implements OnModuleInit { this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; - + try { - this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, { ...body.logMetadata }); - this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); - this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, { ...body.logMetadata }); + this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); const { data, status } = await axios.post(endpoint, body.column_metadata); this.logger.info( `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); return data.map((column) => column.id); } catch (error) { this.logger.error( - `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${error.response?.status - } because of ${error.response?.data || error.message}`, { ...body.logMetadata }); + `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${ + error.response?.status + } because of ${error.response?.data || error.message}`, {...body.logMetadata}); throw new Error(error.response?.data?.message || error.message); } } async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); - this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata }); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; - this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, { ...body.logMetadata }); - this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata }); - this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, { ...body.logMetadata }); + this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); try { const { data, status } = await axios.post(endpoint, body.data_preview); this.logger.info( `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); return data.id; } catch (error) { this.logger.error( - `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${error.response?.status + `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${ + error.response?.status } because of ${error.response?.data || error.message}`, - { ...body.logMetadata }, + {...body.logMetadata}, ); throw new Error(error.response?.data?.message || error.message); } @@ -658,23 +576,6 @@ class CatalogService implements OnModuleInit { ); return res; } - - async createMetadataToPublicAccess(customerId?: string, language?: LanguageEnum) { - if (!customerId) { - throw new NotFoundException(); - } - - const { - customer - } = await this.customerService.getCustomer(customerId); - - return PackTheMetadata({ - customer_id: customer.id, - customer_name: customer.name, - language, - }) - } - } -export { CatalogService }; +export { CatalogService }; \ No newline at end of file diff --git a/src/modules/catalog/controllers/catalog.public.controller.ts b/src/modules/catalog/controllers/catalog.public.controller.ts deleted file mode 100644 index 6d3be81..0000000 --- a/src/modules/catalog/controllers/catalog.public.controller.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { - Controller, - ForbiddenException, - Get, - Headers, - Inject, - Param, - Query, - UseFilters, -} from '@nestjs/common'; -import { - ApiHeaders, - ApiTags, -} from '@nestjs/swagger'; -import { - RequireSomePermission, -} from '../../../decorators/authentication.decorator'; -import { - PERMISSIONS_GROUPS, -} from '../../../authentication/permissions.enum'; -import { CatalogService } from '../catalog.service'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { PackTheMetadata } from 'src/utils/PackTheMetadata'; -import { RequestUser, User } from 'src/decorators/user.decorator'; -import { - IColumnsMetadataResponse, - IDocsResponse, - IPreviewResponse, - -} from '../dtos'; -import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { Language } from 'src/decorators/language.decorator'; -import { LanguageEnum } from 'src/utils/languages.enum'; -import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; - - -@ApiTags('Catalog') -@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) -@Controller('catalog') -@UseFilters(new GrpcToHttpExceptionFilter()) -export class CatalogPublicController { - logger: DadosferaLogger; - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private catalogService: CatalogService, - ) { - this.logger = dadosferaLogger.logger; - } - - private async createAccessToPublicAssets( - user?: RequestUser, - customerId?: string, - language?: LanguageEnum, - ) { - if (user) { - const { customer_name, customer_id, user_id, username } = user; - return PackTheMetadata({ - customer_name, - customer_id, - user_id, - username, - language, - }); - } else { - return await this.catalogService.createMetadataToPublicAccess( - customerId, - language, - ); - } - } - - private checkDefaultPermission(user: RequestUser) { - const is_data_manager = user.permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid, - ); - - const is_get = user.permissions.includes( - PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid, - ); - - if (is_data_manager || is_get) { - return; - } - - throw new ForbiddenException( - 'You do not have permission to access this data asset.', - ); - } - - @Get('data-asset/:id') - async getDataAsset( - @User() user: RequestUser, - @Param('id') id: string, - @Query('c') customerId?: string, - ) { - this.logger.info(`GET /data-asset/${id}`); - try { - if (user) { - return await this.catalogService.getOneDataAssetPrivate(user, id); - } else { - return await this.catalogService.getOneDataAssetPublic(customerId, id); - } - } catch (error) { - this.logger.error(error.message); - throw error; - } - } - - - - @Get('data-asset/:id/columns-metadata') - async getDataAssetColumnsMetadata( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - @Query('c') customerId: string, - ): Promise { - const metadata = await this.createAccessToPublicAssets( - user, - customerId, - language, - ); - this.logger.info(`/catalog - columns-metadata`); - - const columns_metadata = - await this.catalogService.getDatasetColumnsMetadata(id, metadata); - - if (user) { - this.checkDefaultPermission(user); - } - - return { columns_metadata }; - } - - @Get('data-asset/:id/preview') - async getDataAssetPreview( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - @Query('c') customerId: string, - ): Promise { - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`); - - const metadata = await this.createAccessToPublicAssets( - user, - customerId, - language, - ); - - const preview = await this.catalogService.getDatasetPreview(id, metadata); - - if (user) { - this.checkDefaultPermission(user); - } - - return { preview }; - } - - @Get('data-asset/:id/docs') - async getDataAssetDocs( - @User() user: RequestUser, - @Language() language: LanguageEnum, - @Param('id') id: string, - @Query('c') customerId: string, - ): Promise { - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`); - - const metadata = await this.createAccessToPublicAssets( - user, - customerId, - language, - ); - - const docs = await this.catalogService.getDataDocs(id, metadata); - - if (user) { - this.checkDefaultPermission(user); - } - - return { docs }; - } - -}