FEAT: catalog share endpoint for embeds

This commit is contained in:
marcos-silva-rodrigues
2025-05-12 12:15:16 -03:00
parent 10580dfa03
commit b312849fff
8 changed files with 719 additions and 441 deletions
+453 -39
View File
@@ -3873,6 +3873,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3903,6 +3906,11 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
}
]
}
},
@@ -3939,6 +3947,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3970,7 +3981,7 @@
}
},
{
"name": "c",
"name": "shared",
"required": true,
"in": "query",
"schema": {
@@ -3985,6 +3996,14 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
},
"put": {
@@ -4037,6 +4056,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4082,6 +4104,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4122,6 +4147,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4151,14 +4179,6 @@
"schema": {
"type": "string"
}
},
{
"name": "c",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -4175,6 +4195,14 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
@@ -4201,14 +4229,6 @@
"schema": {
"type": "string"
}
},
{
"name": "c",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -4225,6 +4245,14 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
@@ -4251,14 +4279,6 @@
"schema": {
"type": "string"
}
},
{
"name": "c",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -4275,6 +4295,14 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
},
"post": {
@@ -4317,6 +4345,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4355,6 +4386,11 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
}
]
}
},
@@ -4390,6 +4426,11 @@
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
}
]
}
},
@@ -4444,6 +4485,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4499,6 +4543,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4548,6 +4595,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4595,6 +4645,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4634,6 +4687,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4674,6 +4730,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4720,6 +4779,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4767,6 +4829,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4812,6 +4877,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4851,6 +4919,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4883,6 +4954,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4926,6 +5000,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5167,6 +5244,263 @@
]
}
},
"/catalog/data-asset/share/{id}": {
"get": {
"operationId": "ShareController_getShareDataAsset",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Catalog"
]
}
},
"/catalog/data-asset/share/{id}/columns-metadata": {
"get": {
"operationId": "ShareController_getShareDataAssetColumnsMetadata",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IColumnsMetadataResponse"
}
}
}
}
},
"tags": [
"Catalog"
]
}
},
"/catalog/data-asset/share/{id}/preview": {
"get": {
"operationId": "ShareController_getShareDataAssetPreview",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IPreviewResponse"
}
}
}
}
},
"tags": [
"Catalog"
]
}
},
"/catalog/data-asset/share/{id}/docs": {
"get": {
"operationId": "ShareController_getShareDataAssetDocs",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IDocsResponse"
}
}
}
}
},
"tags": [
"Catalog"
]
}
},
"/share": {
"put": {
"operationId": "ShareMetadataController_findOrcreate",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateShareMetadataDto"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"security": [
{
"access-token": []
}
]
},
"patch": {
"operationId": "ShareMetadataController_updateShareType",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UpdateShareMetadataDto"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"security": [
{
"access-token": []
}
]
}
},
"/share/{id}": {
"get": {
"operationId": "ShareMetadataController_get",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"security": [
{
"access-token": []
}
]
}
},
"/terms-of-use/token": {
"get": {
"operationId": "TermsOfUseController_getToken",
@@ -5796,20 +6130,6 @@
]
}
},
"/health": {
"get": {
"operationId": "HealthController_check",
"parameters": [],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Health"
]
}
},
"/network-policy": {
"get": {
"operationId": "NetworkPolicyController_getNetworks",
@@ -5899,6 +6219,59 @@
}
]
}
},
"/assign/public-key": {
"put": {
"operationId": "AssignController_create",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateAssignDto"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"security": [
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/health": {
"get": {
"operationId": "HealthController_check",
"parameters": [],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Health"
]
}
}
},
"info": {
@@ -8419,6 +8792,36 @@
"links"
]
},
"CreateShareMetadataDto": {
"type": "object",
"properties": {
"assetId": {
"type": "string"
},
"proposedId": {
"type": "string"
}
},
"required": [
"assetId",
"proposedId"
]
},
"UpdateShareMetadataDto": {
"type": "object",
"properties": {
"shareId": {
"type": "string"
},
"type": {
"type": "string"
}
},
"required": [
"shareId",
"type"
]
},
"ConnectionTestPingReq": {
"type": "object",
"properties": {
@@ -8779,6 +9182,17 @@
"required": [
"policies"
]
},
"CreateAssignDto": {
"type": "object",
"properties": {
"publicKey": {
"type": "string"
}
},
"required": [
"publicKey"
]
}
}
}
+4 -4
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.23",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1400,9 +1400,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.37.0-beta.23",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz",
"integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==",
"version": "0.0.0",
"resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz",
"integrity": "sha512-NfnvIENsIval+liPIPoVkFjixmKCKu9tRqvXQu3qTk67kZicNBvEOd9UezED1h1a5kX70+CbSR4mlXUZZ7ZVGA==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
+1 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.23",
"@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+6 -2
View File
@@ -29,6 +29,8 @@ import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
@Module({
providers: [
@@ -62,9 +64,11 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule
HealthModule
],
})
export class AppModule {}
@@ -28,12 +28,9 @@ import {
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from '../../../decorators/authentication.decorator';
import {
DADOSFERA_MODULES_KEYS,
PERMISSIONS_GROUPS,
} from '../../../authentication/permissions.enum';
import { CatalogService } from '../catalog.service';
} from '../../decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
@@ -52,7 +49,7 @@ import {
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
} from '../dtos';
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -198,6 +195,209 @@ export class CatalogController {
return res;
}
@Get('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id: string,
@Query('shared') shared?: 'true',
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`GET /data-asset/${id}`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
const user_roles = await this.catalogService.getUserRolesIds(user_id);
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
id,
metadata,
});
has_permission =
is_data_manager ||
data_asset?.owner === username ||
(user_roles as Array<any>).some((r) => data_asset.p_roles.includes(r)) ||
data_asset.p_users.includes(user_id);
if (
shared === 'true' &&
(data_asset.share_type === undefined || data_asset.share_type === 'none')
)
throw new NotFoundException();
if (!has_permission)
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
@ApiInternalOnlyEndpoint()
@Get('data-asset/rls/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetRls(
@User() user: RequestUser,
@Headers() headers,
@Param('id') id: string,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
const user_roles = await this.catalogService.getUserRolesIds(user_id);
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
id,
metadata,
});
if (data_asset?.owner === username) has_permission = true;
for (const role of user_roles) {
if (data_asset.p_roles.includes(role)) has_permission = true;
}
if (data_asset.p_users.includes(user_id)) has_permission = true;
if (is_data_manager || has_permission) {
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
@Get('data-asset/:id/columns-metadata')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - columns-metadata`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const columns_metadata =
await this.catalogService.getDatasetColumnsMetadata(id, metadata);
return { columns_metadata };
}
@Get('data-asset/:id/preview')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username, customer_modules } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
is_mask: customer_modules.some(mod => mod === 'pii')
});
const preview = await this.catalogService.getDatasetPreview(id, metadata);
return { preview };
}
@Get('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const docs = await this.catalogService.getDataDocs(id, metadata);
return { docs };
}
@Put('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
@@ -448,62 +648,6 @@ export class CatalogController {
return response;
}
@ApiInternalOnlyEndpoint()
@Get('data-asset/rls/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetRls(
@User() user: RequestUser,
@Headers() headers,
@Param('id') id: string,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
const user_roles = await this.catalogService.getUserRolesIds(user_id);
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
id,
metadata,
});
if (data_asset?.owner === username) has_permission = true;
for (const role of user_roles) {
if (data_asset.p_roles.includes(role)) has_permission = true;
}
if (data_asset.p_users.includes(user_id)) has_permission = true;
if (is_data_manager || has_permission) {
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
@Post('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) {
@@ -715,32 +859,30 @@ export class CatalogController {
}
@Get('pii-reporter')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@RequireModule(DADOSFERA_MODULES_KEYS.PII)
async getPiiReporter(
@User() user: RequestUser,
@Res() res: Response,
@Query('type') contentType: TypeParser = 'pdf',
) {
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const { file, filename, type } = await this.catalogService.getPiiReporter(
metadata,
contentType,
);
res.setHeader(
'Content-Disposition',
`attachment; filename="${filename}"`,
);
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
// use res.end to send buffer
return res.end(file);
} catch (error) {
console.error(error);
console.error(error)
this.logger.error(error.message);
}
}
}
}
+6 -5
View File
@@ -2,15 +2,15 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './controllers/catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogController } from './catalog.controller';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { CatalogPublicController } from './controllers/catalog.public.controller';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
const client = new CatalogClientConfiguration();
@@ -20,9 +20,10 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule
CustomersModule,
ShareModule,
],
controllers: [CatalogController, CatalogPublicController],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
exports: [CatalogService],
})
+27 -126
View File
@@ -12,11 +12,9 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
import {
BadRequestException,
ForbiddenException,
HttpException,
HttpStatus,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
@@ -37,19 +35,8 @@ import {
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
import { RequestUser } from 'src/decorators/user.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { CustomersService } from '../customers/customers.service';
import { LanguageEnum } from 'src/utils/languages.enum';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -63,7 +50,6 @@ class CatalogService implements OnModuleInit {
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly customerService: CustomersService
) {
this.logger = dadosferaLogger.logger;
}
@@ -248,77 +234,6 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async getOneDataAssetPrivate(user: RequestUser, id: string) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info("getOneDataAssetPrivate: " + user_id + " - " + id)
this.logger.info(`data: `, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get_catalog = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
const user_roles = await this.getUserRolesIds(user_id);
const { data_asset } = await this.getOneDataAsset({
customer_id,
id,
metadata,
});
has_permission =
is_data_manager ||
is_get_catalog ||
data_asset?.owner === username ||
(user_roles as Array<any>).some((r) =>
data_asset.p_roles.includes(r),
) ||
data_asset.p_users.includes(user_id);
if (!has_permission)
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
async getOneDataAssetPublic(customerId: string, id: string) {
this.logger.info("getOneDataAssetPublic: " + customerId + " - " + id)
const metadata = await this.createMetadataToPublicAccess(customerId);
const { data_asset } = await this.getOneDataAsset({
customer_id: customerId,
id,
metadata,
});
this.logger.info("found asset: " + JSON.stringify(data_asset))
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public')
throw new NotFoundException();
return { data_asset };
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -395,7 +310,7 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
@@ -558,26 +473,27 @@ class CatalogService implements OnModuleInit {
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, { ...body.logMetadata });
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, { ...body.logMetadata });
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, { ...body.table_metadata });
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, { ...body.logMetadata });
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
@@ -587,49 +503,51 @@ class CatalogService implements OnModuleInit {
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, { ...body.logMetadata });
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, { ...body.logMetadata });
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${error.response?.status
} because of ${error.response?.data || error.message}`, { ...body.logMetadata });
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, { ...body.logMetadata });
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, { ...body.logMetadata });
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${error.response?.status
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
throw new Error(error.response?.data?.message || error.message);
}
@@ -658,23 +576,6 @@ class CatalogService implements OnModuleInit {
);
return res;
}
async createMetadataToPublicAccess(customerId?: string, language?: LanguageEnum) {
if (!customerId) {
throw new NotFoundException();
}
const {
customer
} = await this.customerService.getCustomer(customerId);
return PackTheMetadata({
customer_id: customer.id,
customer_name: customer.name,
language,
})
}
}
export { CatalogService };
export { CatalogService };
@@ -1,184 +0,0 @@
import {
Controller,
ForbiddenException,
Get,
Headers,
Inject,
Param,
Query,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import {
RequireSomePermission,
} from '../../../decorators/authentication.decorator';
import {
PERMISSIONS_GROUPS,
} from '../../../authentication/permissions.enum';
import { CatalogService } from '../catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog')
@UseFilters(new GrpcToHttpExceptionFilter())
export class CatalogPublicController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogService: CatalogService,
) {
this.logger = dadosferaLogger.logger;
}
private async createAccessToPublicAssets(
user?: RequestUser,
customerId?: string,
language?: LanguageEnum,
) {
if (user) {
const { customer_name, customer_id, user_id, username } = user;
return PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
} else {
return await this.catalogService.createMetadataToPublicAccess(
customerId,
language,
);
}
}
private checkDefaultPermission(user: RequestUser) {
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
@Get('data-asset/:id')
async getDataAsset(
@User() user: RequestUser,
@Param('id') id: string,
@Query('c') customerId?: string,
) {
this.logger.info(`GET /data-asset/${id}`);
try {
if (user) {
return await this.catalogService.getOneDataAssetPrivate(user, id);
} else {
return await this.catalogService.getOneDataAssetPublic(customerId, id);
}
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
@Get('data-asset/:id/columns-metadata')
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('c') customerId: string,
): Promise<IColumnsMetadataResponse> {
const metadata = await this.createAccessToPublicAssets(
user,
customerId,
language,
);
this.logger.info(`/catalog - columns-metadata`);
const columns_metadata =
await this.catalogService.getDatasetColumnsMetadata(id, metadata);
if (user) {
this.checkDefaultPermission(user);
}
return { columns_metadata };
}
@Get('data-asset/:id/preview')
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('c') customerId: string,
): Promise<IPreviewResponse> {
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`);
const metadata = await this.createAccessToPublicAssets(
user,
customerId,
language,
);
const preview = await this.catalogService.getDatasetPreview(id, metadata);
if (user) {
this.checkDefaultPermission(user);
}
return { preview };
}
@Get('data-asset/:id/docs')
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('c') customerId: string,
): Promise<IDocsResponse> {
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`);
const metadata = await this.createAccessToPublicAssets(
user,
customerId,
language,
);
const docs = await this.catalogService.getDataDocs(id, metadata);
if (user) {
this.checkDefaultPermission(user);
}
return { docs };
}
}