Files
maestro/src/modules/catalog/catalog.controller.ts
T

946 lines
25 KiB
TypeScript

import {
BadRequestException,
Body,
Controller,
Delete,
ForbiddenException,
Get,
Headers,
Inject,
NotFoundException,
Param,
Post,
Put,
Query,
UseFilters,
HttpException,
HttpStatus,
Res,
} from '@nestjs/common';
import {
ApiCreatedResponse,
ApiHeaders,
ApiOkResponse,
ApiTags,
} from '@nestjs/swagger';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from '../../decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
GetDatasetCatalogTaskRes,
ICatalogAllRequest,
ICatalogAllResponse,
IColumnsMetadataResponse,
ICreateDataAsset,
IDeleteComment,
IDocsResponse,
IMakeAComment,
IOneDataAsset,
IPreviewResponse,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
import { TypeParser } from 'src/utils/FileParser/parser-types';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
export class CatalogController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogService: CatalogService,
) {
this.logger = dadosferaLogger.logger;
}
@Get()
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async searchCatalog(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
): Promise<ICatalogAllResponse> {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const res = await this.catalogService.searchDataAssets(
query,
metadata,
customer_id,
);
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog/download - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query;
this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
username,
customer_name,
});
if (!pipeline || !object) {
throw new BadRequestException('Query params not provided');
}
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id: undefined,
customer_id,
customer_name,
});
const user_roles = await this.catalogService.getUserRolesIds(user_id);
const { data_asset } =
await this.catalogService.getOneDataAssetByPipelineAndObject({
pipeline: query.pipeline,
object: query.object,
customer_id,
metadata,
});
if (data_asset?.owner === username) has_permission = true;
for (const role of user_roles) {
if (data_asset.p_roles.includes(role)) has_permission = true;
}
if (data_asset.p_users.includes(user_id)) has_permission = true;
if (is_data_manager || has_permission) {
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
@ApiInternalOnlyEndpoint()
@Get('tags')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { user_id, customer, customer_id } = body.info;
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name: customer,
});
const res = await this.catalogService.findAllTags(body, metadata);
return res;
}
@Get('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id: string,
@Query('shared') shared?: 'true',
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`GET /data-asset/${id}`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
const user_roles = await this.catalogService.getUserRolesIds(user_id);
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
id,
metadata,
});
has_permission =
is_data_manager ||
data_asset?.owner === username ||
(user_roles as Array<any>).some((r) => data_asset.p_roles.includes(r)) ||
data_asset.p_users.includes(user_id);
if (
shared === 'true' &&
(data_asset.share_type === undefined || data_asset.share_type === 'none')
)
throw new NotFoundException();
if (!has_permission)
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
@ApiInternalOnlyEndpoint()
@Get('data-asset/rls/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetRls(
@User() user: RequestUser,
@Headers() headers,
@Param('id') id: string,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
const user_roles = await this.catalogService.getUserRolesIds(user_id);
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
id,
metadata,
});
if (data_asset?.owner === username) has_permission = true;
for (const role of user_roles) {
if (data_asset.p_roles.includes(role)) has_permission = true;
}
if (data_asset.p_users.includes(user_id)) has_permission = true;
if (is_data_manager || has_permission) {
delete data_asset.p_roles;
delete data_asset.p_users;
return { data_asset };
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
@Get('data-asset/:id/columns-metadata')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - columns-metadata`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const columns_metadata =
await this.catalogService.getDatasetColumnsMetadata(id, metadata);
return { columns_metadata };
}
@Get('data-asset/:id/preview')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username, customer_modules } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
is_mask: customer_modules.some(mod => mod === 'pii')
});
const preview = await this.catalogService.getDatasetPreview(id, metadata);
return { preview };
}
@Get('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('asset_type') asset_type: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const docs = await this.catalogService.getDataDocs(id, asset_type, metadata);
return { docs };
}
@Put('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async updateDataAsset(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') data_asset_id: string,
@Body() body: IUpdateDataRequest,
): Promise<IOneDataAsset> {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const result = await this.catalogService.updateOneDataAsset({
body,
data_asset_id,
customer_id,
metadata,
});
delete result.data_asset.p_roles;
delete result.data_asset.p_users;
return result;
}
@Post('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async manageDataAssetDocs(
@User() user: RequestUser,
@Headers() headers,
@Param('id') table_id: string,
@Body('docs') docs: string,
@Query('asset_type') asset_type: string,
) {
const { user_id, customer_name, customer_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const body = {
table_id,
docs,
asset_type,
info: {
customer: customer_name,
},
}
const res = await this.catalogService.createDataDocs(body, metadata);
return res;
}
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
async manageDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@Body() body,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.managePermissions(
{ ...body, id },
metadata,
);
return { data_asset: JSON.parse(response.data_asset) };
}
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/revoke-permissions')
async revokeDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@Body() body,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.revokePermissions(
{ ...body, id },
metadata,
);
return { message: response.message };
}
@Post()
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async createDataAsset(
@User() user: RequestUser,
@Body() body: ICreateDataAsset,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const { data_asset } = await this.catalogService.createDataAsset(
body,
metadata,
);
return { data_asset: JSON.parse(data_asset) };
}
@Post('data-asset/:id/comment')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async commentOnDataAsset(
@Param('id') id: string,
@User() user: RequestUser,
@Body() body: IMakeAComment,
) {
const { message } = body;
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.commentOnDataAsset(
{ data_asset_id: id, username, message },
metadata,
);
return response;
}
@Delete('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.deleteDataAsset(
{ id, type: undefined },
metadata,
);
return response;
}
@Delete('data-asset/:id/comment')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async deleteComment(
@Param('id') id: string,
@User() user: RequestUser,
@Body() body: IDeleteComment,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.deleteComment(
{
data_asset_id: id,
comment: { ...body.comment, created_at: undefined, message: undefined },
},
metadata,
);
return response;
}
@Post('dataset-catalog-task')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
)
@ApiCreatedResponse({ type: TriggerCatalogRes })
async triggerCatalog(
@User() user: RequestUser,
@Body() body: TriggerCatalogReq,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const session = await this.catalogService.triggerCatalog(body, metadata);
return { session };
}
@Get('dataset-catalog-task/:session')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
)
@ApiOkResponse({ type: GetDatasetCatalogTaskRes })
async getCatalogTask(
@User() user: RequestUser,
@Param('session') session: string,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getDatasetCatalogTask(
session,
metadata,
);
return response;
}
@Post('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.addRlsRule(body, metadata);
return response;
}
@Get('rls-rule/:id')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getOneRlsRule(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const idInt = parseInt(id);
const response = await this.catalogService.getOneRlsRule(idInt, metadata);
return response;
}
@Get('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getRlsRules(
@User() user: RequestUser,
@Query() query: GetRlsRulesRequest,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getRlsRules(query, metadata);
return response;
}
@Delete('rls-rule/:id')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async removeRlsRule(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const idInt = parseInt(id);
const response = await this.catalogService.removeRlsRule(idInt, metadata);
return response;
}
@Delete('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async batchRemoveRlsRules(
@Query() query: BatchRemoveRlsRulesRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
await this.catalogService.batchRemoveRlsRule(query, metadata);
return { message: 'OK' };
}
@Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getNimbusDashboards(
@User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const dashboards = await this.catalogService.getNimbusDashboards(
body,
metadata,
);
return JSON.parse(dashboards);
}
@Post('register-dataset')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async registerDatasetWithMetadataRequest(
@Body() body: RegisterDatasetWithMetatadaRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
try {
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.log(
`Request from user ${user_id} for customer ${customer_name}`,
logMetadata,
);
// Create table metadata
const tableMetadataBody = {
table_metadata: body.table_metadata,
info: {
customer: customer_name,
},
logMetadata: logMetadata,
};
const table_metadata_id = await this.catalogService.createTableMetadata(
tableMetadataBody,
);
this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
// Create column metadata
const columnMetadataBody = {
column_metadata: body.column_metadata,
info: {
customer: customer_name,
},
};
this.logger.info(
`Creating column metadata for table ${table_metadata_id}`,
logMetadata,
);
const column_metadata_ids =
await this.catalogService.createColumnMetadata(columnMetadataBody);
// Create data preview
const dataPreviewBody = {
data_preview: body.data_preview,
info: {
customer: customer_name,
},
};
this.logger.debug(
`Creating data preview for table ${table_metadata_id}`,
logMetadata,
);
const data_preview_id = await this.catalogService.createDataPreview(
dataPreviewBody,
);
// Catalog dataset item
this.logger.info(
`Cataloging dataset item for table ${table_metadata_id}`,
logMetadata,
);
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
this.logger.info(
`Dataset registration completed successfully for table ${table_metadata_id}`,
logMetadata,
);
return {
message: 'Dataset registered successfully',
table_metadata_id: table_metadata_id,
column_metadata_ids: column_metadata_ids,
data_preview_id: data_preview_id,
};
} catch (error) {
this.logger.error(
`Failed to register dataset. The following error occurred: ${error.response.data}`,
logMetadata,
);
throw new HttpException(
{
message: 'Ocorreu um erro ao registrar o dataset',
error: error.message,
code: 'REGISTRATION_FAILED',
details: error.message,
},
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
@Get('pii-reporter')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
// use res.end to send buffer
return res.end(file);
} catch (error) {
console.error(error)
this.logger.error(error.message);
}
}
}