diff --git a/docsfera.json b/docsfera.json index 2fe5b31..89ce74b 100644 --- a/docsfera.json +++ b/docsfera.json @@ -946,7 +946,7 @@ ], "responses": { "200": { - "description": "List of users of a customer", + "description": "", "content": { "application/json": { "schema": { @@ -2626,7 +2626,7 @@ }, "/catalog": { "get": { - "operationId": "CatalogController_catalogAll", + "operationId": "CatalogController_searchCatalog", "parameters": [], "responses": { "200": { @@ -3233,9 +3233,9 @@ } }, "info": { - "title": "Maestro - stg2", + "title": "Maestro - feat/view-table", "description": "Documentation for Maestro gateway", - "version": "0.0.0", + "version": "1.0.0", "contact": {} }, "tags": [], diff --git a/src/main.ts b/src/main.ts index b2c84c6..7235e3b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -55,7 +55,6 @@ function configureSwagger(app: INestApplication) { const document = SwaggerModule.createDocument(app, config); if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document); - writeFileSync( 'docsfera.json', JSON.stringify( diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 6f7675f..4e5b026 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -10,7 +10,6 @@ import { Post, Put, Query, - UnsupportedMediaTypeException, UseFilters, } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; @@ -55,12 +54,12 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) - async catalogAll( + async searchCatalog( @User() user: RequestUser, @Query() query, ): Promise { const { user_id, customer_name, customer_id, username, permissions } = user; - this.logger.info(`/catalog - Catalog all data assets`, { + this.logger.info(`/catalog - searchCatalog`, { user_id, customer_name, }); @@ -80,13 +79,13 @@ export class CatalogController { is_data_manager, }); - const res = await this.catalogService.getAllOpenSearchDataAssets( + const res = await this.catalogService.searchDataAssets( query, metadata, customer_id, ); - return { ...res }; + return res; } @Get('tags') @@ -123,7 +122,6 @@ export class CatalogController { @Param('id') id, ) { const { username, user_id, customer_id, customer_name, permissions } = user; - const [data_asset_type, data_asset_id] = id.split('-'); this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { username, @@ -145,8 +143,7 @@ export class CatalogController { const { data_asset } = await this.catalogService.getOneDataAsset({ customer_id, - data_asset_id, - data_asset_type, + id, metadata, }); @@ -181,7 +178,6 @@ export class CatalogController { @Param('id') id, ) { const { username, user_id, customer_id, customer_name, permissions } = user; - const [data_asset_type, data_asset_id] = id.split('-'); this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, { username, @@ -203,8 +199,7 @@ export class CatalogController { const { data_asset } = await this.catalogService.getOneDataAsset({ customer_id, - data_asset_id, - data_asset_type, + id, metadata, }); @@ -239,17 +234,8 @@ export class CatalogController { @Param('id') id, ): Promise { const { customer_name, customer_id, user_id, username } = user; - const data_asset_type = id.split('-')[0]; - const supported_data_assets = ['dataset']; - - if (!supported_data_assets.includes(data_asset_type)) { - throw new UnsupportedMediaTypeException( - `${data_asset_type} does not have a preview function for the time being.`, - ); - } - - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { + this.logger.info(`/catalog - columns-metadata`, { user_id, customer_name, }); @@ -279,15 +265,6 @@ export class CatalogController { @Param('id') id, ): Promise { const { customer_name, customer_id, user_id, username } = user; - const data_asset_type = id.split('-')[0]; - - const supported_data_assets = ['dataset']; - - if (!supported_data_assets.includes(data_asset_type)) { - throw new UnsupportedMediaTypeException( - `${data_asset_type} does not have a preview function for the time being.`, - ); - } this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, @@ -318,15 +295,6 @@ export class CatalogController { @Param('id') id, ): Promise { const { customer_name, customer_id, user_id, username } = user; - const data_asset_type = id.split('-')[0]; - - const supported_data_assets = ['dataset']; - - if (!supported_data_assets.includes(data_asset_type)) { - throw new UnsupportedMediaTypeException( - `${data_asset_type} does not have a documentation function for the time being.`, - ); - } this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, @@ -506,10 +474,7 @@ export class CatalogController { PERMISSIONS_GROUPS.CATALOG.permissions.DELETE, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) - async deleteDataAsset( - @Param('id') data_asset_id: string, - @User() user: RequestUser, - ) { + async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) { const { customer_id, customer_name, user_id, username } = user; const metadata = PackTheMetadata({ customer_id, @@ -517,10 +482,8 @@ export class CatalogController { user_id, username, }); - - const [type, id] = data_asset_id.split('-'); const response = await this.catalogService.deleteDataAsset( - { id, type }, + { id, type: undefined }, metadata, ); diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index c4c8032..149db9a 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -139,34 +139,12 @@ class CatalogService implements OnModuleInit { return roles_ids; } - async getUsernamesAndRolesByCustomerId(customerId: string) { - const resultUsers = await this.userService.findAllUsersByCustomerId( - customerId, - ); - const resultRoles = await this.roleService.roleSearch( - {}, - { customer_id: customerId }, - ); - - const customer_users = resultUsers.users.map((user) => { - return { - id: user.id, - username: user.username, - }; - }); - - const customer_roles = resultRoles.roles.map((role) => { - return { - id: role.id, - name: role.name, - }; - }); - - return [customer_users, customer_roles]; - } - - async getAllOpenSearchDataAssets(query, metadata, customer_id) { - this.logger.info('CatalogService - getAllOpenSearchDashboard'); + async searchDataAssets( + query: Record, + metadata: Metadata, + customer_id: string, + ) { + this.logger.info('CatalogService - searchDataAssets'); const { search, page, size, sort_by, order, ...filters } = query; @@ -182,102 +160,39 @@ class CatalogService implements OnModuleInit { }, metadata, ), - ).catch((err) => { - throw new HttpException( - err.details, - err.code === 6 ? HttpStatus.CONFLICT : 404, - ); - }); + ); const result = JSON.parse(data_assets); - const [customer_users, customer_roles] = - await this.getUsernamesAndRolesByCustomerId(customer_id); + const response = await this.getAssetsUsersAndRoles( + result.data_assets, + customer_id, + ); - for (const asset of result.data_assets) { - const condition = - asset.owner && !asset.owner.includes('@') ? true : false; - const result = condition - ? await this.userService.findOneById(asset.owner).catch(() => null) - : null; - - const owner = result !== null ? result.user.username : null; - - const roles = []; - const users = []; - - for (const role of customer_roles) { - for (const role_id of asset.roles) { - if (role.id === role_id) roles.push(role); - } - } - - for (const user of customer_users) { - for (const user_id of asset.users) { - if (user.id === user_id) users.push(user); - } - } - - Object.assign(asset, { - ...asset, - owner, - roles, - users, - }); - } - - return { data_assets: result.data_assets, total }; + return { data_assets: response, total }; } - async getOneDataAsset({ - data_asset_id, - data_asset_type, - customer_id, - metadata, + async getOneDataAsset(data: { + id: string; + customer_id: string; + metadata: Metadata; }) { + const { customer_id, id, metadata } = data; const { data_asset } = await lastValueFrom( this.catalogReadService.GetOneDataAsset( - { id: data_asset_id, type: data_asset_type }, + { id, type: undefined }, metadata, ), ); - const opensearchTable = JSON.parse(data_asset); + let asset = JSON.parse(data_asset); + asset = { + ...asset, + p_roles: asset.roles, + p_users: asset.users, + }; + asset = await this.getAssetsUsersAndRoles([asset], customer_id); - const result = opensearchTable.owner - ? await this.userService - .findOneById(opensearchTable.owner) - .catch(() => null) - : null; - - const owner = result !== null ? result.user.username : null; - - const [customer_users, customer_roles] = - await this.getUsernamesAndRolesByCustomerId(customer_id); - - const users = []; - const roles = []; - - for (const role of customer_roles) { - for (const role_id of opensearchTable.roles) { - if (role.id === role_id) roles.push(role); - } - } - - for (const user of customer_users) { - for (const user_id of opensearchTable.users) { - if (user.id === user_id) users.push(user); - } - } - - Object.assign(opensearchTable, { - p_roles: opensearchTable.roles, - p_users: opensearchTable.users, - roles, - users, - owner, - }); - - return { data_asset: opensearchTable }; + return { data_asset: asset[0] }; } async updateOneDataAsset({ data_asset_id, customer_id, body, metadata }) { @@ -287,66 +202,30 @@ class CatalogService implements OnModuleInit { metadata, ), ); + let asset = JSON.parse(data_asset); + asset = { + ...asset, + p_roles: asset.roles, + p_users: asset.users, + }; + asset = await this.getAssetsUsersAndRoles([asset], customer_id); - const opensearchTable = JSON.parse(data_asset); - - const result = opensearchTable.owner - ? await this.userService - .findOneById(opensearchTable.owner) - .catch(() => null) - : null; - - const owner = result !== null ? result.user.username : null; - - const [customer_users, customer_roles] = - await this.getUsernamesAndRolesByCustomerId(customer_id); - - const users = []; - const roles = []; - - for (const role of customer_roles) { - for (const role_id of opensearchTable.roles) { - if (role.id === role_id) roles.push(role); - } - } - - for (const user of customer_users) { - for (const user_id of opensearchTable.users) { - if (user.id === user_id) users.push(user); - } - } - - Object.assign(opensearchTable, { - p_roles: opensearchTable.roles, - p_users: opensearchTable.users, - roles, - users, - owner, - }); - - return { data_asset: opensearchTable }; + return { data_asset: asset[0] }; } - async getDataDocs(data_asset_id: string, metadata: Metadata) { - const [data_asset_type, id] = data_asset_id.split('-'); - + async getDataDocs(id: string, metadata: Metadata) { const { documentation } = await lastValueFrom( - this.catalogReadService.GetDatasetDoc( - { id, type: data_asset_type }, - metadata, - ), + this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), ); console.log(documentation); const docs = JSON.parse(documentation); return docs; } - async getDatasetPreview(data_asset_id: string, metadata: Metadata) { - const [data_asset_type, id] = data_asset_id.split('-'); - + async getDatasetPreview(id: string, metadata: Metadata) { const { preview } = await lastValueFrom( this.catalogReadService.GetDatasetPreview( - { id, type: data_asset_type }, + { id, type: undefined }, metadata, ), ); @@ -354,12 +233,10 @@ class CatalogService implements OnModuleInit { return result; } - async getDatasetColumnsMetadata(data_asset_id: string, metadata: Metadata) { - const [data_asset_type, id] = data_asset_id.split('-'); - + async getDatasetColumnsMetadata(id: string, metadata: Metadata) { const { columns_metadata } = await lastValueFrom( this.catalogReadService.GetDatasetColumnsMetadata( - { id, type: data_asset_type }, + { id, type: undefined }, metadata, ), ); @@ -393,6 +270,36 @@ class CatalogService implements OnModuleInit { return response; } + async getAssetsUsersAndRoles(data_assets: Array, customer_id: string) { + const { users: customer_users } = + await this.userService.findAllUsersByCustomerId(customer_id); + const { roles: customer_roles } = await this.roleService.roleSearch( + {}, + { customer_id }, + ); + return data_assets.map((data_asset) => { + const owner = customer_users.find( + (u) => u.id === data_asset.owner, + )?.username; + + const roles = []; + const users = []; + for (const role_id of data_asset.roles) { + const role = customer_roles.find((r) => r.id === role_id); + if (role) roles.push({ id: role.id, name: role.name }); + } + for (const user_id of data_asset.users) { + const user = customer_users.find((r) => r.id === user_id); + if (user) users.push({ id: user.id, username: user.username }); + } + return { + ...data_asset, + roles, + users, + owner, + } as typeof data_asset; + }); + } } export { CatalogService }; diff --git a/src/modules/connector/client.service.ts b/src/modules/connector/client.service.ts index bac423c..a5f14d4 100644 --- a/src/modules/connector/client.service.ts +++ b/src/modules/connector/client.service.ts @@ -89,9 +89,9 @@ export class ConnectorClientService implements OnModuleInit { const connectorErrors = connectorsResponse .filter((i) => i.update.error) .map((i) => ({ - id: i.update._id, - reason: i.update.error, - status: i.error.status, + id: i.update?._id, + reason: i.update?.error, + status: i.error?.status, })); const connectorCreated = connectorsResponse .filter((i) => i.update.result === 'created') diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 7162314..030c6a1 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,9 +1,4 @@ -import { - BadRequestException, - Inject, - InternalServerErrorException, - OnModuleInit, -} from '@nestjs/common'; +import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { Messages, @@ -23,6 +18,8 @@ import { InputsService } from '../inputs/inputs.service'; import { RequestUser } from 'src/authentication/user.decorator'; import { TransformationsService } from '../transformations/transformations.service'; import { getObjValueFromPath } from 'src/utils/ObjValueFromPath'; +import ErrorCodes from 'src/utils/errorCodes'; +import ErrorBuilder from 'src/utils/ErrorBuilder'; export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; @@ -99,7 +96,7 @@ export class PipelinesService implements OnModuleInit { const [errorType, message] = error.details.split('|'); throw new BadRequestException(message); } - throw new InternalServerErrorException(error.details); + throw new ErrorBuilder(error.details); }); this.logger.info('Done'); @@ -164,45 +161,53 @@ export class PipelinesService implements OnModuleInit { customer_id: user.customer_id, customer: user.customer_name, }; + let existsInProduct = true; const { pipeline } = await lastValueFrom( this.pipelineReadService.PipelineV2FindOne({ id }, metadata), - ); + ).catch((e) => { + if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false; + else throw new ErrorBuilder(e.details); + + return { pipeline: undefined }; + }); await lastValueFrom( this.pipelineWriteService.PipelineV2Remove({ id }, metadata), ); - //{pipeline:{tables: {tables: [], input_id: ''}}} - const input = pipeline.config.tables - ? JSON.parse(pipeline.config.tables) - : null; - if (input) - await this.inputsService - .remove({ - id: input.input_id, - info, - }) - .catch((error) => - this.logger.error('Could not delete input', { - data: { input, error }, - }), - ); - - const transformations: { id: string }[] = pipeline.transformations - ? JSON.parse(pipeline.transformations) - : null; - if (transformations && transformations.length) - for (const transformation of transformations) { - await this.transformationsService + if (existsInProduct) { + //{pipeline:{tables: {tables: [], input_id: ''}}} + const input = pipeline.config.tables + ? JSON.parse(pipeline.config.tables) + : null; + if (input) + await this.inputsService .remove({ - id: transformation.id, + id: input.input_id, info, }) .catch((error) => - this.logger.error('Could not delete transformation', { - data: { transformation, error }, + this.logger.error('Could not delete input', { + data: { input, error }, }), ); - } + + const transformations: { id: string }[] = pipeline.transformations + ? JSON.parse(pipeline.transformations) + : null; + if (transformations && transformations.length) + for (const transformation of transformations) { + await this.transformationsService + .remove({ + id: transformation.id, + info, + }) + .catch((error) => + this.logger.error('Could not delete transformation', { + data: { transformation, error }, + }), + ); + } + } } async findOneProperties(id: string, metadata: Metadata) { diff --git a/src/modules/users/users.controller.ts b/src/modules/users/users.controller.ts index 325953e..336d8f0 100644 --- a/src/modules/users/users.controller.ts +++ b/src/modules/users/users.controller.ts @@ -52,7 +52,6 @@ import { Metadata } from '@grpc/grpc-js'; import ErrorBuilder from 'src/utils/ErrorBuilder'; import ErrorCodes from 'src/utils/errorCodes'; -// TODO GET de hierarquias e do PATCH em usuário @ApiTags('Users') @Controller('users') @ApiHeader({ @@ -75,10 +74,6 @@ export class UsersController { } @Get() - @ApiOkResponse({ - type: GetAllUsersByCustomerIdRes, - description: 'List of users of a customer', - }) async getAllUsersByCustomerId( @User() user: RequestUser, @Headers('dadosfera-lang') language,