Merge branch 'main' into feat/embed-data-asset

This commit is contained in:
Gabriel Rosa
2022-12-06 09:09:43 -03:00
7 changed files with 125 additions and 256 deletions
+9 -46
View File
@@ -10,7 +10,6 @@ import {
Post,
Put,
Query,
UnsupportedMediaTypeException,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
@@ -55,12 +54,12 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async catalogAll(
async searchCatalog(
@User() user: RequestUser,
@Query() query,
): Promise<ICatalogAllResponse> {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog - Catalog all data assets`, {
this.logger.info(`/catalog - searchCatalog`, {
user_id,
customer_name,
});
@@ -80,13 +79,13 @@ export class CatalogController {
is_data_manager,
});
const res = await this.catalogService.getAllOpenSearchDataAssets(
const res = await this.catalogService.searchDataAssets(
query,
metadata,
customer_id,
);
return { ...res };
return res;
}
@Get('tags')
@@ -123,7 +122,6 @@ export class CatalogController {
@Param('id') id,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const [data_asset_type, data_asset_id] = id.split('-');
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
@@ -145,8 +143,7 @@ export class CatalogController {
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
data_asset_id,
data_asset_type,
id,
metadata,
});
@@ -181,7 +178,6 @@ export class CatalogController {
@Param('id') id,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const [data_asset_type, data_asset_id] = id.split('-');
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
@@ -203,8 +199,7 @@ export class CatalogController {
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
data_asset_id,
data_asset_type,
id,
metadata,
});
@@ -239,17 +234,8 @@ export class CatalogController {
@Param('id') id,
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
const data_asset_type = id.split('-')[0];
const supported_data_assets = ['dataset'];
if (!supported_data_assets.includes(data_asset_type)) {
throw new UnsupportedMediaTypeException(
`${data_asset_type} does not have a preview function for the time being.`,
);
}
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
this.logger.info(`/catalog - columns-metadata`, {
user_id,
customer_name,
});
@@ -279,15 +265,6 @@ export class CatalogController {
@Param('id') id,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username } = user;
const data_asset_type = id.split('-')[0];
const supported_data_assets = ['dataset'];
if (!supported_data_assets.includes(data_asset_type)) {
throw new UnsupportedMediaTypeException(
`${data_asset_type} does not have a preview function for the time being.`,
);
}
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -318,15 +295,6 @@ export class CatalogController {
@Param('id') id,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
const data_asset_type = id.split('-')[0];
const supported_data_assets = ['dataset'];
if (!supported_data_assets.includes(data_asset_type)) {
throw new UnsupportedMediaTypeException(
`${data_asset_type} does not have a documentation function for the time being.`,
);
}
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -506,10 +474,7 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async deleteDataAsset(
@Param('id') data_asset_id: string,
@User() user: RequestUser,
) {
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
@@ -517,10 +482,8 @@ export class CatalogController {
user_id,
username,
});
const [type, id] = data_asset_id.split('-');
const response = await this.catalogService.deleteDataAsset(
{ id, type },
{ id, type: undefined },
metadata,
);
+70 -163
View File
@@ -139,34 +139,12 @@ class CatalogService implements OnModuleInit {
return roles_ids;
}
async getUsernamesAndRolesByCustomerId(customerId: string) {
const resultUsers = await this.userService.findAllUsersByCustomerId(
customerId,
);
const resultRoles = await this.roleService.roleSearch(
{},
{ customer_id: customerId },
);
const customer_users = resultUsers.users.map((user) => {
return {
id: user.id,
username: user.username,
};
});
const customer_roles = resultRoles.roles.map((role) => {
return {
id: role.id,
name: role.name,
};
});
return [customer_users, customer_roles];
}
async getAllOpenSearchDataAssets(query, metadata, customer_id) {
this.logger.info('CatalogService - getAllOpenSearchDashboard');
async searchDataAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
this.logger.info('CatalogService - searchDataAssets');
const { search, page, size, sort_by, order, ...filters } = query;
@@ -182,102 +160,39 @@ class CatalogService implements OnModuleInit {
},
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
);
const result = JSON.parse(data_assets);
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const response = await this.getAssetsUsersAndRoles(
result.data_assets,
customer_id,
);
for (const asset of result.data_assets) {
const condition =
asset.owner && !asset.owner.includes('@') ? true : false;
const result = condition
? await this.userService.findOneById(asset.owner).catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const roles = [];
const users = [];
for (const role of customer_roles) {
for (const role_id of asset.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of asset.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(asset, {
...asset,
owner,
roles,
users,
});
}
return { data_assets: result.data_assets, total };
return { data_assets: response, total };
}
async getOneDataAsset({
data_asset_id,
data_asset_type,
customer_id,
metadata,
async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id: data_asset_id, type: data_asset_type },
{ id, type: undefined },
metadata,
),
);
const opensearchTable = JSON.parse(data_asset);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
const result = opensearchTable.owner
? await this.userService
.findOneById(opensearchTable.owner)
.catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const users = [];
const roles = [];
for (const role of customer_roles) {
for (const role_id of opensearchTable.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of opensearchTable.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(opensearchTable, {
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
roles,
users,
owner,
});
return { data_asset: opensearchTable };
return { data_asset: asset[0] };
}
async updateOneDataAsset({ data_asset_id, customer_id, body, metadata }) {
@@ -287,66 +202,30 @@ class CatalogService implements OnModuleInit {
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
const opensearchTable = JSON.parse(data_asset);
const result = opensearchTable.owner
? await this.userService
.findOneById(opensearchTable.owner)
.catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const users = [];
const roles = [];
for (const role of customer_roles) {
for (const role_id of opensearchTable.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of opensearchTable.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(opensearchTable, {
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
roles,
users,
owner,
});
return { data_asset: opensearchTable };
return { data_asset: asset[0] };
}
async getDataDocs(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
async getDataDocs(id: string, metadata: Metadata) {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc(
{ id, type: data_asset_type },
metadata,
),
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
async getDatasetPreview(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
async getDatasetPreview(id: string, metadata: Metadata) {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id, type: data_asset_type },
{ id, type: undefined },
metadata,
),
);
@@ -354,12 +233,10 @@ class CatalogService implements OnModuleInit {
return result;
}
async getDatasetColumnsMetadata(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id, type: data_asset_type },
{ id, type: undefined },
metadata,
),
);
@@ -393,6 +270,36 @@ class CatalogService implements OnModuleInit {
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
}
export { CatalogService };
+3 -3
View File
@@ -89,9 +89,9 @@ export class ConnectorClientService implements OnModuleInit {
const connectorErrors = connectorsResponse
.filter((i) => i.update.error)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
id: i.update?._id,
reason: i.update?.error,
status: i.error?.status,
}));
const connectorCreated = connectorsResponse
.filter((i) => i.update.result === 'created')
+39 -34
View File
@@ -1,9 +1,4 @@
import {
BadRequestException,
Inject,
InternalServerErrorException,
OnModuleInit,
} from '@nestjs/common';
import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
Messages,
@@ -23,6 +18,8 @@ import { InputsService } from '../inputs/inputs.service';
import { RequestUser } from 'src/authentication/user.decorator';
import { TransformationsService } from '../transformations/transformations.service';
import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger;
@@ -99,7 +96,7 @@ export class PipelinesService implements OnModuleInit {
const [errorType, message] = error.details.split('|');
throw new BadRequestException(message);
}
throw new InternalServerErrorException(error.details);
throw new ErrorBuilder(error.details);
});
this.logger.info('Done');
@@ -164,45 +161,53 @@ export class PipelinesService implements OnModuleInit {
customer_id: user.customer_id,
customer: user.customer_name,
};
let existsInProduct = true;
const { pipeline } = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
);
).catch((e) => {
if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false;
else throw new ErrorBuilder(e.details);
return { pipeline: undefined };
});
await lastValueFrom(
this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
);
//{pipeline:{tables: {tables: [], input_id: ''}}}
const input = pipeline.config.tables
? JSON.parse(pipeline.config.tables)
: null;
if (input)
await this.inputsService
.remove({
id: input.input_id,
info,
})
.catch((error) =>
this.logger.error('Could not delete input', {
data: { input, error },
}),
);
const transformations: { id: string }[] = pipeline.transformations
? JSON.parse(pipeline.transformations)
: null;
if (transformations && transformations.length)
for (const transformation of transformations) {
await this.transformationsService
if (existsInProduct) {
//{pipeline:{tables: {tables: [], input_id: ''}}}
const input = pipeline.config.tables
? JSON.parse(pipeline.config.tables)
: null;
if (input)
await this.inputsService
.remove({
id: transformation.id,
id: input.input_id,
info,
})
.catch((error) =>
this.logger.error('Could not delete transformation', {
data: { transformation, error },
this.logger.error('Could not delete input', {
data: { input, error },
}),
);
}
const transformations: { id: string }[] = pipeline.transformations
? JSON.parse(pipeline.transformations)
: null;
if (transformations && transformations.length)
for (const transformation of transformations) {
await this.transformationsService
.remove({
id: transformation.id,
info,
})
.catch((error) =>
this.logger.error('Could not delete transformation', {
data: { transformation, error },
}),
);
}
}
}
async findOneProperties(id: string, metadata: Metadata) {
-5
View File
@@ -52,7 +52,6 @@ import { Metadata } from '@grpc/grpc-js';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
// TODO GET de hierarquias e do PATCH em usuário
@ApiTags('Users')
@Controller('users')
@ApiHeader({
@@ -75,10 +74,6 @@ export class UsersController {
}
@Get()
@ApiOkResponse({
type: GetAllUsersByCustomerIdRes,
description: 'List of users of a customer',
})
async getAllUsersByCustomerId(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,