Compare commits

...
17 Commits
Author SHA1 Message Date
Gabriel Amorim c450d16445 Merge pull request #184 from dadosfera/feat/embed-data-asset
Feat/embed data asset
2022-12-07 16:26:49 -03:00
Gabriel Rosa 3acda53e8a npm audit fix 2022-12-07 16:17:21 -03:00
Gabriel Rosa a0f21b763a new protospack version 2022-12-07 16:14:57 -03:00
Gabriel Amorim c83d8ab1b2 Merge branch 'main' into feat/embed-data-asset 2022-12-07 15:48:04 -03:00
Gabriel Amorim 610d08fa11 Merge pull request #185 from dadosfera/fix/data-asset-docs
FIX: removed split on id on catalog POST docs route
2022-12-07 14:55:49 -03:00
Gabriel Rosa 21f68b466d FIX: removed split on id on catalog POST docs route 2022-12-07 14:49:58 -03:00
Gabriel Rosa 6fdd73c58c removing treatment of undefined embed field 2022-12-07 08:36:21 -03:00
Gabriel Rosa 1c0377a4fa FIX: treat empty embed on update 2022-12-06 11:04:18 -03:00
Gabriel Rosa 64816e5578 FIX: treat empty embed 2022-12-06 10:07:30 -03:00
Gabriel Rosa f9064417ce docsfera.json 2022-12-06 09:11:32 -03:00
Gabriel Rosa 0b8bf9a535 Merge branch 'main' into feat/embed-data-asset 2022-12-06 09:09:43 -03:00
Gabriel Amorim 6f1e5d08c9 Merge pull request #182 from dadosfera/feat/view-table
Feat/view table
2022-12-05 18:15:05 -03:00
Gabriel Rosa b611c1eb9d Merge branch 'main' into feat/view-table 2022-12-05 18:13:36 -03:00
Gabriel Rosa 81d93ff103 FEAT: embed data asset 2022-12-05 13:22:23 -03:00
Gabriel Rosa d55563f62f FIX: removed split on data_asset id on DELETE route 2022-11-30 16:21:36 -03:00
Gabriel Rosa ba9575c53e FIX: return only owner username 2022-11-30 15:43:30 -03:00
Gabriel Rosa d88b154d43 FIX: new id strategy for data assets 2022-11-30 15:29:31 -03:00
8 changed files with 212 additions and 291 deletions
+35 -4
View File
@@ -2626,7 +2626,7 @@
},
"/catalog": {
"get": {
"operationId": "CatalogController_catalogAll",
"operationId": "CatalogController_searchCatalog",
"parameters": [],
"responses": {
"200": {
@@ -2837,7 +2837,16 @@
},
"post": {
"operationId": "CatalogController_manageDataAssetDocs",
"parameters": [],
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
@@ -3233,9 +3242,9 @@
}
},
"info": {
"title": "Maestro - stg2",
"title": "Maestro - feat/embed-data-asset",
"description": "Documentation for Maestro gateway",
"version": "0.0.0",
"version": "1.0.0",
"contact": {}
},
"tags": [],
@@ -5282,6 +5291,17 @@
"username"
]
},
"EmbedObject": {
"type": "object",
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
]
},
"IDataAsset": {
"type": "object",
"properties": {
@@ -5380,10 +5400,14 @@
"items": {
"$ref": "#/components/schemas/IComment"
}
},
"embed": {
"$ref": "#/components/schemas/EmbedObject"
}
},
"required": [
"id",
"display_name",
"pipeline_id",
"user_id",
"description",
@@ -5392,6 +5416,7 @@
"owner",
"roles",
"users",
"created_at",
"comments"
]
},
@@ -5621,6 +5646,9 @@
"items": {
"type": "string"
}
},
"embed": {
"$ref": "#/components/schemas/EmbedObject"
}
},
"required": [
@@ -5657,6 +5685,9 @@
},
"location": {
"type": "string"
},
"embed": {
"$ref": "#/components/schemas/EmbedObject"
}
},
"required": [
+54 -61
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.21.0",
"@dadosfera/protospack-v2": "3.22.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -36,7 +36,7 @@
"mixpanel": "^0.17.0",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
@@ -1727,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"version": "3.22.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
"integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -3699,8 +3699,9 @@
},
"node_modules/asap": {
"version": "2.0.6",
"dev": true,
"license": "MIT"
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
"dev": true
},
"node_modules/asn1.js": {
"version": "5.4.1",
@@ -4705,9 +4706,10 @@
}
},
"node_modules/dezalgo": {
"version": "1.0.3",
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
"integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
"dev": true,
"license": "ISC",
"dependencies": {
"asap": "^2.0.0",
"wrappy": "1"
@@ -5902,23 +5904,28 @@
}
},
"node_modules/formidable": {
"version": "2.0.1",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.1.tgz",
"integrity": "sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"dezalgo": "1.0.3",
"hexoid": "1.0.0",
"once": "1.4.0",
"qs": "6.9.3"
"dezalgo": "^1.0.4",
"hexoid": "^1.0.0",
"once": "^1.4.0",
"qs": "^6.11.0"
},
"funding": {
"url": "https://ko-fi.com/tunnckoCore/commissions"
}
},
"node_modules/formidable/node_modules/qs": {
"version": "6.9.3",
"version": "6.11.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dev": true,
"license": "BSD-3-Clause",
"dependencies": {
"side-channel": "^1.0.4"
},
"engines": {
"node": ">=0.6"
},
@@ -8713,24 +8720,11 @@
}
},
"node_modules/passport-forcedotcom": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.1.5.tgz",
"integrity": "sha512-raDRQqBJtTEHDxLkxLG+9sYE0xpPy9PqriMPbMy9jmpIcU2+mOq9PgczvUPrU8UjfcZp3gqkDHqujcUWiGOVNQ==",
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.2.0.tgz",
"integrity": "sha512-gRI5+zQK9iVAZ1hJgt3TkWLeDw8SYrNWUsmVk0d0RyvUviQ4mT52x3JbYRiyZefoeID8afmLsJH8VOHjWdPf3Q==",
"dependencies": {
"passport-oauth2": "1.3.x"
},
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/passport-forcedotcom/node_modules/passport-oauth2": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/passport-oauth2/-/passport-oauth2-1.3.0.tgz",
"integrity": "sha512-AisRXcC7Vx8WSx5C9O8UNib/Y91/IOphs5tImlOMs7T+QTEJdmS+OxgXSskbnDdGQT/8mU5T3yMT6136rBjYXg==",
"dependencies": {
"oauth": "0.9.x",
"passport-strategy": "1.x.x",
"uid2": "0.0.x"
"passport-oauth2": "^1.6.1"
},
"engines": {
"node": ">= 0.4.0"
@@ -12330,9 +12324,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"version": "3.22.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
"integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -13675,6 +13669,8 @@
},
"asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
"dev": true
},
"asn1.js": {
@@ -14372,7 +14368,9 @@
"dev": true
},
"dezalgo": {
"version": "1.0.3",
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
"integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
"dev": true,
"requires": {
"asap": "^2.0.0",
@@ -15214,18 +15212,25 @@
}
},
"formidable": {
"version": "2.0.1",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.1.tgz",
"integrity": "sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==",
"dev": true,
"requires": {
"dezalgo": "1.0.3",
"hexoid": "1.0.0",
"once": "1.4.0",
"qs": "6.9.3"
"dezalgo": "^1.0.4",
"hexoid": "^1.0.0",
"once": "^1.4.0",
"qs": "^6.11.0"
},
"dependencies": {
"qs": {
"version": "6.9.3",
"dev": true
"version": "6.11.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dev": true,
"requires": {
"side-channel": "^1.0.4"
}
}
}
},
@@ -17093,23 +17098,11 @@
}
},
"passport-forcedotcom": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.1.5.tgz",
"integrity": "sha512-raDRQqBJtTEHDxLkxLG+9sYE0xpPy9PqriMPbMy9jmpIcU2+mOq9PgczvUPrU8UjfcZp3gqkDHqujcUWiGOVNQ==",
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.2.0.tgz",
"integrity": "sha512-gRI5+zQK9iVAZ1hJgt3TkWLeDw8SYrNWUsmVk0d0RyvUviQ4mT52x3JbYRiyZefoeID8afmLsJH8VOHjWdPf3Q==",
"requires": {
"passport-oauth2": "1.3.x"
},
"dependencies": {
"passport-oauth2": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/passport-oauth2/-/passport-oauth2-1.3.0.tgz",
"integrity": "sha512-AisRXcC7Vx8WSx5C9O8UNib/Y91/IOphs5tImlOMs7T+QTEJdmS+OxgXSskbnDdGQT/8mU5T3yMT6136rBjYXg==",
"requires": {
"oauth": "0.9.x",
"passport-strategy": "1.x.x",
"uid2": "0.0.x"
}
}
"passport-oauth2": "^1.6.1"
}
},
"passport-google-oauth20": {
+2 -2
View File
@@ -28,7 +28,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.21.0",
"@dadosfera/protospack-v2": "3.22.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -52,7 +52,7 @@
"mixpanel": "^0.17.0",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
+14 -49
View File
@@ -10,7 +10,7 @@ import {
Post,
Put,
Query,
UnsupportedMediaTypeException,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
@@ -33,9 +33,11 @@ import {
IPreviewResponse,
IUpdateDataRequest,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@ApiTags('Catalog')
@Controller('catalog')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
export class CatalogController {
logger: DadosferaLogger;
@@ -52,12 +54,12 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async catalogAll(
async searchCatalog(
@User() user: RequestUser,
@Query() query,
): Promise<ICatalogAllResponse> {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog - Catalog all data assets`, {
this.logger.info(`/catalog - searchCatalog`, {
user_id,
customer_name,
});
@@ -77,13 +79,13 @@ export class CatalogController {
is_data_manager,
});
const res = await this.catalogService.getAllOpenSearchDataAssets(
const res = await this.catalogService.searchDataAssets(
query,
metadata,
customer_id,
);
return { ...res };
return res;
}
@Get('tags')
@@ -120,7 +122,6 @@ export class CatalogController {
@Param('id') id,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const [data_asset_type, data_asset_id] = id.split('-');
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
@@ -142,8 +143,7 @@ export class CatalogController {
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
data_asset_id,
data_asset_type,
id,
metadata,
});
@@ -178,7 +178,6 @@ export class CatalogController {
@Param('id') id,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const [data_asset_type, data_asset_id] = id.split('-');
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
@@ -200,8 +199,7 @@ export class CatalogController {
const { data_asset } = await this.catalogService.getOneDataAsset({
customer_id,
data_asset_id,
data_asset_type,
id,
metadata,
});
@@ -236,17 +234,8 @@ export class CatalogController {
@Param('id') id,
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
const data_asset_type = id.split('-')[0];
const supported_data_assets = ['dataset'];
if (!supported_data_assets.includes(data_asset_type)) {
throw new UnsupportedMediaTypeException(
`${data_asset_type} does not have a preview function for the time being.`,
);
}
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
this.logger.info(`/catalog - columns-metadata`, {
user_id,
customer_name,
});
@@ -276,15 +265,6 @@ export class CatalogController {
@Param('id') id,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username } = user;
const data_asset_type = id.split('-')[0];
const supported_data_assets = ['dataset'];
if (!supported_data_assets.includes(data_asset_type)) {
throw new UnsupportedMediaTypeException(
`${data_asset_type} does not have a preview function for the time being.`,
);
}
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -315,15 +295,6 @@ export class CatalogController {
@Param('id') id,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
const data_asset_type = id.split('-')[0];
const supported_data_assets = ['dataset'];
if (!supported_data_assets.includes(data_asset_type)) {
throw new UnsupportedMediaTypeException(
`${data_asset_type} does not have a documentation function for the time being.`,
);
}
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -384,11 +355,10 @@ export class CatalogController {
async manageDataAssetDocs(
@User() user: RequestUser,
@Headers() headers,
@Param('id') data_asset_id,
@Param('id') table_id: string,
@Body('docs') docs: string,
) {
const { user_id, customer_name } = user;
const [, id] = data_asset_id.split('-');
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -396,7 +366,7 @@ export class CatalogController {
});
const res = await this.catalogService.createDataDocs({
table_id: id,
table_id,
docs,
info: {
customer: customer_name,
@@ -503,10 +473,7 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async deleteDataAsset(
@Param('id') data_asset_id: string,
@User() user: RequestUser,
) {
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
@@ -514,10 +481,8 @@ export class CatalogController {
user_id,
username,
});
const [type, id] = data_asset_id.split('-');
const response = await this.catalogService.deleteDataAsset(
{ id, type },
{ id, type: undefined },
metadata,
);
+81 -170
View File
@@ -18,6 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { IUpdateDataRequest } from './dtos';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -60,15 +61,11 @@ class CatalogService implements OnModuleInit {
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
return lastValueFrom(
this.catalogWriteService.CreateDataAsset(data, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
);
}
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
@@ -144,34 +141,12 @@ class CatalogService implements OnModuleInit {
return roles_ids;
}
async getUsernamesAndRolesByCustomerId(customerId: string) {
const resultUsers = await this.userService.findAllUsersByCustomerId(
customerId,
);
const resultRoles = await this.roleService.roleSearch(
{},
{ customer_id: customerId },
);
const customer_users = resultUsers.users.map((user) => {
return {
id: user.id,
username: user.username,
};
});
const customer_roles = resultRoles.roles.map((role) => {
return {
id: role.id,
name: role.name,
};
});
return [customer_users, customer_roles];
}
async getAllOpenSearchDataAssets(query, metadata, customer_id) {
this.logger.info('CatalogService - getAllOpenSearchDashboard');
async searchDataAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
this.logger.info('CatalogService - searchDataAssets');
const { search, page, size, sort_by, order, ...filters } = query;
@@ -187,171 +162,79 @@ class CatalogService implements OnModuleInit {
},
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
);
const result = JSON.parse(data_assets);
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const response = await this.getAssetsUsersAndRoles(
result.data_assets,
customer_id,
);
for (const asset of result.data_assets) {
const condition =
asset.owner && !asset.owner.includes('@') ? true : false;
const result = condition
? await this.userService.findOneById(asset.owner).catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const roles = [];
const users = [];
for (const role of customer_roles) {
for (const role_id of asset.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of asset.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(asset, {
...asset,
owner,
roles,
users,
});
}
return { data_assets: result.data_assets, total };
return { data_assets: response, total };
}
async getOneDataAsset({
data_asset_id,
data_asset_type,
customer_id,
metadata,
async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id: data_asset_id, type: data_asset_type },
{ id, type: undefined },
metadata,
),
);
const opensearchTable = JSON.parse(data_asset);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
const result = opensearchTable.owner
? await this.userService
.findOneById(opensearchTable.owner)
.catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const users = [];
const roles = [];
for (const role of customer_roles) {
for (const role_id of opensearchTable.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of opensearchTable.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(opensearchTable, {
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
roles,
users,
owner,
});
return { data_asset: opensearchTable };
return { data_asset: asset[0] };
}
async updateOneDataAsset({ data_asset_id, customer_id, body, metadata }) {
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
body: IUpdateDataRequest;
metadata: Metadata;
}) {
const { body, customer_id, data_asset_id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: data_asset_id, changes: JSON.stringify(body) },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
const opensearchTable = JSON.parse(data_asset);
const result = opensearchTable.owner
? await this.userService
.findOneById(opensearchTable.owner)
.catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const users = [];
const roles = [];
for (const role of customer_roles) {
for (const role_id of opensearchTable.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of opensearchTable.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(opensearchTable, {
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
roles,
users,
owner,
});
return { data_asset: opensearchTable };
return { data_asset: asset[0] };
}
async getDataDocs(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
async getDataDocs(id: string, metadata: Metadata) {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc(
{ id, type: data_asset_type },
metadata,
),
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
async getDatasetPreview(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
async getDatasetPreview(id: string, metadata: Metadata) {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id, type: data_asset_type },
{ id, type: undefined },
metadata,
),
);
@@ -359,12 +242,10 @@ class CatalogService implements OnModuleInit {
return result;
}
async getDatasetColumnsMetadata(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id, type: data_asset_type },
{ id, type: undefined },
metadata,
),
);
@@ -398,6 +279,36 @@ class CatalogService implements OnModuleInit {
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
}
export { CatalogService };
+14 -5
View File
@@ -1,6 +1,10 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export class EmbedObject {
@ApiProperty()
url: string;
}
export class IRole {
@ApiProperty()
name: string;
@@ -49,8 +53,8 @@ export class IDataAsset {
data_encrypted?: string;
@ApiPropertyOptional()
num_rows?: number;
@ApiPropertyOptional()
display_name?: string;
@ApiProperty()
display_name: string;
@ApiProperty()
pipeline_id: string;
@ApiProperty()
@@ -75,12 +79,14 @@ export class IDataAsset {
parameters?: string[];
@ApiPropertyOptional()
embedding_params?: object;
@ApiPropertyOptional()
created_at?: string;
@ApiProperty()
created_at: string;
@ApiPropertyOptional()
updated_at?: string;
@ApiProperty({ type: [IComment] })
comments: IComment[];
@ApiPropertyOptional()
embed?: EmbedObject;
}
export class IOneDataAsset {
@@ -127,8 +133,9 @@ export class IUpdateDataRequest {
description: string;
@ApiProperty()
tags: string[];
@ApiPropertyOptional()
embed: EmbedObject;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
@@ -140,6 +147,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
external_url: string;
@ApiProperty()
location: string;
@ApiPropertyOptional()
embed: EmbedObject;
}
export class IPreview {
+8
View File
@@ -292,6 +292,14 @@ export function EnrichErrorCode(code: string) {
message: 'Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.CATALOG.DATA_ASSET_NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Não encontramos o ativo solicitado',
message:
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.INTERNAL:
case ErrorCodes.UNKNOWN:
default:
+4
View File
@@ -67,6 +67,9 @@ const CONNECTION = {
const TRANSFORMATION = {
NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
};
const CATALOG = {
DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
};
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
@@ -82,6 +85,7 @@ const ErrorCodes = {
INPUT,
TRANSFORMATION,
CONNECTION,
CATALOG,
};
export default ErrorCodes;