Merge pull request #146 from dadosfera/feat/openSearchCatalog

Feat/open search catalog

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.18.7",
+        "@dadosfera/protospack-v2": "3.18.8",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1726,9 +1726,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.18.7",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.7.tgz",
-      "integrity": "sha512-MloW230CUyK+g+HFkSv1OuaJNT5mfEGRh7tpZjjKxO2Qxm4eKkmI9uh/cNXCDzUpRBFXzBJ+5w12bxw2UnMRSw==",
+      "version": "3.18.8",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.8.tgz",
+      "integrity": "sha512-pzvuIzCrIf/ZG//lwMdCxtZ7w/ZhiLFBA3VWyN9uM7AjeqVl7LllHKV08Fw6YjNNGBmQF3DLJQTK8kMCQja4hA==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12306,9 +12306,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.18.7",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.7.tgz",
-      "integrity": "sha512-MloW230CUyK+g+HFkSv1OuaJNT5mfEGRh7tpZjjKxO2Qxm4eKkmI9uh/cNXCDzUpRBFXzBJ+5w12bxw2UnMRSw==",
+      "version": "3.18.8",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.8.tgz",
+      "integrity": "sha512-pzvuIzCrIf/ZG//lwMdCxtZ7w/ZhiLFBA3VWyN9uM7AjeqVl7LllHKV08Fw6YjNNGBmQF3DLJQTK8kMCQja4hA==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -27,7 +27,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.18.7",
+    "@dadosfera/protospack-v2": "3.18.8",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/authentication/permissions.enum.ts

@@ -325,6 +325,16 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Eliminar atributos',
         },
       },
+      DATA_MANAGER: {
+        seqid: 42,
+        claim: 'catalog:data_manager',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Vizualizar todos os recursos do catálogo.',
+          'en-us': 'View all data assets.',
+          'es-es': 'Ver todos los activos de datos',
+        },
+      },
     },
   },
 

src/modules/catalog/catalog.controller.ts

@@ -3,6 +3,7 @@ import {
   Controller,
   Delete,
   Get,
+  HttpException,
   Inject,
   Param,
   Post,
@@ -93,12 +94,16 @@ export class CatalogController {
 
   @Get('all')
   async catalogAll(@User() user: RequestUser, @Body() body, @Query() params) {
-    const { user_id, customer_name, customer_id, username } = user;
+    const { user_id, customer_name, customer_id, username, permissions } = user;
     this.logger.info(`/catalog - ON CATALOG ALL ROUTE`, {
       user: body.info.user_id,
       customer: body.info.customer,
     });
 
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+
     const roles = await this.catalogService.getUserRolesIds(user_id);
 
     const metadata = PackTheMetadata({
@@ -107,11 +112,13 @@ export class CatalogController {
       customer_name,
       username,
       roles,
+      is_data_manager,
     });
 
     const res = await this.catalogService.getAllOpenSearchDataAssets(
       params,
       metadata,
+      customer_id,
     );
 
     return res.data_assets;
@@ -166,7 +173,11 @@ export class CatalogController {
       customer: body.info.customer,
     });
 
-    const { username, user_id, customer_id, customer_name } = user;
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
     const metadata = PackTheMetadata({
       username,
       user_id,
@@ -174,13 +185,35 @@ export class CatalogController {
       customer_name,
     });
 
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
     const res = await this.catalogService.getOneDashboardMetabase(
       id,
       body,
       metadata,
+      customer_id,
     );
 
-    return res;
+    const dashboard = res.dashboard;
+
+    if (dashboard.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (dashboard.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (dashboard.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete res[0].p_roles;
+      delete res[0].p_users;
+      return res;
+    }
+
+    throw new HttpException(
+      'You do not have permission to access this data asset.',
+      403,
+    );
   }
 
   @Get('table-metadata')
@@ -193,7 +226,11 @@ export class CatalogController {
       user: user.user_id,
       customer: user.customer_name,
     });
-    const { user_id, customer_name, customer_id } = user;
+    const { user_id, customer_name, customer_id, permissions, username } = user;
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
     const metadata = PackTheMetadata({
       user_id,
       customer_id,
@@ -204,12 +241,33 @@ export class CatalogController {
       const res = await this.catalogService.getAllTableMetadata(body);
       return res;
     } else {
+      const user_roles = await this.catalogService.getUserRolesIds(user_id);
       const res = await this.catalogService.getOneTableMetadata(
         body,
         query,
         metadata,
+        customer_id,
+      );
+      const table = res[0];
+
+      if (table.owner === username) has_permission = true;
+
+      for (const role of user_roles) {
+        if (table.p_roles.includes(role)) has_permission = true;
+      }
+
+      if (table.p_users.includes(user_id)) has_permission = true;
+
+      if (is_data_manager || has_permission) {
+        delete res[0].p_roles;
+        delete res[0].p_users;
+        return res;
+      }
+
+      throw new HttpException(
+        'You do not have permission to access this data asset.',
+        403,
       );
-      return res;
     }
   }
 

src/modules/catalog/catalog.module.ts

@@ -8,6 +8,7 @@ import { CatalogClientConfiguration } from './catalog-client';
 import { ClientsModule } from '@nestjs/microservices';
 import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { UsersModule } from '../users/users.module';
+import { RolesModule } from '../roles/roles.module';
 
 const client = new CatalogClientConfiguration();
 
@@ -16,6 +17,7 @@ const client = new CatalogClientConfiguration();
     ClientsModule.register([client.providerOptions]),
     OldPipelineModule,
     UsersModule,
+    RolesModule,
   ],
   controllers: [CatalogController],
   providers: [CatalogService, DadosferaLogger],

src/modules/catalog/catalog.service.ts

@@ -14,6 +14,7 @@ import axios from 'axios';
 import { lastValueFrom } from 'rxjs';
 import { CatalogClientConfiguration } from './catalog-client';
 import { UsersService } from '../users/users.service';
+import { RolesService } from '../roles/roles.service';
 
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
@@ -25,6 +26,7 @@ class CatalogService implements OnModuleInit {
     @Inject(CatalogClientConfiguration.name)
     private readonly grpcClient: ClientGrpc,
     private readonly userService: UsersService,
+    private readonly roleService: RolesService,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -117,25 +119,86 @@ class CatalogService implements OnModuleInit {
     return roles_ids;
   }
 
+  async getUsernamesAndRolesByCustomerId(customerId: string) {
+    const resultUsers = await this.userService.findAllUsersByCustomerId(
+      customerId,
+    );
+    const resultRoles = await this.roleService.roleSearch(
+      {},
+      { customer_id: customerId },
+    );
+
+    const customer_users = resultUsers.users.map((user) => {
+      return {
+        id: user.id,
+        username: user.username,
+      };
+    });
+
+    const customer_roles = resultRoles.roles.map((role) => {
+      return {
+        id: role.id,
+        name: role.name,
+      };
+    });
+
+    return [customer_users, customer_roles];
+  }
+
   async getAllOpenSearchDataAssets(
     data: Messages.GetAllDataAssetsRequest,
     metadata,
+    customer_id,
   ) {
     this.logger.info('CatalogService - getAllOpenSearchDashboard');
 
     const response = await lastValueFrom(
       this.catalogReadService.GetAllDataAssets(data, metadata),
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return JSON.parse(res.data_assets);
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
+
+    const data_assets = JSON.parse(response.data_assets);
+
+    const [customer_users, customer_roles] =
+      await this.getUsernamesAndRolesByCustomerId(customer_id);
+
+    for (const asset of data_assets.tables) {
+      const condition =
+        asset.owner && !asset.owner.includes('@') ? true : false;
+      const result = condition
+        ? await this.userService.findOneById(asset.owner).catch(() => null)
+        : null;
+
+      const owner = result !== null ? result.user.username : null;
+
+      const roles = [];
+      const users = [];
+
+      for (const role of customer_roles) {
+        for (const role_id of asset.roles) {
+          if (role.id === role_id) roles.push(role);
+        }
+      }
+
+      for (const user of customer_users) {
+        for (const user_id of asset.users) {
+          if (user.id === user_id) users.push(user);
+        }
+      }
+
+      Object.assign(asset, {
+        ...asset,
+        owner,
+        roles,
+        users,
       });
+    }
 
-    for (const asset of response.tables) {
+    for (const asset of data_assets.dashboards) {
       const condition =
         asset.owner && !asset.owner.includes('@') ? true : false;
       const result = condition
@@ -147,22 +210,10 @@ class CatalogService implements OnModuleInit {
       Object.assign(asset, { ...asset, owner });
     }
 
-    for (const asset of response.dashboards) {
-      const condition =
-        asset.owner && !asset.owner.includes('@') ? true : false;
-      const result = condition
-        ? await this.userService.findOneById(asset.owner).catch(() => null)
-        : null;
-
-      const owner = result !== null ? result.user.username : null;
-
-      Object.assign(asset, { ...asset, owner });
-    }
-
-    return { data_assets: response };
+    return { data_assets };
   }
 
-  async getOneDashboardMetabase(id, body, metadata) {
+  async getOneDashboardMetabase(id, body, metadata, customer_id) {
     const nimbusUrl = this._getNimbusUrl(body);
     const { data } = await axios.get(
       `${nimbusUrl}/api/catalog/dashboard-metabase/${id}`,
@@ -176,7 +227,7 @@ class CatalogService implements OnModuleInit {
     );
     const opensearchTable = JSON.parse(data_asset);
 
-    const result = opensearchTable.user_id
+    const result = opensearchTable.owner
       ? await this.userService
           .findOneById(opensearchTable.owner)
           .catch(() => null)
@@ -184,12 +235,35 @@ class CatalogService implements OnModuleInit {
 
     const owner = result !== null ? result.user.username : null;
 
-    return {
-      ...data,
+    const [customer_users, customer_roles] =
+      await this.getUsernamesAndRolesByCustomerId(customer_id);
+
+    const users = [];
+    const roles = [];
+
+    for (const role of customer_roles) {
+      for (const role_id of opensearchTable.roles) {
+        if (role.id === role_id) roles.push(role);
+      }
+    }
+
+    for (const user of customer_users) {
+      for (const user_id of opensearchTable.users) {
+        if (user.id === user_id) users.push(user);
+      }
+    }
+
+    Object.assign(data.dashboard, {
       tags: opensearchTable.tags,
       description: opensearchTable.description || null,
+      p_roles: opensearchTable.roles,
+      p_users: opensearchTable.users,
+      roles,
+      users,
       owner,
-    };
+    });
+
+    return data;
   }
 
   async getAllTableMetadata(body) {
@@ -200,7 +274,7 @@ class CatalogService implements OnModuleInit {
     return data;
   }
 
-  async getOneTableMetadata(body, params, metadata) {
+  async getOneTableMetadata(body, params, metadata, customer_id) {
     const nimbusUrl = this._getNimbusUrl(body);
     const { data } = await axios.get(
       `${nimbusUrl}/api/catalog/table-metadata/`,
@@ -214,7 +288,7 @@ class CatalogService implements OnModuleInit {
         ),
       );
       const opensearchTable = JSON.parse(data_asset);
-      const result = opensearchTable.user_id
+      const result = opensearchTable.owner
         ? await this.userService
             .findOneById(opensearchTable.owner)
             .catch(() => null)
@@ -222,11 +296,33 @@ class CatalogService implements OnModuleInit {
 
       const owner = result !== null ? result.user.username : null;
 
+      const [customer_users, customer_roles] =
+        await this.getUsernamesAndRolesByCustomerId(customer_id);
+
+      const users = [];
+      const roles = [];
+
+      for (const role of customer_roles) {
+        for (const role_id of opensearchTable.roles) {
+          if (role.id === role_id) roles.push(role);
+        }
+      }
+
+      for (const user of customer_users) {
+        for (const user_id of opensearchTable.users) {
+          if (user.id === user_id) users.push(user);
+        }
+      }
+
       return [
         {
           ...data[0],
           tags: opensearchTable.tags,
           table_description: opensearchTable.description,
+          p_roles: opensearchTable.roles,
+          p_users: opensearchTable.users,
+          users,
+          roles,
           owner,
         },
       ];

src/modules/permissions/permissions.controller.ts

@@ -36,7 +36,6 @@ export class PermissionsController {
 
   @Get()
   @ApiOkResponse({ type: GetPublicPermissionsRes })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
   getPublicPermissions(
     @User() user: RequestUser,
     @Headers('dadosfera-lang') language,

src/modules/roles/dto/entities.ts

@@ -36,9 +36,9 @@ export enum TrueOrFalseEnum {
 }
 export class GetRolesByCustomerReq {
   @ApiPropertyOptional()
-  permissionId: string[];
+  permissionId?: string[];
   @ApiPropertyOptional({ enum: TrueOrFalseEnum })
-  getUsers: TrueOrFalseEnum;
+  getUsers?: TrueOrFalseEnum;
 }
 
 export class GetRolesByCustomerRes {

src/modules/roles/roles.controller.ts

@@ -69,7 +69,6 @@ export class RolesController {
 
   @Get()
   @ApiOkResponse({ type: GetRolesByCustomerRes })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
   async searchRoles(
     @User() user: RequestUser,
     @Query() filters: GetRolesByCustomerReq,

src/modules/roles/roles.service.ts

@@ -72,7 +72,7 @@ export class RolesService {
 
   async roleSearch(
     filters: GetRolesByCustomerReq,
-    { customer_id, access_token }: RequestUser,
+    { customer_id, access_token }: any,
   ) {
     const meta = new Metadata();
     meta.add('access_token', access_token);

src/modules/users/users.controller.ts

@@ -69,7 +69,6 @@ export class UsersController {
   }
 
   @Get()
-  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
   @ApiOkResponse({
     type: GetAllUsersByCustomerIdRes,
     description: 'List of users of a customer',

src/utils/ PackTheMetadata.ts

@@ -8,6 +8,7 @@ interface IMetadata {
   language?: string;
   details?: string;
   roles?: string[];
+  is_data_manager?: boolean;
 }
 
 export function PackTheMetadata(info: IMetadata): Metadata {