Merge branch 'main' into feat/rest-api

src/modules/catalog/catalog.controller.ts

@@ -14,8 +14,8 @@ import {
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthenticateCondition,
   Authenticated,
+  RequireSomePermission,
 } from '../../authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
@@ -34,36 +34,9 @@ import {
   IUpdateDataRequest,
 } from './dtos';
 
-const catalogPermissions = PERMISSIONS_GROUPS.CATALOG.permissions;
 @ApiTags('Catalog')
 @Controller('catalog')
 @Authenticated()
-@AuthenticateCondition((req, user: RequestUser) => {
-  if (user.permissions.includes(catalogPermissions.DATA_MANAGER.seqid))
-    return true;
-
-  if (req.path.includes('/manage-permissions')) return true;
-  let action: keyof typeof catalogPermissions;
-
-  switch (req.method) {
-    case 'POST':
-    case 'PUT':
-      action = 'UPDATE';
-      break;
-
-    case 'DELETE':
-      if (req.path.includes('/comment')) action = 'UPDATE';
-      else action = req.method;
-      break;
-
-    default:
-      action = req.method as 'GET';
-  }
-
-  return user.permissions.includes(
-    PERMISSIONS_GROUPS.CATALOG.permissions[action].seqid,
-  );
-})
 export class CatalogController {
   logger: DadosferaLogger;
   constructor(
@@ -75,6 +48,10 @@ export class CatalogController {
   }
 
   @Get()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async catalogAll(
     @User() user: RequestUser,
     @Query() query,
@@ -110,6 +87,10 @@ export class CatalogController {
   }
 
   @Get('tags')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async findAllTags(@Body() body) {
     this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
       user: body.info.user_id,
@@ -129,6 +110,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAsset(
     @User() user: RequestUser,
     @Headers() headers,
@@ -183,6 +168,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/rls/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetRls(
     @User() user: RequestUser,
     @Headers() headers,
@@ -237,6 +226,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/columns-metadata')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
     @Headers('Dadosfera-Lang') language,
@@ -273,6 +266,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/preview')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetPreview(
     @User() user: RequestUser,
     @Headers('Dadosfera-lang') language,
@@ -308,6 +305,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetDocs(
     @User() user: RequestUser,
     @Headers('Dadosfera-Lang') language,
@@ -343,6 +344,10 @@ export class CatalogController {
   }
 
   @Put('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async updateDataAsset(
     @User() user: RequestUser,
     @Headers('Dadosfera-Lang') language,
@@ -372,6 +377,10 @@ export class CatalogController {
   }
 
   @Post('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async manageDataAssetDocs(
     @User() user: RequestUser,
     @Headers() headers,
@@ -440,6 +449,10 @@ export class CatalogController {
   }
 
   @Post()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async createDataAsset(
     @User() user: RequestUser,
     @Body() body: ICreateDataAsset,
@@ -460,6 +473,10 @@ export class CatalogController {
   }
 
   @Post('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async commentOnDataAsset(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -482,6 +499,10 @@ export class CatalogController {
   }
 
   @Delete('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async deleteDataAsset(
     @Param('id') data_asset_id: string,
     @User() user: RequestUser,
@@ -504,6 +525,10 @@ export class CatalogController {
   }
 
   @Delete('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async deleteComment(
     @Param('id') id: string,
     @User() user: RequestUser,

src/modules/inputs/dtos/input.model.ts

@@ -1,106 +1,5 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
-import { Info } from 'protospack/dist/lib/interfaces';
 
-export class GoogleAnalyticsClientSecrets {
-  @ApiProperty()
-  type: string;
-  @ApiProperty()
-  project_id: string;
-  @ApiProperty()
-  private_key_id: string;
-  @ApiProperty()
-  private_key: string;
-  @ApiProperty()
-  client_email: string;
-  @ApiProperty()
-  client_id: string;
-  @ApiProperty()
-  auth_uri: string;
-  @ApiProperty()
-  token_uri: string;
-  @ApiProperty()
-  auth_provider_x509_cert_url: string;
-  @ApiProperty()
-  client_x509_cert_url: string;
-}
-export class OauthObject {
-  @ApiPropertyOptional()
-  get_tokens_url: string;
-  @ApiPropertyOptional()
-  get_tokens_url_params: string;
-  @ApiPropertyOptional()
-  get_tokens_set_response: Record<string, any>;
-  @ApiPropertyOptional()
-  content_type: string;
-}
-export class InputOptions {
-  @ApiPropertyOptional()
-  oauth: OauthObject;
-  @ApiPropertyOptional()
-  skip_select_columns: true;
-  @ApiPropertyOptional()
-  skip_select_entities: false;
-  @ApiPropertyOptional()
-  skip_transformation: true;
-}
-export class FileFormatParams {
-  @ApiProperty()
-  sep: string;
-  @ApiProperty()
-  header: boolean;
-  @ApiProperty()
-  encoding: string;
-}
-
-export class AuthParameters {
-  @ApiProperty()
-  aws_access_key_id: string;
-  @ApiProperty()
-  aws_secret_access_key: string;
-}
-export class CredentialsJdbc {
-  @ApiPropertyOptional()
-  jdbc_user: string;
-  @ApiPropertyOptional()
-  jdbc_password: string;
-  @ApiPropertyOptional()
-  database: string;
-  @ApiPropertyOptional()
-  endpoint: string;
-  @ApiPropertyOptional()
-  port: string;
-  @ApiPropertyOptional()
-  engine: string;
-  @ApiPropertyOptional()
-  schema: string;
-}
-export class Credentials extends CredentialsJdbc {
-  @ApiProperty()
-  connection_type: string;
-
-  @ApiPropertyOptional()
-  client_aws_access_key_id: string;
-  @ApiPropertyOptional()
-  client_aws_secret_access_key: string;
-  @ApiPropertyOptional()
-  client_bucket: string;
-  @ApiPropertyOptional()
-  file_to_extract: string;
-  @ApiPropertyOptional()
-  file_format_params: FileFormatParams;
-
-  @ApiPropertyOptional()
-  view_id: string;
-  @ApiPropertyOptional()
-  client_secrets: GoogleAnalyticsClientSecrets;
-  @ApiPropertyOptional()
-  start_date: string;
-  @ApiPropertyOptional()
-  end_date: string;
-
-  @ApiPropertyOptional()
-  oauth_code: string;
-}
 export class Column {
   @ApiProperty()
   name: string;
@@ -114,6 +13,14 @@ export class TableColumns {
   columns: string[];
   @ApiProperty()
   references: Column[];
+  @ApiProperty()
+  type: string;
+}
+export class AvailableEntity {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  replication_methods: string[];
 }
 
 export class Input {
@@ -133,26 +40,6 @@ export class Input {
   created_at: string;
   @ApiProperty()
   updated_at: string;
-
-  //deprecated attributes (for retro compatibility)
-  @ApiPropertyOptional({ deprecated: true })
-  category: string;
-  @ApiPropertyOptional({ deprecated: true })
-  credentials: Credentials;
-  @ApiPropertyOptional({ deprecated: true })
-  options: InputOptions;
-  @ApiPropertyOptional({ deprecated: true })
-  info: Info;
-  @ApiPropertyOptional({ deprecated: true })
-  cron: string;
-  @ApiPropertyOptional({ deprecated: true })
-  source_bucket: string;
-  @ApiPropertyOptional({ deprecated: true })
-  source_prefix: string;
-  @ApiPropertyOptional({ deprecated: true })
-  auth_parameters: AuthParameters;
-  @ApiPropertyOptional({ deprecated: true })
-  file_format_params: FileFormatParams;
 }
 export class GetAvailableEntitiesReq {
   @ApiProperty()
@@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq {
 }
 
 export class GetAvailableEntitiesRes {
-  @ApiProperty()
-  entities: string[];
+  @ApiProperty({ type: [AvailableEntity] })
+  entities: AvailableEntity[];
 }
 
-export class TestConnectionGetColumnsReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  tables: string[];
-  @ApiPropertyOptional()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  id: string;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionGetColumnsRes {
-  @ApiProperty({ type: [TableColumns] })
-  tables: TableColumns[];
-}
-
-export class TestConnectionReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionRes {
-  @ApiProperty()
-  connection_state: boolean;
-  @ApiProperty()
-  total_entities: number;
-  @ApiProperty()
-  database_tables: string[];
-}
 export class CreateInputReq extends OmitType(Input, [
   'id',
   'created_at',

src/modules/inputs/inputs.controller.ts

@@ -19,16 +19,10 @@ import {
   GetAvailableEntitiesReq,
   GetAvailableEntitiesRes,
   Input,
-  TestConnectionGetColumnsReq,
-  TestConnectionGetColumnsRes,
-  TestConnectionReq,
-  TestConnectionRes,
 } from './dtos/input.model';
 import { UpdateInputRequest } from './dtos/old_interfaces';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
-import ErrorBuilder from 'src/utils/ErrorBuilder';
-import ErrorCodes from 'src/utils/errorCodes';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 
 @ApiTags('Inputs')
@@ -75,42 +69,19 @@ export class InputsController {
   @Get('available-entities/:plugin')
   @ApiOkResponse({ type: GetAvailableEntitiesRes })
   async getAvailableEntities(
-    @Body() body,
     @Param() params: GetAvailableEntitiesReq,
+    @User() user: RequestUser,
   ) {
-    const { info } = body;
     const { plugin } = params;
-    return await this.inputService.getAvailableEntities({ info, plugin });
-  }
-
-  @Post('/test-connection')
-  @ApiOkResponse({ type: TestConnectionRes })
-  async testConnection(@Body() data: TestConnectionReq) {
-    this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
-      user: data.info.user_id,
-      customer: data.info.customer,
-    });
-
-    const response = await this.inputService.testConnection(data);
-
-    return response;
-  }
-
-  @Post('/test-connection/get-columns')
-  @ApiOkResponse({ type: TestConnectionGetColumnsRes })
-  async getColumns(@Body() data: TestConnectionGetColumnsReq) {
-    throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED);
-    this.logger.info(
-      `/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
-      {
-        user: data.info.user_id,
-        customer: data.info.customer,
+    const availableEntities = await this.inputService.getAvailableEntities({
+      info: {
+        customer: user.customer_name,
+        customer_id: user.customer_id,
+        user_id: user.user_id,
       },
-    );
-
-    const response = await this.inputService.getColumns(data);
-
-    return response;
+      plugin,
+    });
+    return availableEntities;
   }
 
   @Post()

src/modules/inputs/inputs.service.ts

@@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
 import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
 import { Input } from '@dadosfera/protospack-v2';
 import {
+  GetAvailableEntitiesRequest,
   InputCreateGenericRequest,
   InputCreateS3Request,
   InputNewCreateRequest,
-  TestConnectionGetColumnsRequest,
   TestConnectionRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -88,32 +88,6 @@ export class InputsService {
 
       return testConnectionResponse;
     },
-    getColumns: async (data: TestConnectionGetColumnsRequest) => {
-      this.logger.info('InputClientService - TestConnection/Get-Columns');
-      const getColumnsResponse = await new Promise((resolve, reject) => {
-        this.inputReadService.GetColumns(data).subscribe({
-          next(x) {
-            resolve(objectCamelToSnake(x));
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // console.log('done');
-          },
-        });
-      })
-        .then((res) => {
-          this.logger.info('Done');
-          return res;
-        })
-        .catch((err) => {
-          this.logger.error(err.message);
-          throw new Error(err);
-        });
-
-      return getColumnsResponse;
-    },
     getAvailableEntities: async (data) => {
       this.logger.info('InputClientService - GetAvailableEntities');
       data;
@@ -188,12 +162,9 @@ export class InputsService {
     // this.validateCron(data);
     const { body, info } = data;
 
-    if (body.info) delete body.info;
-
-    const inputCreateGenericRequest: any | InputCreateGenericRequest = {
+    const inputCreateGenericRequest: InputCreateGenericRequest = {
       input: {
         ...body,
-        credentials: body.credentials && JSON.stringify(body.credentials),
       },
       info,
     };
@@ -205,8 +176,8 @@ export class InputsService {
     return { input: adjustedInput };
   }
 
-  async getAvailableEntities(data): Promise<{ entities: string[] }> {
-    return await this.OLD_inputClient.getAvailableEntities(data);
+  async getAvailableEntities(data: GetAvailableEntitiesRequest) {
+    return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
   }
 
   async findAll(info: Info) {
@@ -263,17 +234,6 @@ export class InputsService {
     }
   }
 
-  async getColumns(data) {
-    try {
-      const testConnectionGetColumnsResponse =
-        await this.OLD_inputClient.getColumns(data);
-
-      return testConnectionGetColumnsResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
   generateInputS3Payload(payload) {
     const { credentials, plugin, cron } = payload;
     if (!credentials) return payload;

src/modules/mixpanel/mixpanel.controller.ts

@@ -1,7 +1,9 @@
 import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
 import { init } from 'mixpanel';
+import { Authenticated } from 'src/authentication/authentication.decorator';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 
+@Authenticated()
 @Controller('trackEvent')
 export class MixpanelController {
   constructor(