Merge pull request #194 from dadosfera/feature/download-logs

feature/download-logs

README.md

@@ -290,7 +290,7 @@ Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
 ### Making a Pull Request
 1. Commit your changes
 2. Open the Pull Request on GitHub
-3. Send Pull Request link in microsfera Google Chat Group for review and possible approval
+3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
 
 ## 🛠️ Built with
 Some technologies used in this project:

docsfera.json

@@ -79,6 +79,16 @@
               }
             }
           ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AuthRefreshAccessTokenReq"
+                }
+              }
+            }
+          },
           "responses": {
             "200": {
               "description": "",
@@ -264,6 +274,20 @@
           ]
         }
       },
+      "/auth/verify-access-token": {
+        "get": {
+          "operationId": "AuthController_verifyAccessToken",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Auth"
+          ]
+        }
+      },
       "/connectors": {
         "post": {
           "operationId": "ConnectorController_uploadConnector",
@@ -2296,6 +2320,20 @@
           ]
         }
       },
+      "/pipelinesV2/download-logs": {
+        "get": {
+          "operationId": "PipelinesController_downloadLogs",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/pipelinesV2/{id}/config": {
         "get": {
           "operationId": "PipelinesController_getPipelineproperties",
@@ -2310,6 +2348,20 @@
           ]
         }
       },
+      "/pipelinesV2/{id}/objects": {
+        "get": {
+          "operationId": "PipelinesController_getPipelineObjects",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/pipelinesV2/{id}": {
         "get": {
           "operationId": "PipelinesController_findOne",
@@ -2381,6 +2433,37 @@
           ]
         }
       },
+      "/pipelinesV2/upload": {
+        "post": {
+          "operationId": "PipelinesController_uploadFile",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "multipart/form-data": {
+                "schema": {
+                  "$ref": "#/components/schemas/IUploadCSVFile"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/inputs/available-entities/{plugin}": {
         "get": {
           "operationId": "InputsController_getAvailableEntities",
@@ -2667,6 +2750,20 @@
           ]
         }
       },
+      "/catalog/data-asset": {
+        "get": {
+          "operationId": "CatalogController_findByPipelineAndObject",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Catalog"
+          ]
+        }
+      },
       "/catalog/tags": {
         "get": {
           "operationId": "CatalogController_findAllTags",
@@ -3242,7 +3339,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/embed-data-asset",
+      "title": "Maestro - feature/download-logs",
       "description": "Documentation for Maestro gateway",
       "version": "1.0.0",
       "contact": {}
@@ -3428,6 +3525,21 @@
             "mfaStatus"
           ]
         },
+        "AuthRefreshAccessTokenReq": {
+          "type": "object",
+          "properties": {
+            "refreshToken": {
+              "type": "string"
+            },
+            "customerName": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "refreshToken",
+            "customerName"
+          ]
+        },
         "CreateConnectorDto": {
           "type": "object",
           "properties": {
@@ -5107,6 +5219,38 @@
             "updated_at"
           ]
         },
+        "IUploadCSVFile": {
+          "type": "object",
+          "properties": {
+            "file": {
+              "type": "string",
+              "format": "binary"
+            },
+            "name": {
+              "type": "string"
+            },
+            "description": {
+              "type": "string"
+            },
+            "encoding": {
+              "type": "string"
+            },
+            "sep": {
+              "type": "string"
+            },
+            "header": {
+              "type": "boolean"
+            }
+          },
+          "required": [
+            "file",
+            "name",
+            "description",
+            "encoding",
+            "sep",
+            "header"
+          ]
+        },
         "AvailableEntity": {
           "type": "object",
           "properties": {

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.22.0",
+        "@dadosfera/protospack-v2": "3.27.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.22.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
-      "integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
+      "version": "3.27.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
+      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12324,9 +12324,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.22.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
-      "integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
+      "version": "3.27.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
+      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.22.0",
+    "@dadosfera/protospack-v2": "3.27.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/modules/auth/auth.controller.ts

@@ -7,10 +7,10 @@ import {
   HttpStatus,
   Inject,
   UseFilters,
+  Get,
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthRefreshAccessTokenRequest,
   AuthChangePasswordRequest,
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
@@ -21,13 +21,21 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/authentication/authentication.decorator';
 import { AuthClientService } from './auth.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { Metadata } from '@grpc/grpc-js';
-import { AuthSignInReq, AuthSignInRes } from './dtos/login';
+import {
+  AuthRefreshAccessTokenReq,
+  AuthSignInReq,
+  AuthSignInRes,
+} from './dtos/login';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @ApiTags('Auth')
 @UseFilters(new GrpcToHttpExceptionFilter())
@@ -73,12 +81,16 @@ export class AuthController {
   @Post('refresh-access-token')
   @HttpCode(HttpStatus.OK)
   async refreshAccessToken(
-    @Body() { refreshToken }: AuthRefreshAccessTokenRequest,
+    @Body()
+    { refreshToken, customerName: customer_name }: AuthRefreshAccessTokenReq,
     @Headers('Dadosfera-Lang') language: string,
   ) {
     this.logger.info('/auth - RefreshAccessToken');
-    const metadata = new Metadata();
-    metadata.add('language', language || 'pt-br');
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      language,
+    });
 
     return this.authClient.refreshAccessToken({ refreshToken }, metadata);
   }
@@ -185,4 +197,11 @@ export class AuthController {
 
     return this.authClient.verifyTotp({ accessToken, totp });
   }
+
+  @Authenticated()
+  @Get('verify-access-token')
+  @HttpCode(HttpStatus.OK)
+  verifyAccessToken() {
+    return { access_token_status: 'valid' };
+  }
 }

src/modules/auth/dtos/login.ts

@@ -97,3 +97,10 @@ export class AuthSignInRes implements AuthSignInResponse {
   @ApiPropertyOptional()
   termsOfUse?: TermsOfUseStatus;
 }
+
+export class AuthRefreshAccessTokenReq {
+  @ApiProperty()
+  refreshToken: string;
+  @ApiProperty()
+  customerName: string;
+}

src/modules/catalog/catalog.controller.ts

@@ -88,6 +88,60 @@ export class CatalogController {
     return res;
   }
 
+  @Get('data-asset')
+  async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+    const { pipeline, object } = query;
+    this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
+      username,
+      customer_name,
+    });
+
+    if (!pipeline || !object) {
+      throw new HttpException('Query params not provided', 400);
+    }
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id: undefined,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } =
+      await this.catalogService.getOneDataAssetByPipelineAndObject({
+        pipeline: query.pipeline,
+        object: query.object,
+        customer_id,
+        metadata,
+      });
+
+    if (data_asset?.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (data_asset.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (data_asset.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      return { data_asset };
+    }
+
+    throw new HttpException(
+      'You do not have permission to access this data asset.',
+      403,
+    );
+  }
+
   @Get('tags')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,

src/modules/catalog/catalog.service.ts

@@ -197,6 +197,30 @@ class CatalogService implements OnModuleInit {
     return { data_asset: asset[0] };
   }
 
+  async getOneDataAssetByPipelineAndObject(data: {
+    customer_id: string;
+    pipeline: string;
+    object: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, metadata, pipeline, object } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAssetByPipelineAndObject(
+        { pipeline, object },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
   async updateOneDataAsset(data: {
     data_asset_id: string;
     customer_id: string;

src/modules/connection/connection.controller.ts

@@ -164,11 +164,15 @@ export class ConnectionController {
     @Headers('Dadosfera-Lang') language,
     @User() user: RequestUser,
     @Param('id') id,
-    @Query('details') details,
+    @Query() queries,
   ): Promise<ConnectionDetailsRes> {
     this.logger.info('/connections - Get Connection Details');
     if (!language) language = 'en-us';
 
+    console.log(queries);
+
+    const { details, sensitive } = queries;
+
     const { user_id, customer_id, customer_name, username } = user;
     const metadata = PackTheMetadata({
       user_id,
@@ -177,6 +181,7 @@ export class ConnectionController {
       username,
       details,
       language,
+      sensitive,
     });
 
     const response: any = await this.clientService.getConnectionDetails({

src/modules/pipelinesV2/interfaces.ts

@@ -66,3 +66,23 @@ export interface IGetPipelineLogsRequest {
   id: string;
   details: string;
 }
+
+export class IUploadCSVFile {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  encoding: string;
+
+  @ApiProperty()
+  sep: string;
+
+  @ApiProperty()
+  header: boolean;
+}

src/modules/pipelinesV2/pipelines.controller.ts

@@ -13,13 +13,20 @@ import {
   HttpCode,
   HttpStatus,
   Patch,
+  UseInterceptors,
+  UploadedFile,
+  HttpException,
 } from '@nestjs/common';
 import {
+  ApiConsumes,
   ApiCreatedResponse,
   ApiNoContentResponse,
   ApiTags,
 } from '@nestjs/swagger';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
+import {
+  AuthenticateCondition,
+  RequireAllPermissions,
+} from 'src/authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -28,8 +35,13 @@ import { RequestUser, User } from 'src/authentication/user.decorator';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
-import { ICreatePipelineV2Req, IPipelineV2 } from './interfaces';
+import {
+  ICreatePipelineV2Req,
+  IPipelineV2,
+  IUploadCSVFile,
+} from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { FileInterceptor } from '@nestjs/platform-express';
 
 @UseFilters(new GrpcToHttpExceptionFilter())
 @ApiTags('PipelinesV2')
@@ -121,6 +133,39 @@ export class PipelinesController {
     };
   }
 
+  @Get('/download-logs')
+  async downloadLogs(
+    @User() user: RequestUser,
+    @Headers('Dadosfera-Lang') language,
+    @Query() query,
+  ) {
+    this.logger.info('PipelinesController - downloadLogs', { user });
+
+    if (!language) language = 'en-us';
+
+    if (!query) {
+      throw new HttpException('Query params not provided', 400);
+    }
+
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const url = await this.pipelinesClientService.downloadLogs(
+      query.pipeline_run_id,
+      metadata,
+    );
+
+    return {
+      url,
+    };
+  }
+
   @Get(':id/config')
   async getPipelineproperties(
     @Headers('Dadosfera-Lang') language,
@@ -135,6 +180,20 @@ export class PipelinesController {
     return this.pipelinesClientService.findOneProperties(id, metadata);
   }
 
+  @Get(':id/objects')
+  async getPipelineObjects(
+    @Headers('Dadosfera-Lang') language,
+    @User() user: RequestUser,
+    @Param('id') id,
+  ) {
+    this.logger.info('PipelinesController - getPipelineObjects', { user });
+    const metadata = PackTheMetadata({
+      ...user,
+      language: language || 'pt-br',
+    });
+    return this.pipelinesClientService.findOneObjects(id, metadata);
+  }
+
   @Get('/:id')
   async findOne(
     @Headers('Dadosfera-Lang') language,
@@ -240,4 +299,74 @@ export class PipelinesController {
     await this.pipelinesClientService.remove({ id, metadata, user });
     this.logger.info('PipelinesController - delete: OK');
   }
+
+  @Post('/upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(
+    FileInterceptor('file', { limits: { fileSize: 50 * 1000 * 1000 + 1 } }),
+  )
+  async uploadFile(
+    @User() user: RequestUser,
+    @UploadedFile() file,
+    @Body() body: IUploadCSVFile,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const parts_of_file_name = file.originalname.split('.');
+    const file_format = parts_of_file_name.pop();
+    const file_name = parts_of_file_name.join('.');
+    let name = `${new Date().getTime()}_${file_name}`;
+    body.name ? (name = `${new Date().getTime()}_${body.name}`) : name;
+
+    const source_prefix = `${user.customer_name}/${name}.${file_format}`;
+
+    const response = await this.pipelinesClientService.uploadFile(
+      {
+        file,
+        name,
+      },
+      metadata,
+    );
+
+    if (!response.url) {
+      this.logger.info('pipeline/upload - Failed File pipeline');
+      throw new HttpException(
+        'Upload failed, try again in a few minutes, if the problem persists, contact support.',
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
+
+    const { sep, header, encoding, description } = body;
+
+    const upload_pipeline = {
+      connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
+      connector_name: 'Amazon S3',
+      connector_plugin: 'aws_s3',
+      connector_version: '1.0.0',
+      image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
+      name: body.name || file_name,
+      description,
+      transformations_ids: [],
+      tags: [],
+      cron: '@once',
+      config: { cron: '@once', tables: [] },
+      properties: {
+        engine: 'csv',
+        source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
+        source_prefix,
+        file_format_params: { sep, encoding, header: Boolean(header) },
+        is_a_upload_csv: true,
+      },
+      input_id: undefined,
+    };
+
+    const pipeline = await this.pipelinesClientService.create(
+      upload_pipeline,
+      metadata,
+    );
+
+    return pipeline;
+  }
 }

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,4 +1,10 @@
-import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common';
+import {
+  BadRequestException,
+  HttpException,
+  HttpStatus,
+  Inject,
+  OnModuleInit,
+} from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
   Messages,
@@ -243,4 +249,48 @@ export class PipelinesService implements OnModuleInit {
       },
     };
   }
+
+  async findOneObjects(id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - FindOneObjects');
+
+    const { objects } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2FindObjects({ id }, metadata),
+    );
+    this.logger.info('Done');
+
+    return { objects: JSON.parse(objects) };
+  }
+
+  async uploadFile(uploadFile, metadata) {
+    const body: Messages.PipelineV2UploadFileRequest = {
+      file: {
+        buffer: uploadFile.file.buffer,
+        mimetypes: uploadFile.file.mimetype,
+      },
+      name: uploadFile.name,
+    };
+
+    return lastValueFrom(
+      this.pipelineWriteService.PipelineV2UploadFile(body, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+
+  async downloadLogs(pipeline_run_id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - DownloadLogs');
+
+    const { url } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2DownloadLogs(
+        { pipeline_run_id },
+        metadata,
+      ),
+    );
+    this.logger.info('Done');
+
+    return url;
+  }
 }

src/modules/users/users.service.ts

@@ -122,6 +122,7 @@ export class UsersService implements OnModuleInit {
     if (roleNames) {
       await this.setRoles({ roleNames, userId: id }, customerId);
     }
+
     const { user } = await lastValueFrom(
       this.usersClientService.UserUpdate({
         name,
@@ -131,6 +132,7 @@ export class UsersService implements OnModuleInit {
         hierarchy,
         jobTitle,
         email,
+        metabaseUserId: undefined,
       }),
     );
     return { user: this.adjustUsersPayload([user])[0] };

src/utils/ PackTheMetadata.ts

@@ -7,6 +7,7 @@ interface IMetadata {
   user_id?: string;
   language?: string;
   details?: string;
+  sensitive?: string;
   roles?: string[];
   is_data_manager?: boolean;
 }