Merge pull request #194 from dadosfera/feature/download-logs

feature/download-logs

docsfera.json

@@ -79,6 +79,16 @@
               }
             }
           ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AuthRefreshAccessTokenReq"
+                }
+              }
+            }
+          },
           "responses": {
             "200": {
               "description": "",
@@ -2310,6 +2320,20 @@
           ]
         }
       },
+      "/pipelinesV2/download-logs": {
+        "get": {
+          "operationId": "PipelinesController_downloadLogs",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/pipelinesV2/{id}/config": {
         "get": {
           "operationId": "PipelinesController_getPipelineproperties",
@@ -2324,6 +2348,20 @@
           ]
         }
       },
+      "/pipelinesV2/{id}/objects": {
+        "get": {
+          "operationId": "PipelinesController_getPipelineObjects",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/pipelinesV2/{id}": {
         "get": {
           "operationId": "PipelinesController_findOne",
@@ -2712,6 +2750,20 @@
           ]
         }
       },
+      "/catalog/data-asset": {
+        "get": {
+          "operationId": "CatalogController_findByPipelineAndObject",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Catalog"
+          ]
+        }
+      },
       "/catalog/tags": {
         "get": {
           "operationId": "CatalogController_findAllTags",
@@ -3287,7 +3339,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/upload-csv",
+      "title": "Maestro - feature/download-logs",
       "description": "Documentation for Maestro gateway",
       "version": "1.0.0",
       "contact": {}
@@ -3473,6 +3525,21 @@
             "mfaStatus"
           ]
         },
+        "AuthRefreshAccessTokenReq": {
+          "type": "object",
+          "properties": {
+            "refreshToken": {
+              "type": "string"
+            },
+            "customerName": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "refreshToken",
+            "customerName"
+          ]
+        },
         "CreateConnectorDto": {
           "type": "object",
           "properties": {

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.23.0",
+        "@dadosfera/protospack-v2": "3.27.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.23.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.23.0.tgz",
-      "integrity": "sha512-pe2s7aJYaNlUhQ+QoMGkJoFjTS8Dh+xSgUClSJuCsTia17oidRAucfpeMUX9ws1a3+tDrXTpH519CxeHzuZQPA==",
+      "version": "3.27.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
+      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12324,9 +12324,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.23.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.23.0.tgz",
-      "integrity": "sha512-pe2s7aJYaNlUhQ+QoMGkJoFjTS8Dh+xSgUClSJuCsTia17oidRAucfpeMUX9ws1a3+tDrXTpH519CxeHzuZQPA==",
+      "version": "3.27.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
+      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.23.0",
+    "@dadosfera/protospack-v2": "3.27.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/modules/auth/auth.controller.ts

@@ -11,7 +11,6 @@ import {
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthRefreshAccessTokenRequest,
   AuthChangePasswordRequest,
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
@@ -31,7 +30,12 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { Metadata } from '@grpc/grpc-js';
-import { AuthSignInReq, AuthSignInRes } from './dtos/login';
+import {
+  AuthRefreshAccessTokenReq,
+  AuthSignInReq,
+  AuthSignInRes,
+} from './dtos/login';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @ApiTags('Auth')
 @UseFilters(new GrpcToHttpExceptionFilter())
@@ -77,12 +81,16 @@ export class AuthController {
   @Post('refresh-access-token')
   @HttpCode(HttpStatus.OK)
   async refreshAccessToken(
-    @Body() { refreshToken }: AuthRefreshAccessTokenRequest,
+    @Body()
+    { refreshToken, customerName: customer_name }: AuthRefreshAccessTokenReq,
     @Headers('Dadosfera-Lang') language: string,
   ) {
     this.logger.info('/auth - RefreshAccessToken');
-    const metadata = new Metadata();
-    metadata.add('language', language || 'pt-br');
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      language,
+    });
 
     return this.authClient.refreshAccessToken({ refreshToken }, metadata);
   }

src/modules/auth/dtos/login.ts

@@ -97,3 +97,10 @@ export class AuthSignInRes implements AuthSignInResponse {
   @ApiPropertyOptional()
   termsOfUse?: TermsOfUseStatus;
 }
+
+export class AuthRefreshAccessTokenReq {
+  @ApiProperty()
+  refreshToken: string;
+  @ApiProperty()
+  customerName: string;
+}

src/modules/catalog/catalog.controller.ts

@@ -88,6 +88,60 @@ export class CatalogController {
     return res;
   }
 
+  @Get('data-asset')
+  async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+    const { pipeline, object } = query;
+    this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
+      username,
+      customer_name,
+    });
+
+    if (!pipeline || !object) {
+      throw new HttpException('Query params not provided', 400);
+    }
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id: undefined,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } =
+      await this.catalogService.getOneDataAssetByPipelineAndObject({
+        pipeline: query.pipeline,
+        object: query.object,
+        customer_id,
+        metadata,
+      });
+
+    if (data_asset?.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (data_asset.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (data_asset.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      return { data_asset };
+    }
+
+    throw new HttpException(
+      'You do not have permission to access this data asset.',
+      403,
+    );
+  }
+
   @Get('tags')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,

src/modules/catalog/catalog.service.ts

@@ -197,6 +197,30 @@ class CatalogService implements OnModuleInit {
     return { data_asset: asset[0] };
   }
 
+  async getOneDataAssetByPipelineAndObject(data: {
+    customer_id: string;
+    pipeline: string;
+    object: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, metadata, pipeline, object } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAssetByPipelineAndObject(
+        { pipeline, object },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
   async updateOneDataAsset(data: {
     data_asset_id: string;
     customer_id: string;

src/modules/connection/connection.controller.ts

@@ -164,11 +164,15 @@ export class ConnectionController {
     @Headers('Dadosfera-Lang') language,
     @User() user: RequestUser,
     @Param('id') id,
-    @Query('details') details,
+    @Query() queries,
   ): Promise<ConnectionDetailsRes> {
     this.logger.info('/connections - Get Connection Details');
     if (!language) language = 'en-us';
 
+    console.log(queries);
+
+    const { details, sensitive } = queries;
+
     const { user_id, customer_id, customer_name, username } = user;
     const metadata = PackTheMetadata({
       user_id,
@@ -177,6 +181,7 @@ export class ConnectionController {
       username,
       details,
       language,
+      sensitive,
     });
 
     const response: any = await this.clientService.getConnectionDetails({

src/modules/pipelinesV2/pipelines.controller.ts

@@ -15,6 +15,7 @@ import {
   Patch,
   UseInterceptors,
   UploadedFile,
+  HttpException,
 } from '@nestjs/common';
 import {
   ApiConsumes,
@@ -132,6 +133,39 @@ export class PipelinesController {
     };
   }
 
+  @Get('/download-logs')
+  async downloadLogs(
+    @User() user: RequestUser,
+    @Headers('Dadosfera-Lang') language,
+    @Query() query,
+  ) {
+    this.logger.info('PipelinesController - downloadLogs', { user });
+
+    if (!language) language = 'en-us';
+
+    if (!query) {
+      throw new HttpException('Query params not provided', 400);
+    }
+
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const url = await this.pipelinesClientService.downloadLogs(
+      query.pipeline_run_id,
+      metadata,
+    );
+
+    return {
+      url,
+    };
+  }
+
   @Get(':id/config')
   async getPipelineproperties(
     @Headers('Dadosfera-Lang') language,
@@ -146,6 +180,20 @@ export class PipelinesController {
     return this.pipelinesClientService.findOneProperties(id, metadata);
   }
 
+  @Get(':id/objects')
+  async getPipelineObjects(
+    @Headers('Dadosfera-Lang') language,
+    @User() user: RequestUser,
+    @Param('id') id,
+  ) {
+    this.logger.info('PipelinesController - getPipelineObjects', { user });
+    const metadata = PackTheMetadata({
+      ...user,
+      language: language || 'pt-br',
+    });
+    return this.pipelinesClientService.findOneObjects(id, metadata);
+  }
+
   @Get('/:id')
   async findOne(
     @Headers('Dadosfera-Lang') language,
@@ -265,8 +313,15 @@ export class PipelinesController {
   ) {
     this.logger.info('/upload - Upload Connector Route');
     const metadata = PackTheMetadata({ ...user });
-    let name = `${new Date().getTime()}_${file.originalname.split('.')[0]}`;
+
+    const parts_of_file_name = file.originalname.split('.');
+    const file_format = parts_of_file_name.pop();
+    const file_name = parts_of_file_name.join('.');
+    let name = `${new Date().getTime()}_${file_name}`;
     body.name ? (name = `${new Date().getTime()}_${body.name}`) : name;
+
+    const source_prefix = `${user.customer_name}/${name}.${file_format}`;
+
     const response = await this.pipelinesClientService.uploadFile(
       {
         file,
@@ -275,9 +330,15 @@ export class PipelinesController {
       metadata,
     );
 
-    const { sep, header, encoding, description } = body;
+    if (!response.url) {
+      this.logger.info('pipeline/upload - Failed File pipeline');
+      throw new HttpException(
+        'Upload failed, try again in a few minutes, if the problem persists, contact support.',
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
 
-    const [file_name, file_format] = file.originalname.split('.');
+    const { sep, header, encoding, description } = body;
 
     const upload_pipeline = {
       connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
@@ -294,7 +355,7 @@ export class PipelinesController {
       properties: {
         engine: 'csv',
         source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
-        source_prefix: `${user.customer_name}/${name}.${file_format}`,
+        source_prefix,
         file_format_params: { sep, encoding, header: Boolean(header) },
         is_a_upload_csv: true,
       },

src/modules/pipelinesV2/pipelines.service.ts

@@ -250,6 +250,17 @@ export class PipelinesService implements OnModuleInit {
     };
   }
 
+  async findOneObjects(id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - FindOneObjects');
+
+    const { objects } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2FindObjects({ id }, metadata),
+    );
+    this.logger.info('Done');
+
+    return { objects: JSON.parse(objects) };
+  }
+
   async uploadFile(uploadFile, metadata) {
     const body: Messages.PipelineV2UploadFileRequest = {
       file: {
@@ -268,4 +279,18 @@ export class PipelinesService implements OnModuleInit {
       );
     });
   }
+
+  async downloadLogs(pipeline_run_id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - DownloadLogs');
+
+    const { url } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2DownloadLogs(
+        { pipeline_run_id },
+        metadata,
+      ),
+    );
+    this.logger.info('Done');
+
+    return url;
+  }
 }

src/modules/users/users.service.ts

@@ -122,6 +122,7 @@ export class UsersService implements OnModuleInit {
     if (roleNames) {
       await this.setRoles({ roleNames, userId: id }, customerId);
     }
+
     const { user } = await lastValueFrom(
       this.usersClientService.UserUpdate({
         name,
@@ -131,6 +132,7 @@ export class UsersService implements OnModuleInit {
         hierarchy,
         jobTitle,
         email,
+        metabaseUserId: undefined,
       }),
     );
     return { user: this.adjustUsersPayload([user])[0] };

src/utils/ PackTheMetadata.ts

@@ -7,6 +7,7 @@ interface IMetadata {
   user_id?: string;
   language?: string;
   details?: string;
+  sensitive?: string;
   roles?: string[];
   is_data_manager?: boolean;
 }