Merge pull request #194 from dadosfera/feature/download-logs

feature/download-logs

docsfera.json

@@ -2320,6 +2320,20 @@
           ]
         }
       },
+      "/pipelinesV2/download-logs": {
+        "get": {
+          "operationId": "PipelinesController_downloadLogs",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/pipelinesV2/{id}/config": {
         "get": {
           "operationId": "PipelinesController_getPipelineproperties",
@@ -2334,6 +2348,20 @@
           ]
         }
       },
+      "/pipelinesV2/{id}/objects": {
+        "get": {
+          "operationId": "PipelinesController_getPipelineObjects",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/pipelinesV2/{id}": {
         "get": {
           "operationId": "PipelinesController_findOne",
@@ -2722,6 +2750,20 @@
           ]
         }
       },
+      "/catalog/data-asset": {
+        "get": {
+          "operationId": "CatalogController_findByPipelineAndObject",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Catalog"
+          ]
+        }
+      },
       "/catalog/tags": {
         "get": {
           "operationId": "CatalogController_findAllTags",
@@ -3297,7 +3339,7 @@
       }
     },
     "info": {
-      "title": "Maestro - fix/refresh-token",
+      "title": "Maestro - feature/download-logs",
       "description": "Documentation for Maestro gateway",
       "version": "1.0.0",
       "contact": {}

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.24.0",
+        "@dadosfera/protospack-v2": "3.27.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.24.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.24.0.tgz",
-      "integrity": "sha512-CZmRK2shS8y7t6F2e1PSe8T+GgQJVXQPNYmcBqeWfb/ZmfL9zS2dDCACqbJpzBg2nmEneznYtpTw4nfH8n3utw==",
+      "version": "3.27.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
+      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12324,9 +12324,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.24.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.24.0.tgz",
-      "integrity": "sha512-CZmRK2shS8y7t6F2e1PSe8T+GgQJVXQPNYmcBqeWfb/ZmfL9zS2dDCACqbJpzBg2nmEneznYtpTw4nfH8n3utw==",
+      "version": "3.27.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
+      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.24.0",
+    "@dadosfera/protospack-v2": "3.27.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/modules/catalog/catalog.controller.ts

@@ -88,6 +88,60 @@ export class CatalogController {
     return res;
   }
 
+  @Get('data-asset')
+  async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+    const { pipeline, object } = query;
+    this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
+      username,
+      customer_name,
+    });
+
+    if (!pipeline || !object) {
+      throw new HttpException('Query params not provided', 400);
+    }
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id: undefined,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } =
+      await this.catalogService.getOneDataAssetByPipelineAndObject({
+        pipeline: query.pipeline,
+        object: query.object,
+        customer_id,
+        metadata,
+      });
+
+    if (data_asset?.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (data_asset.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (data_asset.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      return { data_asset };
+    }
+
+    throw new HttpException(
+      'You do not have permission to access this data asset.',
+      403,
+    );
+  }
+
   @Get('tags')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,

src/modules/catalog/catalog.service.ts

@@ -197,6 +197,30 @@ class CatalogService implements OnModuleInit {
     return { data_asset: asset[0] };
   }
 
+  async getOneDataAssetByPipelineAndObject(data: {
+    customer_id: string;
+    pipeline: string;
+    object: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, metadata, pipeline, object } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAssetByPipelineAndObject(
+        { pipeline, object },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
   async updateOneDataAsset(data: {
     data_asset_id: string;
     customer_id: string;

src/modules/pipelinesV2/pipelines.controller.ts

@@ -133,6 +133,39 @@ export class PipelinesController {
     };
   }
 
+  @Get('/download-logs')
+  async downloadLogs(
+    @User() user: RequestUser,
+    @Headers('Dadosfera-Lang') language,
+    @Query() query,
+  ) {
+    this.logger.info('PipelinesController - downloadLogs', { user });
+
+    if (!language) language = 'en-us';
+
+    if (!query) {
+      throw new HttpException('Query params not provided', 400);
+    }
+
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const url = await this.pipelinesClientService.downloadLogs(
+      query.pipeline_run_id,
+      metadata,
+    );
+
+    return {
+      url,
+    };
+  }
+
   @Get(':id/config')
   async getPipelineproperties(
     @Headers('Dadosfera-Lang') language,
@@ -147,6 +180,20 @@ export class PipelinesController {
     return this.pipelinesClientService.findOneProperties(id, metadata);
   }
 
+  @Get(':id/objects')
+  async getPipelineObjects(
+    @Headers('Dadosfera-Lang') language,
+    @User() user: RequestUser,
+    @Param('id') id,
+  ) {
+    this.logger.info('PipelinesController - getPipelineObjects', { user });
+    const metadata = PackTheMetadata({
+      ...user,
+      language: language || 'pt-br',
+    });
+    return this.pipelinesClientService.findOneObjects(id, metadata);
+  }
+
   @Get('/:id')
   async findOne(
     @Headers('Dadosfera-Lang') language,

src/modules/pipelinesV2/pipelines.service.ts

@@ -250,6 +250,17 @@ export class PipelinesService implements OnModuleInit {
     };
   }
 
+  async findOneObjects(id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - FindOneObjects');
+
+    const { objects } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2FindObjects({ id }, metadata),
+    );
+    this.logger.info('Done');
+
+    return { objects: JSON.parse(objects) };
+  }
+
   async uploadFile(uploadFile, metadata) {
     const body: Messages.PipelineV2UploadFileRequest = {
       file: {
@@ -268,4 +279,18 @@ export class PipelinesService implements OnModuleInit {
       );
     });
   }
+
+  async downloadLogs(pipeline_run_id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - DownloadLogs');
+
+    const { url } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2DownloadLogs(
+        { pipeline_run_id },
+        metadata,
+      ),
+    );
+    this.logger.info('Done');
+
+    return url;
+  }
 }

src/modules/users/users.service.ts

@@ -122,6 +122,7 @@ export class UsersService implements OnModuleInit {
     if (roleNames) {
       await this.setRoles({ roleNames, userId: id }, customerId);
     }
+
     const { user } = await lastValueFrom(
       this.usersClientService.UserUpdate({
         name,
@@ -131,6 +132,7 @@ export class UsersService implements OnModuleInit {
         hierarchy,
         jobTitle,
         email,
+        metabaseUserId: undefined,
       }),
     );
     return { user: this.adjustUsersPayload([user])[0] };