Merge pull request #220 from dadosfera/feat/new-customer-monitoring

FEAT: new customer monitoring dashboard

.eslintrc.js

@@ -8,7 +8,7 @@ module.exports = {
   extends: [
     'eslint:recommended',
     'plugin:@typescript-eslint/recommended',
-    'plugin:prettier/recommended',
+    'prettier',
   ],
   root: true,
   env: {

.github/workflows/deploy-manually.yml

@@ -13,6 +13,11 @@ on:
           - stg
           - stg2
           - prd
+      push_to_dockerhub:
+        description: "Push image to Dockerhub?"
+        required: true
+        type: boolean
+        default: false
 
 jobs:
   extract_environment:
@@ -39,15 +44,15 @@ jobs:
       new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - if: github.event_name != 'workflow_dispatch'
         name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v4
         id: semantic
         with:
           extra_plugins: |
-            conventional-changelog-eslint
+            conventional-changelog-eslint@4.0.0
           branches: |
             [
               'main'
@@ -76,7 +81,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Update Pip
         run: |
@@ -95,14 +100,14 @@ jobs:
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v4
         id: aws
         with:
           aws-region: us-east-1
 
       - name: Login to AWS ECR
         id: login_ecr
-        uses: aws-actions/amazon-ecr-login@v1
+        uses: aws-actions/amazon-ecr-login@v2
 
       - name: Build, Tag, and Push Image to AWS ECR
         env:
@@ -113,6 +118,23 @@ jobs:
           docker-compose -f build.docker-compose.yml build
           docker-compose -f build.docker-compose.yml push
 
+      - name: Login to Docker Hub
+        if: ${{inputs.push_to_dockerhub}}
+        uses: docker/login-action@v2
+        with:
+          username: dadosfera
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build, Tag, and Push Image to Dockerhub
+        if: ${{inputs.push_to_dockerhub}}
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+        run: |
+          docker-compose -f build.docker-compose.dockerhub.yml build
+          docker-compose -f build.docker-compose.dockerhub.yml push
+
       - name: Create ZIP file to Deploy AWS Beanstalk
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -146,7 +168,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Extract Docs BlockId and PageId
         env:

.vscode/launch.json

@@ -12,6 +12,7 @@
         "start:debug"
       ],
       "runtimeExecutable": "npm",
+      "runtimeVersion": "18.10.0",
       "skipFiles": [
         "<node_internals>/**"
       ],

LICENSE

@@ -0,0 +1,10 @@
+LICENSE
+Copyright (C) 2023 Dadosfera
+
+All rights reserved.
+
+This software and related documentation are provided under a license agreement containing restrictions on use and disclosure and are protected by intellectual property laws. Except as expressly permitted in your license agreement or allowed by law, you may not use, copy, reproduce, translate, broadcast, modify, license, transmit, distribute, exhibit, perform, publish, or display any part, in any form, or by any means.
+
+Reverse engineering, disassembly, or decompilation of this software, unless required by law for interoperability, is prohibited.
+
+If you have any questions about this, please contact Dadosfera.ai at legal@dadosfera.ai

build.docker-compose.dockerhub.yml

@@ -0,0 +1,5 @@
+version: "3.8"
+services:
+  maestro:
+    build: .
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

docker-compose.dockerhub.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+services:
+  maestro:
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+    container_name: maestro
+    restart: always
+    ports:
+      - 3333:3333
+    env_file:
+      - .env
+
+  node_exporter:
+    image: ${NODE_EXPORTER_URL}
+    container_name: node_exporter
+    restart: always
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host

docsfera.external.json

@@ -490,6 +490,45 @@
           ]
         }
       },
+      "/pipelinesV2/monitoring-dashboard": {
+        "get": {
+          "operationId": "PipelinesController_getMonitoringDashboard",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/pipelinesV2": {
         "post": {
           "operationId": "PipelinesController_create",
@@ -1752,7 +1791,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/generate-token",
+      "title": "Maestro - feat/new-customer-monitoring",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}

docsfera.json

@@ -2663,6 +2663,45 @@
           ]
         }
       },
+      "/pipelinesV2/monitoring-dashboard": {
+        "get": {
+          "operationId": "PipelinesController_getMonitoringDashboard",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/pipelinesV2": {
         "post": {
           "operationId": "PipelinesController_create",
@@ -5060,6 +5099,28 @@
           ]
         }
       },
+      "/customers/monitoring-dashboard": {
+        "get": {
+          "operationId": "CustomersController_getCustomerMonitoringDashboard",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/health": {
         "get": {
           "operationId": "HealthController_check",
@@ -5076,7 +5137,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/generate-token",
+      "title": "Maestro - feat/new-customer-monitoring",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}

package.json

@@ -29,7 +29,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.31.0",
+    "@dadosfera/protospack-v2": "3.33.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/cli": "^9.4.2",
@@ -80,11 +80,10 @@
     "@types/passport-google-oauth20": "^2.0.11",
     "@types/passport-oauth2": "^1.4.11",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.29.0",
-    "@typescript-eslint/parser": "^5.29.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.18.0",
     "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-prettier": "^4.0.0",
     "jest": "^27.5.1",
     "nock": "^13.2.7",
     "prettier": "^2.7.1",

src/authentication/permissions.enum.ts

@@ -67,7 +67,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   PIPELINE: {
     title: {
       'pt-br': 'Coletar | Pipelines',
@@ -117,7 +116,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   CONNECTION: {
     title: {
       'pt-br': 'Coletar | Fontes de dados',
@@ -157,7 +155,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   NETWORK_CONFIG: {
     title: {
       'pt-br': 'Coletar | Redes',
@@ -187,7 +184,6 @@ export const PERMISSIONS_GROUPS = {
       // },
     },
   },
-
   OUTPUT: {
     title: {
       'pt-br': 'Output',
@@ -237,7 +233,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   TRANSFORMATIONS: {
     title: {
       'pt-br': 'Micro-transformações',
@@ -287,7 +282,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   CATALOG: {
     title: {
       'pt-br': 'Explorar | Catálogo',
@@ -368,7 +362,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   CONNECTORS: {
     title: {
       'pt-br': 'Conectores',
@@ -418,7 +411,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   SNOWFLAKE: {
     title: {
       'pt-br': 'Explorar | Consolidar',
@@ -438,7 +430,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   ZENDESK: {
     title: {
       'pt-br': 'Zendesk',
@@ -458,7 +449,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   DATAVIZ: {
     title: {
       'pt-br': 'Analisar | Visualização',
@@ -497,7 +487,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   MODULES: {
     title: {
       'pt-br': 'Módulos da Dadosfera',
@@ -594,6 +583,25 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
+  CUSTOMER: {
+    title: {
+      'pt-br': 'Organização',
+      'en-us': 'Organization',
+      'es-es': 'Organización',
+    },
+    permissions: {
+      MONITORING_DASHBOARD: {
+        seqid: 47,
+        claim: 'customer:monitoring-dashboard',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Ver o dashboard de monitoramento',
+          'en-us': 'View the monitoring dashboard',
+          'es-es': 'Ver el dashboard de monitoreo',
+        },
+      },
+    },
+  },
 };
 export interface DadosferaModule {
   name: string;

src/modules/auth/auth.service.ts

@@ -164,6 +164,8 @@ export class AuthClientService implements OnModuleInit {
 
   async oauthSignIn(data: { username: string; token: string }) {
     const { username, token } = data;
-    return lastValueFrom(this.authService.AuthOauthSignIn({ token, username }));
+    return lastValueFrom(
+      this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
+    );
   }
 }

src/modules/customers/customers.controller.ts

@@ -22,6 +22,7 @@ import { CustomersService } from './customers.service';
 import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import type { StringValue } from 'ms';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @ApiInternalOnlyController()
 @ApiTags('Customers')
@@ -71,4 +72,17 @@ export class CustomersController {
     };
     return this.customersService.generateToken(exp, data);
   }
+
+  @Get('monitoring-dashboard')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
+  )
+  async getCustomerMonitoringDashboard(
+    @User() user: RequestUser,
+  ): Promise<{ url: string }> {
+    this.logger.info('getCustomerMonitoringDashboard');
+
+    const metadata = PackTheMetadata(user);
+    return this.customersService.getMonitoringDashboardUrl(metadata);
+  }
 }

src/modules/customers/customers.module.ts

@@ -4,11 +4,18 @@ import { ClientsModule } from '@nestjs/microservices';
 import { DucClient } from '../duc/client.config';
 import { CustomersController } from './customers.controller';
 import { CustomersService } from './customers.service';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
 
-const client = new DucClient();
+const ducClient = new DucClient();
+const piFactoryClient = new PipelinesClientConfiguration();
 
 @Module({
-  imports: [ClientsModule.register([client.providerOptions])],
+  imports: [
+    ClientsModule.register([
+      ducClient.providerOptions,
+      piFactoryClient.providerOptions,
+    ]),
+  ],
   controllers: [CustomersController],
   providers: [CustomersService, DadosferaLogger],
   exports: [CustomersService],

src/modules/customers/customers.service.ts

@@ -22,20 +22,34 @@ import {
 } from '@aws-sdk/client-secrets-manager';
 import getEnv from 'src/utils/getEnv';
 import { logger } from 'elastic-apm-node';
+import {
+  ReadService,
+  ProtoServices as PipelineProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
+import { Metadata } from '@grpc/grpc-js';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
 
 // This function will accept any string, which may result in a bug.
 @Injectable()
 export class CustomersService implements OnModuleInit {
   private customerService: CustomersProtoService;
 
+  private pipelineReadService: ReadService.PipelineV2ReadService;
+
   constructor(
     @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly pipelinesGrpcClient: ClientGrpc,
   ) {}
 
   onModuleInit() {
     this.customerService = this.grpcClient.getService<CustomersProtoService>(
       ProtoServices.CustomersProtoService,
     );
+    this.pipelineReadService =
+      this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
+        PipelineProtoServices.PipelineV2ReadService,
+      );
   }
 
   async getLinks(customerId: string) {
@@ -120,4 +134,21 @@ export class CustomersService implements OnModuleInit {
     // const decoded = jwt.decode(jwt_token, { complete: true });
     return jwt_token;
   }
+  async getMonitoringDashboardUrl(metadata: Metadata) {
+    logger.info('CustomersService - getMonitoringDashboardUrl');
+
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '45',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    logger.info('Done');
+
+    return res;
+  }
 }

src/modules/pipelinesV2/pipelines.controller.ts

@@ -14,6 +14,7 @@ import {
   Patch,
   HttpException,
   BadRequestException,
+  CacheTTL,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -86,6 +87,18 @@ export class PipelinesController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @Get('monitoring-dashboard')
+  async getMonitoringDashboard(@User() user: RequestUser) {
+    this.logger.info('PipelinesController - getMonitoringDashboard', { user });
+
+    const metadata = PackTheMetadata(user);
+
+    const response =
+      await this.pipelinesClientService.getMonitoringDashboardUrl(metadata);
+
+    return response;
+  }
+
   @Post()
   @ApiCreatedResponse({ type: IPipelineV2 })
   async create(

src/modules/pipelinesV2/pipelines.service.ts

@@ -321,4 +321,22 @@ export class PipelinesService implements OnModuleInit {
       );
     });
   }
+
+  async getMonitoringDashboardUrl(metadata: Metadata) {
+    this.logger.info('PipelinesClientService - getMonitoringDashboardUrl');
+
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '83',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    this.logger.info('Done');
+
+    return res;
+  }
 }