From 0c333b476a02f5d9e745c9c978c8d45fb3d81b26 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Thu, 20 Jun 2024 10:50:07 -0300 Subject: [PATCH 1/4] FEAT: new customer monitoring --- docsfera.json | 24 +++++++++++++- src/authentication/permissions.enum.ts | 30 +++++++++++------- src/modules/customers/customers.controller.ts | 12 +++++++ src/modules/customers/customers.module.ts | 11 +++++-- src/modules/customers/customers.service.ts | 31 +++++++++++++++++++ 5 files changed, 94 insertions(+), 14 deletions(-) diff --git a/docsfera.json b/docsfera.json index d809980..c300887 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5099,6 +5099,28 @@ ] } }, + "/customers/monitoring-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMonitoringDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/health": { "get": { "operationId": "HealthController_check", @@ -5115,7 +5137,7 @@ } }, "info": { - "title": "Maestro - feat/pipeline-monitoring-dashboard", + "title": "Maestro - main", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 64e7a26..5116cfe 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -67,7 +67,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - PIPELINE: { title: { 'pt-br': 'Coletar | Pipelines', @@ -117,7 +116,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - CONNECTION: { title: { 'pt-br': 'Coletar | Fontes de dados', @@ -157,7 +155,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - NETWORK_CONFIG: { title: { 'pt-br': 'Coletar | Redes', @@ -187,7 +184,6 @@ export const PERMISSIONS_GROUPS = { // }, }, }, - OUTPUT: { title: { 'pt-br': 'Output', @@ -237,7 +233,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - TRANSFORMATIONS: { title: { 'pt-br': 'Micro-transformações', @@ -287,7 +282,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - CATALOG: { title: { 'pt-br': 'Explorar | Catálogo', @@ -368,7 +362,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - CONNECTORS: { title: { 'pt-br': 'Conectores', @@ -418,7 +411,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - SNOWFLAKE: { title: { 'pt-br': 'Explorar | Consolidar', @@ -438,7 +430,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - ZENDESK: { title: { 'pt-br': 'Zendesk', @@ -458,7 +449,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - DATAVIZ: { title: { 'pt-br': 'Analisar | Visualização', @@ -497,7 +487,6 @@ export const PERMISSIONS_GROUPS = { }, }, }, - MODULES: { title: { 'pt-br': 'Módulos da Dadosfera', @@ -594,6 +583,25 @@ export const PERMISSIONS_GROUPS = { }, }, }, + CUSTOMER: { + title: { + 'pt-br': 'Organização', + 'en-us': 'Organization', + 'es-es': 'Organización', + }, + permissions: { + MONITORING_DASHBOARD: { + seqid: 47, + claim: 'customer:monitoring-dashboard', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Ver o dashboard de monitoramento', + 'en-us': 'View the monitoring dashboard', + 'es-es': 'Ver el dashboard de monitoreo', + }, + }, + }, + }, }; export interface DadosferaModule { name: string; diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index f88a677..d40de80 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -22,6 +22,7 @@ import { CustomersService } from './customers.service'; import { CustomerDto, CustomerLinksResponse } from './dtos/customers'; import { RequestUser, User } from 'src/decorators/user.decorator'; import type { StringValue } from 'ms'; +import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; @ApiInternalOnlyController() @ApiTags('Customers') @@ -71,4 +72,15 @@ export class CustomersController { }; return this.customersService.generateToken(exp, data); } + + @Get('monitoring-dashboard') + @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN) + async getCustomerMonitoringDashboard( + @User() user: RequestUser, + ): Promise<{ url: string }> { + this.logger.info('getCustomerMonitoringDashboard'); + + const metadata = PackTheMetadata(user); + return this.customersService.getMonitoringDashboardUrl(metadata); + } } diff --git a/src/modules/customers/customers.module.ts b/src/modules/customers/customers.module.ts index 56a285f..4704e22 100644 --- a/src/modules/customers/customers.module.ts +++ b/src/modules/customers/customers.module.ts @@ -4,11 +4,18 @@ import { ClientsModule } from '@nestjs/microservices'; import { DucClient } from '../duc/client.config'; import { CustomersController } from './customers.controller'; import { CustomersService } from './customers.service'; +import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client'; -const client = new DucClient(); +const ducClient = new DucClient(); +const piFactoryClient = new PipelinesClientConfiguration(); @Module({ - imports: [ClientsModule.register([client.providerOptions])], + imports: [ + ClientsModule.register([ + ducClient.providerOptions, + piFactoryClient.providerOptions, + ]), + ], controllers: [CustomersController], providers: [CustomersService, DadosferaLogger], exports: [CustomersService], diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index ea9a93b..5102db5 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -22,20 +22,34 @@ import { } from '@aws-sdk/client-secrets-manager'; import getEnv from 'src/utils/getEnv'; import { logger } from 'elastic-apm-node'; +import { + ReadService, + ProtoServices as PipelineProtoServices, +} from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; +import { Metadata } from '@grpc/grpc-js'; +import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client'; // This function will accept any string, which may result in a bug. @Injectable() export class CustomersService implements OnModuleInit { private customerService: CustomersProtoService; + private pipelineReadService: ReadService.PipelineV2ReadService; + constructor( @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + @Inject(PipelinesClientConfiguration.name) + private readonly pipelinesGrpcClient: ClientGrpc, ) {} onModuleInit() { this.customerService = this.grpcClient.getService( ProtoServices.CustomersProtoService, ); + this.pipelineReadService = + this.pipelinesGrpcClient.getService( + PipelineProtoServices.PipelineV2ReadService, + ); } async getLinks(customerId: string) { @@ -120,4 +134,21 @@ export class CustomersService implements OnModuleInit { // const decoded = jwt.decode(jwt_token, { complete: true }); return jwt_token; } + async getMonitoringDashboardUrl(metadata: Metadata) { + logger.info('CustomersService - getMonitoringDashboardUrl'); + + const res = await lastValueFrom( + this.pipelineReadService.PipelineV2GetDashboardUrl( + { + dashboard_id: '45', + exp: '15m', + metabase_customer_name: 'dadosferatech', + }, + metadata, + ), + ); + logger.info('Done'); + + return res; + } } From 8b6cd8a88ed60e401ad114433cb46ee48574e3e9 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Fri, 21 Jun 2024 09:21:27 -0300 Subject: [PATCH 2/4] FEAT: customer monitoring dashboard --- docsfera.external.json | 41 ++++++++++++++++++- docsfera.json | 2 +- src/modules/customers/customers.controller.ts | 4 +- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docsfera.external.json b/docsfera.external.json index 5823882..eb7906d 100644 --- a/docsfera.external.json +++ b/docsfera.external.json @@ -490,6 +490,45 @@ ] } }, + "/pipelinesV2/monitoring-dashboard": { + "get": { + "operationId": "PipelinesController_getMonitoringDashboard", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2": { "post": { "operationId": "PipelinesController_create", @@ -1752,7 +1791,7 @@ } }, "info": { - "title": "Maestro - feat/generate-token", + "title": "Maestro - feat/new-customer-monitoring", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} diff --git a/docsfera.json b/docsfera.json index c300887..8719479 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5137,7 +5137,7 @@ } }, "info": { - "title": "Maestro - main", + "title": "Maestro - feat/new-customer-monitoring", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index d40de80..66a9fe9 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -74,7 +74,9 @@ export class CustomersController { } @Get('monitoring-dashboard') - @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN) + @RequireAllPermissions( + PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD, + ) async getCustomerMonitoringDashboard( @User() user: RequestUser, ): Promise<{ url: string }> { From 2f30837fede7e48b2792241637f7b6bc25f74d3b Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Fri, 28 Jun 2024 16:47:45 -0300 Subject: [PATCH 3/4] FIX: use docker compose instead of docker-compose --- .github/workflows/deploy-manually.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-manually.yml b/.github/workflows/deploy-manually.yml index cb90dd9..58187c8 100644 --- a/.github/workflows/deploy-manually.yml +++ b/.github/workflows/deploy-manually.yml @@ -87,10 +87,6 @@ jobs: run: | python3 -m pip install --upgrade pip - - name: Install Docker Compose - run: | - python3 -m pip install docker-compose --upgrade - - name: Install AWS CLI run: | python3 -m pip install awscli --upgrade @@ -115,8 +111,8 @@ jobs: IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} run: | - docker-compose -f build.docker-compose.yml build - docker-compose -f build.docker-compose.yml push + docker compose -f build.docker-compose.yml build + docker compose -f build.docker-compose.yml push - name: Login to Docker Hub if: ${{inputs.push_to_dockerhub}} @@ -132,8 +128,8 @@ jobs: IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} run: | - docker-compose -f build.docker-compose.dockerhub.yml build - docker-compose -f build.docker-compose.dockerhub.yml push + docker compose -f build.docker-compose.dockerhub.yml build + docker compose -f build.docker-compose.dockerhub.yml push - name: Create ZIP file to Deploy AWS Beanstalk env: From 78ee64fc4539313aad4850227f134a1a6d3b29a3 Mon Sep 17 00:00:00 2001 From: Gabriel Rosa Date: Fri, 28 Jun 2024 16:55:31 -0300 Subject: [PATCH 4/4] FIX: login aws on test --- .github/workflows/test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76b8c70..d2049ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,13 @@ jobs: runs-on: self-hosted steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Configure AWS + uses: aws-actions/configure-aws-credentials@v4 + id: aws + with: + aws-region: us-east-1 - name: Build env: