Merge pull request #252 from dadosfera/feature/new-monitoring-dash

FEAT: update monitoring dash

.github/workflows/deploy-k8s.yml

@@ -0,0 +1,73 @@
+name: Deploy K8S Modifications
+
+on:
+  push:
+    branches:
+      - main
+      - beta
+
+jobs:
+  extract_environment:
+    runs-on: ubuntu-22.04
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
+          fi
+        id: extract_environment
+
+  helmfile-deploy:
+    needs: [extract_environment]
+    runs-on: [self-hosted, "prd-azure"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install Azure ClI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - uses: azure/login@v2
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+
+      - name: Run Helmfile Apply
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: helmfile -f helmfiles/${ENV}.yaml sync

README.md

@@ -2,7 +2,6 @@
   <image src="./assets/maestro.svg" style="width:10rem">
 </p>
 
-
 # Maestro
 
 Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.

docsfera.json

@@ -5431,117 +5431,6 @@
           ]
         }
       },
-      "/open-data/sharing-ocean-data": {
-        "post": {
-          "operationId": "OpenDataController_createUser",
-          "parameters": [
-            {
-              "name": "dadosfera-lang",
-              "in": "header",
-              "required": false,
-              "schema": {
-                "enum": [
-                  "pt-br",
-                  "en-us"
-                ],
-                "type": "string"
-              }
-            },
-            {
-              "name": "language",
-              "required": true,
-              "in": "query",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "responses": {
-            "200": {
-              "description": ""
-            }
-          },
-          "tags": [
-            "OpenData"
-          ]
-        }
-      },
-      "/customers/{id}/theme": {
-        "post": {
-          "operationId": "ThemeController_saveCustomertheme",
-          "parameters": [
-            {
-              "name": "id",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "requestBody": {
-            "required": true,
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/CustomerThemeRequest"
-                }
-              }
-            }
-          },
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "$ref": "#/components/schemas/CustomerThemeResponse"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Theme"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        },
-        "get": {
-          "operationId": "ThemeController_getCustomerTheme",
-          "parameters": [
-            {
-              "name": "id",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "$ref": "#/components/schemas/CustomerThemeResponse"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Theme"
-          ]
-        }
-      },
       "/health": {
         "get": {
           "operationId": "HealthController_check",
@@ -8345,59 +8234,6 @@
           "required": [
             "links"
           ]
-        },
-        "CustomerThemeRequest": {
-          "type": "object",
-          "properties": {
-            "displayName": {
-              "type": "string"
-            },
-            "backgroundColor": {
-              "type": "string"
-            },
-            "textColor": {
-              "type": "string"
-            }
-          },
-          "required": [
-            "displayName",
-            "backgroundColor",
-            "textColor"
-          ]
-        },
-        "CustomerTheme": {
-          "type": "object",
-          "properties": {
-            "backgroundColor": {
-              "type": "string"
-            },
-            "textColor": {
-              "type": "string"
-            },
-            "logoWhite": {
-              "type": "string"
-            },
-            "logoBlack": {
-              "type": "string"
-            }
-          },
-          "required": [
-            "backgroundColor",
-            "textColor",
-            "logoWhite",
-            "logoBlack"
-          ]
-        },
-        "CustomerThemeResponse": {
-          "type": "object",
-          "properties": {
-            "theme": {
-              "$ref": "#/components/schemas/CustomerTheme"
-            }
-          },
-          "required": [
-            "theme"
-          ]
         }
       }
     }

environment.d.ts

@@ -12,8 +12,6 @@ declare global {
       INTERNAL_SWAGGER: 'true' | 'false';
 
       AWS_REGION: string;
-      OPEN_GROUP_ID: string;
-      OPEN_CUSTOMER_ID: string;
     }
   }
 }

helmfiles/prd.yaml

@@ -14,7 +14,3 @@ charts:
         value: in-factory.dadosfera.ai
       - name: maestro.tr_factory_url
         value: in-factory.dadosfera.ai
-      - name: maestro.open_customer_id
-        value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
-      - name: maestro.open_group_id
-        value: 401573bb-334f-44b2-b30e-88d4cea31ae9

helmfiles/stg.yaml

@@ -5,18 +5,12 @@ charts:
     - ../maestro/values.yaml
     set:
       - name: maestro.duc_url
-        value: duc.stg.dadosfera.ai
+        value: duc-temp.dadosfera.ai
       - name: hostname
-        value: maestro.stg.dadosfera.ai
+        value: maestro-temp.dadosfera.ai
       - name: maestro.pi_factory_url
-        value: pi-factory.dadosfera.ai
+        value: pi-factory-temp.dadosfera.ai
       - name: maestro.in_factory_url
-        value: in-factory.dadosfera.ai
+        value: in-factory-temp.dadosfera.ai
       - name: maestro.tr_factory_url
-        value: in-factory.dadosfera.ai
+        value: in-factory-temp.dadosfera.ai
-      - name: maestro.open_customer_id
-        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
-      - name: maestro.open_group_id
-        value: e3f98a2f-7748-4981-8505-7695c8ca8218
-      - name: replicaCount
-        value: 1

maestro/templates/deployment.yaml

@@ -36,11 +36,6 @@ spec:
                     operator: In
                     values:
                       - backend
-      tolerations:
-      - key: "kubernetes.azure.com/scalesetpriority"
-        operator: "Equal"
-        value: "spot"
-        effect: "NoSchedule"
 
       containers:
         - name: maestro

maestro/values.yaml

@@ -40,8 +40,7 @@ maestro:
   sm_oauth_path: prd/root/oauth_applications
   tr_factory_url: in-factory.dadosfera.ai
   upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
-  open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
+
-  open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
 autoscaling:
   enabled: false
   minReplicas: 1

package-lock.json

@@ -12,7 +12,7 @@
         "@aws-sdk/client-secrets-manager": "^3.414.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "3.37.0-beta.3",
+        "@dadosfera/protospack-v2": "3.34.0",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -53,7 +53,7 @@
         "@types/jest": "27.0.2",
         "@types/jsonwebtoken": "^8.5.9",
         "@types/jwk-to-pem": "^2.0.1",
-        "@types/multer": "^1.4.12",
+        "@types/multer": "^1.4.7",
         "@types/node": "^16.18.52",
         "@types/passport-facebook": "^2.1.11",
         "@types/passport-google-oauth20": "^2.0.11",
@@ -1403,10 +1403,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.37.0-beta.3",
+      "version": "3.34.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.3.tgz",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.34.0.tgz",
-      "integrity": "sha512-9YSgzkXVqe9QsBxJjfxYxttAiyCXxgq49Xjs3OvnTLCXRDn3XtTlPaC+yHrTEUgCkQyhlDZuswe6f6oBImqIRQ==",
+      "integrity": "sha512-VUjpoHg5/uNNkg1cTxWbUv+20t5CTAf1tB4dhBmGSYjgb+Efo5gi3aO/p3ajYuzOOMOxF+EhjQVouU6BD3E+Xg==",
-      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"
@@ -3527,11 +3526,10 @@
       "dev": true
     },
     "node_modules/@types/multer": {
-      "version": "1.4.12",
+      "version": "1.4.11",
-      "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz",
+      "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.11.tgz",
-      "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==",
+      "integrity": "sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "@types/express": "*"
       }

package.json

@@ -30,7 +30,7 @@
     "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.37.0-beta.3",
+    "@dadosfera/protospack-v2": "3.34.0",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",
@@ -74,7 +74,7 @@
     "@types/jest": "27.0.2",
     "@types/jsonwebtoken": "^8.5.9",
     "@types/jwk-to-pem": "^2.0.1",
-    "@types/multer": "^1.4.12",
+    "@types/multer": "^1.4.7",
     "@types/node": "^16.18.52",
     "@types/passport-facebook": "^2.1.11",
     "@types/passport-google-oauth20": "^2.0.11",

src/app.module.ts

@@ -26,10 +26,9 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
 import { ProductboardModule } from './modules/productboard/productboard.module';
 import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
 import { CustomersModule } from './modules/customers/customers.module';
-import { OpenDataModule } from './modules/open-data/open-data.module';
-import { ThemeModule } from './modules/theme/theme.module';
 
 @Module({
+  controllers: [],
   providers: [
     DadosferaLogger,
     {
@@ -59,10 +58,8 @@ import { ThemeModule } from './modules/theme/theme.module';
     ProductboardModule,
     MixpanelModule,
     CustomersModule,
-    OpenDataModule,
-    ThemeModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
-    HealthModule
+    HealthModule,
   ],
 })
 export class AppModule {}

src/decorators/set-origin.decorator.ts

@@ -1,32 +0,0 @@
-import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common';
-import { Reflector } from '@nestjs/core';
-import { SetMetadata } from '@nestjs/common';
-
-export const SetOrigin = (origin: string) => SetMetadata('allowedOrigin', origin);
-
-@Injectable()
-export class CORSGuard implements CanActivate {
-  constructor(private reflector: Reflector) {}
-
-  canActivate(context: ExecutionContext): boolean {
-    const response = context.switchToHttp().getResponse();
-    const request = context.switchToHttp().getRequest();
-    const origin = request.headers.origin;
-
-    // Obter a origem permitida através do decorador
-    const allowedOrigin = this.reflector.get<string>('allowedOrigin', context.getHandler());
-    if (process.env.ENV === "local") {
-        return true;
-    }
-
-    // Verifica se a origem da requisição é permitida   
-    if (allowedOrigin && origin !== allowedOrigin) {
-      throw new ForbiddenException('Acesso não permitido pela política CORS');
-    }
-
-    response.setHeader('Access-Control-Allow-Origin', allowedOrigin);
-
-    return true;
-  }
-}
-

src/modules/customers/customers.controller.ts

@@ -1,4 +1,5 @@
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import {
   Body,
   Controller,
@@ -17,6 +18,7 @@ import {
   Authenticated,
   RequireAllPermissions,
 } from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { CustomersService } from './customers.service';
 import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
@@ -26,6 +28,7 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @ApiTags('Customers')
 @Controller('customers')
+@Authenticated()
 @UseFilters(GrpcToHttpExceptionFilter)
 export class CustomersController {
   logger: DadosferaLogger;
@@ -39,7 +42,6 @@ export class CustomersController {
   }
 
   @Get(':id/links')
-  @Authenticated()
   @ApiOkResponse({ type: CustomerLinksResponse })
   async getCustomerLinks(@Param('id') id: string) {
     this.logger.info('getCustomerLinks', { id });
@@ -48,7 +50,6 @@ export class CustomersController {
   }
 
   @Put(':id/links')
-  @Authenticated()
   @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
   @ApiOkResponse()
   @HttpCode(HttpStatus.OK)
@@ -62,7 +63,6 @@ export class CustomersController {
   }
 
   @Get('token')
-  @Authenticated()
   @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
   @ApiProduces('text/plain')
   async getCustomerToken(
@@ -79,7 +79,6 @@ export class CustomersController {
   }
 
   @Get('monitoring-dashboard')
-  @Authenticated()
   @RequireAllPermissions(
     PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
   )

src/modules/customers/customers.service.ts

@@ -28,24 +28,19 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { Metadata } from '@grpc/grpc-js';
 import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 // This function will accept any string, which may result in a bug.
 @Injectable()
 export class CustomersService implements OnModuleInit {
   private customerService: CustomersProtoService;
-  private logger: DadosferaLogger;
+
   private pipelineReadService: ReadService.PipelineV2ReadService;
 
   constructor(
     @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
     @Inject(PipelinesClientConfiguration.name)
     private readonly pipelinesGrpcClient: ClientGrpc,
-    @Inject(DadosferaLogger)
+  ) {}
-    dadosferaLogger: DadosferaLogger,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
 
   onModuleInit() {
     this.customerService = this.grpcClient.getService<CustomersProtoService>(

src/modules/customers/dtos/customers.ts

@@ -20,4 +20,3 @@ export class CustomerLinksResponse {
   @ApiProperty({ type: [CustomerLink] })
   links: CustomerLink[];
 }
-

src/modules/duc/client.config.ts

@@ -29,8 +29,6 @@ export class DucClient {
         objects: true,
         arrays: true,
       },
-      maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem
-      maxReceiveMessageLength: 15 * 1024 * 1024,
     },
   };
 

src/modules/open-data/dto/wordpres-form.ts

@@ -1,48 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-
-export class CreateUserOpenDataDTO {
-  @ApiProperty()
-  firstName: string;
-  @ApiProperty()
-  lastName: string;
-  @ApiProperty()
-  email: string;
-  @ApiProperty()
-  organization: string;
-  @ApiProperty()
-  enquiryType: string;
-}
-
-type FormField = {
-  id: string;
-  type: string;
-  title: string;
-  value: string;
-  raw_value: string;
-  required: string;
-};
-
-type MetaData = {
-  title: string;
-  value: string;
-};
-
-export type WordpressForm = {
-  form: {
-    id: string;
-    name: string;
-  };
-  fields: {
-    [key: string]: FormField;
-  };
-  meta: {
-    date: MetaData;
-    time: MetaData;
-    page_url: MetaData;
-    user_agent: MetaData;
-    remote_ip: MetaData;
-    credit: MetaData;
-  };
-};
-
-

src/modules/open-data/open-data.controller.ts

@@ -1,108 +0,0 @@
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { Body, Controller, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards } from '@nestjs/common';
-import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
-import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
-import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { LanguageEnum } from 'src/utils/languages.enum';
-import { UsersService } from '../users/users.service';
-import { Language } from 'src/decorators/language.decorator';
-import { OpenDataService } from './open-data.service';
-import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
-import { CORSGuard, SetOrigin } from 'src/decorators/set-origin.decorator';
-import { Metadata } from '@grpc/grpc-js';
-import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
-import { request } from 'http';
-import { Request } from 'express';
-
-@Controller('open-data')
-@ApiInternalOnlyController()
-@ApiTags('OpenData')
-@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
-@UseFilters(GrpcToHttpExceptionFilter)
-export class OpenDataController {
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private openDataService: OpenDataService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  @Post("/sharing-ocean-data")
-  // @SetOrigin('devsbm.dadosfera.io')
-  // @UseGuards(CORSGuard)
-  @HttpCode(200)
-  @Header('content-type', 'application/json')
-  @ApiOkResponse()
-  async createUser(
-    @Body()
-    body: WordpressForm,
-    @Query('language')
-    language: string,
-    @Req()
-    request: Request,
-  ) {
-    this.logger.info('createUser for open data'
-        + JSON.stringify({
-          origin: request.headers.origin,
-          language,
-          body
-    }));
-
-    // "401573bb-334f-44b2-b30e-88d4cea31ae9"
-    // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID;
-    // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e""
-    // const OPENDATA_CUSTOMER_ID =  process.env.OPEN_CUSTOMER_ID;
-    const OPENDATA_CUSTOMER_ID = "b3e3dfe5-b992-4586-a73c-c0b0c00f615d";
-
-    this.logger.info("OPENDATA_CUSTOMER_ID: " + process.env.OPEN_CUSTOMER_ID)
-    this.logger.info("OPEN_GROUP_ID: " + process.env.OPEN_GROUP_ID)
-    const roles = ["e3f98a2f-7748-4981-8505-7695c8ca8218"];
-    const metadata = PackTheMetadata({
-      language: language || 'en-us'
-    });
-
-    const data = {}
-
-    try {
-      Object.keys(body.fields)
-        .filter(key => body.fields[key].required === "1")
-        .forEach(key => {
-          const field = body.fields[key]
-          data[field.id] = field.value
-        });
-    } catch (e) {
-      this.logger.error('user data ' + e.message);
-    }
-
-
-    const user: CreateUserOpenDataDTO = {
-      email: data["email"],
-      enquiryType: data["enquiry_type"],
-      firstName: data["first_name"],
-      lastName: data["last_name"],
-      organization: data["organization"]
-    }
-    this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID }));
-
-    try {
-      await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata);
-      this.logger.info('user created with sucessfull data');
-      return {
-        success: true,
-        status: 'success',
-        message: 'user created with succesfull'
-      }
-    } catch (e) {
-      this.logger.error('failed with exception: ' + e.message);
-      return {
-        success: false,
-        status: 'failed',
-        message: e.message
-      };
-    }
-  }
-
-}

src/modules/open-data/open-data.module.ts

@@ -1,22 +0,0 @@
-import { Module } from '@nestjs/common';
-import { OpenDataController } from './open-data.controller';
-import { UsersService } from '../users/users.service';
-import { ClientsModule } from '@nestjs/microservices'
-import { DucClient } from '../duc/client.config';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { RolesModule } from '../roles/roles.module';
-import { PermissionsModule } from '../permissions/permissions.module';
-import { OpenDataService } from './open-data.service';
-
-const client = new DucClient();
-
-@Module({
-  controllers: [OpenDataController],
-  imports: [
-    ClientsModule.register([client.providerOptions]),
-    RolesModule,
-    // PermissionsModule,
-  ],
-  providers: [DadosferaLogger, UsersService, OpenDataService]
-})
-export class OpenDataModule {}

src/modules/open-data/open-data.service.ts

@@ -1,51 +0,0 @@
-import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
-import { lastValueFrom } from 'rxjs';
-import { CreateUserOpenDataDTO } from './dto/wordpres-form';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
-import { DucClient } from '../duc/client.config';
-import { ClientGrpc } from '@nestjs/microservices';
-import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { Metadata } from '@grpc/grpc-js';
-
-@Injectable()
-export class OpenDataService implements OnModuleInit {
-  logger: DadosferaLogger;
-
-  private usersClientService: UsersProtoService;
-  constructor(
-    @Inject(DadosferaLogger)
-    private dadosferaLogger: DadosferaLogger,
-    @Inject(DucClient.name)
-    private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.usersClientService = this.grpcClient.getService(
-      ProtoServices.UsersProtoService,
-    );
-
-  }
-
-  async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) {
-    const body = {
-      email: data.email,
-      name: data.firstName + " " + data.lastName,
-      department: data.organization,
-      jobTitle: data.enquiryType,
-      customerId: customerId,
-      roleIds: roleIds
-    }
-
-    try {
-      await lastValueFrom(
-        this.usersClientService.SimpleUserCreate(body, metadata),
-      );
-      return "User created";
-    } catch(err) {
-      return err;
-    }
-  }
-}

src/modules/pipelinesV2/pipelines.service.ts

@@ -328,7 +328,7 @@ export class PipelinesService implements OnModuleInit {
     const res = await lastValueFrom(
       this.pipelineReadService.PipelineV2GetDashboardUrl(
         {
-          dashboard_id: '83',
+          dashboard_id: '95',
           exp: '15m',
           metabase_customer_name: 'dadosferatech',
         },

src/modules/theme/dtos/customers.ts

@@ -1,47 +0,0 @@
-import { Link, Theme } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
-import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
-
-export class CustomerLink implements Link {
-  @ApiProperty()
-  href: string;
-  @ApiProperty()
-  name: string;
-  @ApiProperty()
-  description: string;
-  @ApiPropertyOptional()
-  iconSrc: string;
-}
-export class CustomerLinkRequest {
-  @ApiProperty({ type: [CustomerLink] })
-  links: CustomerLink[];
-}
-
-export class CustomerLinksResponse {
-  @ApiProperty({ type: [CustomerLink] })
-  links: CustomerLink[];
-}
-
-export class CustomerTheme implements Theme {
-  @ApiProperty()
-  backgroundColor: string;
-  @ApiProperty()
-  textColor: string;
-  @ApiProperty()
-  logoWhite: string;
-  @ApiProperty()
-  logoBlack: string;
-}
-
-export class CustomerThemeResponse {
-  @ApiProperty()
-  theme: CustomerTheme;
-}
-
-export class CustomerThemeRequest {
-  @ApiProperty()
-  displayName: string;
-  @ApiProperty()
-  backgroundColor: string;
-  @ApiProperty()
-  textColor: string;
-}

src/modules/theme/theme.controller.ts

@@ -1,125 +0,0 @@
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import {
-  Body,
-  Controller,
-  Get,
-  HttpException,
-  HttpStatus,
-  Inject,
-  Param,
-  Post,
-  Put,
-  Query,
-  UploadedFiles,
-  UseFilters,
-  UseInterceptors,
-  HttpCode
-} from '@nestjs/common';
-import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import {
-  Authenticated,
-  RequireAllPermissions,
-} from 'src/decorators/authentication.decorator';
-import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers';
-
-import ErrorCodes from 'src/utils/errorCodes';
-import { AnyFilesInterceptor } from '@nestjs/platform-express';
-import { ThemeService } from './theme.service';
-
-@ApiTags('Theme')
-@Controller('customers')
-@UseFilters(GrpcToHttpExceptionFilter)
-export class ThemeController {
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private themeService: ThemeService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-
-  @Post('/:id/theme')
-  @Authenticated()
-  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
-  @ApiOkResponse({ type: CustomerThemeResponse })
-  @UseInterceptors(AnyFilesInterceptor())
-  @HttpCode(HttpStatus.OK)
-  async saveCustomertheme(
-    @Param('id') id: string,
-    @Body() data: CustomerThemeRequest,
-    @UploadedFiles() files: Array<Express.Multer.File>
-  ) {
-
-    this.logger.info('saveCustomertheme' + JSON.stringify({
-      id, 
-    }));
-
-    const logoWhite = files.find(file => file.fieldname === 'logoWhite');
-    const logoBlack = files.find(file => file.fieldname === 'logoBlack');
-
-    this.validFileSize(logoWhite);
-    this.validFileSize(logoBlack);
-    this.validMimeType(logoWhite);
-    this.validMimeType(logoBlack);
-
-    try {
-      const theme = await this.themeService.createThemeByCustomer(id, {
-        ...data,
-        logoWhite,
-        logoBlack
-      });
-      this.logger.info('saveCustomertheme' + JSON.stringify(theme));
-      return { theme: theme };
-    } catch (err) {
-      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
-        this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND');
-        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
-      } else {
-        this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message);
-        return { theme: null };
-      };
-    }
-  }
-
-  private validFileSize(file: Express.Multer.File) {
-    const maxFileSize = 10 * 1024 * 1024; // 10MB
-
-    if (file && file.size > maxFileSize) {
-      throw new HttpException(`O Arquivo ${file.filename} possui mais de 10MB`, HttpStatus.BAD_REQUEST);
-    }
-  }
-
-  private validMimeType(file: Express.Multer.File) {
-    const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png'];
-
-    if (file && !mimeTypesValid.includes(file.mimetype)) {
-      throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg ou png`, HttpStatus.BAD_REQUEST);
-    }
-  }
-
-  @Get('/:id/theme')
-  @ApiOkResponse({ type: CustomerThemeResponse })
-  async getCustomerTheme(@Param('id') id: string) {
-    this.logger.info('getCustomerTheme with id' + id);
-
-    try {
-      const data = await this.themeService.getThemeByCustomer(id);
-      this.logger.info('Success - getCustomerTheme with id'+ id);
-      return data;
-    }catch (err) {
-      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
-        this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');
-        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
-      } else {
-        this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message);
-        return { theme: null };
-      };
-    }
-  }
-
-}

src/modules/theme/theme.module.ts

@@ -1,20 +0,0 @@
-import { Module } from '@nestjs/common';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { ClientsModule } from '@nestjs/microservices';
-import { ThemeController } from './theme.controller';
-import { ThemeService } from './theme.service';
-import { DucClient } from '../duc/client.config';
-
-const ducClient = new DucClient();
-
-@Module({
-  imports: [
-    ClientsModule.register([
-      ducClient.providerOptions,
-    ]),
-  ],
-  controllers: [ThemeController],
-  providers: [ThemeService, DadosferaLogger],
-  exports: [ThemeService],
-})
-export class ThemeModule {}

src/modules/theme/theme.service.ts

@@ -1,133 +0,0 @@
-import {
-  OnModuleInit,
-  Inject,
-  Injectable,
-  HttpException,
-  HttpStatus,
-  InternalServerErrorException,
-} from '@nestjs/common';
-
-import { firstValueFrom, lastValueFrom, ReplaySubject } from 'rxjs';
-import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
-import { DucClient } from '../duc/client.config';
-import { ClientGrpc } from '@nestjs/microservices';
-import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
-import { ThemeProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
-import { ThemeRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
-import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { resolve } from 'path';
-import { Readable } from 'stream';
-
-type Files = {
-  logoWhite: Express.Multer.File,
-  logoBlack: Express.Multer.File,
-}
-
-// This function will accept any string, which may result in a bug.
-@Injectable()
-export class ThemeService implements OnModuleInit {
-  private themeService: ThemeProtoService;
-  private logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.themeService = this.grpcClient.getService<ThemeProtoService>(
-      ProtoServices.ThemeProtoService,
-    );
-  }
-
-  async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) {
-    if (!id) {
-      this.logger.error('Error - saveCustomertheme - not found id:' + id);
-      throw new HttpException(null, HttpStatus.BAD_REQUEST);
-    }
-
-    const customerThemeRequest$ = new ReplaySubject<ThemeRequest>();
-
-    customerThemeRequest$.next({
-      customerId: id,
-      displayName: theme.displayName,
-      backgroundColor: theme.backgroundColor,
-      textColor: theme.textColor,
-      isMetadata: true,
-      filename: '',
-      chunk: Buffer.alloc(0)
-    })
-
-    if(theme.logoBlack) {
-      await this.sendFile(theme.logoBlack, customerThemeRequest$);
-    }
-
-    if(theme.logoWhite) {
-      await this.sendFile(theme.logoWhite, customerThemeRequest$);
-    }
-    customerThemeRequest$.complete();
-
-    const stream = this.themeService.CustomerCreateTheme(customerThemeRequest$);
-
-    return lastValueFrom(stream);
-  }
-
-  async getThemeByCustomer(id: string): Promise<CustomerThemeResponse> {
-    if (!id) {
-      this.logger.error('Error - getCustomerTheme - not found id:' + id);
-      throw new HttpException(null, HttpStatus.BAD_REQUEST);
-    }
-
-    const { theme } = await firstValueFrom(
-      this.themeService.CustomerGetTheme({
-        id
-      }),
-    );
-
-    return {
-      theme
-    }
-  }
-
-   private async sendFile(file: Express.Multer.File, stream$: ReplaySubject<ThemeRequest>) {
-    const bufferStream = new Readable({
-      highWaterMark: 1024 * 1024, // 1 MB por chunk
-      read() {}
-    });
-    bufferStream.push(file.buffer);
-    bufferStream.push(null);
-
-    return new Promise((resolve, reject) => {
-      bufferStream.on('data', (chunk) => {
-        console.log('enviando chunk', file.fieldname)
-        const mimetype = file.mimetype.split('/')[1]; // example image/jpeg
-        const filename = file.fieldname.concat(".", mimetype);
-        stream$.next({
-          customerId: '',
-          displayName: '',
-          backgroundColor: '',
-          textColor: '',
-          isMetadata: false,
-          filename: filename,
-          chunk: chunk
-        });
-      });
-
-      bufferStream.on('end', () => {
-        console.log('terminou de enviar')
-        resolve(file.filename)
-      });
-
-      bufferStream.on('error', (err) => {
-        console.error('Erro no stream:', err);
-        reject(err);
-      });
-    });
-  }
-
-}