FIX: merge with main

Release/11 07

.github/workflows/deploy-manually.yml

@@ -13,6 +13,11 @@ on:
         options:
           - stg
           - prd
+      push_to_dockerhub:
+        description: "Push image to Dockerhub?"
+        required: true
+        type: boolean
+        default: false
 
 jobs:
   extract_environment:
@@ -113,6 +118,23 @@ jobs:
           docker compose -f build.docker-compose.yml build
           docker compose -f build.docker-compose.yml push
 
+      - name: Login to Docker Hub
+        if: ${{inputs.push_to_dockerhub}}
+        uses: docker/login-action@v2
+        with:
+          username: dadosfera
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build, Tag, and Push Image to Dockerhub
+        if: ${{inputs.push_to_dockerhub}}
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+        run: |
+          docker compose -f build.docker-compose.dockerhub.yml build
+          docker compose -f build.docker-compose.dockerhub.yml push
+
       # - name: Create ZIP file to Deploy AWS Beanstalk
       #   env:
       #     ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -144,10 +166,7 @@ jobs:
 
   helmfile-deploy:
     needs: [extract_environment, semantic_release, build_ecr_image]
-    env:
+    runs-on: [self-hosted, "prd-azure"]
-      HOME: /home/runner
-    runs-on: [self-hosted, "prd-oracle"]
-    environment: ${{ needs.extract_environment.outputs.environment }}
 
     steps:
       - name: Checkout code
@@ -158,17 +177,13 @@ jobs:
         with:
           version: 'v3.9.0'
 
-      - name: Install OCI CLI
+      - name: Install Azure ClI
         run: |
-          bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
-          echo "$HOME/bin" >> $GITHUB_PATH
 
-      - name: Configure OCI CLI
+      - uses: azure/login@v2
-        run: |
+        with:
-          mkdir -p ~/.oci || true
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
-          echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
-          echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem 
-          chmod 600 ~/.oci/oci_api_key.pem
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -179,31 +194,24 @@ jobs:
         run: |
           wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
           tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
-          sudo mv helmfile /usr/local/bin/
+          mv helmfile /usr/local/bin/
           helmfile --version
 
       - name: Install Helm Diff Plugin
         run: helm plugin install https://github.com/databus23/helm-diff || true
 
-      - name: Authenticate with OKE cluster
+      - name: Setup kubectl
-        env:
+        uses: azure/setup-kubectl@v1
-          ENV: ${{ needs.extract_environment.outputs.environment }}
+        with:
-          STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
+          version: 'v1.30.1'
-          PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
-        run: |
-          if [ "$ENV" = "stg" ]; then
-            CLUSTER_ID=$STG_CLUSTER_ID
-          elif [ "$ENV" = "prd" ]; then
-            CLUSTER_ID=$PRD_CLUSTER_ID
-          else
-            echo "Unknown environment: $ENV"
-            exit 1
-          fi
 
-          oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
 
       - name: Run Helmfile Apply
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
-        run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+        run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG

.github/workflows/validate-k8s.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - beta
+      - stg
 
 jobs:
   extract_environment:
@@ -21,13 +21,17 @@ jobs:
           fi
         id: extract_environment
 
-  helmfile-check:
+  helmfile-deploy:
-    env:
-      HOME: /home/runner
     needs: [extract_environment]
-    environment: ${{ needs.extract_environment.outputs.environment }}
+    runs-on: [self-hosted, "prd-azure"]
-    runs-on: [self-hosted, "prd-oracle"]
+
     steps:
+      - name: Summary
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: |
+          echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\`  para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
+
       - name: Checkout code
         uses: actions/checkout@v3
 
@@ -36,28 +40,13 @@ jobs:
         with:
           version: 'v3.9.0'
 
-      - name: Determine DNS_HOST based on environment
+      - name: Install Azure ClI
-        id: set_dns
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
         run: |
-          if [ "$ENV" = "prd" ]; then
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
-            echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
-          elif [ "$ENV" = "stg" ]; then
-            echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
-          fi
 
-      - name: Install OCI CLI
+      - uses: azure/login@v2
-        run: |
+        with:
-          bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
-          echo "$HOME/bin" >> $GITHUB_PATH
-
-      - name: Configure OCI CLI
-        run: |
-          mkdir -p ~/.oci || true
-          echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
-          echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
-          chmod 600 ~/.oci/oci_api_key.pem
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -68,35 +57,23 @@ jobs:
         run: |
           wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
           tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
-          sudo mv helmfile /usr/local/bin/
+          mv helmfile /usr/local/bin/
           helmfile --version
 
       - name: Install Helm Diff Plugin
         run: helm plugin install https://github.com/databus23/helm-diff || true
 
-      - name: Authenticate with OKE cluster
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-          STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
-          PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
-        run: |
-          if [ "$ENV" = "stg" ]; then
-            CLUSTER_ID=$STG_CLUSTER_ID
-          elif [ "$ENV" = "prd" ]; then
-            CLUSTER_ID=$PRD_CLUSTER_ID
-          else
-            echo "Unknown environment: $ENV"
-            exit 1
-          fi
-          
-          oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT 
-
       - name: Setup kubectl
         uses: azure/setup-kubectl@v1
         with:
           version: 'v1.30.1'
 
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+
       - name: Run Helmfile Diff
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
-        run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+        run: helmfile -f helmfiles/${ENV}.yaml diff

deploy/helmfiles/stg.yaml

@@ -1,58 +0,0 @@
-charts:
-  - name: maestro
-    chart: ../helm-chart
-    values:
-    - ../helm-chart/values.yaml
-    set:
-      - name: maestro.duc_url
-        value: duc.stg.dadosfera.ai
-      - name: hostname
-        value: maestro.stg.dadosfera.ai
-      - name: maestro.pi_factory_url
-        value: pi-factory.stg.dadosfera.ai
-      - name: maestro.in_factory_url
-        value: in-factory.stg.dadosfera.ai
-      - name: maestro.tr_factory_url
-        value: in-factory.stg.dadosfera.ai
-      - name: maestro.open_customer_id
-        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
-      - name: maestro.open_group_id
-        value: e3f98a2f-7748-4981-8505-7695c8ca8218
-      - name: replicaCount
-        value: 1
-  
-    # Environment to test Network Policies
-  # - name: private-maestro
-  #   chart: ../helm-chart
-  #   values:
-  #   - ../helm-chart/values.yaml
-  #   set:
-  #     - name: app_name
-  #       value: maestro-private
-  #     - name: maestro.env
-  #       value: stg
-  #     - name: maestro.duc_url
-  #       value: duc.stg.dadosfera.ai
-  #     - name: hostname
-  #       value: private-maestro.stg.dadosfera.ai
-  #     - name: maestro.pi_factory_url
-  #       value: pi-factory.dadosfera.ai
-  #     - name: maestro.in_factory_url
-  #       value: in-factory.stg.dadosfera.ai
-  #     - name: maestro.tr_factory_url
-  #       value: in-factory.dadosfera.ai
-  #     - name: maestro.open_customer_id
-  #       value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
-  #     - name: maestro.open_group_id
-  #       value: e3f98a2f-7748-4981-8505-7695c8ca8218
-  #     # Customer id
-  #     - name: maestro.dedicated_proxy
-  #       value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
-  #     - name: replicaCount
-  #       value: 1
-  #     - name: affinity
-  #       value: null
-  #     - name: resources
-  #       value: null
-  #     - name: maestro.restricted_ip
-  #       value: "57.151.113.140/30"

helmfiles/prd.yaml

@@ -1,8 +1,8 @@
 charts:
   - name: maestro
-    chart: ../helm-chart
+    chart: ../maestro
     values:
-    - ../helm-chart/values.yaml
+    - ../maestro/values.yaml
     set:
       - name: app_name
         value: maestro
@@ -24,9 +24,9 @@ charts:
         value: 2
   
   - name: unimed-maestro
-    chart: ../helm-chart
+    chart: ../maestro
     values:
-    - ../helm-chart/values.yaml
+    - ../maestro/values.yaml
     set:
       - name: app_name
         value: maestro-unimed
@@ -50,4 +50,4 @@ charts:
       - name: replicaCount
         value: 1
       - name: maestro.restricted_ip
-        value: "177.52.172.0/24,57.151.113.140/30"
+        value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30"

helmfiles/stg.yaml

@@ -0,0 +1,52 @@
+charts:
+  - name: maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: maestro.duc_url
+        value: duc.stg.dadosfera.ai
+      - name: hostname
+        value: maestro.stg.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.stg.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: e3f98a2f-7748-4981-8505-7695c8ca8218
+      - name: replicaCount
+        value: 1
+  
+    # Environment to test Network Policies
+  - name: private-maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: app_name
+        value: maestro-private
+      - name: maestro.duc_url
+        value: duc.stg.dadosfera.ai
+      - name: hostname
+        value: private-maestro.stg.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.stg.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: e3f98a2f-7748-4981-8505-7695c8ca8218
+      # Customer id
+      - name: maestro.dedicated_proxy
+        value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
+      - name: replicaCount
+        value: 1
+      - name: maestro.restricted_ip
+        value: "57.151.113.140/30"

maestro/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

maestro/templates/deployment.yaml

@@ -27,10 +27,15 @@ spec:
         - name: {{ .Values.imagePullSecrets }}
       nodeSelector:
         "beta.kubernetes.io/os": linux
-      {{- if .Values.affinity }}
       affinity:
-        {{- toYaml .Values.affinity | nindent 8 }}
+        nodeAffinity:
-      {{- end }}
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: application
+                    operator: In
+                    values:
+                      - general
 
       tolerations:
       - key: "kubernetes.azure.com/scalesetpriority"
@@ -43,10 +48,13 @@ spec:
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           ports:
             - containerPort: {{ .Values.containerPort }}
-          {{- if .Values.resources }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            requests:
-          {{- end }}
+              cpu: {{ .Values.resources.requests.cpu }}
+              memory: {{ .Values.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.resources.limits.cpu }}
+              memory: {{ .Values.resources.limits.memory }}
           env:
             - name: AWS_IDENTITY_POOL_ID
               value: {{ .Values.maestro.aws_identity_pool_id }}

maestro/values.yaml

@@ -54,13 +54,3 @@ autoscaling:
   maxReplicas: 100
   targetCPUUtilizationPercentage: 80
   targetMemoryUtilizationPercentage: 80
-
-affinity:
-  nodeAffinity:
-    requiredDuringSchedulingIgnoredDuringExecution:
-      nodeSelectorTerms:
-        - matchExpressions:
-            - key: name
-              operator: In
-              values:
-                - general

src/@types/express-session.d.ts

@@ -1,8 +0,0 @@
-import 'express-session';
-
-declare module 'express-session' {
-  interface SessionData {
-    state: string | undefined;
-    code_verifier: string | undefined;
-  }
-} 

src/app.module.ts

@@ -30,9 +30,6 @@ import { OpenDataModule } from './modules/open-data/open-data.module';
 import { ThemeModule } from './modules/theme/theme.module';
 import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
 import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
-import { AssignModule } from './modules/assign/assign.module';
-import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
-import { ApiKeyModule } from './modules/api-key/api-key.module';
 
 @Module({
   providers: [
@@ -66,15 +63,10 @@ import { ApiKeyModule } from './modules/api-key/api-key.module';
     CustomersModule,
     OpenDataModule,
     ThemeModule,
-    NetworkPolicyModule,
-    AssignModule,
-    ShareMetadataModule,
-    NetworkPolicyModule,
-    ApiKeyModule,
-    IdentityProviderModule,
-    NetworkPolicyModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
+    IdentityProviderModule,
+    NetworkPolicyModule
   ],
 })
 export class AppModule {}

src/authentication/authentication.guard.spec.ts

@@ -99,6 +99,7 @@ describe('authentication.guard', () => {
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
     customer_name: 'dadosfera',
     customer_tier: 'BASIC',
+    customer_modules: []
   };
 
   beforeAll(async () => {

src/authentication/authentication.guard.ts

@@ -18,7 +18,6 @@ import {
 import { RequestUser } from '../decorators/user.decorator';
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';
-import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 @Injectable()
 export class AuthenticationGuard
@@ -33,7 +32,6 @@ export class AuthenticationGuard
     dadosferaLogger: DadosferaLogger,
     private reflector: Reflector,
     private authClient: AuthClientService,
-    private apiKeyService: ApiKeyService
   ) {
     this.pems = new Map();
     this.logger = dadosferaLogger.logger;
@@ -51,40 +49,20 @@ export class AuthenticationGuard
     });
   }
 
-  async canActivate(ctx: ExecutionContext): Promise<boolean> {
+  canActivate(ctx: ExecutionContext): boolean {
     const authFunctions = this.reflector.getAllAndMerge<
       AuthenticationFunction[]
     >(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
     const mustBeAuthenticated = authFunctions.length > 0;
 
+    const request = ctx.switchToHttp().getRequest();
+    const accessToken = this.validateToken(request, mustBeAuthenticated);
+
     if (!mustBeAuthenticated) {
       // no need to be authenticated
       return true;
     }
 
-    const request = ctx.switchToHttp().getRequest();
-    const apiKey = request.get('X-api-key');
-    if (apiKey) {
-      const {
-        api_key
-      } = await this.apiKeyService.get(apiKey);
-
-      request.user = {
-        user_id: api_key.user_id,
-        username: api_key.username,
-        permissions: api_key.permissions,
-        customer_id: api_key.customer_id,
-        customer_name: api_key.customer_name,
-        customer_tier: api_key.customer_tier,
-        customer_modules: api_key.customer_modules,
-        access_token: apiKey,
-      };
-
-      return true;
-    }
-
-    const accessToken = this.validateToken(request, mustBeAuthenticated);
-
     if (!accessToken) {
       // couldn't load valid token
       throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
@@ -143,10 +121,10 @@ export class AuthenticationGuard
     }
 
     // Bloquear o customer de acesso o maestro publico
-    // const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
+    const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
-    // if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
+    if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
-    //   throw new ForbiddenException();
+      throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
-    // }
+    }
 
     request.accessTokenPayload = accessTokenPayload;
     request.user = {

src/authentication/permissions.enum.ts

@@ -614,8 +614,7 @@ export const DADOSFERA_MODULES_KEYS = {
   LOG_DASHBOARD: 'logs-dashboard',
   ACCESS_DASHBOARD: 'access-dashboard',
   DANGER_ZONE: 'danger-zone',
-  PII: 'pii',
+  PII: 'pii'
-  PUBLIC_ASSIGN_EMBED: 'public-assign-embed',
 }
 
 export const DADOSFERA_MODULES: Array<DadosferaModule> = [

src/decorators/user.decorator.spec.ts

@@ -52,6 +52,7 @@ describe('user.decorator', () => {
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
     customer_name: 'dadosfera',
     customer_tier: 'BASIC',
+    customer_modules: [],
     access_token: '',
   };
 

src/main.ts

@@ -9,7 +9,7 @@ import { AppModule } from './app.module';
 import { writeFileSync } from 'fs';
 import { execSync } from 'child_process';
 import { INestApplication } from '@nestjs/common';
-import session from 'express-session';
+
 async function bootstrap() {
   DadosferaLogger.setupLogger({
     serviceName: 'maestro',
@@ -28,7 +28,7 @@ async function bootstrap() {
   });
   app.use(helmet());
 
-  if (process.env.ENV !== 'local') {
+  if (process.env.ENV === 'prd') {
     app.use('/catalog/register-dataset', json({ limit: '10mb' }));
     app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
   }

src/modules/api-key/api-key.controller.ts

@@ -1,68 +0,0 @@
-import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
-import { ApiKeyService } from './api-key.service';
-import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
-import { Authenticated } from 'src/decorators/authentication.decorator';
-import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
-import { LanguageEnum } from 'src/utils/languages.enum';
-import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { RequestUser, User } from 'src/decorators/user.decorator';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-@Controller('api-key')
-@Authenticated()
-@ApiTags('ApiKey')
-@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
-@UseFilters(new GrpcToHttpExceptionFilter())
-export class ApiKeyController {
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private readonly apiKeyService: ApiKeyService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  @Post()
-  @ApiResponse({ type: CreateApiKeyResponseDto })
-  async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
-    this.logger.info('POST /api-key', {
-      permissions: createApiKeyDto.permissions,
-      method: 'create'
-    });
-    const result = await this.apiKeyService.create(createApiKeyDto, user);
-    this.logger.info('POST /api-key success', { 
-      id: result.id,
-      method: 'create'
-    });
-    return result;
-  }
-
-  @Get()
-  @ApiResponse({ type: [ApiKeyBaseResponseDto] })
-  async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
-    this.logger.info('GET /api-key', {
-      method: 'findAll'
-    });
-    const result = await this.apiKeyService.findAll(user);
-    this.logger.info('GET /api-key success', { 
-      count: result.length,
-      method: 'findAll'
-    });
-    return result;
-  }
-
-  @Delete(':id')
-  async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
-    this.logger.info('DELETE /api-key/:id', { 
-      id,
-      method: 'remove'
-    });
-    await this.apiKeyService.remove(id, user);
-    this.logger.info('DELETE /api-key/:id success', { 
-      id,
-      method: 'remove'
-    });
-  }
-}

src/modules/api-key/api-key.module.ts

@@ -1,18 +0,0 @@
-import { Module } from '@nestjs/common';
-import { ApiKeyService } from './api-key.service';
-import { ApiKeyController } from './api-key.controller';
-import { ClientsModule } from '@nestjs/microservices';
-import { DucClient } from '../duc/client.config';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-
-const ducClient = new DucClient();
-
-@Module({
-  imports: [
-    ClientsModule.register([ducClient.providerOptions])
-  ],
-  controllers: [ApiKeyController],
-  providers: [ApiKeyService, DadosferaLogger],
-  exports: [ApiKeyService]
-})
-export class ApiKeyModule {}

src/modules/api-key/api-key.service.ts

@@ -1,50 +0,0 @@
-import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
-import { RequestUser } from 'src/decorators/user.decorator';
-import { DucClient } from '../duc/client.config';
-import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
-import { lastValueFrom } from 'rxjs';
-import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
-
-@Injectable()
-export class ApiKeyService implements OnModuleInit {
-  private apiKeyService: ApiKeyWriteProtoService;
-
-  constructor(
-    @Inject(DucClient.name) private readonly client: ClientGrpc,
-  ) {}
-
-  onModuleInit() {
-    this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
-  }
-
-  create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
-    const metadata = PackTheMetadata(user);
-
-    return lastValueFrom(this.apiKeyService.CreateApiKey({
-      permissions: createApiKeyDto.permissions
-    }, metadata));
-  }
-
-  async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
-    const metadata = PackTheMetadata(user);
-    console.log(metadata)
-
-    const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
-    return data.api_keys;
-  }
-
-  async remove(id: string, user: RequestUser) {
-    const metadata = PackTheMetadata(user);
-
-    await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
-  }
-
-  async get(key: string) {
-    const metadata = PackTheMetadata({});
-
-    return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
-  }
-}

src/modules/api-key/dto/api-key.dto.ts

@@ -1,39 +0,0 @@
-import { ApiProperty } from '@nestjs/swagger';
-import { IsArray, IsNumber } from 'class-validator';
-
-export class PermissionDto {
-  @ApiProperty({ type: Number })
-  id: number;
-
-  @ApiProperty({ type: String })
-  name: string;
-}
-
-export class ApiKeyBaseResponseDto {
-  @ApiProperty({ type: String, format: 'uuid' })
-  id: string;
-
-  @ApiProperty({ type: String })
-  key_mask: string;
-
-  @ApiProperty({ type: [PermissionDto] })
-  permissions: PermissionDto[];
-
-  @ApiProperty({ type: String, format: 'date-time' })
-  created_at: string;
-
-  @ApiProperty({ type: String })
-  created_by: string;
-}
-
-export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
-  @ApiProperty({ type: String })
-  key: string;
-}
-
-export class CreateApiKeyDto {
-  @ApiProperty({ type: [Number], description: 'Array of permission IDs' })
-  @IsArray()
-  @IsNumber({}, { each: true })
-  permissions: number[];
-} 

src/modules/assign/assign.controller.ts

@@ -1,33 +0,0 @@
-import { Controller, Post, Body, Put, Get} from '@nestjs/common';
-import { AssignService } from './assign.service';
-import { CreateAssignDto } from './dto/create-assign.dto';
-import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
-import { RequestUser, User } from 'src/decorators/user.decorator';
-import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
-
-@Controller('assign')
-@Authenticated()
-export class AssignController {
-  constructor(private readonly assignService: AssignService) {}
-
-  @Put('/public-key')
-  @RequireSomePermission(
-    PERMISSIONS_GROUPS.USERS.permissions.ADMIN
-  )
-  @RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED)
-  create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
-    const metadata = PackTheMetadata(user);
-    return this.assignService.create(createAssignDto, metadata);
-  }
-
-  @Get('/public-key')
-  @RequireSomePermission(
-    PERMISSIONS_GROUPS.USERS.permissions.ADMIN
-  )
-  @RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED)
-  async get(@User() user: RequestUser) {
-    const metadata = PackTheMetadata(user);
-    return await this.assignService.get(metadata);
-  }
-}

src/modules/assign/assign.module.ts

@@ -1,15 +0,0 @@
-import { Module } from '@nestjs/common';
-import { AssignService } from './assign.service';
-import { AssignController } from './assign.controller';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { ClientsModule } from '@nestjs/microservices';
-import { DucClient } from '../duc/client.config';
-
-const client = new DucClient();
-
-@Module({
-  imports: [ClientsModule.register([client.providerOptions])],
-  controllers: [AssignController],
-  providers: [AssignService, DadosferaLogger]
-})
-export class AssignModule {}

src/modules/assign/assign.service.ts

@@ -1,38 +0,0 @@
-import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
-import { CreateAssignDto } from './dto/create-assign.dto';
-import { Metadata } from '@grpc/grpc-js';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { ClientGrpc } from '@nestjs/microservices';
-import { DucClient } from 'src/modules/duc/client.config';
-import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
-
-@Injectable()
-export class AssignService implements OnModuleInit {
-
-  ducService: AssingProtoService;
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.ducService =this.grpcClient.getService<AssingProtoService>(
-      ProtoServices.AssingProtoService,
-    );
-  }
-
-  async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
-    const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
-    return data;
-  }
-
-  async get(metadata: Metadata) {
-    return await lastValueFrom(this.ducService.GetAssignPublicKey(metadata))
-  }
-}

src/modules/assign/dto/create-assign.dto.ts

@@ -1,3 +0,0 @@
-export class CreateAssignDto {
-    publicKey: string;
-}

src/modules/auth/auth.controller.ts

@@ -45,7 +45,7 @@ import {
   AuthSignInRes,
   BulkEditRequest,
 } from './dtos/login';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { AuthGuard } from '@nestjs/passport';
 import { Request } from 'express';
 import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';

src/modules/auth/passport-strategies/google-strategy.ts

@@ -26,7 +26,6 @@ export class GoogleLoginStrategy extends PassportStrategy(
       callbackURL: oauthSecrets['google-login'].redirect_uri,
       scope: ['email', 'profile', 'openid'],
     };
-    console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
     const verify = (
       accessToken: string,
       refreshToken: string,

src/modules/catalog/catalog.controller.ts

@@ -32,7 +32,7 @@ import {
 import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import {
   BatchRemoveRlsRulesRequest,

src/modules/catalog/catalog.module.ts

@@ -3,15 +3,12 @@ import { Module } from '@nestjs/common';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import { CatalogController } from './catalog.controller';
+import { CatalogService } from './catalog.service';
 import { CatalogClientConfiguration } from './catalog-client';
 import { ClientsModule } from '@nestjs/microservices';
 import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { UsersModule } from '../users/users.module';
 import { RolesModule } from '../roles/roles.module';
-import { CustomersModule } from '../customers/customers.module';
-import { ShareModule } from './share/share.module';
-import { CatalogService } from './catalog.service';
-import { MixpanelModule } from '../mixpanel/mixpanel.module';
 
 const client = new CatalogClientConfiguration();
 
@@ -21,8 +18,6 @@ const client = new CatalogClientConfiguration();
     OldPipelineModule,
     UsersModule,
     RolesModule,
-    CustomersModule,
-    ShareModule,
   ],
   controllers: [CatalogController],
   providers: [CatalogService, DadosferaLogger],

src/modules/catalog/catalog.service.ts

@@ -36,6 +36,12 @@ import {
   GetRlsRulesRequest,
   PiiMetadata,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+import { writeFileSync } from 'fs';
+import path from 'path';
+import { HtmlParser } from 'src/utils/FileParser/html-parser';
+import { PiiDto } from './dtos/pii.dto';
+import { CsvParser } from 'src/utils/FileParser/csv-parser';
+import { PDFParser } from 'src/utils/FileParser/pdf-parser';
 import { TypeParser } from 'src/utils/FileParser/parser-types';
 import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
 
@@ -339,7 +345,7 @@ class CatalogService implements OnModuleInit {
     const { documentation } = await lastValueFrom(
       this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
     );
- 
+    console.log(documentation);
     const docs = JSON.parse(documentation);
     return docs;
   }

src/modules/catalog/share/share.controller.ts

@@ -1,90 +0,0 @@
-import {
-  Controller,
-  Get,
-  Inject,
-  Param,
-  Req,
-  UseFilters,
-} from '@nestjs/common';
-import {
-  ApiHeaders,
-  ApiTags,
-} from '@nestjs/swagger';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { RequestUser, User } from 'src/decorators/user.decorator';
-import {
-  IColumnsMetadataResponse,
-  IDocsResponse,
-  IPreviewResponse,
-
-} from '../dtos';
-import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { Language } from 'src/decorators/language.decorator';
-import { LanguageEnum } from 'src/utils/languages.enum';
-import { ShareService } from './share.service';
-import { Request } from 'express';
-
-@ApiTags('Catalog')
-@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
-@Controller('catalog/data-asset/share')
-@UseFilters(new GrpcToHttpExceptionFilter())
-export class ShareController {
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private catalogShareService: ShareService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  @Get('/:id')
-  async getShareDataAsset(
-    @Param('id') id: string,
-    @User() user: RequestUser,
-    @Req() request: Request
-  ) {
-    this.logger.info(`GET //:id`);
-    return await this.catalogShareService.getOneDataAssetPublic(id, user, request);
-  }
-
-  @Get('/:id/columns-metadata')
-  async getShareDataAssetColumnsMetadata(
-    @Language() language: LanguageEnum,
-    @Param('id') id: string,
-    @User() user: RequestUser
-  ): Promise<IColumnsMetadataResponse> {
-    this.logger.info(`GET /:id/columns-metadata`);
-
-    const columns_metadata =
-      await this.catalogShareService.getDatasetColumnsMetadata(id, user);
-
-
-    return { columns_metadata };
-  }
-
-  @Get('/:id/preview')
-  async getShareDataAssetPreview(
-    @Language() language: LanguageEnum,
-    @Param('id') id: string,
-    @User() user: RequestUser
-  ): Promise<IPreviewResponse> {
-    this.logger.info(`GET /:id/preview`);
-    const preview = await this.catalogShareService.getDatasetPreview(id, user);
-
-    return { preview };
-  }
-
-  @Get('/:id/docs')
-  async getShareDataAssetDocs(
-    @Language() language: LanguageEnum,
-    @Param('id') id: string,
-    @User() user: RequestUser
-  ): Promise<IDocsResponse> {
-    this.logger.info(`GET /:id/docs`);
-    const docs = await this.catalogShareService.getDataDocs(id, user);
-
-    return { docs };
-  }
-
-}

src/modules/catalog/share/share.module.ts

@@ -1,28 +0,0 @@
-import { Module } from "@nestjs/common";
-import { CatalogClientConfiguration } from "../catalog-client";
-import { ClientsModule } from "@nestjs/microservices";
-import { RolesModule } from "src/modules/roles/roles.module";
-import { UsersModule } from "src/modules/users/users.module";
-import { CustomersModule } from "src/modules/customers/customers.module";
-import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
-import { ShareController } from "./share.controller";
-import DadosferaLogger from "@dadosfera/dadosfera-logs";
-import { ShareService } from "./share.service";
-import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
-
-const client = new CatalogClientConfiguration();
-
-@Module({
-  imports: [
-    ClientsModule.register([client.providerOptions]),
-    UsersModule,
-    RolesModule,
-    CustomersModule,
-    ShareMetadataModule,
-    MixpanelModule
-  ],
-  controllers: [ShareController],
-  providers: [ShareService, DadosferaLogger],
-  exports: [ShareModule],
-})
-export class ShareModule {}

src/modules/catalog/share/share.service.ts

@@ -1,247 +0,0 @@
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import {
-  ProtoServices,
-  ReadService,
-} from '@dadosfera/protospack-v2/dist/lib/Catalog';
-import {
-  ForbiddenException,
-  Inject,
-  NotFoundException,
-  OnModuleInit,
-} from '@nestjs/common';
-import { CatalogClientConfiguration } from '../catalog-client';
-import { ClientGrpc } from '@nestjs/microservices';
-import { UsersService } from 'src/modules/users/users.service';
-import { RolesService } from 'src/modules/roles/roles.service';
-import { RequestUser } from 'src/decorators/user.decorator';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { Metadata } from '@grpc/grpc-js';
-import { lastValueFrom } from 'rxjs';
-import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
-import { isJWT } from 'class-validator';
-import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
-import { Request } from 'express';
-
-export class ShareService implements OnModuleInit {
-  catalogReadService: ReadService.CatalogReadServices;
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject(CatalogClientConfiguration.name)
-    private readonly grpcClient: ClientGrpc,
-    private readonly userService: UsersService,
-    private readonly roleService: RolesService,
-    private readonly shareMetadataService: ShareMetadataService,
-    private readonly mixpanelService: MixpanelService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.catalogReadService =
-      this.grpcClient.getService<ReadService.CatalogReadServices>(
-        ProtoServices.CatalogReadServices,
-      );
-  }
-
-  async getDatasetColumnsMetadata(id: string, user: RequestUser) {
-    const shareMetadata = await this.getShareMetadata(id, user);
-    const metadata = PackTheMetadata({
-      customer_id: shareMetadata.customerId,
-      customer_name: shareMetadata.customerName,
-    });
-    const { columns_metadata } = await lastValueFrom(
-      this.catalogReadService.GetDatasetColumnsMetadata(
-        { id: shareMetadata.assetId, type: undefined },
-        metadata,
-      ),
-    );
-    const result = JSON.parse(columns_metadata);
-    return result;
-  }
-
-  async getDatasetPreview(id: string, user: RequestUser) {
-    const shareMetadata = await this.getShareMetadata(id, user);
-    const metadata = PackTheMetadata({
-      customer_id: shareMetadata.customerId,
-      customer_name: shareMetadata.customerName,
-    });
-    const { preview } = await lastValueFrom(
-      this.catalogReadService.GetDatasetPreview(
-        { id: shareMetadata.assetId, type: undefined },
-        metadata,
-      ),
-    );
-    const result = JSON.parse(preview);
-    return result;
-  }
-
-  async getOneDataAssetPublic(id: string, user: RequestUser, request: Request) {
-    this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
-      id,
-      userId: user?.user_id,
-      customerId: user?.customer_id
-    }))
-    try {
-      const shareMetadata = await this.getShareMetadata(id, user);
-
-      const mixpanelTracker = {
-        asset: shareMetadata.assetId,
-        type: isJWT(id) ? 'assigned' : shareMetadata.type,
-        customer: shareMetadata.customerName
-      }
-
-      if (user) {
-        await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
-      } else {
-        await this.mixpanelService.trackShare(request, mixpanelTracker);
-      }
-
-      this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
-      const metadata = PackTheMetadata({
-        customer_id: shareMetadata.customerId,
-        customer_name: shareMetadata.customerName,
-      });
-  
-      const { data_asset } = await this.getOneDataAsset({
-        customer_id: shareMetadata.customerId,
-        id: shareMetadata.assetId,
-        metadata,
-      });
-      this.logger.info('found asset: ' + JSON.stringify(data_asset));
-      delete data_asset.p_roles;
-      delete data_asset.p_users;
-      data_asset.share_type = 'public';
-      if (data_asset.share_type !== 'public') throw new NotFoundException();
-  
-      return { data_asset };
-    } catch (error) {
-      this.logger.error(error);
-      throw error;
-    }
-
-  }
-
-  private async getOneDataAsset(data: {
-    id: string;
-    customer_id: string;
-    metadata: Metadata;
-  }) {
-    const { customer_id, id, metadata } = data;
-    const { data_asset } = await lastValueFrom(
-      this.catalogReadService.GetOneDataAsset(
-        { id, type: undefined },
-        metadata,
-      ),
-    );
-    let asset = JSON.parse(data_asset);
-    asset = {
-      ...asset,
-      p_roles: asset.roles,
-      p_users: asset.users,
-    };
-    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
-
-    return { data_asset: asset[0] };
-  }
-
-  async getDataDocs(id: string, user: RequestUser) {
-    const shareMetadata = await this.getShareMetadata(id, user);
-    const metadata = PackTheMetadata({
-      customer_id: shareMetadata.customerId,
-      customer_name: shareMetadata.customerName,
-    });
-
-    const { documentation } = await lastValueFrom(
-      this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
-    );
-    console.log(documentation);
-    const docs = JSON.parse(documentation);
-    return docs;
-  }
-
-  private async getAssetsUsersAndRoles(
-    data_assets: Array<any>,
-    customer_id: string,
-  ) {
-    const { users: customer_users } =
-      await this.userService.findAllUsersByCustomerId(customer_id);
-    const { roles: customer_roles } = await this.roleService.roleSearch(
-      {},
-      { customer_id },
-    );
-    return data_assets.map((data_asset) => {
-      const owner = customer_users.find(
-        (u) => u.id === data_asset.owner,
-      )?.username;
-
-      const roles = [];
-      const users = [];
-      for (const role_id of data_asset.roles) {
-        const role = customer_roles.find((r) => r.id === role_id);
-        if (role) roles.push({ id: role.id, name: role.name });
-      }
-      for (const user_id of data_asset.users) {
-        const user = customer_users.find((r) => r.id === user_id);
-        if (user) users.push({ id: user.id, username: user.username });
-      }
-      return {
-        ...data_asset,
-        roles,
-        users,
-        owner,
-      } as typeof data_asset;
-    });
-  }
-
-  // private async validateShareAssign(token: string) {
-  //   const tokenDecoded = jwt.decode(token, {
-  //     complete: true,
-  //   });
-
-  //   jwt.verify(token, this.pemValue, {
-  //     algorithms: ['RS256'],
-  //   });
-
-  //   const shareId = (tokenDecoded.payload as JwtPayload).sub;
-  //   const metadata = PackTheMetadata({});
-  //   return await this.shareMetadataService.get(shareId, metadata);
-  // }
-
-  private async getShareMetadata(id: string, user: RequestUser) {
-    const metadata = PackTheMetadata({});
-    this.logger.info('GET share metadata')
-    const info = await this.shareMetadataService.get(id, metadata);
-    if (isJWT(id) && info ){
-      return info;
-    }
-
-    if (info.type === 'private') {
-      if (!user) {
-        throw new ForbiddenException(
-          'You do not have permission to access this data asset.',
-        );
-      }
-
-      const is_data_manager = user.permissions.includes(
-        PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
-      );
-
-      const is_get = user.permissions.includes(
-        PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
-      );
-
-      if (is_data_manager || is_get) {
-        return info;
-      }
-
-      throw new ForbiddenException(
-        'You do not have permission to access this data asset.',
-      );
-    }
-    return info;
-  }
-}

src/modules/connection-test/connection-test.service.ts

@@ -20,7 +20,7 @@ import {
   DatabaseConnectionPropertiesDto,
 } from '../connection/dtos/connection';
 import { RequestUser } from 'src/decorators/user.decorator';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @Injectable()
 export class ConnectionTestService {

src/modules/connection/connection.controller.ts

@@ -28,7 +28,7 @@ import {
   UpdateConnectionDto,
 } from './dtos/connection';
 import { CreateConnectionDto } from './dtos/connection';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { Language } from 'src/decorators/language.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';

src/modules/customers/customers.controller.ts

@@ -25,7 +25,7 @@ import { CustomersService } from './customers.service';
 import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import type { StringValue } from 'ms';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { EnforceMfa } from './dtos/enforce-mfa';
 
 @ApiTags('Customers')

src/modules/customers/customers.service.ts

@@ -59,14 +59,6 @@ export class CustomersService implements OnModuleInit {
       );
   }
 
-  async getCustomer(customerId: string) {
-    return await lastValueFrom(
-      this.customerService.CustomerFindOneById({
-        id: customerId
-      })
-    )
-  }
-
   async getLinks(customerId: string) {
     try {
       const result = await lastValueFrom(
@@ -149,7 +141,6 @@ export class CustomersService implements OnModuleInit {
     // const decoded = jwt.decode(jwt_token, { complete: true });
     return jwt_token;
   }
-
   async getMonitoringDashboardUrl(metadata: Metadata) {
     logger.info('CustomersService - getMonitoringDashboardUrl');
 
@@ -167,7 +158,6 @@ export class CustomersService implements OnModuleInit {
 
     return res;
   }
-
   async getLogsDashboardUrl(metadata: Metadata) {
     logger.info('CustomersService - getMixPanelLogsDashboardUrl');
   

src/modules/identity-provider/identity-provider.controller.ts

@@ -17,7 +17,7 @@ import { IdentityProviderService } from './identity-provider.service';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { Language } from 'src/decorators/language.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { ApiOkResponse } from '@nestjs/swagger';
 import {
   CreateIdentityProvider,

src/modules/mixpanel/mixpanel.controller.ts

@@ -3,13 +3,14 @@ import { init } from 'mixpanel';
 import { Authenticated } from 'src/decorators/authentication.decorator';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 import { RequestUser, User } from 'src/decorators/user.decorator';
-import { MixpanelService } from './mixpanel.service';
 
 @ApiInternalOnlyController()
+@Authenticated()
 @Controller('trackEvent')
 export class MixpanelController {
   constructor(
-    private mixpanelService: MixpanelService
+    @Inject('MIXPANEL_TOKEN')
+    private readonly mixpanelToken: string,
   ) {}
   @Post(':id')
   async trackEvent(
@@ -19,11 +20,82 @@ export class MixpanelController {
     @Req() request
   ) {
     delete body.info;
+    const mixpanel = init(this.mixpanelToken);
 
-    await this.mixpanelService.track(id, user, request, body)
+    const separator = user.username.includes('-') ? '-' : '.';
+    const removeValues = [
+      '.dadosferatech.dadosfera',
+      '.demo.dadosfera',
+      '.dadosferademo',
+      '.dadosferarh.dadosfera',
+      '.dadosferatech.dadosfera2',
+      '.dadosferatech.dadosfera',
+      '.dadosfera.fin',
+      '.dadosferafin.dadosfera',
+      '.praxio.dadosfera',
+      '.dadosfera.tech',
+      '.treinamentos@dadosfera.ai',
+      '.dadosfera2',
+      '.treinamentosfera',
+      '.dadosfera',
+    ];
+
+    let username = user.username;
+
+    removeValues.forEach((value) => {
+      username = username.replace(value, '');
+    });
+
+    username = username.split('@')?.[0];
+    username = username.split('+')?.[0];
+
+    let firstName = username
+      .substring(0, username.indexOf(separator))
+      .replace('dadosfera', '');
+    let lastName = username
+      .substring(username.lastIndexOf(separator) + 1)
+      .replace('dadosfera', '');
+
+    if (!firstName) {
+      firstName = lastName;
+      lastName = '';
+    }
+
+    firstName = this.capitalize(firstName);
+    lastName = this.capitalize(lastName);
+
+    await mixpanel.people.set(user.username, {
+      $first_name: firstName,
+      $last_name: lastName,
+      $name: this.getFullName(firstName, lastName),
+      $email: user.username.includes('@')
+        ? user.username
+        : user.username + '@dadosfera.ai',
+      customer_name: user.customer_name
+    });
+
+    await mixpanel.track(id, {
+      distinct_id: user.username,
+      customer: user.customer_name,
+      env: process.env.ENV,
+      $ip: request.ip,
+      $os: request.headers['sec-ch-ua-platform'] || '',
+      $browser: request.headers['user-agent'],
+      ...body,
+    });
 
     return { id, body, user: user.username };
   }
 
+  capitalize(sentence: string): string {
+    if (!sentence) {
+      return '';
+    }
 
+    return sentence[0].toUpperCase() + sentence.substring(1);
+  }
+
+  getFullName(firstName: string, lastName: string) {
+    return `${firstName}${lastName ? ' ' + lastName : ''}`;
+  }
 }

src/modules/mixpanel/mixpanel.module.ts

@@ -1,8 +1,6 @@
 import { Module } from '@nestjs/common';
 import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
 import { MixpanelController } from './mixpanel.controller';
-import { MixpanelService } from './mixpanel.service';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 @Module({
   controllers: [MixpanelController],
@@ -10,12 +8,9 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs';
     {
       provide: 'MIXPANEL_TOKEN',
       useValue: getSecretFromSecretsManager(
-        `prd/root/mixpanel_token`,
+        `${process.env.ENV}/root/mixpanel_token`,
       ),
     },
-    MixpanelService,
-    DadosferaLogger
   ],
-  exports: [MixpanelService]
 })
 export class MixpanelModule {}

src/modules/mixpanel/mixpanel.service.ts

@@ -1,118 +0,0 @@
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { Inject } from '@nestjs/common';
-import { Request } from 'express';
-import mixpanel, { init } from 'mixpanel';
-import { RequestUser } from 'src/decorators/user.decorator';
-
-export class MixpanelService {
-    logger: DadosferaLogger;
-
-  constructor(
-    @Inject('MIXPANEL_TOKEN')
-    private readonly mixpanelToken: string,
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  async track(eventName: string, user: RequestUser, request: Request, body: any) {
-    this.logger.info("track: " + JSON.stringify({
-        eventName,
-        ...body
-    }))
-    const mixpanel = init(this.mixpanelToken);
-    await this.setPeople(user, mixpanel);
-
-    await mixpanel.track(eventName, {
-      distinct_id: user.username,
-      customer: user.customer_name,
-      env: process.env.ENV,
-      $ip: request.ip,
-      $os: request.headers['sec-ch-ua-platform'] || '',
-      $browser: request.headers['user-agent'],
-      ...body,
-    });
-    return;
-  }
-
-  async trackShare(request: Request,  body: any) {
-    this.logger.info("trackShare: " + JSON.stringify(body))
-    const mixpanel = init(this.mixpanelToken);
-
-    await mixpanel.track("share_page", {
-      env: process.env.ENV,
-      $ip: request.ip,
-      $os: request.headers['sec-ch-ua-platform'] || '',
-      $browser: request.headers['user-agent'],
-      ...body,
-    });
-    return;
-  }
-
-  private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
-    const separator = user.username.includes('-') ? '-' : '.';
-    const removeValues = [
-      '.dadosferatech.dadosfera',
-      '.demo.dadosfera',
-      '.dadosferademo',
-      '.dadosferarh.dadosfera',
-      '.dadosferatech.dadosfera2',
-      '.dadosferatech.dadosfera',
-      '.dadosfera.fin',
-      '.dadosferafin.dadosfera',
-      '.praxio.dadosfera',
-      '.dadosfera.tech',
-      '.treinamentos@dadosfera.ai',
-      '.dadosfera2',
-      '.treinamentosfera',
-      '.dadosfera',
-    ];
-
-    let username = user.username;
-
-    removeValues.forEach((value) => {
-      username = username.replace(value, '');
-    });
-
-    username = username.split('@')?.[0];
-    username = username.split('+')?.[0];
-
-    let firstName = username
-      .substring(0, username.indexOf(separator))
-      .replace('dadosfera', '');
-    let lastName = username
-      .substring(username.lastIndexOf(separator) + 1)
-      .replace('dadosfera', '');
-
-    if (!firstName) {
-      firstName = lastName;
-      lastName = '';
-    }
-
-    firstName = this.capitalize(firstName);
-    lastName = this.capitalize(lastName);
-
-    await mixpanel.people.set(user.username, {
-      $first_name: firstName,
-      $last_name: lastName,
-      $name: this.getFullName(firstName, lastName),
-      $email: user.username.includes('@')
-        ? user.username
-        : user.username + '@dadosfera.ai',
-      customer_name: user.customer_name,
-    });
-  }
-
-  private capitalize(sentence: string): string {
-    if (!sentence) {
-      return '';
-    }
-
-    return sentence[0].toUpperCase() + sentence.substring(1);
-  }
-
-  private getFullName(firstName: string, lastName: string) {
-    return `${firstName}${lastName ? ' ' + lastName : ''}`;
-  }
-}

src/modules/network-policy/network-policy.controller.ts

@@ -8,7 +8,7 @@ import {
 import { NetworkPoliciesDTO } from './dto/network-policy.dto';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { NetworkPolicyService } from './network-policy.service';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @Controller('network-policy')
 export class NetworkPolicyController {

src/modules/oauth/oauth.controller.ts

@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
 import { ConnectionClientService } from '../connection/client.service';
 import jwt from 'jsonwebtoken';
 import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 @ApiTags('oauth')
 @Controller('oauth')

src/modules/open-data/open-data.controller.ts

@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
 import { OpenDataService } from './open-data.service';
 import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
 import { Metadata } from '@grpc/grpc-js';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { request } from 'http';
 import { Request } from 'express';
 

src/modules/pipelinesV2/pipelines.controller.ts

@@ -32,7 +32,7 @@ import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { RequestUser, User } from 'src/decorators/user.decorator';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import {

src/modules/share-metadata/dto/create-share-metadata.dto.ts

@@ -1,10 +0,0 @@
-export class CreateShareMetadataDto {
-    assetId: string;
-    proposedId: string;
-    type: string;
-}
-
-export class UpdateShareMetadataDto {
-    shareId: string;
-    type: string;
-}

src/modules/share-metadata/share-metadata.controller.ts

@@ -1,31 +0,0 @@
-import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common';
-import { ShareMetadataService } from './share-metadata.service';
-import { Authenticated } from 'src/decorators/authentication.decorator';
-import { RequestUser, User } from 'src/decorators/user.decorator';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
-import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
-
-@Controller('share')
-@Authenticated()
-export class ShareMetadataController {
-  constructor(private readonly shareMetadataService: ShareMetadataService) {}
-
-  @Put('/')
-  findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) {
-    const metadata = PackTheMetadata(user);
-    return this.shareMetadataService.create(data, metadata);
-  }
-
-  @Patch('/')
-  updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) {
-    const metadata = PackTheMetadata(user);
-    return this.shareMetadataService.updateShare(data, metadata);
-  }
-
-
-  @Get('/:id')
-  get(@Param('id') id: string, @User() user: RequestUser) {
-    const metadata = PackTheMetadata(user);
-    return this.shareMetadataService.get(id, metadata);
-  }
-}

src/modules/share-metadata/share-metadata.module.ts

@@ -1,16 +0,0 @@
-import { Module } from '@nestjs/common';
-import { ShareMetadataService } from './share-metadata.service';
-import { ShareMetadataController } from './share-metadata.controller';
-import { DucClient } from '../duc/client.config';
-import { ClientsModule } from '@nestjs/microservices';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-
-const client = new DucClient();
-
-@Module({
-  imports: [ClientsModule.register([client.providerOptions])],
-  controllers: [ShareMetadataController],
-  providers: [ShareMetadataService, DadosferaLogger],
-  exports: [ShareMetadataService]
-})
-export class ShareMetadataModule {}

src/modules/share-metadata/share-metadata.service.ts

@@ -1,62 +0,0 @@
-import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
-import { Metadata } from '@grpc/grpc-js';
-import { ClientGrpc } from '@nestjs/microservices';
-import { DucClient } from '../duc/client.config';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
-import { lastValueFrom } from 'rxjs';
-import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
-
-@Injectable()
-export class ShareMetadataService implements OnModuleInit {
-  ducService: ShareMetadataProtoService;
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.ducService =this.grpcClient.getService<ShareMetadataProtoService>(
-      ProtoServices.ShareMetadataProtoService,
-    );
-  }
-
-  create(createShareDto: CreateShareMetadataDto, metadata: Metadata) {
-    return lastValueFrom(
-      this.ducService.FindOrCreateShareMetadata(
-        {
-          assetId: createShareDto.assetId,
-          proposedId: createShareDto.proposedId,
-          type: createShareDto.type
-        },
-        metadata,
-      ),
-    );
-  }
-
-  updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) {
-    return lastValueFrom(
-      this.ducService.ChangeShareMetadataType(
-        updateShare,
-        metadata,
-      ),
-    );
-  }
-
-  get(id: string, metadata: Metadata) {
-    return lastValueFrom(
-      this.ducService.GetShareMetadata(
-        {
-          id,
-        },
-        metadata,
-      ),
-    );
-  }
-}

src/modules/users/users.controller.spec.ts

@@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module';
 import { PermissionsModule } from '../permissions/permissions.module';
 
 // const client = new DucClient();
+jest.mock('puppeteer', () => ({
+  launch: jest.fn().mockResolvedValue({
+    newPage: jest.fn().mockResolvedValue({
+      goto: jest.fn(),
+      evaluate: jest.fn(),
+      close: jest.fn()
+    }),
+    close: jest.fn()
+  })
+}));
+
 
 const logger = {
   info: (...args) => args,

src/modules/users/users.controller.ts

@@ -31,7 +31,7 @@ import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
 import ErrorCodes from 'src/utils/errorCodes';
 import { LanguageEnum } from 'src/utils/languages.enum';

src/modules/users/users.service.spec.ts

@@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module';
 
 // const client = new DucClient();
 
+jest.mock('puppeteer', () => ({
+  launch: jest.fn().mockResolvedValue({
+    newPage: jest.fn().mockResolvedValue({
+      goto: jest.fn(),
+      evaluate: jest.fn(),
+      close: jest.fn()
+    }),
+    close: jest.fn()
+  })
+}));
+
 const logger = {
   info: (...args) => args,
   error: (...args) => args,

src/utils/OauthSecrets.ts

@@ -24,21 +24,15 @@ export async function getOauthSecrets() {
   const path = process.env.SM_OAUTH_PATH;
   const secretsManagerClient = new SecretsManagerClient({});
   for (const key in secrets) {
-    console.log(`Fetching secret for ${key} from path ${path}/${key}`);
     const getSecretComand = new GetSecretValueCommand({
       SecretId: `${path}/${key}`,
     });
     const res = await secretsManagerClient
       .send(getSecretComand)
-      .catch((err) => {
+      .catch(() => null);
-        console.error(`Error fetching secret for ${key}:`, err);
-        return null;
-      });
     if (res) {
       secrets[key] = JSON.parse(res.SecretString);
-      console.log(JSON.parse(res.SecretString).client_id)
     } else {
-      console.log("No secret found for", key);
       secrets[key] = {
         client_id: 'id',
         client_secret: 'id',