DOCS: Map maestro-pifactory integration points for migration

Add comprehensive documentation mapping all integration points between
maestro and pi-factory services to facilitate removing the dependency.

Includes:
- 3 gRPC client configurations using PIFACTORY_URL
- 36+ gRPC method calls across Catalog, PipelineV2, and legacy Pipeline services
- 44 REST endpoints that proxy to pi-factory
- Configuration files and environment variables
- Proto package dependencies (protospack and protospack-v2)
- Migration strategy options and recommendations
- Comparison with in-factory migration scope

.eslintrc.js

@@ -8,7 +8,7 @@ module.exports = {
   extends: [
     'eslint:recommended',
     'plugin:@typescript-eslint/recommended',
-    'plugin:prettier/recommended',
+    'prettier',
   ],
   root: true,
   env: {

.github/workflows/deploy-manually.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - beta
   workflow_dispatch:
     inputs:
       environment:
@@ -11,7 +12,6 @@ on:
         type: choice
         options:
           - stg
-          - stg2
           - prd
 
 jobs:
@@ -29,6 +29,8 @@ jobs:
             echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
           elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
             echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
           fi
         id: extract_environment
 
@@ -39,7 +41,7 @@ jobs:
       new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - if: github.event_name != 'workflow_dispatch'
         name: Semantic Release
@@ -50,42 +52,40 @@ jobs:
             conventional-changelog-eslint@4.0.0
           branches: |
             [
-              'main'
+              'main',
+              {
+                name: 'alpha',
+                prerelease: true
+              },
+              {
+                name: 'beta',
+                prerelease: true
+              }
             ]
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  deploy-info:
-    if: ${{ github.event_name == 'workflow_dispatch'}}
+  build_ecr_image:
+    if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
     needs: [extract_environment, semantic_release]
-    runs-on: ubuntu-latest
+    runs-on:
+      [self-hosted, "prd"]
 
     steps:
-      - name: Create summary
+      - name: Printing stats
         env:
           EVENT: ${{ github.event_name }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ENV: ${{ needs.extract_environment.outputs.environment }}
-        run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY
+        run: echo ${GITHUB_REF#refs/heads/}
 
-  deploy:
-    if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
-    needs: [extract_environment, semantic_release]
-    runs-on:
-      [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
-
-    steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Update Pip
         run: |
           python3 -m pip install --upgrade pip
 
-      - name: Install Docker Compose
-        run: |
-          python3 -m pip install docker-compose --upgrade
-
       - name: Install AWS CLI
         run: |
           python3 -m pip install awscli --upgrade
@@ -95,14 +95,14 @@ jobs:
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v4
         id: aws
         with:
           aws-region: us-east-1
 
       - name: Login to AWS ECR
         id: login_ecr
-        uses: aws-actions/amazon-ecr-login@v1
+        uses: aws-actions/amazon-ecr-login@v2
 
       - name: Build, Tag, and Push Image to AWS ECR
         env:
@@ -110,113 +110,43 @@ jobs:
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
-          docker-compose -f build.docker-compose.yml build
-          docker-compose -f build.docker-compose.yml push
+          docker compose -f build.docker-compose.yml build
+          docker compose -f build.docker-compose.yml push
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: dadosfera
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-    
-      - name: Build, Tag, and Push Image to Dockerhub
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
-          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
-        run: |
-            docker-compose -f build.docker-compose.dockerhub.yml build
-            docker-compose -f build.docker-compose.dockerhub.yml push
-  
+      # - name: Create ZIP file to Deploy AWS Beanstalk
+      #   env:
+      #     ENV: ${{ needs.extract_environment.outputs.environment }}
+      #     IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+      #     ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+      #     NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
+      #   run: |
+      #     sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
+      #     sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+      #     sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
+      #     sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
+      #     zip deploy.zip docker-compose.yml  -r .ebextensions
 
-      - name: Create ZIP file to Deploy AWS Beanstalk
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
-          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
-          NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
-        run: |
-          sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
-          sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
-          sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
-          sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
-          zip deploy.zip docker-compose.yml  -r .ebextensions
-
-      - name: Deploy AWS Beanstalk
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-          AWS_REGION: us-east-1
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          eb use $APP_NAME-$ENV
-          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
-          eb deploy
+      # - name: Deploy AWS Beanstalk
+      #   env:
+      #     ENV: ${{ needs.extract_environment.outputs.environment }}
+      #     AWS_REGION: us-east-1
+      #     APP_NAME: ${{ github.event.repository.name }}
+      #   run: |
+      #     eb use $APP_NAME-$ENV
+      #     echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
+      #     eb deploy
 
       - name: Remove Docker's Trash
         if: always()
         run: |
           docker system prune --volumes -a -f
           docker system df
-  api_docs:
-    needs: [extract_environment, semantic_release, deploy]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
 
-      - name: Extract Docs BlockId and PageId
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-          STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
-          STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
-          STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
-          STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
-          PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
-          PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
-        shell: bash
-        run: |
-          if [ $ENV == "stg2" ]; then
-            echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
-            echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
-          elif [ $ENV == "stg" ]; then
-            echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
-            echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
-          elif [ $ENV == "prd" ]; then
-            echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
-            echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
-          fi
-        id: extract_docs_info
-
-      - name: Generate API docs
-        env:
-          DOCS_URL: ${{ secrets.DOCS_URL }}
-          DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
-          DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
-          DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
-          EVENT: ${{ github.event_name }}
-          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
-        run: |
-          if [ ${EVENT} != "workflow_dispatch" ]; then
-            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
-          fi
-          sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
-          sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
-          sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
-          curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
-
-      - name: Generate External API docs
-        env:
-          DOCS_URL: ${{ secrets.DOCS_URL }}
-          DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
-          DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}}
-          DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}}
-          EVENT: ${{ github.event_name }}
-          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
-        run: |
-          if [ ${EVENT} != "workflow_dispatch" ]; then
-            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json
-          fi
-          sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json
-          sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json
-          sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json
-          curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL
+  k8s-deploy:
+    needs: [extract_environment, semantic_release, build_ecr_image]
+    uses: ./.github/workflows/k8s-deploy.yml
+    with:
+      cloud: 'oracle'
+      environment: ${{ needs.extract_environment.outputs.environment }}
+      image: ${{ needs.semantic_release.outputs.new_release_version }}
+    secrets: inherit

.github/workflows/k8s-deploy.yml

@@ -0,0 +1,137 @@
+name : K8s deploy
+
+on:
+  workflow_call:
+    inputs:
+      cloud:
+        description: "Cloud provider for the deployment"
+        required: true
+        default: "azure"
+        type: string
+      environment:
+        description: "Deployment environment"
+        required: true
+        default: "prd"
+        type: string
+      image:
+        description: "Image Tag"
+        required: true
+        type: string
+
+jobs:
+  azure:
+    if: inputs.cloud == 'azure'
+    runs-on: [self-hosted, "prd-azure"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install Azure ClI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - uses: azure/login@v2
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ inputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+      
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+      
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          sudo mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+      
+      - name: Run Helmfile Apply
+        env:
+          ENV: ${{ inputs.environment }}
+          IMAGE_TAG: ${{ inputs.image }}
+        run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+
+  oracle:
+    if: inputs.cloud == 'oracle'
+    runs-on: [self-hosted, "prd-oracle"]
+    env:
+      HOME: /home/runner
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install OCI CLI
+        env:
+          HOME: /home/runner
+        run: |
+          bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+      - name: Configure OCI CLI
+        run: |
+          mkdir -p ~/.oci || true
+          echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
+          echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem 
+          chmod 600 ~/.oci/oci_api_key.pem
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          sudo mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Authenticate with OKE cluster
+        env:
+          ENV: ${{ inputs.environment }}
+          STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
+          PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
+        run: |
+          if [ "$ENV" = "stg" ]; then
+            CLUSTER_ID=$STG_CLUSTER_ID
+          elif [ "$ENV" = "prd" ]; then
+            CLUSTER_ID=$PRD_CLUSTER_ID
+          else
+            echo "Unknown environment: $ENV"
+            exit 1
+          fi
+          
+          oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
+
+      - name: Run Helmfile Apply
+        env:
+          ENV: ${{ inputs.environment }}
+          IMAGE_TAG: ${{ inputs.image }}
+        run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG

.github/workflows/test.yml

@@ -6,23 +6,18 @@ on:
 
 jobs:
   test:
-    runs-on: self-hosted
+    runs-on: [self-hosted, prd]
+    env:
+      APP_NAME: ${{ github.event.repository.name }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build
-        env:
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          docker build -t $APP_NAME --target test .
+        run: docker build -t ${APP_NAME}_teste --target test .
 
       - name: Run Test
-        env:
-          ENV: test
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          docker run --rm --entrypoint="npm" $APP_NAME run test
+        run: docker run ${APP_NAME}_teste
 
       - name: Remove Docker's Trash
         if: always()

.github/workflows/validate-k8s.yml

@@ -0,0 +1,102 @@
+name: Validate K8S Modifications
+
+on:
+  pull_request:
+    branches:
+      - main
+      - beta
+
+jobs:
+  extract_environment:
+    runs-on: ubuntu-22.04
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
+          fi
+        id: extract_environment
+
+  helmfile-check:
+    env:
+      HOME: /home/runner
+    needs: [extract_environment]
+    environment: ${{ needs.extract_environment.outputs.environment }}
+    runs-on: [self-hosted, "prd-oracle"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Determine DNS_HOST based on environment
+        id: set_dns
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: |
+          if [ "$ENV" = "prd" ]; then
+            echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
+          elif [ "$ENV" = "stg" ]; then
+            echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
+          fi
+  
+      - name: Install OCI CLI
+        run: |
+          bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+      - name: Configure OCI CLI
+        run: |
+          mkdir -p ~/.oci || true
+          echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
+          echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
+          chmod 600 ~/.oci/oci_api_key.pem
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          sudo mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Authenticate with OKE cluster
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
+          PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
+        run: |
+          if [ "$ENV" = "stg" ]; then
+            CLUSTER_ID=$STG_CLUSTER_ID
+          elif [ "$ENV" = "prd" ]; then
+            CLUSTER_ID=$PRD_CLUSTER_ID
+          else
+            echo "Unknown environment: $ENV"
+            exit 1
+          fi
+          
+          oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT 
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+
+      - name: Run Helmfile Diff
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: helmfile -f deploy/helmfiles/${ENV}.yaml diff

.nvmrc

@@ -1 +1 @@
-18.10.0
+18.17

.vscode/launch.json

@@ -12,6 +12,7 @@
         "start:debug"
       ],
       "runtimeExecutable": "npm",
+      "runtimeVersion": "18.17",
       "skipFiles": [
         "<node_internals>/**"
       ],

Dockerfile

@@ -1,21 +1,66 @@
-# install all dependencies
-FROM node:18.10-alpine as packages
+FROM node:18.17-alpine AS base_image
+
+FROM base_image AS build_base
 WORKDIR /app
-COPY package.json package-lock.json ./
-RUN apk add --no-cache aws-cli \
- && aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
- && npm install
+RUN apk update
+# needed packages to build dependencies from source
+RUN apk add --no-cache \
+      aws-cli \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont
+COPY package*.json ./
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
+
+# run aws cli without mounting secret, because CI already has AWS credentials
+FROM build_base AS ci_image
+RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+RUN npm ci
+COPY . .
+
 
 # unit test specific build
-FROM node:18.10-alpine as test
-WORKDIR /app
-COPY --from=packages /app/node_modules ./node_modules
-COPY . ./
+FROM ci_image AS test
+ENV DUC_URL=0.0.0.0:50051
+ENTRYPOINT ["npm", "run", "test"]
 
-FROM node:18.10-alpine
-WORKDIR /app
-COPY . /app/
-COPY --from=packages /app/node_modules ./node_modules
+
+# dev build
+FROM build_base AS dev
+RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
+  aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+# flag --build-from-source is required to force-build sqlite3
+RUN npm ci
+COPY . .
+ENTRYPOINT npm run start:dev
+
+
+FROM ci_image AS prod_build
 RUN npm run build
-EXPOSE 3333
-ENTRYPOINT npm run start
+
+
+FROM base_image
+WORKDIR /app
+COPY --from=prod_build /app/dist ./dist
+COPY --from=prod_build /app/node_modules ./node_modules
+COPY --from=prod_build /app/package*.json ./
+RUN apk update
+# needed packages to build dependencies from source
+RUN apk add --no-cache \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+  
+ENTRYPOINT npm run start:prod

MIGRATION_MAP_INFACTORY.md

@@ -0,0 +1,482 @@
+# Maestro ↔ In-Factory Migration Map
+
+This document maps all integration points between Maestro and In-Factory that need to be addressed to remove the dependency.
+
+## Architecture Overview
+
+```mermaid
+graph TD;
+  Frontend<-->Maestro;
+  Maestro<-->DUC;
+  Maestro<-->pi-factory;
+  Maestro<-->in-factory;
+```
+
+Maestro connects to **3 external microservices**:
+- **DUC** (`DUC_URL`) - User management, authentication, permissions
+- **PI-Factory** (`PIFACTORY_URL`) - Pipelines, Catalog services
+- **IN-Factory** (`INFACTORY_URL`) - Connections, Connectors, Inputs, Network Config, Transformations
+
+---
+
+## Summary Table
+
+| Category | Count | Impact Level |
+|----------|-------|--------------|
+| gRPC Client Configurations | 6 | HIGH |
+| NestJS Modules | 6 | HIGH |
+| REST Controllers/Endpoints | 6 | HIGH |
+| Service Classes | 6 | HIGH |
+| Configuration Files | 5 | MEDIUM |
+| Proto Package Dependencies | 2 | HIGH |
+| Environment Variables | 2 | LOW |
+
+---
+
+## 1. gRPC CLIENT CONFIGURATIONS (Files to Migrate)
+
+These files configure gRPC connections to In-Factory services:
+
+### 1.1 Connection Client (`INFACTORY_URL`)
+**File:** `src/modules/connection/client.config.ts`
+```typescript
+// Lines 10-11, 18
+process.env.INFACTORY_URL.startsWith('in-factory:')
+process.env.INFACTORY_URL.includes('0.0.0.0')
+url: process.env.INFACTORY_URL
+```
+**Proto Services Used:**
+- `ConnectionManager.ProtoPackages.WritePackage`
+- `ConnectionManager.ProtoPackages.ReadPackage`
+
+### 1.2 Connector Client (`INFACTORY_URL`)
+**File:** `src/modules/connector/client.config.ts`
+```typescript
+// Lines 10-11, 18
+url: process.env.INFACTORY_URL
+```
+**Proto Services Used:**
+- `ConnectorManager.ProtoPackages.WritePackage`
+- `ConnectorManager.ProtoPackages.ReadPackage`
+
+### 1.3 Inputs Client (`INFACTORY_URL`)
+**File:** `src/modules/inputs/inputs-client.config.ts`
+```typescript
+// Lines 10-11, 17
+url: process.env.INFACTORY_URL
+```
+**Proto Services Used:**
+- `Input.ProtoPackages.WritePackage`
+- `Input.ProtoPackages.ReadPackage`
+
+### 1.4 Network Config Client (`INFACTORY_URL`)
+**File:** `src/modules/network-config/network-config-client.config.ts`
+```typescript
+// Lines 10-11, 19
+url: process.env.INFACTORY_URL
+```
+**Proto Services Used:**
+- `NetworkConfig.ProtoPackages.WritePackage`
+- `NetworkConfig.ProtoPackages.ReadPackage`
+
+### 1.5 Connection Test Client (`INFACTORY_URL`)
+**File:** `src/modules/connection-test/connection-test-client.config.ts`
+```typescript
+// Lines 10-11, 17
+url: process.env.INFACTORY_URL
+```
+**Proto Services Used:**
+- `ConnectionTest.ProtoPackages.ReadPackage`
+
+### 1.6 Transformations Client (`INFACTORY_URL`)
+**File:** `src/modules/transformations/transformations-client.ts`
+```typescript
+// Lines 10-11, 18
+url: process.env.INFACTORY_URL
+```
+**Proto Services Used:**
+- `Transformation.ProtoPackages.WritePackage`
+
+---
+
+## 2. SERVICE CLASSES (Business Logic to Migrate)
+
+### 2.1 Connection Client Service
+**File:** `src/modules/connection/client.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Direction |
+|--------|---------|-----------|
+| `CreateConnection()` | ConnectionManagerWriteServices | Write |
+| `UpdateConnection()` | ConnectionManagerWriteServices | Write |
+| `DeleteConnection()` | ConnectionManagerWriteServices | Write |
+| `GetConnectionDetails()` | ConnectionManagerReadServices | Read |
+| `GetAllConnections()` | ConnectionManagerReadServices | Read |
+| `GetConnectorAvailableConnectionsByCustomer()` | ConnectionManagerReadServices | Read |
+| `ValidatePlatformConnections()` | ConnectionManagerReadServices | Read |
+
+**Key Methods:**
+- `createConnection()`
+- `updateConnection()`
+- `deleteConnection()`
+- `getConnections()`
+- `getConnectionDetails()`
+- `validatePlatformConnections()`
+
+### 2.2 Connector Client Service
+**File:** `src/modules/connector/client.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Direction |
+|--------|---------|-----------|
+| `RegisterConnector()` | ConnectorManagerWriteServices | Write |
+| `UploadFile()` | ConnectorManagerWriteServices | Write |
+| `RegisterMultipleConnectorsWithoutImage()` | ConnectorManagerWriteServices | Write |
+| `UpdateConnectorByID()` | ConnectorManagerWriteServices | Write |
+| `DeleteConnectorById()` | ConnectorManagerWriteServices | Write |
+| `GetConnectors()` | ConnectorManagerReadServices | Read |
+
+### 2.3 Inputs Service
+**File:** `src/modules/inputs/inputs.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Direction |
+|--------|---------|-----------|
+| `CreateInput()` | InputWriteService | Write |
+| `UpdateInput()` | InputWriteService | Write |
+| `DeleteInput()` | InputWriteService | Write |
+| `GetAvailableEntities()` | InputReadService | Read |
+
+### 2.4 Network Config Service
+**File:** `src/modules/network-config/network-config.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Direction |
+|--------|---------|-----------|
+| `NetworkConfigCreate()` | NetworkConfigWriteService | Write |
+| `NetworkConfigUpdate()` | NetworkConfigWriteService | Write |
+| `NetworkConfigDelete()` | NetworkConfigWriteService | Write |
+| `NetworkConfigFindAll()` | NetworkConfigReadService | Read |
+| `NetworkConfigFindOneById()` | NetworkConfigReadService | Read |
+
+### 2.5 Connection Test Service
+**File:** `src/modules/connection-test/connection-test.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Direction |
+|--------|---------|-----------|
+| `TestConnection()` | ConnectionTestReadService | Read |
+
+### 2.6 Transformations Service
+**File:** `src/modules/transformations/client.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Direction |
+|--------|---------|-----------|
+| `CreateTransformation()` | TransformationWriteService | Write |
+| `UpdateTransformation()` | TransformationWriteService | Write |
+| `DeleteTransformation()` | TransformationWriteService | Write |
+| Various read operations | TransformationReadService | Read |
+
+---
+
+## 3. NESTJS MODULES (Module Registration)
+
+These modules register the gRPC clients and export services:
+
+| Module | File | Imports | Exports |
+|--------|------|---------|---------|
+| ConnectionModule | `src/modules/connection/connection.module.ts` | ConnectionClientConfiguration | ConnectionClientService |
+| ConnectorModule | `src/modules/connector/connector.module.ts` | ConnectorClientConfiguration | ConnectorClientService |
+| InputsModule | `src/modules/inputs/inputs.module.ts` | InputsGrpcClient | InputsService |
+| NetworkConfigModule | `src/modules/network-config/network-config.module.ts` | NetworkConfigGrpcClient | NetworkConfigService |
+| ConnectionTestModule | `src/modules/connection-test/connection-test.module.ts` | ConnectionTestClientConfiguration | ConnectionTestService |
+| TransformationsModule | `src/modules/transformations/transformations.module.ts` | TransformationsClientConfiguration | TransformationsService |
+
+**App Module Registration:** `src/app.module.ts` (Lines 15-21, 50-60)
+
+---
+
+## 4. REST CONTROLLERS (API Endpoints to Migrate)
+
+### 4.1 Connection Controller
+**File:** `src/modules/connection/connection.controller.ts`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| POST | `/connections` | Create a new connection |
+| PUT | `/connections/:id` | Update an existing connection |
+| DELETE | `/connections/:id` | Delete a connection |
+| GET | `/connections` | Get all connections |
+| GET | `/connections/:id` | Get connection details |
+| GET | `/connections/available/:connector_id` | Get available connections by connector |
+
+### 4.2 Connector Controller
+**File:** `src/modules/connector/connector.controller.ts`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| POST | `/connectors` | Register a new connector |
+| POST | `/connectors/uploads` | Upload multiple connector files |
+| GET | `/connectors` | List all connectors |
+| PUT | `/connectors/:id` | Update a connector |
+| DELETE | `/connectors/:id` | Delete a connector |
+
+### 4.3 Inputs Controller
+**File:** `src/modules/inputs/inputs.controller.ts`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| GET | `/inputs/available-entities/:plugin` | Get available entities for a plugin |
+| POST | `/inputs` | Create an input |
+| PATCH | `/inputs` | Update an input |
+| DELETE | `/inputs/:id` | Delete an input |
+
+### 4.4 Network Config Controller
+**File:** `src/modules/network-config/network-config.controller.ts`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| GET | `/network-configs` | Get all network configurations |
+| GET | `/network-configs/:id` | Get network config by ID |
+| POST | `/network-configs` | Create network configuration |
+| PUT | `/network-configs/:id` | Update network configuration |
+| DELETE | `/network-configs/:id` | Delete network configuration |
+
+### 4.5 Connection Test Controller
+**File:** `src/modules/connection-test/connection-test.controller.ts`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| POST | `/connection-test` | Test a connection |
+
+### 4.6 Transformations Controller
+**File:** `src/modules/transformations/transformations.controller.ts`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| POST | `/transformations` | Create transformation |
+| GET | `/transformations` | Get transformations |
+| PUT | `/transformations/:id` | Update transformation |
+| DELETE | `/transformations/:id` | Delete transformation |
+
+---
+
+## 5. CONFIGURATION FILES
+
+### 5.1 Helm Chart Values (Production)
+**File:** `deploy/helm-chart/values.yaml`
+```yaml
+# Lines 32, 42
+maestro:
+  in_factory_url: in-factory.dadosfera.ai
+  tr_factory_url: in-factory.dadosfera.ai  # Transformation factory also uses in-factory
+```
+
+### 5.2 Helm Chart Values (Staging)
+**File:** `deploy/helm-chart/values-stg.yaml`
+```yaml
+# Lines 5-6
+maestro:
+  in_factory_url: in-factory.stg.dadosfera.ai
+  tr_factory_url: in-factory.stg.dadosfera.ai
+```
+
+### 5.3 Helmfiles (Production)
+**File:** `deploy/helmfiles/prd.yaml`
+```yaml
+# Lines 15-18, 39-42 (for both maestro and maestro-unimed releases)
+maestro.in_factory_url: in-factory.dadosfera.ai
+maestro.tr_factory_url: in-factory.dadosfera.ai
+```
+
+### 5.4 Deployment Template
+**File:** `deploy/helm-chart/templates/deployment.yaml`
+```yaml
+# Lines 71-72
+- name: INFACTORY_URL
+  value: {{ .Values.maestro.in_factory_url }}
+```
+
+### 5.5 Environment Type Definition
+**File:** `environment.d.ts`
+```typescript
+// Line 9
+INFACTORY_URL: string;
+```
+
+---
+
+## 6. PROTO PACKAGE DEPENDENCIES
+
+### 6.1 Package.json
+**File:** `package.json`
+```json
+{
+  "@dadosfera/protospack": "2.5.3",
+  "@dadosfera/protospack-v2": "3.38.0-beta.14"
+}
+```
+
+### 6.2 Proto Imports from `@dadosfera/protospack-v2` (In-Factory related)
+
+| Import Path | Used In |
+|-------------|---------|
+| `ConnectionManager` | connection/client.config.ts, connection/client.service.ts |
+| `ConnectionManager/interfaces/messages` | connection/client.service.ts, connection/dtos/connection.ts |
+| `ConnectionManager/interfaces/entities` | connection/dtos/connection.ts |
+| `ConnectorManager` | connector/client.config.ts, connector/client.service.ts |
+| `Input` | inputs/inputs-client.config.ts, inputs/inputs.service.ts |
+| `Input/interfaces/messages` | inputs/inputs.service.ts |
+| `Input/interfaces/entities` | inputs/inputs.controller.ts, inputs/inputs.service.ts |
+| `NetworkConfig` | network-config/network-config-client.config.ts, network-config/network-config.service.ts |
+| `NetworkConfig/interfaces/entities` | network-config/dto/network-config.ts |
+| `ConnectionTest` | connection-test/connection-test-client.config.ts, connection-test/connection-test.service.ts |
+| `Transformation` | transformations/transformations-client.ts, transformations/client.service.ts |
+
+---
+
+## 7. MIGRATION STRATEGY OPTIONS
+
+### Option A: Move In-Factory functionality INTO Maestro
+**Pros:**
+- Single service to maintain
+- No network latency for these operations
+- Simpler deployment
+
+**Cons:**
+- Increases Maestro's responsibility/complexity
+- Requires database access from Maestro
+- May require significant refactoring
+
+**Files to create/migrate:**
+1. Database models for connections, connectors, inputs, network-configs, transformations
+2. Repository layer for database operations
+3. Convert gRPC services to internal services
+4. Remove all gRPC client configurations
+
+### Option B: Create REST API wrapper in In-Factory
+**Pros:**
+- Minimal changes to Maestro
+- Can migrate incrementally
+
+**Cons:**
+- Still maintains dependency
+- Additional REST→gRPC translation layer
+
+### Option C: Direct database access from Maestro
+**Pros:**
+- Removes runtime dependency
+- Better performance
+
+**Cons:**
+- Shared database coupling
+- Complex migration
+
+---
+
+## 8. FILES TO MODIFY/DELETE (Summary)
+
+### High Priority - Core Integration Files
+```
+src/modules/connection/client.config.ts          → DELETE or REPLACE
+src/modules/connection/client.service.ts         → REPLACE with local implementation
+src/modules/connector/client.config.ts           → DELETE or REPLACE
+src/modules/connector/client.service.ts          → REPLACE with local implementation
+src/modules/inputs/inputs-client.config.ts       → DELETE or REPLACE
+src/modules/inputs/inputs.service.ts             → REPLACE with local implementation
+src/modules/network-config/network-config-client.config.ts → DELETE or REPLACE
+src/modules/network-config/network-config.service.ts      → REPLACE with local implementation
+src/modules/connection-test/connection-test-client.config.ts → DELETE or REPLACE
+src/modules/connection-test/connection-test.service.ts    → REPLACE with local implementation
+src/modules/transformations/transformations-client.ts     → DELETE or REPLACE
+src/modules/transformations/client.service.ts            → REPLACE with local implementation
+```
+
+### Medium Priority - Module Registration
+```
+src/modules/connection/connection.module.ts      → UPDATE imports
+src/modules/connector/connector.module.ts        → UPDATE imports
+src/modules/inputs/inputs.module.ts              → UPDATE imports
+src/modules/network-config/network-config.module.ts → UPDATE imports
+src/modules/connection-test/connection-test.module.ts → UPDATE imports
+src/modules/transformations/transformations.module.ts → UPDATE imports
+src/app.module.ts                                → UPDATE if module structure changes
+```
+
+### Low Priority - Configuration
+```
+deploy/helm-chart/values.yaml                    → REMOVE in_factory_url, tr_factory_url
+deploy/helm-chart/values-stg.yaml                → REMOVE in_factory_url, tr_factory_url
+deploy/helmfiles/prd.yaml                        → REMOVE in_factory_url references
+deploy/helm-chart/templates/deployment.yaml      → REMOVE INFACTORY_URL env var
+environment.d.ts                                 → REMOVE INFACTORY_URL type
+README.md                                        → UPDATE architecture diagram
+```
+
+---
+
+## 9. DEPENDENCY COUNT BY MODULE
+
+| Module | Files | gRPC Calls | REST Endpoints |
+|--------|-------|------------|----------------|
+| Connection | 4 | 7 | 6 |
+| Connector | 3 | 6 | 5 |
+| Inputs | 3 | 4 | 4 |
+| Network Config | 3 | 5 | 5 |
+| Connection Test | 3 | 1 | 1 |
+| Transformations | 4 | 4+ | 4 |
+| **TOTAL** | **20** | **27+** | **25** |
+
+---
+
+## 10. DATA MODELS (Proto Messages Used)
+
+### Connection Manager
+- `CreateConnectionRequest` / `CreateConnectionResponse`
+- `UpdateConnectionRequest` / `UpdateConnectionResponse`
+- `DeleteConnectionRequest` / `DeleteConnectionResponse`
+- `GetConnectionDetailsRequest` / `GetConnectionDetailsResponse`
+- `GetAllConnectionsRequest` / `GetAllConnectionsResponse`
+- `Connection` (entity)
+- `ConnectionCredential` (entity)
+
+### Connector Manager
+- `RegisterConnectorRequest` / `RegisterConnectorResponse`
+- `UploadFileRequest` / `UploadFileResponse`
+- `GetConnectorsRequest` / `GetConnectorsResponse`
+- `Connector` (entity)
+
+### Input
+- `CreateInputRequest` / `CreateInputResponse`
+- `UpdateInputRequest` / `UpdateInputResponse`
+- `DeleteInputRequest` / `DeleteInputResponse`
+- `GetAvailableEntitiesRequest` / `GetAvailableEntitiesResponse`
+- `Info` (entity)
+
+### Network Config
+- `NetworkConfigCreateRequest` / `NetworkConfigCreateResponse`
+- `NetworkConfigUpdateRequest` / `NetworkConfigUpdateResponse`
+- `NetworkConfigDeleteRequest` / `NetworkConfigDeleteResponse`
+- `NetworkConfigFindAllRequest` / `NetworkConfigFindAllResponse`
+- `NetworkConfig` (entity)
+
+### Connection Test
+- `TestConnectionRequest` / `TestConnectionResponse`
+
+### Transformation
+- `CreateTransformationRequest` / `CreateTransformationResponse`
+- `UpdateTransformationRequest` / `UpdateTransformationResponse`
+- `DeleteTransformationRequest` / `DeleteTransformationResponse`
+
+---
+
+## NEXT STEPS
+
+1. **Decide on migration strategy** (Option A, B, or C)
+2. **Prioritize modules** - Recommend starting with Connection Test (smallest), then Inputs, Network Config, Transformations, Connection, Connector (largest)
+3. **Create database schema** if moving to Option A
+4. **Implement local services** one module at a time
+5. **Update tests** for each migrated module
+6. **Update deployment configs** to remove INFACTORY_URL
+7. **Coordinate with In-Factory team** for data migration

MIGRATION_MAP_INFACTORY_SIDE.md

@@ -0,0 +1,320 @@
+# In-Factory Side - Migration Requirements
+
+This document outlines what needs to be addressed in the **In-Factory** service to remove its coupling with Maestro.
+
+> **Note:** This analysis is based on the Maestro codebase. For a complete analysis, the In-Factory repository should also be reviewed.
+
+---
+
+## Current Architecture (In-Factory → Maestro)
+
+Based on the Maestro codebase analysis, **In-Factory** exposes the following gRPC services that Maestro consumes:
+
+```mermaid
+graph LR;
+  Maestro -->|gRPC| InFactory;
+  subgraph InFactory Services
+    CM[ConnectionManager]
+    ConM[ConnectorManager]
+    IN[Input]
+    NC[NetworkConfig]
+    CT[ConnectionTest]
+    TR[Transformation]
+  end
+  Maestro --> CM;
+  Maestro --> ConM;
+  Maestro --> IN;
+  Maestro --> NC;
+  Maestro --> CT;
+  Maestro --> TR;
+```
+
+---
+
+## gRPC Services Exposed by In-Factory
+
+### 1. ConnectionManager Service
+
+**Package:** `ConnectionManager` from `@dadosfera/protospack-v2`
+
+#### Write Services (`ConnectionManagerWriteServices`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `CreateConnection` | Create a new data connection | Maestro POST /connections |
+| `UpdateConnection` | Update an existing connection | Maestro PUT /connections/:id |
+| `DeleteConnection` | Delete a connection | Maestro DELETE /connections/:id |
+
+#### Read Services (`ConnectionManagerReadServices`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `GetConnectionDetails` | Get details of a single connection | Maestro GET /connections/:id |
+| `GetAllConnections` | List all connections for a customer | Maestro GET /connections |
+| `GetConnectorAvailableConnectionsByCustomer` | Get available connections by connector | Maestro GET /connections/available/:connector_id |
+| `ValidatePlatformConnections` | Validate connections against platform | Maestro internal |
+
+---
+
+### 2. ConnectorManager Service
+
+**Package:** `ConnectorManager` from `@dadosfera/protospack-v2`
+
+#### Write Services (`ConnectorManagerWriteServices`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `RegisterConnector` | Register a new connector | Maestro POST /connectors |
+| `UploadFile` | Upload connector files | Maestro POST /connectors/uploads |
+| `RegisterMultipleConnectorsWithoutImage` | Bulk register connectors | Maestro internal |
+| `UpdateConnectorByID` | Update a connector | Maestro PUT /connectors/:id |
+| `DeleteConnectorById` | Delete a connector | Maestro DELETE /connectors/:id |
+
+#### Read Services (`ConnectorManagerReadServices`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `GetConnectors` | List all connectors | Maestro GET /connectors |
+
+---
+
+### 3. Input Service
+
+**Package:** `Input` from `@dadosfera/protospack-v2`
+
+#### Write Services (`InputWriteService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `CreateInput` | Create a new input configuration | Maestro POST /inputs |
+| `UpdateInput` | Update input configuration | Maestro PATCH /inputs |
+| `DeleteInput` | Delete an input | Maestro DELETE /inputs/:id |
+
+#### Read Services (`InputReadService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `GetAvailableEntities` | Get available entities for a plugin | Maestro GET /inputs/available-entities/:plugin |
+
+---
+
+### 4. NetworkConfig Service
+
+**Package:** `NetworkConfig` from `@dadosfera/protospack-v2`
+
+#### Write Services (`NetworkConfigWriteService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `NetworkConfigCreate` | Create network configuration | Maestro POST /network-configs |
+| `NetworkConfigUpdate` | Update network configuration | Maestro PUT /network-configs/:id |
+| `NetworkConfigDelete` | Delete network configuration | Maestro DELETE /network-configs/:id |
+
+#### Read Services (`NetworkConfigReadService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `NetworkConfigFindAll` | List all network configs | Maestro GET /network-configs |
+| `NetworkConfigFindOneById` | Get network config by ID | Maestro GET /network-configs/:id |
+
+---
+
+### 5. ConnectionTest Service
+
+**Package:** `ConnectionTest` from `@dadosfera/protospack-v2`
+
+#### Read Services (`ConnectionTestReadService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `TestConnection` | Test a connection's connectivity | Maestro POST /connection-test |
+
+---
+
+### 6. Transformation Service
+
+**Package:** `Transformation` from `@dadosfera/protospack-v2`
+
+#### Write Services (`TransformationWriteService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| `CreateTransformation` | Create a transformation | Maestro POST /transformations |
+| `UpdateTransformation` | Update a transformation | Maestro PUT /transformations/:id |
+| `DeleteTransformation` | Delete a transformation | Maestro DELETE /transformations/:id |
+
+#### Read Services (`TransformationReadService`)
+| gRPC Method | Description | Called By |
+|-------------|-------------|-----------|
+| Various read operations | Get transformation details | Maestro GET /transformations |
+
+---
+
+## Proto Package Ownership
+
+The proto definitions are managed in `@dadosfera/protospack-v2`:
+
+```
+@dadosfera/protospack-v2/
+├── dist/lib/
+│   ├── ConnectionManager/
+│   │   ├── interfaces/messages.ts
+│   │   └── interfaces/entities.ts
+│   ├── ConnectorManager/
+│   │   ├── interfaces/messages.ts
+│   │   └── interfaces/entities.ts
+│   ├── Input/
+│   │   ├── interfaces/messages.ts
+│   │   └── interfaces/entities.ts
+│   ├── NetworkConfig/
+│   │   ├── interfaces/messages.ts
+│   │   └── interfaces/entities.ts
+│   ├── ConnectionTest/
+│   │   ├── interfaces/messages.ts
+│   │   └── interfaces/entities.ts
+│   └── Transformation/
+│       ├── interfaces/messages.ts
+│       └── interfaces/entities.ts
+```
+
+---
+
+## In-Factory Migration Options
+
+### Option 1: Expose REST API (Keep In-Factory, Add REST Layer)
+
+**Changes Required in In-Factory:**
+1. Add REST controllers for all services
+2. Implement HTTP endpoints mirroring gRPC methods
+3. Update deployment to expose HTTP port
+4. Create OpenAPI documentation
+
+**Pros:**
+- Minimal architectural changes
+- Can run both gRPC and REST in parallel during migration
+- Maestro can switch to REST calls gradually
+
+**Cons:**
+- Still maintains service dependency
+- Adds another communication layer
+
+---
+
+### Option 2: Move Logic to Maestro (Deprecate In-Factory for these features)
+
+**Changes Required in In-Factory:**
+1. Export database schema/migrations
+2. Document all business logic
+3. Provide data migration scripts
+4. Deprecate gRPC endpoints after migration
+
+**Changes Required in Maestro:**
+1. Create database models
+2. Implement repositories
+3. Create service layer with same business logic
+4. Run data migration
+
+**Pros:**
+- Removes runtime dependency completely
+- Simplifies architecture
+- One less service to maintain
+
+**Cons:**
+- Significant development effort
+- Risk of business logic divergence during migration
+- Database sharing concerns
+
+---
+
+### Option 3: Merge Services (Combine In-Factory into a larger service)
+
+**Changes Required:**
+1. Create new combined service
+2. Migrate both In-Factory and relevant Maestro code
+3. Update all clients
+
+**Pros:**
+- Clean architectural redesign
+- Opportunity to optimize
+
+**Cons:**
+- Largest effort
+- Risk of disruption
+
+---
+
+## Data Migration Considerations
+
+### Entities Managed by In-Factory
+
+Based on proto definitions, In-Factory manages:
+
+1. **Connections**
+   - Connection credentials
+   - Connection metadata
+   - Customer associations
+
+2. **Connectors**
+   - Connector definitions
+   - Connector images/files
+   - Plugin configurations
+
+3. **Inputs**
+   - Input configurations
+   - Entity mappings
+
+4. **Network Configs**
+   - Network configuration settings
+   - Security settings
+
+5. **Transformations**
+   - Transformation definitions
+   - Transformation scripts
+
+### Migration Steps
+1. Export database schema from In-Factory
+2. Create equivalent schema in target database
+3. Write data migration scripts
+4. Validate data integrity
+5. Switch traffic
+6. Decommission old service
+
+---
+
+## Recommended Investigation for In-Factory Team
+
+1. **Check for Maestro dependencies in In-Factory**
+   - Does In-Factory call any Maestro APIs?
+   - Are there any shared databases?
+   - Any shared message queues?
+
+2. **Document database schema**
+   - All tables related to connections, connectors, inputs, network-configs, transformations
+   - Foreign key relationships
+   - Indexes and constraints
+
+3. **List all consumers**
+   - Besides Maestro, who else calls In-Factory?
+   - Are there other internal services?
+   - Any external integrations?
+
+4. **Business logic documentation**
+   - Validation rules
+   - Business constraints
+   - Side effects (events, notifications, etc.)
+
+---
+
+## Timeline Considerations
+
+| Phase | Description | Dependencies |
+|-------|-------------|--------------|
+| Phase 1 | Analysis & Planning | Both teams available |
+| Phase 2 | Schema/API Design | Proto definitions finalized |
+| Phase 3 | Implementation | Development resources |
+| Phase 4 | Data Migration | Database access, downtime window |
+| Phase 5 | Testing | QA resources, test environments |
+| Phase 6 | Cutover | Deployment coordination |
+| Phase 7 | Decommission | Monitoring, rollback plan |
+
+---
+
+## Questions for In-Factory Team
+
+1. What database does In-Factory use? (PostgreSQL, MongoDB, etc.)
+2. Are there any async operations? (message queues, event sourcing)
+3. What is the current data volume for each entity type?
+4. Are there any scheduled jobs or background processes?
+5. What monitoring/alerting is in place?
+6. Are there any data retention policies?
+7. What is the backup/recovery strategy?

MIGRATION_MAP_PIFACTORY.md

@@ -0,0 +1,486 @@
+# Maestro ↔ PI-Factory Migration Map
+
+This document maps all integration points between Maestro and PI-Factory that need to be addressed to remove the dependency.
+
+## Architecture Overview
+
+```mermaid
+graph TD;
+  Frontend<-->Maestro;
+  Maestro<-->DUC;
+  Maestro<-->pi-factory;
+  Maestro<-->in-factory;
+```
+
+PI-Factory (`PIFACTORY_URL`) is responsible for:
+- **Pipeline Management** - Create, read, update, delete pipelines
+- **Catalog Services** - Data asset management, metadata, previews
+- **Platform Interfaces** - Dataset cataloging operations
+
+---
+
+## Summary Table
+
+| Category | Count | Impact Level |
+|----------|-------|--------------|
+| gRPC Client Configurations | 3 | HIGH |
+| NestJS Modules | 3 | HIGH |
+| REST Controllers/Endpoints | 3 (~50 endpoints) | HIGH |
+| Service Classes | 4 | HIGH |
+| Configuration Files | 4 | MEDIUM |
+| Proto Package Dependencies | 2 | HIGH |
+| Environment Variables | 1 | LOW |
+
+---
+
+## 1. gRPC CLIENT CONFIGURATIONS (Files to Migrate)
+
+These files configure gRPC connections to PI-Factory services:
+
+### 1.1 Catalog Client (`PIFACTORY_URL`)
+**File:** `src/modules/catalog/catalog-client.ts`
+```typescript
+// Lines 11-12, 19
+process.env.PIFACTORY_URL.startsWith('pi-factory:')
+process.env.PIFACTORY_URL.includes('0.0.0.0')
+url: process.env.PIFACTORY_URL
+```
+**Proto Services Used:**
+- `Catalog.ProtoPackages.ReadPackage`
+- `Catalog.ProtoPackages.WritePackage`
+- `PlatformInterfaces.ProtoPackages.WritePackage`
+
+### 1.2 Pipelines V2 Client (`PIFACTORY_URL`)
+**File:** `src/modules/pipelinesV2/pipelines-client.ts`
+```typescript
+// Lines 13-14, 21
+url: process.env.PIFACTORY_URL
+```
+**Proto Services Used:**
+- `PipelineV2.ProtoPackages.ReadPackage`
+- `PipelineV2.ProtoPackages.WritePackage`
+
+### 1.3 Pipelines Client (Legacy) (`PIFACTORY_URL`)
+**File:** `src/modules/pipelines/pipelines-client.ts`
+```typescript
+// Lines 10-11, 18
+url: process.env.PIFACTORY_URL
+```
+**Proto Services Used:**
+- `PipelinePackages` from `@dadosfera/protospack`
+
+---
+
+## 2. SERVICE CLASSES (Business Logic to Migrate)
+
+### 2.1 Catalog Service
+**File:** `src/modules/catalog/catalog.service.ts`
+
+**gRPC Services Initialized:**
+- `CatalogReadServices` (from `Catalog.ReadService`)
+- `CatalogWriteServices` (from `Catalog.WriteService`)
+- `PlatformInterfacesWriteServices` (from `PlatformInterfaces.WriteService`)
+
+**gRPC Methods Called:**
+
+| Method | Service | Description |
+|--------|---------|-------------|
+| `GetAllDataAssets()` | CatalogReadServices | Search/list data assets |
+| `GetOneDataAsset()` | CatalogReadServices | Get single data asset by ID |
+| `GetOneDataAssetByPipelineAndObject()` | CatalogReadServices | Get asset by pipeline/object |
+| `GetDatasetDoc()` | CatalogReadServices | Get dataset documentation |
+| `GetDatasetPreview()` | CatalogReadServices | Get data preview |
+| `GetDatasetColumnsMetadata()` | CatalogReadServices | Get column metadata |
+| `GetCustomerTags()` | CatalogReadServices | Get all tags for customer |
+| `GetDatasetCatalogTask()` | CatalogReadServices | Get catalog task status |
+| `GetRlsRules()` | CatalogReadServices | Get RLS rules |
+| `GetOneRlsRule()` | CatalogReadServices | Get single RLS rule |
+| `GetNimbusDashboards()` | CatalogReadServices | Get Nimbus dashboards |
+| `CreateDataAsset()` | CatalogWriteServices | Create new data asset |
+| `UpdateDataAsset()` | CatalogWriteServices | Update existing data asset |
+| `DeleteDataAsset()` | CatalogWriteServices | Delete data asset |
+| `ManagePermission()` | CatalogWriteServices | Manage asset permissions |
+| `RevokePermission()` | CatalogWriteServices | Revoke asset permissions |
+| `MakeAComment()` | CatalogWriteServices | Add comment to asset |
+| `UpdateAComment()` | CatalogWriteServices | Update/delete comment |
+| `TriggerDatasetCataloging()` | CatalogWriteServices | Trigger catalog process |
+| `AddRlsRule()` | CatalogWriteServices | Add RLS rule |
+| `RemoveRlsRule()` | CatalogWriteServices | Remove RLS rule |
+| `RemoveRlsRulesByRlsId()` | CatalogWriteServices | Batch remove by RLS ID |
+| `RemoveRlsRulesByDashboardId()` | CatalogWriteServices | Batch remove by dashboard |
+| `GetPiiReporter()` | CatalogWriteServices | Get PII report data |
+| `CatalogDataAssets()` | PlatformInterfacesWriteServices | Catalog datasets |
+
+**Additional HTTP Calls to Nimbus:**
+- `POST ${nimbusUrl}/api/catalog/data-docs/` - Create data docs
+- `POST ${nimbusUrl}/api/catalog/table-metadata/` - Create table metadata
+- `POST ${nimbusUrl}/api/catalog/column-metadata/` - Create column metadata
+- `POST ${nimbusUrl}/api/catalog/data-preview/` - Create data preview
+
+### 2.2 Pipelines V2 Service
+**File:** `src/modules/pipelinesV2/pipelines.service.ts`
+
+**gRPC Services Initialized:**
+- `PipelineV2ReadService` (from `PipelineV2.ReadService`)
+- `PipelineV2WriteService` (from `PipelineV2.WriteService`)
+
+**gRPC Methods Called:**
+
+| Method | Service | Description |
+|--------|---------|-------------|
+| `PipelineV2Create()` | PipelineV2WriteService | Create new pipeline |
+| `PipelineV2Update()` | PipelineV2WriteService | Update pipeline |
+| `PipelineV2Remove()` | PipelineV2WriteService | Delete pipeline |
+| `PipelineV2UploadFile()` | PipelineV2WriteService | Initialize file upload |
+| `PipelineV2CompleteUploadFile()` | PipelineV2WriteService | Complete file upload |
+| `PipelineV2FindAll()` | PipelineV2ReadService | List all pipelines |
+| `PipelineV2FindOne()` | PipelineV2ReadService | Get single pipeline |
+| `PipelineV2FindObjects()` | PipelineV2ReadService | Get pipeline objects |
+| `PipelineV2DownloadLogs()` | PipelineV2ReadService | Download pipeline logs |
+| `PipelineV2GetDashboardUrl()` | PipelineV2ReadService | Get monitoring dashboard URL |
+
+### 2.3 Pipelines Service (Legacy)
+**File:** `src/modules/pipelines/pipelines.service.ts`
+
+**Uses:** `PipelinesClientService`
+
+**Methods:**
+- `getPipelineStatus()` - Get pipeline execution status
+- `runPipeline()` - Trigger pipeline execution
+
+### 2.4 Pipelines Client Service (Legacy)
+**File:** `src/modules/pipelines/client.service.ts`
+
+**gRPC Methods Called:**
+| Method | Service | Description |
+|--------|---------|-------------|
+| `getPipelineStatus()` | PipelineService | Get pipeline status |
+| `triggerPipeline()` | PipelineService | Trigger pipeline run |
+
+---
+
+## 3. NESTJS MODULES (Module Registration)
+
+| Module | File | Client Configuration | Exports |
+|--------|------|---------------------|---------|
+| CatalogModule | `src/modules/catalog/catalog.module.ts` | CatalogClientConfiguration | CatalogService |
+| PipelinesV2Module | `src/modules/pipelinesV2/pipelines.module.ts` | PipelinesClientConfiguration | PipelinesService |
+| PipelinesModule | `src/modules/pipelines/pipelines.module.ts` | PipelinesClientConfiguration | PipelinesService, PipelinesClientService |
+
+**App Module Registration:** `src/app.module.ts` (Lines 20, 23, 25, 53-54, 59)
+
+---
+
+## 4. REST CONTROLLERS (API Endpoints to Migrate)
+
+### 4.1 Catalog Controller
+**File:** `src/modules/catalog/catalog.controller.ts`
+**Base Path:** `/catalog`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| GET | `/catalog` | Search data assets |
+| GET | `/catalog/download` | Download assets as CSV |
+| GET | `/catalog/data-asset` | Get asset by pipeline/object |
+| GET | `/catalog/data-asset/:id` | Get single data asset |
+| GET | `/catalog/data-asset/rls/:id` | Get data asset RLS info |
+| GET | `/catalog/data-asset/:id/columns-metadata` | Get column metadata |
+| GET | `/catalog/data-asset/:id/preview` | Get data preview |
+| GET | `/catalog/data-asset/:id/docs` | Get documentation |
+| GET | `/catalog/tags` | Get all tags |
+| PUT | `/catalog/data-asset/:id` | Update data asset |
+| PUT | `/catalog/data-asset/:id/manage-permissions` | Manage permissions |
+| PUT | `/catalog/data-asset/:id/revoke-permissions` | Revoke permissions |
+| POST | `/catalog` | Create data asset |
+| POST | `/catalog/data-asset/:id/docs` | Create documentation |
+| POST | `/catalog/data-asset/:id/comment` | Add comment |
+| POST | `/catalog/dataset-catalog-task` | Trigger catalog task |
+| POST | `/catalog/rls-rule` | Add RLS rule |
+| POST | `/catalog/register-dataset` | Register dataset with metadata |
+| DELETE | `/catalog/data-asset/:id` | Delete data asset |
+| DELETE | `/catalog/data-asset/:id/comment` | Delete comment |
+| DELETE | `/catalog/rls-rule/:id` | Remove RLS rule |
+| DELETE | `/catalog/rls-rule` | Batch remove RLS rules |
+| GET | `/catalog/dataset-catalog-task/:session` | Get catalog task status |
+| GET | `/catalog/rls-rule/:id` | Get single RLS rule |
+| GET | `/catalog/rls-rule` | Get RLS rules |
+| GET | `/catalog/nimbus-dashboards` | Get Nimbus dashboards |
+| GET | `/catalog/pii-reporter` | Get PII report |
+
+**Total: 27 endpoints**
+
+### 4.2 Pipelines V2 Controller
+**File:** `src/modules/pipelinesV2/pipelines.controller.ts`
+**Base Path:** `/pipelinesV2`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| GET | `/pipelinesV2/monitoring-dashboard` | Get monitoring dashboard URL |
+| GET | `/pipelinesV2` | List all pipelines |
+| GET | `/pipelinesV2/download-logs` | Download pipeline logs |
+| GET | `/pipelinesV2/:id` | Get single pipeline |
+| GET | `/pipelinesV2/:id/config` | Get pipeline properties |
+| GET | `/pipelinesV2/:id/objects` | Get pipeline objects |
+| GET | `/pipelinesV2/:id/status` | Get pipeline status (legacy) |
+| POST | `/pipelinesV2` | Create pipeline |
+| POST | `/pipelinesV2/init-upload` | Initialize file upload |
+| POST | `/pipelinesV2/complete-upload` | Complete file upload |
+| POST | `/pipelinesV2/file` | Upload file pipeline |
+| POST | `/pipelinesV2/start/:id` | Start pipeline |
+| PATCH | `/pipelinesV2/:id` | Update pipeline |
+| PUT | `/pipelinesV2/:id` | Update pipeline (deprecated) |
+| DELETE | `/pipelinesV2/:id` | Delete pipeline |
+
+**Total: 15 endpoints**
+
+### 4.3 Pipelines Controller (Legacy)
+**File:** `src/modules/pipelines/pipelines.controller.ts`
+**Base Path:** `/pipelines`
+
+| HTTP Method | Endpoint | Description |
+|-------------|----------|-------------|
+| POST | `/pipelines/start/:id` | Start pipeline (deprecated) |
+| GET | `/pipelines/:id/status` | Get pipeline status (deprecated) |
+
+**Total: 2 endpoints (deprecated)**
+
+---
+
+## 5. CONFIGURATION FILES
+
+### 5.1 Helm Chart Values (Production)
+**File:** `deploy/helm-chart/values.yaml`
+```yaml
+# Line 40
+maestro:
+  pi_factory_url: pi-factory.dadosfera.ai
+```
+
+### 5.2 Helm Chart Values (Staging)
+**File:** `deploy/helm-chart/values-stg.yaml`
+```yaml
+# Line 4
+maestro:
+  pi_factory_url: pi-factory.stg.dadosfera.ai
+```
+
+### 5.3 Helmfiles (Production)
+**File:** `deploy/helmfiles/prd.yaml`
+```yaml
+# Lines 14, 38
+maestro.pi_factory_url: pi-factory.dadosfera.ai
+```
+
+### 5.4 Deployment Template
+**File:** `deploy/helm-chart/templates/deployment.yaml`
+```yaml
+# Line 85
+- name: PIFACTORY_URL
+  value: {{ .Values.maestro.pi_factory_url }}
+```
+
+### 5.5 Environment Type Definition
+**File:** `environment.d.ts`
+```typescript
+// Line 10
+PIFACTORY_URL: string;
+```
+
+---
+
+## 6. PROTO PACKAGE DEPENDENCIES
+
+### 6.1 Package.json
+**File:** `package.json`
+```json
+{
+  "@dadosfera/protospack": "2.5.3",      // Legacy pipelines
+  "@dadosfera/protospack-v2": "3.38.0-beta.14"  // PipelineV2, Catalog, PlatformInterfaces
+}
+```
+
+### 6.2 Proto Imports from `@dadosfera/protospack-v2` (PI-Factory related)
+
+| Import Path | Used In |
+|-------------|---------|
+| `Catalog` | catalog/catalog-client.ts, catalog/catalog.service.ts |
+| `Catalog/interfaces/messages` | catalog/catalog.service.ts, catalog/catalog.controller.ts, catalog/dtos |
+| `PlatformInterfaces` | catalog/catalog-client.ts, catalog/catalog.service.ts |
+| `PipelineV2` | pipelinesV2/pipelines-client.ts, pipelinesV2/pipelines.service.ts |
+| `PipelineV2/interfaces/messages` | pipelinesV2/pipelines.service.ts, pipelinesV2/pipelines.controller.ts |
+
+### 6.3 Proto Imports from `@dadosfera/protospack` (Legacy)
+
+| Import Path | Used In |
+|-------------|---------|
+| `PipelinePackages` | pipelines/pipelines-client.ts |
+| `PipelineProtoFilePath` | pipelines/pipelines-client.ts |
+| `PipelineServicesNames` | pipelines/client.service.ts |
+| `PipelinesServiceInterface` | pipelines/client.service.ts |
+
+---
+
+## 7. FILES TO MODIFY/DELETE (Summary)
+
+### High Priority - Core Integration Files
+```
+src/modules/catalog/catalog-client.ts              → DELETE or REPLACE
+src/modules/catalog/catalog.service.ts             → REPLACE with local implementation
+src/modules/pipelinesV2/pipelines-client.ts        → DELETE or REPLACE
+src/modules/pipelinesV2/pipelines.service.ts       → REPLACE with local implementation
+src/modules/pipelines/pipelines-client.ts          → DELETE or REPLACE
+src/modules/pipelines/client.service.ts            → REPLACE with local implementation
+src/modules/pipelines/pipelines.service.ts         → REPLACE with local implementation
+```
+
+### Medium Priority - Module Registration
+```
+src/modules/catalog/catalog.module.ts              → UPDATE imports
+src/modules/pipelinesV2/pipelines.module.ts        → UPDATE imports
+src/modules/pipelines/pipelines.module.ts          → UPDATE imports
+src/app.module.ts                                  → UPDATE if module structure changes
+```
+
+### Low Priority - Configuration
+```
+deploy/helm-chart/values.yaml                      → REMOVE pi_factory_url
+deploy/helm-chart/values-stg.yaml                  → REMOVE pi_factory_url
+deploy/helmfiles/prd.yaml                          → REMOVE pi_factory_url references
+deploy/helm-chart/templates/deployment.yaml        → REMOVE PIFACTORY_URL env var
+environment.d.ts                                   → REMOVE PIFACTORY_URL type
+README.md                                          → UPDATE architecture diagram
+```
+
+---
+
+## 8. DEPENDENCY COUNT BY MODULE
+
+| Module | Files | gRPC Calls | REST Endpoints |
+|--------|-------|------------|----------------|
+| Catalog | 3 | 24+ | 27 |
+| PipelinesV2 | 3 | 10 | 15 |
+| Pipelines (Legacy) | 3 | 2 | 2 |
+| **TOTAL** | **9** | **36+** | **44** |
+
+---
+
+## 9. COMPARISON: PI-Factory vs In-Factory
+
+| Aspect | PI-Factory | In-Factory |
+|--------|------------|------------|
+| Environment Variable | `PIFACTORY_URL` | `INFACTORY_URL` |
+| Modules | 3 | 6 |
+| gRPC Calls | 36+ | 27+ |
+| REST Endpoints | 44 | 25 |
+| Complexity | HIGH | MEDIUM-HIGH |
+| Domain | Pipelines, Catalog | Connections, Connectors, Inputs |
+
+---
+
+## 10. DATA MODELS (Proto Messages Used)
+
+### Catalog Messages
+- `CreateDataAssetRequest` / `CreateDataAssetResponse`
+- `GetAllDataAssetsRequest` / `GetAllDataAssetsResponse`
+- `GetOneDataAssetRequest` / Response
+- `UpdateDataAssetRequest` / Response
+- `DeleteDataAssetRequest` / Response
+- `ManagePermissionRequest` / Response
+- `RevokePermissionRequest` / Response
+- `MakeACommentRequest` / Response
+- `UpdateACommentRequest` / Response
+- `TriggerDatasetCatalogingRequest` / Response
+- `GetDatasetCatalogTaskRequest` / Response
+- `AddRlsRuleRequest` / Response
+- `RemoveRlsRuleRequest` / Response
+- `GetRlsRulesRequest` / Response
+- `GetNimbusDashboardsRequest` / Response
+- `PiiMetadata`
+- `RegisterDatasetWithMetatadaRequest`
+
+### PipelineV2 Messages
+- `PipelineV2CreateRequest` / `PipelineV2CreateResponse`
+- `PipelineV2FindAllRequest` / `PipelineV2FindAllResponse`
+- `PipelineV2FindOneRequest` / `PipelineV2FindOneResponse`
+- `PipelineV2UpdateRequest` / `PipelineV2UpdateResponse`
+- `PipelineV2RemoveRequest` / Response
+- `PipelineV2UploadFileRequest` / Response
+- `PipelineV2CompleteUploadFileRequest` / Response
+- `PipelineV2FindObjectsRequest` / Response
+- `PipelineV2DownloadLogsRequest` / Response
+- `PipelineV2GetDashboardUrlRequest` / Response
+
+### Platform Interfaces Messages
+- `CatalogDataAssetsRequest` / Response
+
+---
+
+## 11. MIGRATION STRATEGY OPTIONS
+
+### Option A: Move PI-Factory functionality INTO Maestro
+**Pros:**
+- Single service to maintain
+- No network latency for these operations
+- Simpler deployment
+
+**Cons:**
+- Significantly increases Maestro's responsibility
+- Requires database access from Maestro
+- Large refactoring effort (44 endpoints)
+
+### Option B: Create REST API wrapper in PI-Factory
+**Pros:**
+- Minimal changes to Maestro
+- Can migrate incrementally
+
+**Cons:**
+- Still maintains dependency
+- Additional REST→gRPC translation layer
+
+### Option C: Direct database access from Maestro
+**Pros:**
+- Removes runtime dependency
+- Better performance
+
+**Cons:**
+- Shared database coupling
+- Complex migration
+
+---
+
+## 12. RECOMMENDED MIGRATION ORDER
+
+Given the complexity, we recommend migrating in this order:
+
+1. **Legacy Pipelines** (2 endpoints, deprecated) - Lowest risk
+2. **PipelinesV2** (15 endpoints) - Core pipeline functionality
+3. **Catalog** (27 endpoints) - Most complex, migrate last
+
+### Phase 1: Legacy Pipelines (Deprecated)
+- Remove `/pipelines/start/:id`
+- Remove `/pipelines/:id/status`
+- Update all clients to use `/pipelinesV2/*` endpoints
+
+### Phase 2: PipelinesV2
+- Migrate pipeline CRUD operations
+- Migrate file upload functionality
+- Migrate monitoring dashboard
+
+### Phase 3: Catalog
+- Migrate data asset CRUD
+- Migrate permissions management
+- Migrate RLS rules
+- Migrate dataset registration
+- Migrate PII reporting
+
+---
+
+## NEXT STEPS
+
+1. **Decide on migration strategy** (Option A, B, or C)
+2. **Deprecate legacy pipelines module** first
+3. **Create database schema** if moving to Option A
+4. **Implement local services** one module at a time
+5. **Update tests** for each migrated module
+6. **Update deployment configs** to remove PIFACTORY_URL
+7. **Coordinate with PI-Factory team** for data migration

README.md

@@ -2,52 +2,57 @@
   <image src="./assets/maestro.svg" style="width:10rem">
 </p>
 
+
 # Maestro
 
-Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
+Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
 
-## 🚀 Starting
+```mermaid
+graph TD;
+  Frontend<-->Maestro;
+  Maestro<-->duc;
+  Maestro<-->pi-factory;
+  Maestro<-->in-factory;
+```
 
-These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
+É uma API REST, desenvolvida em NodeJs utilizando o Framework [NestJs](https://docs.nestjs.com/).
 
-### 📋 Requirements
+## 🚀 Iniciando
 
-- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
-- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
-  - Create your Access Key on the "Security credentials" menu
-  - Set the Access Key on your local development machine
-- Request access to the dev, stg and prd VPNs
+Estas instruções permitirão que você obtenha uma cópia funcional do projeto em sua máquina local para desenvolvimento e testes.
 
-### 🔧 Installation<a id="installation"></a>
+### 📋 Requisitos
 
-- Clone the repository
+- [NodeJS v18.17 / NPM v9.6.7](https://nodejs.org/pt-br/download/)
 
-  - SSH
+> Dica: Utilize [NVM](https://github.com/nvm-sh/nvm) para gerenciar facilmente as versões do node
 
-    ```
-    git clone git@github.com:dadosfera/maestro.git
-    ```
+- Solicite acesso à conta de desenvolvimento do AWS Console para **todos os serviços necessários**
+  - Crie sua Access Key no menu "Security credentials"
+  - Defina a Access Key em sua máquina de desenvolvimento local
+- Solicite acesso às VPNs de stg e prd
 
-    or
+### 🔧 Instalação<a id="installation"></a>
 
-  - HTTPS
-    ```
-    git clone https://github.com/dadosfera/maestro.git
-    ```
+- Clone o repositório
 
-- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
+  ```sh
+  git clone git@github.com:dadosfera/maestro.git
+  ```
+
+- Selecione a versão correta do node (opcional, apenas se estiver usando o [NVM](https://github.com/nvm-sh/nvm)):
 
   ```sh
   nvm use
   ```
 
-- Install the project dependencies:
+- Instale as dependências do projeto:
 
   ```sh
   npm i
   ```
 
-- Setup the following enviroment variables:
+- Configure as seguintes variáveis de ambiente:
 
   ```
   ENV=
@@ -58,42 +63,41 @@ These instructions will allow you to get a working copy of the project on your l
   SM_OAUTH_PATH=
   ```
 
-- Start the server:
+- Inicie o servidor:
 
   ```sh
-  # dev mode
   npm run start:dev
-
-  # or in debug mode
-  npm run start:debug
   ```
 
-  The service should start successfully.
+  > Se preferir, utilize o debbuger do VSCode apertando F5
 
-## 📄 Documentation
+  O serviço deve iniciar com sucesso.
 
-NestJs makes it easy to document each route using decorators on all requests and responses properties. It automatically generates a swagger for given information and provides a route to access it http://localhost:3333/api.
-For more info check the [official documentation](https://docs.nestjs.com/openapi/introduction).
+## 📄 Documentação
 
-### - Multiple documentations
+O NestJs facilita a documentação de cada rota usando decoradores em todas as propriedades de solicitações e respostas. Ele gera automaticamente um swagger para as informações fornecidas e fornece uma rota para acessá-lo em http://localhost:3333/api. Para mais informações, consulte a [documentação oficial](https://docs.nestjs.com/openapi/introduction).
 
-We are currently generating **2 different** documentations: Internal and External.
+### - Documentações múltiplas
 
-All routes that have the decorator `@ApiInternalOnly()` will not be visible on the **External** API swagger.
+Atualmente, estamos gerando **2 documentações diferentes**: Interna e Externa.
 
-- When you start the application with `npm run start` it will serve and generate the swagger JSON of the **External** API
-- When you start the application with `npm run start:internal` it will serve and generate the swagger JSON of the **Internal** API
-- When you run `npm run docs` it will generate the swagger JSON of both **Internal** and **External** API, and save them to `docsfera.json` and `docsfera.external.json` respectively;
+Todas as rotas que têm o decorador `@ApiInternalOnly()` não serão visíveis no swagger da API **Externa**.
 
-It is important to run `npm run docs` before every deploy so we can always have the most updated docs published.
+- Quando você inicia a aplicação com `npm run start:dev`, ela servirá e gerará o JSON do swagger da API **Interna**
 
-## Authentication decorators
+- Quando você executa `npm run docs`, ele gerará o JSON do swagger de ambas as APIs **Interna** e **Externa** e os salvará em `docsfera.json` e `docsfera.external.json`, respectivamente;
 
-Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
+> Link para documentação interna: https://dadosfera.github.io/docsfera
+
+É importante executar `npm run docs` antes de cada implantação para que sempre tenhamos as documentações mais atualizadas publicadas.
+
+## Decoradores de autenticação
+
+O Maestro possui utilitários para facilitar a autenticação do usuário em todos os controladores e rotas. Os seguintes decoradores estão disponíveis:
 
 ### `@Authenticated`
 
-If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
+Se o usuário precisar estar autenticado para fazer uma solicitação, podemos usar o decorador `@Authenticated` no controlador ou rota, conforme necessário.
 
 ```ts
 import { Authenticated } from '../../authentication/authentication.decorator';
@@ -105,7 +109,7 @@ class FooController {
 
   @Get('bar')
   async getBar() {
-    this.logger.info('user is authenticated!');
+    this.logger.info('usuário está autenticado!');
 
     return { authenticated: true };
   }
@@ -122,14 +126,14 @@ class FooController {
   @Get('bar')
   @Authenticated()
   async getBar() {
-    this.logger.info('user is authenticated!');
+    this.logger.info('usuário está autenticado!');
 
     return { authenticated: true };
   }
 
   @Post('bar')
   async postBar() {
-    this.logger.info('user is NOT authenticated!');
+    this.logger.info('usuário NÃO está autenticado!');
 
     return { authenticated: false };
   }
@@ -138,7 +142,7 @@ class FooController {
 
 ### `@RequireAllPermissions`
 
-This decorator requires that **all permissions** listed are granted to the requesting user.
+Este decorador exige que **todas as permissões** listadas sejam concedidas ao usuário solicitante.
 
 ```ts
 import { RequireAllPermissions } from '../../authentication/authentication.decorator';
@@ -158,7 +162,7 @@ class FooController {
 
 ### `@RequireSomePermission`
 
-In the following case, the user is required to have **at least one** listed permission.
+No caso seguinte, é necessário que o usuário tenha **pelo menos uma** das permissões listadas.
 
 ```ts
 import { RequireSomePermission } from '../../authentication/authentication.decorator';
@@ -178,18 +182,18 @@ class FooController {
 
 ### `@AuthenticateCondition`
 
-If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
+Se for necessária uma verificação de autenticação mais complicada, podemos usar o decorador `@AuthenticateCondition` para defini-la. A função personalizada deve retornar `true` para autenticar a solicitação.
 
-In the following example:
+No exemplo a seguir:
 
-- all routes on the `FooController` controller can only be requested from localhost
-- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
+- todas as rotas no controlador `FooController` só podem ser solicitadas a partir do localhost
+- `POST /foo/bar` só pode ser solicitado a partir do localhost **e** por usuários do cliente com id `111...eef`
 
 ```ts
 import { AuthenticateCondition } from '../../authentication/authentication.decorator';
 
 @Controller('foo')
-// allow requests only from localhost
+// permitir solicitações apenas do localhost
 @AuthenticateCondition((request: Request) => request.ip === '::ffff:127.0.0.1')
 class FooController {
   /* ... */
@@ -200,7 +204,7 @@ class FooController {
   }
 
   @Post('bar')
-  // allow requests only from a specific customer
+  // permitir solicitações apenas de um cliente específico
   @AuthenticateCondition(
     (request: Request, user: RequestUser) =>
       user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
@@ -211,11 +215,11 @@ class FooController {
 }
 ```
 
-### Note on authentication decorators
+### Nota sobre decoradores de autenticação
 
-- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
+- O método antigo de autenticação colocava os dados do usuário no campo `request.body.info`, o que impõe certos problemas em relação ao corpo da solicitação, pois esses dados devem vir do frontend sem qualquer modificação pelo Maestro. Agora, esse uso está ⚠️ **DESCONTINUADO** ⚠️. Estamos trabalhando para migrar para o decorador de parâmetro `@User`. O método antigo está funcionando enquanto a migração está em andamento.
 
-- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (_and wouldn't like to..._) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
+- Os decoradores de autenticação podem ser usados juntos e todos eles **devem** passar para que a solicitação seja autenticada, mas, no caso geral, você não precisa (_e não gostaria de..._) usar todos eles juntos, pois você pode codificar toda a lógica de autenticação no decorador `@AuthenticateCondition`.
 
   ```ts
   import {
@@ -251,17 +255,17 @@ class FooController {
   }
   ```
 
-- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
+- Se `@RequireAllPermissions` e `@RequireSomePermission` forem usados com **apenas uma única permissão**, eles apresentam o **exatamente mesmo comportamento**.
 
   ```ts
-  // same behavior
+  // mesmo comportamento
   @RequireAllPermissions(Permissions.BAR.MANAGE)
   @RequireSomePermission(Permissions.BAR.MANAGE)
   ```
 
-## `@User` parameter decorator
+## Decorador de parâmetro `@User`
 
-The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
+Os dados do usuário solicitante podem ser obtidos usando o decorador de parâmetro @User, como no exemplo a seguir:
 
 ```ts
 import { User, RequestUser } from '../../authentication/user.decorator';
@@ -279,7 +283,7 @@ class FooController {
 }
 ```
 
-If the user is required to be logged in, set `required` to `true`, as you would want in the `change-password` operation:
+Se o usuário precisar estar logado, defina `required` como `true`, como por exemplo:
 
 ```ts
 import { User, RequestUser } from '../../authentication/user.decorator';
@@ -305,46 +309,46 @@ class UserController {
 }
 ```
 
-## 📦 Development
+## 📦 Desenvolvimento
 
-### ⌨️ Coding Style
+### ⌨️ Estilo de Codificação
 
-By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
+Por padrão, usamos [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) com configurações padrão.
 
-**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
+**Recomendamos usar o Visual Studio Code e instalar as extensões recomendadas para facilitar o processo de desenvolvimento.**
 
-### Commits pattern
+### Padrão de commits
 
-Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
+Nosso pipeline de fluxo de trabalho segue as especificações de commits convencionais ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) para liberar versões conforme necessário.
 
-Format: `<type>[optional scope]: <description>`
+Formato: `<type>[optional scope]: <description>`
 
-Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
+Exemplo: `FIX: ensure Range headers adhere more closely to RFC 2616`
 
-### Branching naming convention
+### Convenção de nomenclatura de branchs
 
-- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
+- **Feature**: Quaisquer alterações de código para um novo módulo ou caso de uso devem ser feitas em uma branch de feature. Esta branch é criada com base na branch `main`. Quando todas as alterações estiverem concluídas, será necessário um Pull Request/Merge Request para colocar todas essas alterações de volta na branch `main`. Exemplos: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
 
-  **It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (\_) could be used to separate the ID and description.**
+  **Recomenda-se usar todas as letras em minúsculas e hífen (-) para separar palavras, a menos que seja um nome ou ID de item específico. O sublinhado (\_) pode ser usado para separar o ID e a descrição.**
 
-- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
+- **Bug Fix**: Se as alterações de código feitas na branch de feature foram rejeitadas após um lançamento, sprint ou demo, quaisquer correções necessárias após isso devem ser feitas na branch de correção de bug. Exemplos: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
 
-- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
+- **Hot Fix**: Se houver necessidade de corrigir um bloqueador, fazer um patch temporário, aplicar uma mudança crítica de framework ou configuração que deva ser tratada imediatamente, ela deve ser criada como um Hotfix. Exemplos: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
 
-- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
+- **Experimental**: Uma branch para experimentar. Qualquer nova feature ou ideia que não faça parte de um lançamento ou sprint. Exemplo: `experimental/dark-theme-support`.
 
-### Making a Pull Request
+### Fazendo um Pull Request
 
-1. Commit your changes
-2. Open the Pull Request on GitHub
-3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
+1. Comite suas alterações
+2. Abra o Pull Request no GitHub
+3. Envie o link do Pull Request no grupo de chat do Google da Microsfera para revisão e possível aprovação
 
-## 🛠️ Built with
+## 🛠️ Construído com
 
-Some technologies used in this project:
+Algumas tecnologias usadas neste projeto:
 
-- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
+- [NestJS](https://docs.nestjs.com) - Framework para construir aplicações NodeJS eficientes e escaláveis no lado do servidor
 
-## ⚙️ Back-end Architecture
+## ⚙️ Arquitetura de Back-end
 
-The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
+A arquitetura pode ser encontrada neste [link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).

build.docker-compose.dockerhub.yml

@@ -1,5 +1,4 @@
-version: "3.8"
 services:
   maestro:
     build: .
-    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

build.docker-compose.yml

@@ -1,5 +1,4 @@
-version: "3.8"
 services:
   maestro:
     build: .
-    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd:${IMAGE_TAG}

deploy/helm-chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: maestro
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

deploy/helm-chart/templates/deployment.yaml

@@ -0,0 +1,126 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.app_name }}
+  namespace: applications
+  labels:
+    app: {{ .Values.app_name }}
+
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ .Values.app_name }}
+  
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.app_name }}
+    
+    spec:
+      imagePullSecrets:
+        - name: {{ .Values.imagePullSecrets }}
+      nodeSelector:
+        "beta.kubernetes.io/os": linux
+      {{- if .Values.affinity }}
+      affinity:
+        {{- toYaml .Values.affinity | nindent 8 }}
+      {{- end }}
+
+      tolerations:
+      - key: "kubernetes.azure.com/scalesetpriority"
+        operator: "Equal"
+        value: "spot"
+        effect: "NoSchedule"
+
+      containers:
+        - name: maestro
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+          ports:
+            - containerPort: {{ .Values.containerPort }}
+          {{- if .Values.resources }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          {{- end }}
+          env:
+            - name: AWS_IDENTITY_POOL_ID
+              value: {{ .Values.maestro.aws_identity_pool_id }}
+            - name: AWS_REGION
+              value: "us-east-1"
+            - name: BASE_HOST
+              value: "maestro_prd"
+            - name: BUCKET_CUSTOMER_CSV_ASSETS
+              value: {{ .Values.maestro.bucket_customer_csv_assets }}
+            - name: CONNECTORS_INDEX
+              value: "connectors"
+            - name: DUC_URL
+              value: {{ .Values.maestro.duc_url }}
+            - name: ELASTIC_APM_ENVIRONMENT
+              value: {{ .Values.maestro.env }}
+            - name: ELASTIC_APM_SERVER_URL
+              value: "https://apm-server.dadosfera.ai"
+            - name: ELASTIC_APM_SERVICE_NAME
+              value: {{ .Values.maestro.apm_service }}
+            - name: ENV
+              value: {{ .Values.maestro.env }}
+            - name: INFACTORY_URL
+              value: {{ .Values.maestro.in_factory_url }}
+            - name: LOGGER_GELF_HOST
+              value: "logstash-pipelines.dadosfera.ai"
+            - name: LOGGER_GELF_PORT
+              value: "{{ .Values.maestro.logger_gelf_port }}"
+            - name: LOGGER_CONSOLE_EXTRA
+              value: "true"
+            - name: NIMBUS_BASE_URL
+              value: "http://nimbus-api"
+            - name: NPM_TOKEN
+              value: {{ .Values.maestro.npm_token }}
+            - name: PB_TOKEN_PATH
+              value: {{ .Values.maestro.pb_token_path }}
+            - name: PIFACTORY_URL
+              value: {{ .Values.maestro.pi_factory_url }}
+            - name: SM_OAUTH_PATH
+              value: {{ .Values.maestro.sm_oauth_path }}
+            - name: TRFACTORY_URL
+              value: {{ .Values.maestro.tr_factory_url }}
+            - name: UPLOAD_FILE_AGENT_CONNECTION
+              value: {{ .Values.maestro.upload_file_agent_connection }}
+            - name: OPEN_CUSTOMER_ID
+              value: {{ .Values.maestro.open_customer_id }}
+            - name: OPEN_GROUP_ID
+              value: {{ .Values.maestro.open_group_id }}
+            - name: DEDICATED_PROXY
+              value: {{ .Values.maestro.dedicated_proxy }}
+            - name: COOKIE_SECRET
+              value: {{ .Values.maestro.cookie_secret }}
+            - name: REDIS_DATABASE
+              value: "{{ .Values.maestro.redis_database }}"
+            - name: REDIS_HOST
+              value: {{ .Values.maestro.redis_host }}
+            - name: REDIS_PORT
+              value: "{{ .Values.maestro.redis_port }}"
+            - name: JWT_PRIVATE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: prd-duc
+                  key: jwt_token
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: prd-{{ .Values.app_name }}
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: prd-{{ .Values.app_name }}
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: AWS_DEFAULT_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: prd-{{ .Values.app_name }}
+                  key: AWS_DEFAULT_REGION

deploy/helm-chart/templates/ingress-private.yaml

@@ -0,0 +1,41 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip 
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/server-snippet: |
+      underscores_in_headers on;
+      ignore_invalid_headers on;
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
+    nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
+    nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
+    {{- if .Values.maestro.restricted_ip}}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
+    {{- end }}
+      
+  generation: 1
+  labels:
+    app: {{ .Values.app_name }}
+  {{- if .Values.maestro.dedicated_proxy}}
+  name: open-data-{{ .Values.app_name }}
+  {{- else }}
+  name: open-data
+  {{- end }}
+  namespace: applications
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{ .Values.hostname }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: {{ .Values.app_name }}
+            port:
+              number: {{ .Values.ingress.port }}
+        path: /open-data/sharing-ocean-data
+        pathType: Prefix

deploy/helm-chart/templates/ingress.yaml

@@ -0,0 +1,36 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/server-snippet: |
+      underscores_in_headers on;
+      ignore_invalid_headers on;
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
+    nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
+    nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
+    {{- if .Values.maestro.restricted_ip}}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
+    {{- end }}
+      
+  generation: 1
+  labels:
+    app: {{ .Values.app_name }}
+  name: {{ .Values.app_name }}
+  namespace: applications
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{ .Values.hostname }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: {{ .Values.app_name }}
+            port:
+              number: {{ .Values.ingress.port }}
+        path: /
+        pathType: Prefix

deploy/helm-chart/templates/secret.yaml

@@ -0,0 +1,40 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: prd-{{ .Values.app_name }}
+  namespace: applications
+  labels:
+    app: {{ .Values.app_name }}
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: secretsmanager-prd
+    kind: SecretStore
+  target:
+    name: prd-{{ .Values.app_name }}
+    creationPolicy: Owner
+  data:
+  - secretKey: AWS_ACCESS_KEY_ID
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_ACCESS_KEY_ID
+
+  - secretKey: AWS_SECRET_ACCESS_KEY
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_SECRET_ACCESS_KEY
+
+  - secretKey: AWS_DEFAULT_REGION
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_DEFAULT_REGION
+
+  - secretKey: jwt_token
+    remoteRef:
+      key: {{ .Values.maestro.jwt_token_secret_id }}
+      version: "AWSCURRENT"
+      property: token
+

deploy/helm-chart/templates/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.app_name }}
+  namespace: applications
+  labels:
+    app: {{ .Values.app_name }}
+
+spec:
+  type: ClusterIP
+  ports:
+  - name: {{ .Values.app_name }}
+    protocol: TCP
+    port: {{ .Values.service.port }}
+    targetPort: {{ .Values.service.targetPort }}
+  selector:
+    app: {{ .Values.app_name }}
+

deploy/helm-chart/values-stg.yaml

@@ -0,0 +1,16 @@
+maestro:
+  env: stg
+  duc_url: duc.stg.dadosfera.ai
+  pi_factory_url: pi-factory.stg.dadosfera.ai
+  in_factory_url: in-factory.stg.dadosfera.ai
+  tr_factory_url: in-factory.stg.dadosfera.ai
+  open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+  open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
+  cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
+  redis_database: "1"
+
+hostname: maestro.stg.dadosfera.ai
+
+replicaCount: 1
+
+affinity: null

deploy/helm-chart/values.yaml

@@ -0,0 +1,67 @@
+# Default values for metabase.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 3
+hostname: maestro.dadosfera.ai
+image:
+  repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: 1.56.0
+app_name: maestro
+containerPort: 3333
+imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
+service:
+  type: ClusterIP
+  port: 3333
+  targetPort: 3333
+ingress:
+  enabled: false
+  port: 3333
+resources:
+  requests:
+    cpu: 100m
+    memory: 1500Mi
+  limits:
+    cpu: 2000m
+    memory: 2Gi
+maestro:
+  aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
+  duc_url: duc.dadosfera.ai
+  in_factory_url: in-factory.dadosfera.ai
+  bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153"
+  env: prd
+  apm_service: maestro
+  jwt_token_secret_id: prd/root/jwt_token
+  logger_gelf_port: "1026"
+  npm_token: npm_Xp17h3daMkORcZ55NY95Ez4gRfdzsQ3UvINP
+  pb_token_path: prd/root/productboard_token
+  pi_factory_url: pi-factory.dadosfera.ai
+  sm_oauth_path: prd/root/oauth_applications
+  tr_factory_url: in-factory.dadosfera.ai
+  upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
+  open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
+  open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
+  dedicated_proxy: ""
+  restricted_ip: ""
+  redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
+  redis_port: "6379"
+  redis_database: "0"
+  cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  targetMemoryUtilizationPercentage: 80
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: name
+              operator: In
+              values:
+                - product

deploy/helmfiles/prd.yaml

@@ -0,0 +1,56 @@
+releases:
+  - name: maestro
+    chart: ../helm-chart
+    values:
+    - ../helm-chart/values.yaml
+    set:
+      - name: app_name
+        value: maestro
+      - name: maestro.duc_url
+        value: duc.dadosfera.ai
+      - name: hostname
+        value: maestro.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
+      - name: replicaCount
+        value: 2
+  
+  - name: unimed-maestro
+    chart: ../helm-chart
+    values:
+    - ../helm-chart/values.yaml
+    set:
+      - name: app_name
+        value: maestro-unimed
+      - name: maestro.duc_url
+        value: duc.dadosfera.ai
+      - name: hostname
+        value: maestro-unimed.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
+      # Customer id
+      - name: maestro.dedicated_proxy
+        value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
+      - name: replicaCount
+        value: 1
+        # 10.70.0.0/16 internal network
+        # 137.131.167.254/32 loadbalancer
+        # 159.112.184.81/32 cluster ip for the uptime request ingest
+      - name: maestro.restricted_ip
+        value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"

deploy/helmfiles/stg.yaml

@@ -0,0 +1,30 @@
+charts:
+  - name: maestro
+    chart: ../helm-chart
+    values:
+    - ../helm-chart/values.yaml
+    - ../helm-chart/values-stg.yaml
+      
+  
+    # Environment to test Network Policies
+  - name: private-maestro
+    chart: ../helm-chart
+    values:
+    - ../helm-chart/values.yaml
+    - ../helm-chart/values-stg.yaml
+    set:
+      - name: app_name
+        value: maestro-private
+      - name: hostname
+        value: private-maestro.stg.dadosfera.ai
+      # Customer id
+      - name: maestro.dedicated_proxy
+        value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
+      - name: replicaCount
+        value: 1
+      - name: affinity
+        value: null
+      - name: resources
+        value: null
+      - name: maestro.restricted_ip
+        value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"

deprecated/docker-compose.dockerhub.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   maestro:
     image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

deprecated/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   maestro:
     image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}

docsfera.external.json

@@ -133,7 +133,32 @@
         },
         "get": {
           "operationId": "ConnectionController_getAllConnections",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -490,6 +515,45 @@
           ]
         }
       },
+      "/pipelinesV2/monitoring-dashboard": {
+        "get": {
+          "operationId": "PipelinesController_getMonitoringDashboard",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/pipelinesV2": {
         "post": {
           "operationId": "PipelinesController_create",
@@ -1736,6 +1800,118 @@
           ]
         }
       },
+      "/customers/{id}/links": {
+        "get": {
+          "operationId": "CustomersController_getCustomerLinks",
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/CustomerLinksResponse"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        },
+        "put": {
+          "operationId": "CustomersController_setCustomerLinks",
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/CustomerLinkRequest"
+                }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/customers/token": {
+        "get": {
+          "operationId": "CustomersController_getCustomerToken",
+          "parameters": [
+            {
+              "name": "exp",
+              "required": true,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "text/plain": {
+                  "schema": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/health": {
         "get": {
           "operationId": "HealthController_check",
@@ -1752,7 +1928,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/generate-token",
+      "title": "Maestro - feat/new-customer-monitoring",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
@@ -2033,6 +2209,9 @@
         "ConnectionToCatalogDto": {
           "type": "object",
           "properties": {
+            "id": {
+              "type": "string"
+            },
             "name": {
               "type": "string"
             },
@@ -3124,6 +3303,56 @@
             "updated_at",
             "updated_by"
           ]
+        },
+        "CustomerLink": {
+          "type": "object",
+          "properties": {
+            "href": {
+              "type": "string"
+            },
+            "name": {
+              "type": "string"
+            },
+            "description": {
+              "type": "string"
+            },
+            "iconSrc": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "href",
+            "name",
+            "description"
+          ]
+        },
+        "CustomerLinksResponse": {
+          "type": "object",
+          "properties": {
+            "links": {
+              "type": "array",
+              "items": {
+                "$ref": "#/components/schemas/CustomerLink"
+              }
+            }
+          },
+          "required": [
+            "links"
+          ]
+        },
+        "CustomerLinkRequest": {
+          "type": "object",
+          "properties": {
+            "links": {
+              "type": "array",
+              "items": {
+                "$ref": "#/components/schemas/CustomerLink"
+              }
+            }
+          },
+          "required": [
+            "links"
+          ]
         }
       }
     }

environment.d.ts

@@ -12,6 +12,10 @@ declare global {
       INTERNAL_SWAGGER: 'true' | 'false';
 
       AWS_REGION: string;
+      OPEN_GROUP_ID: string;
+      OPEN_CUSTOMER_ID: string;
+      DEDICATED_PROXY: string;
+      COOKIE_SECRET: string;
     }
   }
 }

nest-cli.json

@@ -4,6 +4,7 @@
   "compilerOptions": {
     "assets": [
       "**/*.proto",
+      "assets/**/*",
       {
         "include": "i18n/**/*",
         "watchAssets": true

package.json

@@ -6,10 +6,10 @@
   "private": true,
   "license": "UNLICENSED",
   "engines": {
-    "node": "18.10.0"
+    "node": "18.17"
   },
   "scripts": {
-    "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
+    "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
     "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
     "prebuild": "rimraf dist",
     "build": "nest build",
@@ -20,80 +20,91 @@
     "start:prod": "node dist/main",
     "docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
-    "test": "jest",
+    "test": "jest --detectOpenHandles --forceExit",
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
-    "@aws-sdk/client-secrets-manager": "^3.112.0",
+    "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.31.0",
-    "@grpc/grpc-js": "^1.6.7",
-    "@grpc/proto-loader": "^0.6.13",
-    "@nestjs/cli": "^9.4.2",
-    "@nestjs/common": "^9.4.0",
-    "@nestjs/config": "^2.3.1",
-    "@nestjs/core": "^9.4.0",
+    "@dadosfera/protospack": "2.5.3",
+    "@dadosfera/protospack-v2": "3.38.0-beta.14",
+    "@grpc/grpc-js": "^1.9.3",
+    "@grpc/proto-loader": "^0.7.9",
+    "@nestjs/cli": "^9.5.0",
+    "@nestjs/common": "^9.4.3",
+    "@nestjs/config": "^2.3.4",
+    "@nestjs/core": "^9.4.3",
     "@nestjs/mapped-types": "^1.2.2",
-    "@nestjs/microservices": "^9.4.0",
+    "@nestjs/microservices": "^9.4.3",
     "@nestjs/passport": "^9.0.3",
-    "@nestjs/platform-express": "^9.4.0",
-    "@nestjs/schematics": "^9.1.0",
+    "@nestjs/platform-express": "^9.4.3",
+    "@nestjs/schematics": "^9.2.0",
     "@nestjs/swagger": "^6.3.0",
-    "@nestjs/testing": "^9.4.0",
+    "@nestjs/testing": "^9.4.3",
     "axios": "^0.27.2",
+    "cache-manager": "^5.1.4",
+    "cache-manager-ioredis-yet": "^1.1.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
-    "cron-parser": "^4.4.0",
+    "cookie-parser": "^1.4.7",
+    "cron-parser": "^4.9.0",
+    "csv": "^6.3.11",
     "dotenv": "^14.3.2",
-    "elastic-apm-node": "^3.36.0",
-    "helmet": "^5.1.0",
-    "jsonwebtoken": "^9.0.0",
+    "elastic-apm-node": "^3.50.0",
+    "handlebars": "^4.7.8",
+    "helmet": "^5.1.1",
+    "jsonwebtoken": "^9.0.2",
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
     "ms": "^3.0.0-canary.1",
+    "openid-client": "^5.7.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
     "passport-forcedotcom": "^0.2.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-hubspot-oauth2": "^1.0.3",
     "passport-mailchimp": "^1.1.0",
-    "protospack": "2.5.2",
+    "puppeteer": "^24.7.2",
+    "redis": "^4.5.1",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",
-    "swagger-ui-express": "^4.4.0"
+    "swagger-ui-express": "^4.6.3"
   },
   "overrides": {
     "multer": "1.4.5-lts.1"
   },
   "devDependencies": {
-    "@types/express": "^4.17.13",
+    "@types/cache-manager": "^4.0.6",
+    "@types/cookie-parser": "^1.4.9",
+    "@types/express": "^4.17.17",
+    "@types/express-session": "^1.18.1",
     "@types/jest": "27.0.2",
-    "@types/jsonwebtoken": "^8.5.8",
+    "@types/jsonwebtoken": "^8.5.9",
     "@types/jwk-to-pem": "^2.0.1",
-    "@types/multer": "^1.4.7",
-    "@types/node": "^16.11.41",
+    "@types/multer": "^1.4.12",
+    "@types/node": "^16.18.52",
     "@types/passport-facebook": "^2.1.11",
     "@types/passport-google-oauth20": "^2.0.11",
-    "@types/passport-oauth2": "^1.4.11",
+    "@types/passport-oauth2": "^1.4.12",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.29.0",
-    "@typescript-eslint/parser": "^5.29.0",
-    "eslint": "^8.18.0",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-prettier": "^4.0.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "eslint": "^8.49.0",
+    "eslint-config-prettier": "^8.10.0",
+    "eslint-plugin-prettier": "^4.2.1",
     "jest": "^27.5.1",
-    "nock": "^13.2.7",
-    "prettier": "^2.7.1",
+    "nock": "^13.3.3",
+    "prettier": "^2.8.8",
     "source-map-support": "^0.5.20",
-    "supertest": "^6.2.3",
+    "supertest": "^6.3.3",
     "ts-jest": "^27.1.5",
-    "ts-loader": "^9.3.1",
-    "ts-node": "^10.8.1",
-    "tsconfig-paths": "^3.14.1",
-    "typescript": "^4.6.3"
+    "ts-loader": "^9.4.4",
+    "ts-node": "^10.9.1",
+    "tsconfig-paths": "^3.14.2",
+    "typescript": "^4.9.5"
   }
 }

src/app.module.ts

@@ -26,9 +26,15 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
 import { ProductboardModule } from './modules/productboard/productboard.module';
 import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
 import { CustomersModule } from './modules/customers/customers.module';
+import { OpenDataModule } from './modules/open-data/open-data.module';
+import { ThemeModule } from './modules/theme/theme.module';
+import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
+import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
+import { AssignModule } from './modules/assign/assign.module';
+import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
+import { ApiKeyModule } from './modules/api-key/api-key.module';
 
 @Module({
-  controllers: [],
   providers: [
     DadosferaLogger,
     {
@@ -58,6 +64,15 @@ import { CustomersModule } from './modules/customers/customers.module';
     ProductboardModule,
     MixpanelModule,
     CustomersModule,
+    OpenDataModule,
+    ThemeModule,
+    NetworkPolicyModule,
+    AssignModule,
+    ShareMetadataModule,
+    NetworkPolicyModule,
+    ApiKeyModule,
+    IdentityProviderModule,
+    NetworkPolicyModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
   ],

src/assets/template/pii-reporter.html

@@ -0,0 +1,130 @@
+<!DOCTYPE html>
+<html lang="pt-br">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Dadosfera Relatório de PII</title>
+    <link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
+    <style>
+        @page {
+            size: A4 landscape; /* Alterado para paisagem (landscape) */
+            margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
+        }
+        body {
+            font-family: 'Quicksand', sans-serif;
+            color: #5c5c5c;
+            margin: 0;
+            padding: 10px;
+            font-size: 12px; /* Reduzindo o tamanho da fonte */
+        }
+        .container {
+            margin: 0;
+            width: 100%;
+        }
+        .header {
+            display: flex;
+            align-items: center;
+            margin-bottom: 20px;
+        }
+        .logo {
+            max-width: 150px; /* Reduzida para economizar espaço */
+            height: auto;
+        }
+        h1 {
+            color: #0d003b;
+            font-weight: 700;
+            margin-left: 20px;
+            font-size: 24px; /* Tamanho ajustado */
+        }
+        table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 15px;
+            table-layout: fixed; /* Importante: define larguras fixas */
+            box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+            border: 1px solid #d0d0d0;
+        }
+        th {
+            background-color: #1700a2;
+            color: white;
+            font-weight: bold;
+            text-align: left;
+            padding: 8px 10px;
+            border: 1px solid #3a26b8;
+            font-size: 11px; /* Tamanho ajustado */
+            word-wrap: break-word; /* Permite quebra de palavras */
+            overflow-wrap: break-word;
+        }
+        td {
+            padding: 6px 10px;
+            border: 1px solid #d0d0d0;
+            font-size: 11px; /* Tamanho ajustado */
+            word-wrap: break-word; /* Permite quebra de palavras */
+            overflow-wrap: break-word;
+        }
+        /* Definindo larguras específicas para cada coluna */
+        th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
+        th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
+        th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
+        th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
+        th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
+        th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
+        
+        tr:nth-child(even) {
+            background-color: #f9f9f9;
+        }
+        tr:nth-child(odd) {
+            background-color: white;
+        }
+        .info-section {
+            margin-top: 20px;
+            color: #5c5c5c;
+        }
+        .timestamp {
+            font-style: italic;
+            text-align: right;
+            margin-top: 15px;
+            font-size: 0.9em;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
+            <h1>Relatório de PII</h1>
+        </div>
+
+        <div class="info-section">
+            <p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
+        </div>
+
+        <table>
+            <thead>
+                <tr>
+                    <th>Database</th>
+                    <th>Schema</th>
+                    <th>Tabela</th>
+                    <th>Coluna</th>
+                    <th>Tipo de Dado</th>
+                    <th>Regras PII</th>
+                </tr>
+            </thead>
+            <tbody>
+                {{#each dados}}
+                <tr>
+                    <td>{{database_name}}</td>
+                    <td>{{table_schema}}</td>
+                    <td>{{table_name}}</td>
+                    <td>{{column_name}}</td>
+                    <td>{{data_type}}</td>
+                    <td>{{pii_rules}}</td>
+                </tr>
+                {{/each}}
+            </tbody>
+        </table>
+
+        <p class="timestamp">Gerado em: {{dataGeracao}}</p>
+    </div>
+</body>
+</html>

src/authentication/authentication.guard.spec.ts

@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
 import { AuthClientService } from '../modules/auth/auth.service';
 
 import ErrorCodes from '../utils/errorCodes';
+import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 const logger = {
   info: (...args) => args,
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
     customer_name: 'dadosfera',
     customer_tier: 'BASIC',
+    customer_modules: []
   };
 
   beforeAll(async () => {
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
           provide: APP_GUARD,
           useClass: AuthenticationGuard,
         },
+        {
+          provide: ApiKeyService,
+          useValue: {
+            get: () => Promise.resolve(null)
+          }
+        }
       ],
       controllers: [NoClassAuthController, ClassAuthConditionController],
     }).compile();
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
   NoClassAuthTest(null, null);
   ClassAuthConditionTest(null, null);
 
-  const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
-  NoClassAuthTest(tokenZ, ['zendesk']);
-  ClassAuthConditionTest(tokenZ, ['zendesk']);
+  // const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
+  // NoClassAuthTest(tokenZ, ['zendesk']);
+  // ClassAuthConditionTest(tokenZ, ['zendesk']);
 
-  const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
-  NoClassAuthTest(tokenM, ['metabase']);
-  ClassAuthConditionTest(tokenM, ['metabase']);
+  // const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
+  // NoClassAuthTest(tokenM, ['metabase']);
+  // ClassAuthConditionTest(tokenM, ['metabase']);
 
-  const tokenZM = CreateToken([
-    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
-  ]);
-  NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
-  ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
+  // const tokenZM = CreateToken([
+  //   PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
+  //   PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
+  // ]);
+  // NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
+  // ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
 });

src/authentication/authentication.guard.ts

@@ -4,6 +4,7 @@ import {
   OnApplicationBootstrap,
   ExecutionContext,
   Inject,
+  ForbiddenException,
 } from '@nestjs/common';
 import { Reflector } from '@nestjs/core';
 import assert from 'assert';
@@ -17,6 +18,7 @@ import {
 import { RequestUser } from '../decorators/user.decorator';
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';
+import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 @Injectable()
 export class AuthenticationGuard
@@ -31,6 +33,7 @@ export class AuthenticationGuard
     dadosferaLogger: DadosferaLogger,
     private reflector: Reflector,
     private authClient: AuthClientService,
+    private apiKeyService: ApiKeyService
   ) {
     this.pems = new Map();
     this.logger = dadosferaLogger.logger;
@@ -48,20 +51,40 @@ export class AuthenticationGuard
     });
   }
 
-  canActivate(ctx: ExecutionContext): boolean {
+  async canActivate(ctx: ExecutionContext): Promise<boolean> {
     const authFunctions = this.reflector.getAllAndMerge<
       AuthenticationFunction[]
     >(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
     const mustBeAuthenticated = authFunctions.length > 0;
 
-    const request = ctx.switchToHttp().getRequest();
-    const accessToken = this.validateToken(request, mustBeAuthenticated);
-
     if (!mustBeAuthenticated) {
       // no need to be authenticated
       return true;
     }
 
+    const request = ctx.switchToHttp().getRequest();
+    const apiKey = request.get('X-api-key');
+    if (apiKey) {
+      const {
+        api_key
+      } = await this.apiKeyService.get(apiKey);
+
+      request.user = {
+        user_id: api_key.user_id,
+        username: api_key.username,
+        permissions: api_key.permissions,
+        customer_id: api_key.customer_id,
+        customer_name: api_key.customer_name,
+        customer_tier: api_key.customer_tier,
+        customer_modules: api_key.customer_modules,
+        access_token: apiKey,
+      };
+
+      return true;
+    }
+
+    const accessToken = this.validateToken(request, mustBeAuthenticated);
+
     if (!accessToken) {
       // couldn't load valid token
       throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
@@ -113,6 +136,18 @@ export class AuthenticationGuard
       return false;
     }
 
+    // Bloquear outros customer de usar o maestor dedicado
+    const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
+    if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
+      throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
+    }
+
+    // Bloquear o customer de acesso o maestro publico
+    const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
+    if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
+      throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
+    }
+
     request.accessTokenPayload = accessTokenPayload;
     request.user = {
       user_id: accessTokenPayload.user_id,
@@ -121,6 +156,7 @@ export class AuthenticationGuard
       customer_id: accessTokenPayload.customer_id,
       customer_name: accessTokenPayload.customer_name,
       customer_tier: accessTokenPayload.customer_tier,
+      customer_modules: accessTokenPayload.customer_modules,
       access_token: accessToken,
     };
     // TODO: for backwards compatibility. remove in the future

src/authentication/extract-user.ts

@@ -0,0 +1,20 @@
+import jwt, { JwtPayload } from 'jsonwebtoken';
+
+export function extractUserFrom(aRawJwt: string) {
+  const decodedToken = jwt.decode(aRawJwt, {
+    complete: true,
+  });
+
+  const payload = decodedToken.payload as JwtPayload;
+
+  return {
+    user_id: payload.user_id,
+    username: payload.username,
+    permissions: payload.permissions,
+    customer_id: payload.customer_id,
+    customer_name: payload.customer_name,
+    customer_tier: payload.customer_tier,
+    customer_modules: payload.customer_modules,
+    access_token: aRawJwt,
+  }
+}

src/authentication/permissions.enum.ts

@@ -67,7 +67,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   PIPELINE: {
     title: {
       'pt-br': 'Coletar | Pipelines',
@@ -117,7 +116,44 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
+  IMPORT_FILES: {
+    title: {
+      'pt-br': 'Coletar | Importar arquivos',
+      'en-us': 'Collect | Import files',
+      'es-es': 'Colecta | Importar archivos',
+    },
+    permissions: {
+      VIEW: {
+        seqid: 48,
+        claim: 'import-file:view',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Importar arquivos',
+          'en-us': 'Import files',
+          'es-es': 'Importar archivos',
+        },
+      },
+    },
+  },
+   AI_CHAT: {
+    title: {
+      'pt-br': 'AutodriveDDF',
+      'en-us': 'AutodriveDDF',
+      'es-es': 'AutodriveDDF',
+    },
+    permissions: {
+      VIEW: {
+        seqid: 49,
+        claim: 'ai-chat:view',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'AutodriveDDF',
+          'en-us': 'AutodriveDDF',
+          'es-es': 'AutodriveDDF',
+        },
+      },
+    },
+  },
   CONNECTION: {
     title: {
       'pt-br': 'Coletar | Fontes de dados',
@@ -157,7 +193,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   NETWORK_CONFIG: {
     title: {
       'pt-br': 'Coletar | Redes',
@@ -187,7 +222,6 @@ export const PERMISSIONS_GROUPS = {
       // },
     },
   },
-
   OUTPUT: {
     title: {
       'pt-br': 'Output',
@@ -237,7 +271,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   TRANSFORMATIONS: {
     title: {
       'pt-br': 'Micro-transformações',
@@ -287,7 +320,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   CATALOG: {
     title: {
       'pt-br': 'Explorar | Catálogo',
@@ -346,16 +378,6 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
         },
       },
-      EMBED_ANALYTICS: {
-        seqid: 44,
-        claim: 'catalog:embed',
-        usage: PermissionUsages.INTERNAL,
-        name: {
-          'pt-br': 'Acessar Módulo de Incorporação de Ativos',
-          'en-us': 'Access Embedding analytics Module',
-          'es-es': 'Acceder al Módulo de Incorporación de Activos',
-        },
-      },
       TRIGGER_CATALOG_TASK: {
         seqid: 45,
         claim: 'catalog:trigger-task',
@@ -368,7 +390,44 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
+  LINEAGE: {
+    title: {
+      'pt-br': 'Explorar | Linhagem',
+      'en-us': 'Explore | Lineage',
+      'es-es': 'Explorar | Linaje',
+    },
+    permissions: {
+      VIEW: {
+        seqid: 50,
+        claim: 'lineage:view',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar ao módulo de Linhagem',
+          'en-us': 'Access to Lineage module',
+          'es-es': 'Acceda al módulo de Linaje',
+        },
+      }
+    },
+  },
+  EMBED: {
+    title: {
+      'pt-br': 'Analisar | Incorporação',
+      'en-us': 'Analyze | Embedding',
+      'es-es': 'Analizar | Incorporación',
+    },
+    permissions: {
+      EMBED_ANALYTICS: {
+        seqid: 44,
+        claim: 'catalog:embed',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Módulo de Incorporação de Ativos',
+          'en-us': 'Access Embedding analytics Module',
+          'es-es': 'Acceder al Módulo de Incorporación de Activos',
+        },
+      },
+    }
+  },
   CONNECTORS: {
     title: {
       'pt-br': 'Conectores',
@@ -418,7 +477,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   SNOWFLAKE: {
     title: {
       'pt-br': 'Explorar | Consolidar',
@@ -438,7 +496,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   ZENDESK: {
     title: {
       'pt-br': 'Zendesk',
@@ -458,7 +515,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   DATAVIZ: {
     title: {
       'pt-br': 'Analisar | Visualização',
@@ -497,7 +553,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   MODULES: {
     title: {
       'pt-br': 'Módulos da Dadosfera',
@@ -594,6 +649,25 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
+  CUSTOMER: {
+    title: {
+      'pt-br': 'Organização',
+      'en-us': 'Organization',
+      'es-es': 'Organización',
+    },
+    permissions: {
+      MONITORING_DASHBOARD: {
+        seqid: 47,
+        claim: 'customer:monitoring-dashboard',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Ver o dashboard de monitoramento',
+          'en-us': 'View the monitoring dashboard',
+          'es-es': 'Ver el dashboard de monitoreo',
+        },
+      },
+    },
+  },
 };
 export interface DadosferaModule {
   name: string;
@@ -601,6 +675,16 @@ export interface DadosferaModule {
   key: string;
   permissionSeqId: number;
 }
+
+export const DADOSFERA_MODULES_KEYS = {
+  LOG_DASHBOARD: 'logs-dashboard',
+  ACCESS_DASHBOARD: 'access-dashboard',
+  DANGER_ZONE: 'danger-zone',
+  PII: 'pii',
+  EMBED: 'embedded-analytics',
+  EMBED_ASSIGNED: 'embed-assigned',
+}
+
 export const DADOSFERA_MODULES: Array<DadosferaModule> = [
   {
     name: 'Intelligence Module',

src/decorators/authentication.decorator.ts

@@ -25,6 +25,14 @@ export function RequireSomePermission(
   );
 }
 
+export function RequireModule(
+  key: string
+) {
+  return createAuthenticatedDecorator((_, user: RequestUser) =>
+    user.customer_modules.some(module => module === key),
+  );
+}
+
 export function AuthenticateCondition(func: AuthenticationFunction) {
   return createAuthenticatedDecorator(func);
 }

src/decorators/user.decorator.spec.ts

@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
 import { AuthClientService } from '../modules/auth/auth.service';
 import ErrorCodes from '../utils/errorCodes';
 import { User } from './user.decorator';
+import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 const logger = {
   info: (...args) => args,
@@ -52,6 +53,7 @@ describe('user.decorator', () => {
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
     customer_name: 'dadosfera',
     customer_tier: 'BASIC',
+    customer_modules: [],
     access_token: '',
   };
 
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
           provide: APP_GUARD,
           useClass: AuthenticationGuard,
         },
+        {
+          provide: ApiKeyService,
+          useValue: {
+            get: () => Promise.resolve(null)
+          }
+        }
       ],
       controllers: [UserController],
     }).compile();
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
 
   const token = CreateToken();
   fakeUserPayload.access_token = token;
-  UserTest(token);
+  // UserTest(token);
 });

src/decorators/user.decorator.ts

@@ -11,6 +11,7 @@ export interface RequestUser {
   customer_name: string;
   customer_tier: string;
   access_token: string;
+  customer_modules: string[];
 }
 
 export const User: (options?: { required?: boolean }) => ParameterDecorator =

src/main.ts

@@ -3,11 +3,14 @@ import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import helmet from 'helmet';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { json, urlencoded } from 'express';
 
 import { AppModule } from './app.module';
 import { writeFileSync } from 'fs';
 import { execSync } from 'child_process';
 import { INestApplication } from '@nestjs/common';
+import cookieParser from 'cookie-parser';
+
 async function bootstrap() {
   DadosferaLogger.setupLogger({
     serviceName: 'maestro',
@@ -15,16 +18,43 @@ async function bootstrap() {
   });
   const logger = new DadosferaLogger();
 
+  const corsOrigins = [];
+
+  if (process.env.ENV === 'local') {
+    corsOrigins.push('http://localhost:4200');
+  } else {
+    corsOrigins.push( 
+      'https://app.stg.dadosfera.ai',
+      'https://app.dadosfera.ai',
+      'https://private-frontend.stg.dadosfera.ai',
+      'https://unimed.dadosfera.ai',
+      'https://boston-scientific.dadosfera.ai',
+      'https://plataforma.dadosfera.ai'
+    );
+  }
+
   const app = await NestFactory.create(AppModule, {
     logger,
     cors: {
-      origin: '*',
+      origin: corsOrigins,
       methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
       preflightContinue: false,
       optionsSuccessStatus: 204,
+      credentials: true,
     },
   });
+
   app.use(helmet());
+  app.use(cookieParser(process.env.COOKIE_SECRET));
+
+  if (process.env.ENV !== 'local') {
+    app.use('/catalog/register-dataset', json({ limit: '10mb' }));
+    app.use(
+      '/catalog/register-dataset',
+      urlencoded({ extended: true, limit: '10mb' }),
+    );
+  }
+
   configureSwagger(app);
   await app.listen(3333);
   if (process.env.KILL_AFTER_START) await app.close();

src/modules/api-key/api-key.controller.ts

@@ -0,0 +1,68 @@
+import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
+import { ApiKeyService } from './api-key.service';
+import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+@Controller('api-key')
+@Authenticated()
+@ApiTags('ApiKey')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
+@UseFilters(new GrpcToHttpExceptionFilter())
+export class ApiKeyController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private readonly apiKeyService: ApiKeyService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post()
+  @ApiResponse({ type: CreateApiKeyResponseDto })
+  async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
+    this.logger.info('POST /api-key', {
+      permissions: createApiKeyDto.permissions,
+      method: 'create'
+    });
+    const result = await this.apiKeyService.create(createApiKeyDto, user);
+    this.logger.info('POST /api-key success', { 
+      id: result.id,
+      method: 'create'
+    });
+    return result;
+  }
+
+  @Get()
+  @ApiResponse({ type: [ApiKeyBaseResponseDto] })
+  async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
+    this.logger.info('GET /api-key', {
+      method: 'findAll'
+    });
+    const result = await this.apiKeyService.findAll(user);
+    this.logger.info('GET /api-key success', { 
+      count: result.length,
+      method: 'findAll'
+    });
+    return result;
+  }
+
+  @Delete(':id')
+  async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
+    this.logger.info('DELETE /api-key/:id', { 
+      id,
+      method: 'remove'
+    });
+    await this.apiKeyService.remove(id, user);
+    this.logger.info('DELETE /api-key/:id success', { 
+      id,
+      method: 'remove'
+    });
+  }
+}

src/modules/api-key/api-key.module.ts

@@ -0,0 +1,18 @@
+import { Module } from '@nestjs/common';
+import { ApiKeyService } from './api-key.service';
+import { ApiKeyController } from './api-key.controller';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+const ducClient = new DucClient();
+
+@Module({
+  imports: [
+    ClientsModule.register([ducClient.providerOptions])
+  ],
+  controllers: [ApiKeyController],
+  providers: [ApiKeyService, DadosferaLogger],
+  exports: [ApiKeyService]
+})
+export class ApiKeyModule {}

src/modules/api-key/api-key.service.ts

@@ -0,0 +1,50 @@
+import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
+import { RequestUser } from 'src/decorators/user.decorator';
+import { DucClient } from '../duc/client.config';
+import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { lastValueFrom } from 'rxjs';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+
+@Injectable()
+export class ApiKeyService implements OnModuleInit {
+  private apiKeyService: ApiKeyWriteProtoService;
+
+  constructor(
+    @Inject(DucClient.name) private readonly client: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
+  }
+
+  create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
+    const metadata = PackTheMetadata(user);
+
+    return lastValueFrom(this.apiKeyService.CreateApiKey({
+      permissions: createApiKeyDto.permissions
+    }, metadata));
+  }
+
+  async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
+    const metadata = PackTheMetadata(user);
+    console.log(metadata)
+
+    const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
+    return data.api_keys;
+  }
+
+  async remove(id: string, user: RequestUser) {
+    const metadata = PackTheMetadata(user);
+
+    await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
+  }
+
+  async get(key: string) {
+    const metadata = PackTheMetadata({});
+
+    return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
+  }
+}

src/modules/api-key/dto/api-key.dto.ts

@@ -0,0 +1,39 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsArray, IsNumber } from 'class-validator';
+
+export class PermissionDto {
+  @ApiProperty({ type: Number })
+  id: number;
+
+  @ApiProperty({ type: String })
+  name: string;
+}
+
+export class ApiKeyBaseResponseDto {
+  @ApiProperty({ type: String, format: 'uuid' })
+  id: string;
+
+  @ApiProperty({ type: String })
+  key_mask: string;
+
+  @ApiProperty({ type: [PermissionDto] })
+  permissions: PermissionDto[];
+
+  @ApiProperty({ type: String, format: 'date-time' })
+  created_at: string;
+
+  @ApiProperty({ type: String })
+  created_by: string;
+}
+
+export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
+  @ApiProperty({ type: String })
+  key: string;
+}
+
+export class CreateApiKeyDto {
+  @ApiProperty({ type: [Number], description: 'Array of permission IDs' })
+  @IsArray()
+  @IsNumber({}, { each: true })
+  permissions: number[];
+} 

src/modules/assign/assign.controller.ts

@@ -0,0 +1,37 @@
+import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
+import { AssignService } from './assign.service';
+import { CreateAssignDto } from './dto/create-assign.dto';
+import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+
+@Controller('assign')
+@Authenticated()
+export class AssignController {
+  constructor(private readonly assignService: AssignService) {}
+
+  @Put('/public-key')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.USERS.permissions.ADMIN
+  )
+  @RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
+  create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
+    const metadata = PackTheMetadata(user);
+    return this.assignService.create(createAssignDto, metadata);
+  }
+
+  @Get('/public-key')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.USERS.permissions.ADMIN
+  )
+  @RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
+  async get(@User() user: RequestUser) {
+    const metadata = PackTheMetadata(user);
+    try {
+      return await this.assignService.get(metadata);
+    } catch (error) {
+      throw new NotFoundException(error.message)
+    }
+  }
+}

src/modules/assign/assign.module.ts

@@ -0,0 +1,15 @@
+import { Module } from '@nestjs/common';
+import { AssignService } from './assign.service';
+import { AssignController } from './assign.controller';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+
+const client = new DucClient();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [AssignController],
+  providers: [AssignService, DadosferaLogger]
+})
+export class AssignModule {}

src/modules/assign/assign.service.ts

@@ -0,0 +1,38 @@
+import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
+import { CreateAssignDto } from './dto/create-assign.dto';
+import { Metadata } from '@grpc/grpc-js';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { ClientGrpc } from '@nestjs/microservices';
+import { DucClient } from 'src/modules/duc/client.config';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+
+@Injectable()
+export class AssignService implements OnModuleInit {
+
+  ducService: AssingProtoService;
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.ducService =this.grpcClient.getService<AssingProtoService>(
+      ProtoServices.AssingProtoService,
+    );
+  }
+
+  async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
+    const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
+    return data;
+  }
+
+  async get(metadata: Metadata) {
+    return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
+  }
+}

src/modules/assign/dto/create-assign.dto.ts

@@ -0,0 +1,3 @@
+export class CreateAssignDto {
+    publicKey: string;
+}

src/modules/auth/auth.controller.ts

@@ -12,6 +12,7 @@ import {
   Redirect,
   Req,
   Param,
+  Res,
 } from '@nestjs/common';
 import {
   ApiHeaders,
@@ -26,7 +27,7 @@ import {
   AuthConfirmResetPasswordRequest,
   AuthEnableTotpMfaRequest,
   AuthDisableTotpMfaRequest,
-  AuthVerifyTotpMfaRequest,
+  AuthVerifyTotpMfaRequest
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -43,16 +44,23 @@ import {
   AuthRefreshAccessTokenRes,
   AuthSignInReq,
   AuthSignInRes,
+  BulkEditRequest,
 } from './dtos/login';
-import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 import { AuthGuard } from '@nestjs/passport';
-import { Request } from 'express';
+import { Request, Response } from 'express';
 import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
-import jwt from 'jsonwebtoken';
+import jwt, { JwtPayload } from 'jsonwebtoken';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
+type CookiesValues = {
+  accessToken?: string;
+  refreshToken?: string;
+  userId?: string
+}
+
 @ApiTags('Auth')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
@@ -85,10 +93,66 @@ export class AuthController {
   async signIn(
     @Body() { username, password, totp }: AuthSignInReq,
     @Language() language: LanguageEnum,
-  ): Promise<AuthSignInRes> {
-    this.logger.info('/auth - SignIn');
-    const metadata = PackTheMetadata({ language });
-    return this.authClient.signIn({ username, password, totp }, metadata);
+    @Res() res: Response,
+  ) {
+    try {
+      this.logger.info('/auth - SignIn');
+      const metadata = PackTheMetadata({ language });
+      this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
+      const data = await this.authClient.signIn({ username, password, totp }, metadata);
+
+      if (data.tokens) {
+        this.addTokenInCookie(res, {
+          accessToken: data.tokens.accessToken,
+          refreshToken: data.tokens.refreshToken,
+          userId: data.user.id
+        });
+      }
+
+
+      return res.send(data);
+    } catch (error) {
+      this.logger.error('/auth - SignIn - ERROR', error);
+      throw error;
+    }
+
+  }
+
+  @Post('sign-out')
+  @HttpCode(HttpStatus.NO_CONTENT)
+  async signOut(
+    @Language() language: LanguageEnum,
+    @Res() res: Response,
+  ) {
+    try {
+      this.logger.info('/auth - SignOut');
+      const exp = 1000 * 60 * 3;
+
+      res.cookie('ddf-auth', '', {
+        domain: 'dadosfera.local',
+        maxAge: Date.now() - exp,
+        expires: new Date(),
+        httpOnly: true,
+        secure: true,
+        sameSite: 'none', // Necessário para cookies em requisições cross-site
+      });
+
+      res.cookie('ddf-refresh-auth', '', {
+        domain: 'dadosfera.local',
+        maxAge: Date.now() - exp,
+        expires: new Date(),
+        httpOnly: true,
+        secure: true,
+        sameSite: 'none', // Necessário para cookies em requisições cross-site
+      });
+
+      this.logger.info('Clean cookie sessions');
+
+      return res.send();
+    } catch (error) {
+      this.logger.error('/auth - SignIn - ERROR', error);
+    }
+
   }
 
   @Post('refresh-access-token')
@@ -97,16 +161,26 @@ export class AuthController {
   async refreshAccessToken(
     @Body() body: AuthRefreshAccessTokenReq,
     @Language() language: LanguageEnum,
+    @Headers('origin') origin: string,
+    @Res() res: Response,
   ) {
     this.logger.info('/auth - RefreshAccessToken');
-    const { refreshToken, customerName: customer_name } = body;
+    const frontHost = origin.replace(/^https?:\/\//, '');
+    const { refreshToken, userId } = body;
 
     const metadata = PackTheMetadata({
-      customer_name,
       language,
+      custom_host: frontHost,
     });
 
-    return this.authClient.refreshAccessToken({ refreshToken }, metadata);
+    const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
+
+    this.addTokenInCookie(res, {
+      accessToken: data.accessToken,
+      userId
+    });
+
+    return res.send(data);
   }
 
   @ApiInternalOnlyEndpoint()
@@ -177,16 +251,23 @@ export class AuthController {
   @ApiInternalOnlyEndpoint()
   @Post('confirm-reset-password')
   @HttpCode(HttpStatus.OK)
-  async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
+  async confirmResetPassword(
+    @Body() body: AuthConfirmResetPasswordRequest,
+    @Headers('origin') origin: string,
+  ) {
     this.logger.info('/auth - confirm-reset-password');
-
+    const frontHost = origin.replace(/^https?:\/\//, '');
+    const metadata = PackTheMetadata({ custom_host: frontHost });
     const { username, code, newPassword } = body;
 
-    return this.authClient.confirmResetPassword({
-      username,
-      code,
-      newPassword,
-    });
+    return this.authClient.confirmResetPassword(
+      {
+        username,
+        code,
+        newPassword,
+      },
+      metadata,
+    );
   }
 
   @ApiInternalOnlyEndpoint()
@@ -344,4 +425,152 @@ export class AuthController {
 
     return { token, email, url, language };
   }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('users/block')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async blockUsers(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Body() body: BulkEditRequest,
+  ) {
+    this.logger.info('blockUsers - Starting request');
+
+    try {
+      const metadata = PackTheMetadata(user);
+
+      this.logger.debug('Calling blockUsers service', {
+        metadata: {
+          access_token: metadata.get('access_token'),
+          language: metadata.get('language'),
+        },
+      });
+
+      const result = await this.authClient.blockUsers(body.users, metadata);
+      this.logger.info('blockUsers - Success', { result });
+      return result;
+    } catch (error) {
+      this.logger.error('blockUsers - Error', {
+        error: error.message,
+        stack: error.stack,
+      });
+      throw error;
+    }
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('users/unblock')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async unblockUsers(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Body() body: BulkEditRequest,
+  ) {
+    this.logger.info('unblockUsers');
+    const metadata = PackTheMetadata(user);
+
+    return this.authClient.unblockUsers(body.users, metadata);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('users/reset')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async resetUsers(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Body() body: BulkEditRequest,
+  ) {
+    this.logger.info('resetUsers');
+    const metadata = PackTheMetadata(user);
+
+    return this.authClient.resetUsers(body.users, metadata);
+  }
+
+  @Get('me')
+  async getMe(@Req() req: Request, @Res() res: Response) {
+    this.logger.info('GET /auth/me ')
+    // Lê cookies
+    const accessToken = req.cookies['ddf-auth'];
+    const userId = req.cookies['ddf-user-id'];
+
+    this.logger.info('Has cookie: ' + Boolean(accessToken))
+    try {
+      // Decodifica e valida o JWT de acesso
+      const userDto = await this.authClient.extractUserFrom(accessToken);
+      return res.status(200).json(userDto);
+    } catch (err) {
+      this.logger.error(err.message);
+      const refreshToken = req.cookies['ddf-refresh-auth'];
+
+      this.logger.info('Token is invalid')
+      this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
+      // Se access token inválido, tenta refresh
+      if (!refreshToken || !userId) {
+        this.logger.error('Invalid refresh token or customer name');
+        return res.status(401).json({ error: 'Not authenticated' });
+      }
+      try {
+        // Chama refreshAccessToken
+        const metadata = PackTheMetadata({
+        });
+        this.logger.info('Call Refresh Token')
+        const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
+        this.logger.info('Finish Refresh Token')
+        // Retorna novo access token e dados mínimos
+        this.addTokenInCookie(res, {
+          accessToken: data.accessToken,
+          userId
+        });
+        // Decodifica novo token
+        const userDto = await this.authClient.extractUserFrom(accessToken);
+        return res.status(200).json(userDto);
+      } catch (refreshErr) {
+        this.logger.error(refreshErr)
+        return res.status(401).json({ error: 'Not authenticated' });
+      }
+    }
+  }
+
+  private addTokenInCookie(res: Response, data: CookiesValues) {
+    let exp = 1000 * 60 * 5; // 5 minutes
+
+    if (data.accessToken) {
+      const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
+      exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
+
+      this.logger.info('Set Cookie ddf-auth')
+      res.cookie('ddf-auth', data.accessToken, {
+        domain: '.dadosfera.ai',
+        maxAge: exp,
+        httpOnly: true,
+        secure: true,
+        sameSite: 'none', // Necessário para cookies em requisições cross-site
+      });
+    }
+
+    if (data.refreshToken) {
+      this.logger.info('Set Cookie ddf-refresh-auth')
+      res.cookie('ddf-refresh-auth', data.refreshToken, {
+        domain: '.dadosfera.ai',
+        maxAge: exp,
+        httpOnly: true,
+        secure: true,
+        sameSite: 'none', // Necessário para cookies em requisições cross-site
+      });
+    }
+
+    if (data.userId) {
+      this.logger.info('Set Cookie ddf-refresh-auth')
+      res.cookie('ddf-user-id', data.userId, {
+        domain: '.dadosfera.ai',
+        maxAge: exp,
+        httpOnly: true,
+        secure: true,
+        sameSite: 'none', // Necessário para cookies em requisições cross-site
+      });
+    }
+  }
 }

src/modules/auth/auth.service.ts

@@ -1,10 +1,10 @@
-import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
+import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { lastValueFrom } from 'rxjs';
 
 import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService, UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
 import {
   AuthSnowflakeSignInRequest,
   AuthSignInRequest,
@@ -17,15 +17,22 @@ import {
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
   AuthConfirmResetPasswordRequest,
+  AuthSignInResponse,
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { DucClient } from '../duc/client.config';
 import { Metadata } from '@grpc/grpc-js';
+import { BulkEditResponse, UserDTO } from './dtos/login';
+import jwt from 'jsonwebtoken';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 
 @Injectable()
 export class AuthClientService implements OnModuleInit {
+
   logger: DadosferaLogger;
 
   private authService: AuthServiceInterface;
+  private userService: UsersProtoService;
+
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
@@ -38,6 +45,10 @@ export class AuthClientService implements OnModuleInit {
     this.authService = this.grpcClient.getService<AuthServiceInterface>(
       ProtoServices.AuthProtoService,
     );
+
+    this.userService = this.grpcClient.getService<UsersProtoService>(
+      ProtoServices.UsersProtoService,
+    );
   }
 
   async getPublicKeys() {
@@ -52,25 +63,59 @@ export class AuthClientService implements OnModuleInit {
     return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
   }
 
+  checkDedicatedProxy({
+    customer
+  }: AuthSignInResponse) {
+    const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
+    this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
+    this.logger.info('Customer ID: ' + customer.id);
+
+    if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
+      throw new ForbiddenException();
+    }
+
+    // Bloquear o customer de acesso o maestro publico
+    this.logger.info('Check if customer have network policy: ' + customer.modules);
+    const hasNetworkPolicyModule = customer.modules.includes('network-policy');
+    if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
+      throw new ForbiddenException();
+    }
+  }
+
   async signIn(
     { username, password, totp }: AuthSignInRequest,
     metadata: Metadata,
   ) {
     this.logger.info('SignIn');
 
-    return lastValueFrom(
-      this.authService.AuthSignIn({ username, password, totp }, metadata),
-    );
+    let result: AuthSignInResponse;
+
+    try {
+      result = await lastValueFrom(
+        this.authService.AuthSignIn({ username, password, totp }, metadata),
+      );
+
+    } catch (error) {
+      this.logger.error('SignIn - Error during sign-in');
+      this.logger.error(error);
+      throw error;
+    }
+
+    if (result.customer) {
+      this.checkDedicatedProxy(result);
+    }
+
+    return result
   }
 
   async refreshAccessToken(
-    { refreshToken }: AuthRefreshAccessTokenRequest,
+    { refreshToken, userId }: AuthRefreshAccessTokenRequest,
     metadata: Metadata,
   ) {
     this.logger.info('RefreshAccessToken');
 
     return lastValueFrom(
-      this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
+      this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
     );
   }
 
@@ -112,19 +157,21 @@ export class AuthClientService implements OnModuleInit {
     );
   }
 
-  async confirmResetPassword({
-    username,
-    code,
-    newPassword,
-  }: AuthConfirmResetPasswordRequest) {
+  async confirmResetPassword(
+    { username, code, newPassword }: AuthConfirmResetPasswordRequest,
+    metadata: Metadata,
+  ) {
     this.logger.info('confirmResetPassword');
 
     return lastValueFrom(
-      this.authService.AuthConfirmResetPassword({
-        username,
-        code,
-        newPassword,
-      }),
+      this.authService.AuthConfirmResetPassword(
+        {
+          username,
+          code,
+          newPassword,
+        },
+        metadata,
+      ),
     );
   }
 
@@ -164,6 +211,123 @@ export class AuthClientService implements OnModuleInit {
 
   async oauthSignIn(data: { username: string; token: string }) {
     const { username, token } = data;
-    return lastValueFrom(this.authService.AuthOauthSignIn({ token, username }));
+    return lastValueFrom(
+      this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
+    );
   }
+
+  async blockUsers(
+    users: string[],
+    metadata: Metadata,
+  ): Promise<BulkEditResponse> {
+    this.logger.info('blockUsers - Service starting');
+
+    try {
+      this.logger.debug('Calling BlockUser gRPC method', {
+        metadata: {
+          access_token: metadata.get('access_token'),
+          language: metadata.get('language'),
+        },
+      });
+
+      const response = await lastValueFrom<BulkEditResponse>(
+        this.authService.BlockUser({ users }, metadata),
+      );
+
+      this.logger.info('blockUsers - Service success', { response });
+      return response;
+    } catch (error) {
+      this.logger.error('blockUsers - Service error', {
+        error: error.message,
+        stack: error.stack,
+      });
+      throw error;
+    }
+  }
+
+  async unblockUsers(
+    users: string[],
+    metadata: Metadata,
+  ): Promise<BulkEditResponse> {
+    this.logger.info('unblockUsers');
+    return await lastValueFrom(
+      this.authService.UnblockUser({ users }, metadata),
+    );
+  }
+
+  async resetUsers(
+    users: string[],
+    metadata: Metadata,
+  ): Promise<BulkEditResponse> {
+    this.logger.info('resetUsers');
+
+    try {
+      this.logger.debug('Calling ResetUser gRPC method', {
+        metadata: {
+          access_token: metadata.get('access_token'),
+          language: metadata.get('language'),
+        },
+      });
+
+      const response = await lastValueFrom<BulkEditResponse>(
+        this.authService.ResetUser({ users }, metadata),
+      );
+
+      this.logger.info('resetUsers - Success', { response });
+      return response;
+    } catch (error) {
+      this.logger.error('resetUsers - Error', {
+        error: error.message,
+        stack: error.stack,
+      });
+      throw error;
+    }
+  }
+
+  private async getUser(id: string, metadata: Metadata) {
+    this.logger.info('getUser');
+    return await lastValueFrom(
+      this.userService.UserFindOneById({ id }, metadata),
+    );
+  }
+
+  public async extractUserFrom(token: string) {
+      const decoded: any = token && jwt.decode(token, { complete: true });
+      if (!decoded) throw new Error('Invalid token');
+
+      const { kid } = decoded.header;
+      // Busca a chave pública
+      const { keys } = await this.getPublicKeys();
+      const pemValue = keys.find((k) => k.kid === kid)?.pem;
+      if (!pemValue)
+          throw new Error('Public key not found');
+      jwt.verify(token, pemValue);
+
+      const payload = decoded.payload;
+      const metadata = PackTheMetadata({
+        customer_id: payload.customer_id
+      })
+
+      const {
+        user
+      } = await this.getUser(
+        payload.user_id,
+        metadata
+      );
+
+      const userDto: UserDTO = {
+          id: user.id,
+          name: user.username,
+          jobTitle: user?.jobTitle || null,
+          department: user?.department || null,
+          hierarchy: user?.hierarchy || null,
+          customer: {
+            id: payload.customer_id,
+            name: payload.customer_name,
+            tier: payload.customer_tier,
+          }
+        };
+
+      return userDto;
+    }
 }

src/modules/auth/dtos/login.ts

@@ -67,16 +67,28 @@ export class AuthUser {
 export class AuthCustomer {
   @ApiProperty()
   modules: string[];
+
   @ApiProperty()
   id: string;
+
   @ApiProperty()
   name: string;
+
+  @ApiProperty()
+  displayName: string;
   @ApiProperty()
   tier: string;
+
   @ApiProperty()
   scheduleLimit: string;
+
   @ApiProperty()
   links: Link[];
+
+  @ApiProperty()
+  themeEnabled: boolean;
+  @ApiProperty()
+  enforceMfa: boolean;
 }
 
 export class AuthSignInReq implements AuthSignInRequest {
@@ -110,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
   @ApiProperty()
   refreshToken: string;
   @ApiProperty()
-  customerName: string;
+  userId: string;
 }
 export class AuthRefreshAccessTokenRes {
   @ApiProperty()
@@ -118,3 +130,26 @@ export class AuthRefreshAccessTokenRes {
   @ApiProperty()
   accessToken: string;
 }
+
+export interface BulkEditRequest {
+  users: string[];
+}
+
+export interface BulkEditResponse {
+  message: string;
+  successfulUsers: string[];
+  failedUsers: string[];
+}
+
+export type UserDTO = {
+  id: string,
+  name: string,
+  jobTitle?: string,
+  department?: string,
+  hierarchy?: string,
+  customer: {
+    id: string,
+    name: string,
+    tier: string,
+  }
+}

src/modules/auth/passport-strategies/google-strategy.ts

@@ -26,6 +26,7 @@ export class GoogleLoginStrategy extends PassportStrategy(
       callbackURL: oauthSecrets['google-login'].redirect_uri,
       scope: ['email', 'profile', 'openid'],
     };
+    console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
     const verify = (
       accessToken: string,
       refreshToken: string,

src/modules/catalog/catalog-client.ts

@@ -5,8 +5,11 @@ import {
 } from '@nestjs/microservices';
 import { credentials } from '@grpc/grpc-js';
 import { Catalog } from '@dadosfera/protospack-v2';
+import { PlatformInterfaces } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  process.env.PIFACTORY_URL.includes('0.0.0.0');
 
 export class CatalogClientConfiguration {
   public name = 'CatalogClientConfiguration';
@@ -17,11 +20,13 @@ export class CatalogClientConfiguration {
       package: [
         Catalog.ProtoPackages.ReadPackage,
         Catalog.ProtoPackages.WritePackage,
+        PlatformInterfaces.ProtoPackages.WritePackage
       ],
       credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         Catalog.ProtoPaths.ReadFilePath,
         Catalog.ProtoPaths.WriteFilePath,
+        PlatformInterfaces.ProtoPaths.WriteFilePath
       ],
       loader: {
         keepCase: true,

src/modules/catalog/catalog.controller.ts

@@ -13,6 +13,9 @@ import {
   Put,
   Query,
   UseFilters,
+  HttpException,
+  HttpStatus,
+  Res,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -23,14 +26,16 @@ import {
 import {
   Authenticated,
   RequireAllPermissions,
+  RequireModule,
   RequireSomePermission,
 } from '../../decorators/authentication.decorator';
-import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import {
+  BatchRemoveRlsRulesRequest,
   GetDatasetCatalogTaskRes,
   ICatalogAllRequest,
   ICatalogAllResponse,
@@ -49,6 +54,14 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
 import { Language } from 'src/decorators/language.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+import {
+  AddRlsRuleRequest,
+  GetNimbusDashboardsRequest,
+  GetRlsRulesRequest,
+  RegisterDatasetWithMetatadaRequest,
+} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+import { Response } from 'express';
+import { TypeParser } from 'src/utils/FileParser/parser-types';
 
 @ApiTags('Catalog')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -104,6 +117,52 @@ export class CatalogController {
     return res;
   }
 
+  @Get('/download')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async dowloadAsserts(
+    @User() user: RequestUser,
+    @Query() query: ICatalogAllRequest,
+    @Res() res: Response
+  ) {
+    const { user_id, customer_name, customer_id, username, permissions } = user;
+    this.logger.info(`/catalog/download - searchCatalog`, {
+      user_id,
+      customer_name,
+    });
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+
+    const roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+      roles,
+      is_data_manager,
+    });
+
+    const {
+      file,
+      filename
+    } = await this.catalogService.downloadAssets(
+      query,
+      metadata,
+      customer_id,
+    );
+
+    res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+    res.setHeader('Content-Type', 'text/csv');
+
+    res.end(file);
+  }
+
   @ApiInternalOnlyEndpoint()
   @Get('data-asset')
   async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
@@ -334,7 +393,7 @@ export class CatalogController {
     @Language() language: LanguageEnum,
     @Param('id') id: string,
   ): Promise<IPreviewResponse> {
-    const { customer_name, customer_id, user_id, username } = user;
+    const { customer_name, customer_id, user_id, username, customer_modules } = user;
 
     this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
       user_id,
@@ -347,6 +406,7 @@ export class CatalogController {
       user_id,
       username,
       language,
+      is_mask: customer_modules.some(mod => mod === 'pii')
     });
 
     const preview = await this.catalogService.getDatasetPreview(id, metadata);
@@ -633,4 +693,242 @@ export class CatalogController {
 
     return response;
   }
-}
+
+  @Post('rls-rule')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.addRlsRule(body, metadata);
+
+    return response;
+  }
+
+  @Get('rls-rule/:id')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  async getOneRlsRule(@Param('id') id: string, @User() user: RequestUser) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const idInt = parseInt(id);
+    const response = await this.catalogService.getOneRlsRule(idInt, metadata);
+
+    return response;
+  }
+
+  @Get('rls-rule')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  async getRlsRules(
+    @User() user: RequestUser,
+    @Query() query: GetRlsRulesRequest,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.getRlsRules(query, metadata);
+
+    return response;
+  }
+
+  @Delete('rls-rule/:id')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  async removeRlsRule(@Param('id') id: string, @User() user: RequestUser) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const idInt = parseInt(id);
+    const response = await this.catalogService.removeRlsRule(idInt, metadata);
+
+    return response;
+  }
+
+  @Delete('rls-rule')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  async batchRemoveRlsRules(
+    @Query() query: BatchRemoveRlsRulesRequest,
+    @User() user: RequestUser,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    await this.catalogService.batchRemoveRlsRule(query, metadata);
+
+    return { message: 'OK' };
+  }
+
+  @Get('nimbus-dashboards')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  async getNimbusDashboards(
+    @User() user: RequestUser,
+    @Body() body: GetNimbusDashboardsRequest,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const dashboards = await this.catalogService.getNimbusDashboards(
+      body,
+      metadata,
+    );
+
+    return JSON.parse(dashboards);
+  }
+
+  @Post('register-dataset')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async registerDatasetWithMetadataRequest(
+    @Body() body: RegisterDatasetWithMetatadaRequest,
+    @User() user: RequestUser,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const logMetadata = {
+      customer_name: customer_name,
+      user_id: user_id,
+      method: 'POST',
+      path: '/catalog/register-dataset',
+    };
+    try {
+      const metadata = PackTheMetadata({
+        customer_id,
+        customer_name,
+        user_id,
+        username,
+      });
+
+      this.logger.log(
+        `Request from user ${user_id} for customer ${customer_name}`,
+        logMetadata,
+      );
+
+      // Create table metadata
+      const tableMetadataBody = {
+        table_metadata: body.table_metadata,
+        info: {
+          customer: customer_name,
+        },
+        logMetadata: logMetadata,
+      };
+
+      const table_metadata_id = await this.catalogService.createTableMetadata(
+        tableMetadataBody,
+      );
+      this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
+
+      // Create column metadata
+      const columnMetadataBody = {
+        column_metadata: body.column_metadata,
+        info: {
+          customer: customer_name,
+        },
+      };
+      this.logger.info(
+        `Creating column metadata for table ${table_metadata_id}`,
+        logMetadata,
+      );
+      const column_metadata_ids =
+        await this.catalogService.createColumnMetadata(columnMetadataBody);
+
+      // Create data preview
+      const dataPreviewBody = {
+        data_preview: body.data_preview,
+        info: {
+          customer: customer_name,
+        },
+      };
+      this.logger.debug(
+        `Creating data preview for table ${table_metadata_id}`,
+        logMetadata,
+      );
+      const data_preview_id = await this.catalogService.createDataPreview(
+        dataPreviewBody,
+      );
+
+      // Catalog dataset item
+      this.logger.info(
+        `Cataloging dataset item for table ${table_metadata_id}`,
+        logMetadata,
+      );
+      await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
+      this.logger.info(
+        `Dataset registration completed successfully for table ${table_metadata_id}`,
+        logMetadata,
+      );
+      return {
+        message: 'Dataset registered successfully',
+        table_metadata_id: table_metadata_id,
+        column_metadata_ids: column_metadata_ids,
+        data_preview_id: data_preview_id,
+      };
+    } catch (error) {
+      this.logger.error(
+        `Failed to register dataset. The following error occurred: ${error.response.data}`,
+        logMetadata,
+      );
+
+      throw new HttpException(
+        {
+          message: 'Ocorreu um erro ao registrar o dataset',
+          error: error.message,
+          code: 'REGISTRATION_FAILED',
+          details: error.message,
+        },
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
+  }
+
+  @Get('pii-reporter')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.USERS.permissions.ADMIN
+  )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.PII
+  )
+  async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
+    this.logger.info('GET pii-reporter');
+    const metadata = PackTheMetadata(user);
+
+    try {
+      const {
+        file,
+        filename,
+        type
+      } = await this.catalogService.getPiiReporter(metadata, contentType);
+      res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+      res.setHeader('Content-Type', type);
+
+      // use res.end to send buffer
+      return res.end(file);
+    } catch (error) {
+      console.error(error)
+      this.logger.error(error.message);
+    }
+  }
+}

src/modules/catalog/catalog.module.ts

@@ -3,12 +3,15 @@ import { Module } from '@nestjs/common';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import { CatalogController } from './catalog.controller';
-import { CatalogService } from './catalog.service';
 import { CatalogClientConfiguration } from './catalog-client';
 import { ClientsModule } from '@nestjs/microservices';
 import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { UsersModule } from '../users/users.module';
 import { RolesModule } from '../roles/roles.module';
+import { CustomersModule } from '../customers/customers.module';
+import { ShareModule } from './share/share.module';
+import { CatalogService } from './catalog.service';
+import { MixpanelModule } from '../mixpanel/mixpanel.module';
 
 const client = new CatalogClientConfiguration();
 
@@ -18,6 +21,8 @@ const client = new CatalogClientConfiguration();
     OldPipelineModule,
     UsersModule,
     RolesModule,
+    CustomersModule,
+    ShareModule,
   ],
   controllers: [CatalogController],
   providers: [CatalogService, DadosferaLogger],

src/modules/catalog/catalog.service.ts

@@ -6,6 +6,12 @@ import {
   Messages,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog';
 import {
+  Messages as PlatformInterfaceMessages,
+  WriteService as PlatformInterfaceWriteService,
+  ProtoServices as PlatformInterfacesProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
+import {
+  BadRequestException,
   HttpException,
   HttpStatus,
   Inject,
@@ -18,11 +24,25 @@ import { CatalogClientConfiguration } from './catalog-client';
 import { UsersService } from '../users/users.service';
 import { RolesService } from '../roles/roles.service';
 import { Metadata } from '@grpc/grpc-js';
-import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
+import {
+  AssetReporter,
+  BatchRemoveRlsRulesRequest,
+  IUpdateDataRequest,
+  TriggerCatalogReq,
+} from './dtos';
+import {
+  AddRlsRuleRequest,
+  GetNimbusDashboardsRequest,
+  GetRlsRulesRequest,
+  PiiMetadata,
+} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+import { TypeParser } from 'src/utils/FileParser/parser-types';
+import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
 
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
+  platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
   logger: any;
   constructor(
     @Inject(DadosferaLogger)
@@ -44,9 +64,14 @@ class CatalogService implements OnModuleInit {
       this.grpcClient.getService<WriteService.CatalogWriteServices>(
         ProtoServices.CatalogWriteServices,
       );
+    this.platformWriteService =
+      this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
+        PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
+      );
   }
 
   _getNimbusUrl(body) {
+    this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
     if (process.env.ENV === 'prd') {
@@ -59,6 +84,42 @@ class CatalogService implements OnModuleInit {
     )}.dadosfera.ai`;
   }
 
+  async getPiiReporter(metadata: Metadata, type: TypeParser) {
+    this.logger.info('getPiiReporter: ' + type)
+    try {
+      const {
+        data
+      } = await lastValueFrom(
+        this.catalogWriteService.GetPiiReporter({}, metadata)
+      )
+      this.logger.info("Finish grpc call")
+
+      const parser = ParserBuilder.build<PiiMetadata>(type);
+
+      this.logger.info('parser file to: ' + type)
+      const file = await parser.parse(data)
+      this.logger.info('finish parser')
+      const mimeTypes: Record<TypeParser, string> = {
+        'csv': 'text/csv',
+        'html': 'text/html',
+        'pdf': 'application/pdf'
+      }
+
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const filename = `relatorio-pii-${timestamp}.${type}`;
+
+      return {
+        file,
+        filename: filename,
+        type: mimeTypes[type]
+      }
+    } catch (error) {
+      this.logger.error(error.message);
+      throw error;
+    }
+
+  }
+
   async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
     if (!data.embed) data.embed = undefined;
@@ -136,9 +197,11 @@ class CatalogService implements OnModuleInit {
   async getUserRolesIds(userId: string) {
     const result = await this.userService.findOneById(userId).catch(() => null);
 
-    const roles_ids = result.user.roles.map((role) => role.id);
+    if (result) {
+      return result.user.roles.map((role) => role.id);
+    }
 
-    return roles_ids;
+    return [];
   }
 
   async searchDataAssets(
@@ -174,6 +237,34 @@ class CatalogService implements OnModuleInit {
     return { data_assets: response, total };
   }
 
+  async downloadAssets(
+    query: Record<string, any>,
+    metadata: Metadata,
+    customer_id: string,
+  ) {
+    const data = await this.searchDataAssets(query, metadata, customer_id);
+
+    const formatData = data.data_assets.map(asset => ({
+      id: asset.id,
+      display_name: asset.display_name,
+      data_asset_type: asset.data_asset_type,
+      created_at: asset.created_at,
+      tags: '[' + asset.tags.join(', ') + ']'
+    }))
+
+    const parser = ParserBuilder.build<AssetReporter>('csv');
+
+    const file = await parser.parse(formatData);
+
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const filename = `dadosfera_assets_${timestamp}.csv`;
+
+    return {
+      file,
+      filename
+    }
+  }
+
   async getOneDataAsset(data: {
     id: string;
     customer_id: string;
@@ -250,7 +341,7 @@ class CatalogService implements OnModuleInit {
     const { documentation } = await lastValueFrom(
       this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
     );
-    console.log(documentation);
+ 
     const docs = JSON.parse(documentation);
     return docs;
   }
@@ -317,11 +408,14 @@ class CatalogService implements OnModuleInit {
 
       const roles = [];
       const users = [];
-      for (const role_id of data_asset.roles) {
+      const data_asset_roles = data_asset?.roles || []
+      for (const role_id of data_asset_roles) {
         const role = customer_roles.find((r) => r.id === role_id);
         if (role) roles.push({ id: role.id, name: role.name });
       }
-      for (const user_id of data_asset.users) {
+
+      const data_asset_users = data_asset?.users || []
+      for (const user_id of data_asset_users) {
         const user = customer_users.find((r) => r.id === user_id);
         if (user) users.push({ id: user.id, username: user.username });
       }
@@ -333,6 +427,7 @@ class CatalogService implements OnModuleInit {
       } as typeof data_asset;
     });
   }
+
   async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
     const { session } = await lastValueFrom(
       this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -345,6 +440,176 @@ class CatalogService implements OnModuleInit {
     );
     return res;
   }
+
+  async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogWriteService.AddRlsRule(data, metadata),
+    );
+    return res;
+  }
+
+  async removeRlsRule(id: number, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogWriteService.RemoveRlsRule({ id }, metadata),
+    );
+    return res;
+  }
+
+  async batchRemoveRlsRule(
+    query: BatchRemoveRlsRulesRequest,
+    metadata: Metadata,
+  ) {
+    const { id_rls, nimbus_dashboard_id } = query;
+
+    if (id_rls && nimbus_dashboard_id) {
+      throw new BadRequestException(
+        "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
+      );
+    }
+    if (id_rls) {
+      await lastValueFrom(
+        this.catalogWriteService.RemoveRlsRulesByRlsId({ id_rls }, metadata),
+      );
+    } else if (nimbus_dashboard_id) {
+      await lastValueFrom(
+        this.catalogWriteService.RemoveRlsRulesByDashboardId(
+          { nimbus_dashboard_id: parseInt(nimbus_dashboard_id) },
+          metadata,
+        ),
+      );
+    }
+    return 'OK';
+  }
+
+  async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetRlsRules(data, metadata),
+    );
+    return res.rls_rules;
+  }
+
+  async getOneRlsRule(id: number, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetOneRlsRule({ id }, metadata),
+    );
+    return res.rls_rule;
+  }
+
+  async getNimbusDashboards(
+    data: GetNimbusDashboardsRequest,
+    metadata: Metadata,
+  ) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetNimbusDashboards(data, metadata),
+    );
+    return res.dashboards;
+  }
+
+  async createTableMetadata(body: any): Promise<number> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+
+    const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
+
+    this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
+
+    try {
+      const { data, status } = await axios.post(endpoint, {...body.table_metadata});
+
+      this.logger.info(
+        `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.id;
+    } catch (error) {
+      this.logger.error(
+        `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
+          error.response?.status
+        } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async createColumnMetadata(body: any): Promise<number[]> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
+    const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
+
+ 
+
+    try {
+      this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
+      this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+      this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
+      const { data, status } = await axios.post(endpoint, body.column_metadata);
+
+      this.logger.info(
+        `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.map((column) => column.id);
+    } catch (error) {
+      this.logger.error(
+        `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
+          error.response?.status
+        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async createDataPreview(body: any): Promise<number> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+    const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
+
+    this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
+
+    try {
+      const { data, status } = await axios.post(endpoint, body.data_preview);
+
+      this.logger.info(
+        `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.id;
+    } catch (error) {
+      this.logger.error(
+        `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
+          error.response?.status
+        } because of ${error.response?.data || error.message}`,
+        {...body.logMetadata},
+      );
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
+    const customer_name_raw = metadata.get('customer_name');
+
+    const customer_name = customer_name_raw?.[0]?.toString();
+    if (!customer_name) {
+      throw new BadRequestException('Customer name not found in metadata');
+    }
+    const res = await lastValueFrom(
+      this.platformWriteService.CatalogDataAssets(
+        {
+          data_assets: [
+            {
+              data_asset_id: table_metadata_id.toString(),
+              customer_name: customer_name,
+              data_asset_type: 'dataset',
+            },
+          ],
+        },
+        metadata,
+      ),
+    );
+    return res;
+  }
 }
 
-export { CatalogService };
+export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -312,3 +312,19 @@ export class GetDatasetCatalogTaskRes {
   @ApiProperty()
   updated_by: string;
 }
+
+export class BatchRemoveRlsRulesRequest {
+  @ApiPropertyOptional()
+  nimbus_dashboard_id?: string;
+
+  @ApiPropertyOptional()
+  id_rls?: string;
+}
+
+export type AssetReporter = {
+  id: string;
+  display_name: string;
+  data_asset_type: string;
+  created_at: string;
+  tags: string;
+}

src/modules/catalog/dtos/pii.dto.ts

@@ -0,0 +1,8 @@
+export class PiiDto {
+  database_name: string;
+  table_schema: string;
+  table_name: string;
+  column_name: string;
+  data_type: string;
+  pii_rules: string;
+}

src/modules/catalog/share/share.controller.ts

@@ -0,0 +1,89 @@
+import {
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Req,
+  UseFilters,
+} from '@nestjs/common';
+import {
+  ApiHeaders,
+  ApiTags,
+} from '@nestjs/swagger';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import {
+  IColumnsMetadataResponse,
+  IDocsResponse,
+  IPreviewResponse,
+
+} from '../dtos';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ShareService } from './share.service';
+import { Request } from 'express';
+
+@ApiTags('Catalog')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
+@Controller('catalog/data-asset/share')
+@UseFilters(new GrpcToHttpExceptionFilter())
+export class ShareController {
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private catalogShareService: ShareService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get('/:id')
+  async getShareDataAsset(
+    @Param('id') id: string,
+    @Req() request: Request
+  ) {
+    this.logger.info(`GET //:id`);
+    return await this.catalogShareService.getOneDataAssetPublic(id, request);
+  }
+
+  @Get('/:id/columns-metadata')
+  async getShareDataAssetColumnsMetadata(
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @Req() request: Request
+  ): Promise<IColumnsMetadataResponse> {
+    this.logger.info(`GET /:id/columns-metadata`);
+
+    const columns_metadata =
+      await this.catalogShareService.getDatasetColumnsMetadata(id, request);
+
+
+    return { columns_metadata };
+  }
+
+  @Get('/:id/preview')
+  async getShareDataAssetPreview(
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @Req() request: Request
+  ): Promise<IPreviewResponse> {
+    this.logger.info(`GET /:id/preview`);
+    const preview = await this.catalogShareService.getDatasetPreview(id, request);
+
+    return { preview };
+  }
+
+  @Get('/:id/docs')
+  async getShareDataAssetDocs(
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @Req() request: Request
+  ): Promise<IDocsResponse> {
+    this.logger.info(`GET /:id/docs`);
+    const docs = await this.catalogShareService.getDataDocs(id, request);
+
+    return { docs };
+  }
+
+}

src/modules/catalog/share/share.module.ts

@@ -0,0 +1,30 @@
+import { Module } from "@nestjs/common";
+import { CatalogClientConfiguration } from "../catalog-client";
+import { ClientsModule } from "@nestjs/microservices";
+import { RolesModule } from "src/modules/roles/roles.module";
+import { UsersModule } from "src/modules/users/users.module";
+import { CustomersModule } from "src/modules/customers/customers.module";
+import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
+import { ShareController } from "./share.controller";
+import DadosferaLogger from "@dadosfera/dadosfera-logs";
+import { ShareService } from "./share.service";
+import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
+import { AuthModule } from "src/modules/auth/auth.module";
+
+const client = new CatalogClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    UsersModule,
+    RolesModule,
+    CustomersModule,
+    ShareMetadataModule,
+    MixpanelModule,
+    AuthModule
+  ],
+  controllers: [ShareController],
+  providers: [ShareService, DadosferaLogger],
+  exports: [ShareModule],
+})
+export class ShareModule {}

src/modules/catalog/share/share.service.ts

@@ -0,0 +1,275 @@
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import {
+  ProtoServices,
+  ReadService,
+} from '@dadosfera/protospack-v2/dist/lib/Catalog';
+import {
+  ForbiddenException,
+  Inject,
+  NotFoundException,
+  OnModuleInit,
+} from '@nestjs/common';
+import { CatalogClientConfiguration } from '../catalog-client';
+import { ClientGrpc } from '@nestjs/microservices';
+import { UsersService } from 'src/modules/users/users.service';
+import { RolesService } from 'src/modules/roles/roles.service';
+import { RequestUser } from 'src/decorators/user.decorator';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { Metadata } from '@grpc/grpc-js';
+import { lastValueFrom } from 'rxjs';
+import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
+import { isJWT } from 'class-validator';
+import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
+import { Request } from 'express';
+import jwt from 'jsonwebtoken';
+import { AuthClientService } from 'src/modules/auth/auth.service';
+
+
+export class ShareService implements OnModuleInit {
+  catalogReadService: ReadService.CatalogReadServices;
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject(CatalogClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+    private readonly userService: UsersService,
+    private readonly roleService: RolesService,
+    private readonly shareMetadataService: ShareMetadataService,
+    private readonly mixpanelService: MixpanelService,
+    private authClient: AuthClientService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.catalogReadService =
+      this.grpcClient.getService<ReadService.CatalogReadServices>(
+        ProtoServices.CatalogReadServices,
+      );
+  }
+
+  async getDatasetColumnsMetadata(id: string, request: Request) {
+    const shareMetadata = await this.getShareMetadata(id, request);
+    const metadata = PackTheMetadata({
+      customer_id: shareMetadata.customerId,
+      customer_name: shareMetadata.customerName,
+    });
+    const { columns_metadata } = await lastValueFrom(
+      this.catalogReadService.GetDatasetColumnsMetadata(
+        { id: shareMetadata.assetId, type: undefined },
+        metadata,
+      ),
+    );
+    const result = JSON.parse(columns_metadata);
+    return result;
+  }
+
+  async getDatasetPreview(id: string, request: Request) {
+    const shareMetadata = await this.getShareMetadata(id, request);
+    const metadata = PackTheMetadata({
+      customer_id: shareMetadata.customerId,
+      customer_name: shareMetadata.customerName,
+    });
+    const { preview } = await lastValueFrom(
+      this.catalogReadService.GetDatasetPreview(
+        { id: shareMetadata.assetId, type: undefined },
+        metadata,
+      ),
+    );
+    const result = JSON.parse(preview);
+    return result;
+  }
+
+  async getOneDataAssetPublic(id: string, request: Request) {
+    this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
+      id
+    }))
+    try {
+    const user = await this.getUserFromRequest(request);
+
+      const shareMetadata = await this.getShareMetadata(id, request);
+
+      const mixpanelTracker = {
+        asset: shareMetadata.assetId,
+        type: isJWT(id) ? 'assigned' : shareMetadata.type,
+        customer: shareMetadata.customerName
+      }
+
+      if (user) {
+        await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
+      } else {
+        await this.mixpanelService.trackShare(request, mixpanelTracker);
+      }
+
+      this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
+      const metadata = PackTheMetadata({
+        customer_id: shareMetadata.customerId,
+        customer_name: shareMetadata.customerName,
+      });
+  
+      const { data_asset } = await this.getOneDataAsset({
+        customer_id: shareMetadata.customerId,
+        id: shareMetadata.assetId,
+        metadata,
+      });
+      this.logger.info('found asset: ' + JSON.stringify(data_asset));
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      data_asset.share_type = 'public';
+      if (data_asset.share_type !== 'public') throw new NotFoundException();
+  
+      return { data_asset };
+    } catch (error) {
+      this.logger.error(error);
+      throw error;
+    }
+
+  }
+
+  private async getOneDataAsset(data: {
+    id: string;
+    customer_id: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, id, metadata } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAsset(
+        { id, type: undefined },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
+  async getDataDocs(id: string, request: Request) {
+    const shareMetadata = await this.getShareMetadata(id, request);
+    const metadata = PackTheMetadata({
+      customer_id: shareMetadata.customerId,
+      customer_name: shareMetadata.customerName,
+    });
+
+    const { documentation } = await lastValueFrom(
+      this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
+    );
+    console.log(documentation);
+    const docs = JSON.parse(documentation);
+    return docs;
+  }
+
+  private async getAssetsUsersAndRoles(
+    data_assets: Array<any>,
+    customer_id: string,
+  ) {
+    const { users: customer_users } =
+      await this.userService.findAllUsersByCustomerId(customer_id);
+    const { roles: customer_roles } = await this.roleService.roleSearch(
+      {},
+      { customer_id },
+    );
+    return data_assets.map((data_asset) => {
+      const owner = customer_users.find(
+        (u) => u.id === data_asset.owner,
+      )?.username;
+
+      const roles = [];
+      const users = [];
+      for (const role_id of data_asset.roles) {
+        const role = customer_roles.find((r) => r.id === role_id);
+        if (role) roles.push({ id: role.id, name: role.name });
+      }
+      for (const user_id of data_asset.users) {
+        const user = customer_users.find((r) => r.id === user_id);
+        if (user) users.push({ id: user.id, username: user.username });
+      }
+      return {
+        ...data_asset,
+        roles,
+        users,
+        owner,
+      } as typeof data_asset;
+    });
+  }
+
+
+  private async getShareMetadata(id: string, request: Request) {
+    const metadata = PackTheMetadata({});
+    this.logger.info('GET share metadata')
+    const info = await this.shareMetadataService.get(id, metadata);
+    if (isJWT(id) && info ){
+      return info;
+    }
+
+    const user = await this.getUserFromRequest(request);
+
+    if (info.type === 'private') {
+      if (!user) {
+        throw new ForbiddenException(
+          'You do not have permission to access this data asset.',
+        );
+      }
+
+      const is_data_manager = user.permissions.includes(
+        PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+      );
+
+      const is_get = user.permissions.includes(
+        PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
+      );
+
+      if (is_data_manager || is_get) {
+        return info;
+      }
+
+      throw new ForbiddenException(
+        'You do not have permission to access this data asset.',
+      );
+    }
+    return info;
+  }
+
+   private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
+    const accessToken = request.get('Authorization');
+    if (accessToken) {
+      const accessTokenDecoded: any = jwt.decode(accessToken, {
+        complete: true,
+      });
+
+      const { kid } = accessTokenDecoded.header;
+
+      const { keys } = await this.authClient.getPublicKeys();
+
+      const pemValue = keys.find((key) => key.kid === kid);
+
+      if (!pemValue) {
+        return null;
+      }
+
+      jwt.verify(accessToken, pemValue.pem);
+      const accessTokenPayload = accessTokenDecoded.payload;
+
+      return {
+        user_id: accessTokenPayload.user_id,
+        username: accessTokenPayload.username,
+        permissions: accessTokenPayload.permissions,
+        customer_id: accessTokenPayload.customer_id,
+        customer_name: accessTokenPayload.customer_name,
+        customer_tier: accessTokenPayload.customer_tier,
+        customer_modules: accessTokenPayload.customer_modules,
+        access_token: accessToken,
+      };
+    }
+
+    return null;
+  }
+}

src/modules/connection-test/connection-test-client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionTest } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectionTestClientConfiguration {
   private config: GrpcOptions = {

src/modules/connection-test/connection-test.service.ts

@@ -20,7 +20,7 @@ import {
   DatabaseConnectionPropertiesDto,
 } from '../connection/dtos/connection';
 import { RequestUser } from 'src/decorators/user.decorator';
-import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 
 @Injectable()
 export class ConnectionTestService {

src/modules/connection/client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectionClientConfiguration {
   public name = 'ConnectionClientConfiguration';

src/modules/connection/connection.controller.ts

@@ -24,10 +24,11 @@ import {
   ConnectionDetailsRes,
   ConnectionRes,
   ConnectionsRes,
+  GetAllConnectionsReq,
   UpdateConnectionDto,
 } from './dtos/connection';
 import { CreateConnectionDto } from './dtos/connection';
-import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { Language } from 'src/decorators/language.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
@@ -138,7 +139,7 @@ export class ConnectionController {
   async getAllConnections(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Query() queries,
+    @Query() queries: GetAllConnectionsReq,
   ): Promise<ConnectionsRes> {
     this.logger.info('/connections - Get All Connection');
 

src/modules/connection/dtos/connection.ts

@@ -9,6 +9,7 @@ import {
   ConnectionToCatalog,
 } from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
 import {
+  GetAllConnectionRequest,
   GetAllConnectionResponse,
   GetConnectionDetailsResponse,
   GetConnectionResponse,
@@ -21,7 +22,7 @@ export type ConnectionCredentialsType =
   | 'oauth'
   | 'api_key'
   | 'service_account'
-  | 'headers_authF';
+  | 'headers_auth';
 export interface ConnectionApiConnection {
   config_id: string;
   plugin: string;
@@ -94,7 +95,6 @@ export class ConnectionDto implements Connection {
 
 export class ConnectionToCatalogDto implements ConnectionToCatalog {
   // ---Automatically generated information - will not be sent by the frontend--- //
-  id: string;
   customer_id: string;
   updated_at: string;
   created_at: string;
@@ -104,6 +104,9 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
   customer_name: string;
 
   // ---Information sent by the frontend--- //
+  @ApiPropertyOptional()
+  id: string;
+
   @ApiProperty()
   name: string;
 
@@ -147,6 +150,15 @@ export class UpdateConnectionDto extends PickType(ConnectionDto, [
   'properties',
 ]) {}
 
+export class GetAllConnectionsReq implements GetAllConnectionRequest {
+  @ApiPropertyOptional()
+  search?: string;
+  @ApiPropertyOptional()
+  page?: string;
+  @ApiPropertyOptional()
+  size?: string;
+}
+
 export class ConnectionsRes implements GetAllConnectionResponse {
   @ApiProperty({ type: [ConnectionToCatalogDto] })
   connections: ConnectionToCatalogDto[];

src/modules/connector/client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectorManager } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectorClientConfiguration {
   public name = 'ConnectorClientConfiguration';

src/modules/customers/customers.controller.ts

@@ -1,32 +1,35 @@
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import {
   Body,
   Controller,
   Get,
+  HttpCode,
+  HttpStatus,
   Inject,
   Param,
+  Post,
   Put,
   Query,
   UseFilters,
 } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import {
   Authenticated,
   RequireAllPermissions,
+  RequireModule,
+  RequireSomePermission,
 } from 'src/decorators/authentication.decorator';
-import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { CustomersService } from './customers.service';
-import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
+import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import type { StringValue } from 'ms';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { EnforceMfa } from './dtos/enforce-mfa';
 
-@ApiInternalOnlyController()
 @ApiTags('Customers')
 @Controller('customers')
-@Authenticated()
 @UseFilters(GrpcToHttpExceptionFilter)
 export class CustomersController {
   logger: DadosferaLogger;
@@ -39,26 +42,42 @@ export class CustomersController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @Post(':id/mfa')
+  @Authenticated()
+  @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @RequireModule(DADOSFERA_MODULES_KEYS.DANGER_ZONE)
+  async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) {
+    this.logger.info('enableMfaEnforce', { id });
+    return await this.customersService.enableEnforceMfa(id, data.enabled);
+  }
+
   @Get(':id/links')
-  async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
+  @Authenticated()
+  @ApiOkResponse({ type: CustomerLinksResponse })
+  async getCustomerLinks(@Param('id') id: string) {
     this.logger.info('getCustomerLinks', { id });
     const links = await this.customersService.getLinks(id);
     return { links };
   }
 
   @Put(':id/links')
+  @Authenticated()
   @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
-  setCustomerLinks(
-    @Body() body: CustomerDto,
-    @Param('id') id,
-  ): Promise<IdResponse> {
+  @ApiOkResponse()
+  @HttpCode(HttpStatus.OK)
+  async setCustomerLinks(
+    @Body() body: CustomerLinkRequest,
+    @Param('id') id: string,
+  ) {
     const { links } = body;
     this.logger.info('setCustomerLinks', { id, links });
-    return this.customersService.setLinks(id, links);
+    await this.customersService.setLinks(id, links);
   }
 
   @Get('token')
+  @Authenticated()
   @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
+  @ApiProduces('text/plain')
   async getCustomerToken(
     @Query('exp') exp: StringValue,
     @User() user: RequestUser,
@@ -71,4 +90,50 @@ export class CustomersController {
     };
     return this.customersService.generateToken(exp, data);
   }
+
+  @Get('monitoring-dashboard')
+  @Authenticated()
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
+  )
+  async getCustomerMonitoringDashboard(
+    @User() user: RequestUser,
+  ): Promise<{ url: string }> {
+    this.logger.info('getCustomerMonitoringDashboard');
+
+    const metadata = PackTheMetadata(user);
+    return this.customersService.getMonitoringDashboardUrl(metadata);
+  }
+
+  @Get('logs-dashboard')
+  @Authenticated()
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
+  )
+  @RequireModule(DADOSFERA_MODULES_KEYS.LOG_DASHBOARD)
+  async getCustomerMixPanelLogsDashboard(
+    @User() user: RequestUser,
+  ): Promise<{ url: string }> {
+    this.logger.info('getLogsDashboardUrl');
+    const metadata = PackTheMetadata(user);
+  
+    const result = await this.customersService.getLogsDashboardUrl(metadata);
+    return result;
+  }
+
+  @Get('access-dashboard')
+  @Authenticated()
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
+  )
+  @RequireModule(DADOSFERA_MODULES_KEYS.ACCESS_DASHBOARD)
+  async getAccessDashboard(
+    @User() user: RequestUser,
+  ): Promise<{ url: string }> {
+    this.logger.info('getAccessDashboard');
+    const metadata = PackTheMetadata(user);
+  
+    const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
+    return result;
+  }
 }

src/modules/customers/customers.module.ts

@@ -4,11 +4,18 @@ import { ClientsModule } from '@nestjs/microservices';
 import { DucClient } from '../duc/client.config';
 import { CustomersController } from './customers.controller';
 import { CustomersService } from './customers.service';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
 
-const client = new DucClient();
+const ducClient = new DucClient();
+const piFactoryClient = new PipelinesClientConfiguration();
 
 @Module({
-  imports: [ClientsModule.register([client.providerOptions])],
+  imports: [
+    ClientsModule.register([
+      ducClient.providerOptions,
+      piFactoryClient.providerOptions,
+    ]),
+  ],
   controllers: [CustomersController],
   providers: [CustomersService, DadosferaLogger],
   exports: [CustomersService],

src/modules/customers/customers.service.ts

@@ -5,6 +5,7 @@ import {
   HttpException,
   HttpStatus,
   InternalServerErrorException,
+  ForbiddenException,
 } from '@nestjs/common';
 
 import { firstValueFrom, lastValueFrom } from 'rxjs';
@@ -22,20 +23,48 @@ import {
 } from '@aws-sdk/client-secrets-manager';
 import getEnv from 'src/utils/getEnv';
 import { logger } from 'elastic-apm-node';
+import {
+  ReadService,
+  ProtoServices as PipelineProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
+import { Metadata } from '@grpc/grpc-js';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 // This function will accept any string, which may result in a bug.
 @Injectable()
 export class CustomersService implements OnModuleInit {
+
   private customerService: CustomersProtoService;
+  private logger: DadosferaLogger;
+  private pipelineReadService: ReadService.PipelineV2ReadService;
 
   constructor(
     @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
-  ) {}
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly pipelinesGrpcClient: ClientGrpc,
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
 
   onModuleInit() {
     this.customerService = this.grpcClient.getService<CustomersProtoService>(
       ProtoServices.CustomersProtoService,
     );
+    this.pipelineReadService =
+      this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
+        PipelineProtoServices.PipelineV2ReadService,
+      );
+  }
+
+  async getCustomer(customerId: string) {
+    return await lastValueFrom(
+      this.customerService.CustomerFindOneById({
+        id: customerId
+      })
+    )
   }
 
   async getLinks(customerId: string) {
@@ -120,4 +149,78 @@ export class CustomersService implements OnModuleInit {
     // const decoded = jwt.decode(jwt_token, { complete: true });
     return jwt_token;
   }
+
+  async getMonitoringDashboardUrl(metadata: Metadata) {
+    logger.info('CustomersService - getMonitoringDashboardUrl');
+
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '98',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    logger.info('Done');
+
+    return res;
+  }
+
+  async getLogsDashboardUrl(metadata: Metadata) {
+    logger.info('CustomersService - getMixPanelLogsDashboardUrl');
+  
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '103',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    logger.info('Done');
+  
+    return res;
+  }
+
+  async getAccessDashboardUrl(customerName: string, metadata: Metadata) {
+    /*
+     * TODO(Refactor): dar um jeito de exibir o dash da sbm diferente dos outros customer
+     * pois o signicado de department para sbm significa as instituições do usuários
+     */
+    if (customerName !== 'sbmoffshorecom') {
+      throw new ForbiddenException();
+    }
+    logger.info('CustomersService - getAccessDashboardUrl');
+  
+    const res = await this.getDashboardUrl('105', metadata);
+    logger.info('Done');
+  
+    return res;
+  }
+
+  private async getDashboardUrl(dashboardId: string, metadata: Metadata) {
+    return await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: dashboardId,
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata
+      )
+    );
+  }
+
+  async enableEnforceMfa(id: string, enabled: boolean) {
+    return await lastValueFrom(
+      this.customerService.CustomerUpdateEnforceMfa({
+        customerId: id,
+        enforceMfa: enabled
+      })
+    )
+  }
 }

src/modules/customers/dtos/customers.ts

@@ -1,12 +1,23 @@
 import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
-import { ApiProperty } from '@nestjs/swagger';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
-export class CustomerDto {
+export class CustomerLink implements Link {
   @ApiProperty()
-  links: Link[];
+  href: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+  @ApiPropertyOptional()
+  iconSrc: string;
+}
+export class CustomerLinkRequest {
+  @ApiProperty({ type: [CustomerLink] })
+  links: CustomerLink[];
 }
 
 export class CustomerLinksResponse {
-  @ApiProperty()
-  links: Link[];
+  @ApiProperty({ type: [CustomerLink] })
+  links: CustomerLink[];
 }
+

src/modules/customers/dtos/enforce-mfa.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class EnforceMfa {
+    @ApiProperty()
+    enabled: boolean
+}

src/modules/duc/client.config.ts

@@ -9,7 +9,9 @@ import {
   ProtoPaths,
 } from '@dadosfera/protospack-v2/dist/lib/Duc';
 
-const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.DUC_URL.startsWith('duc:') ||
+  process.env.DUC_URL.includes('0.0.0.0');
 
 export class DucClient {
   public name = 'DucClient';
@@ -27,6 +29,8 @@ export class DucClient {
         objects: true,
         arrays: true,
       },
+      maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem
+      maxReceiveMessageLength: 15 * 1024 * 1024,
     },
   };
 

src/modules/identity-provider/dto/identity-provider.dto.ts

@@ -0,0 +1,47 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class CreateIdentityProvider {
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  clientId: string;
+
+  @ApiProperty()
+  clientSecret: string;
+
+  @ApiProperty()
+  issuerUrl: string;
+
+  @ApiProperty()
+  permissions: number[];
+}
+
+
+export class IdentityProviderResponse {
+
+  @ApiProperty()
+  id: string;
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  clientId: string;
+
+  @ApiProperty()
+  issueUrl: string;
+
+  @ApiProperty()
+  permissions: {
+    id: number;
+    name: string;
+  }[];
+}
+
+export class IdentityProviderListResponse {
+
+  @ApiProperty()
+  providers: IdentityProviderResponse[]
+}

src/modules/identity-provider/dto/sso-signin.dto.ts

@@ -0,0 +1,10 @@
+export class SsoSignInDto {
+    readonly nonce: string;
+    readonly codeVerifier: string;
+    readonly state: string;
+    readonly id: string;
+    readonly clientId: string;
+    readonly clientSecret: string;
+    readonly issuerUrl: string;
+    readonly redirectUrls: string[];
+}

src/modules/identity-provider/identity-provider.controller.ts

@@ -0,0 +1,246 @@
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpCode,
+  HttpStatus,
+  Inject,
+  Param,
+  Post,
+  Put,
+  Redirect,
+  Req,
+} from '@nestjs/common';
+import { IdentityProviderService } from './identity-provider.service';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { ApiOkResponse } from '@nestjs/swagger';
+import {
+  CreateIdentityProvider,
+  IdentityProviderListResponse,
+  IdentityProviderResponse,
+} from './dto/identity-provider.dto';
+import { Request } from 'express';
+import ErrorCodes from 'src/utils/errorCodes';
+import {
+  Authenticated,
+  RequireModule,
+  RequireSomePermission,
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+
+@Controller('identity-providers')
+export class IdentityProviderController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private identityProviderService: IdentityProviderService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post()
+  @HttpCode(HttpStatus.OK)
+  @ApiOkResponse({ type: IdentityProviderResponse })
+  @Authenticated()
+  @RequireModule('sso')
+  @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  async addIdentityProvider(
+    @User() user: RequestUser,
+    @Body() body: CreateIdentityProvider,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('POST /identity-providers');
+
+    const metadata = PackTheMetadata({
+      ...user,
+      language,
+    });
+
+    return await this.identityProviderService.create(body, metadata);
+  }
+
+  @Get()
+  @HttpCode(HttpStatus.OK)
+  @ApiOkResponse({ type: IdentityProviderListResponse })
+  @Authenticated()
+  @RequireModule('sso')
+  @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  async getProviders(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('GET identity-providers');
+    const metadata = PackTheMetadata({
+      ...user,
+      language,
+    });
+
+    const result = await this.identityProviderService.getList(metadata);
+    return result;
+  }
+
+  @Delete(':id')
+  @HttpCode(HttpStatus.NO_CONTENT)
+  @RequireModule('sso')
+  @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  async deleteIdentityProvider(
+    @Param('id') id: string,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('DELETE /identity-providers');
+
+    const metadata = PackTheMetadata({
+      ...user,
+    });
+
+    return await this.identityProviderService.deleteIdentityProvider(
+      id,
+      metadata,
+    );
+  }
+
+  @Put(':id')
+  @HttpCode(HttpStatus.OK)
+  @Authenticated()
+  @RequireModule('sso')
+  @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  async updateIdentityProviders(
+    @Param('id') id: string,
+    @Body() body: CreateIdentityProvider,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('PUT /identity-providers');
+
+    const metadata = PackTheMetadata({
+      ...user,
+    });
+
+    return await this.identityProviderService.updateIdentityProviders(
+      id,
+      body,
+      metadata,
+    );
+  }
+
+  @Post('/callback')
+  @HttpCode(HttpStatus.OK)
+  async callbackIdp(
+    @Req() req: Request,
+    @Language() language: LanguageEnum,
+    @Body()
+    body: {
+      state: string;
+      code: string;
+    },
+  ) {
+    this.logger.info('GET /identity-providers/callback');
+    const { code, state } = body;
+
+    if (!code) {
+      this.logger.error('No code received from IDP');
+      throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
+    }
+
+    if (!state) {
+      this.logger.error('No state received from IDP');
+      throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
+    }
+
+    try {
+      const origin = req.headers['origin'] as string;
+      this.logger.info('Header Origin: ' + origin);
+
+      const lang =
+        language.substring(0, 2) + language.substring(2).toUpperCase();
+      const callbackUrl =
+        process.env.ENV !== 'prd'
+          ? `${origin}/auth/callback`
+          : `${origin}/${lang}/auth/callback`;
+
+      this.logger.info('Callback URL: ' + callbackUrl);
+      return await this.identityProviderService.getTokenByIdp(
+        code,
+        state,
+        callbackUrl,
+      );
+    } catch (error) {
+      this.logger.error(error);
+      throw error;
+    }
+  }
+
+  @Get('/links')
+  @HttpCode(HttpStatus.OK)
+  async providerLinks(@Req() req: Request) {
+    this.logger.info('GET /identity-providers/links');
+
+    try {
+      const frontDomain = req.headers['origin'] as string;
+      this.logger.info('Header Origin: ' + frontDomain);
+
+      if (!frontDomain) {
+        this.logger.info('Not found front domain');
+        throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
+      }
+
+      const result =
+        await this.identityProviderService.identityProvidersLinksPerDomain(
+          frontDomain,
+        );
+      return result;
+    } catch (error) {
+      this.logger.error(error);
+      throw error;
+    }
+  }
+
+  @Get(':id')
+  @HttpCode(HttpStatus.OK)
+  @Redirect()
+  async loginIdp(
+    @Param('id') id: string,
+    @Req() req: Request,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('GET /identity-providers/:id');
+
+    try {
+      const frontDomain =
+        (req.headers['origin'] as string) || (req.headers['referer'] as string);
+      this.logger.info(`Front domain: ${frontDomain}`);
+      const host =
+        frontDomain.lastIndexOf('/') !== -1
+          ? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
+          : frontDomain;
+
+      const lang =
+        language.substring(0, 2) + language.substring(2).toUpperCase();
+      const callbackUrl =
+        process.env.ENV !== 'prd'
+          ? `${host}/auth/callback`
+          : `${host}/${lang}/auth/callback`;
+
+      this.logger.info('Callback URL: ' + callbackUrl);
+      const redirectUrl =
+        await this.identityProviderService.loginIdentityProvider(
+          id,
+          callbackUrl,
+        );
+
+      this.logger.info(`Redirecting to: ${redirectUrl}`);
+      return {
+        url: redirectUrl,
+      };
+    } catch (error) {
+      this.logger.error(error);
+      throw error;
+    }
+  }
+}

src/modules/identity-provider/identity-provider.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { IdentityProviderController } from './identity-provider.controller';
+import { IdentityProviderService  } from './identity-provider.service';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { ServicesModule } from 'src/services/service.module';
+
+const client = new DucClient();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
+  controllers: [IdentityProviderController],
+  providers: [IdentityProviderService, DadosferaLogger],
+})
+export class IdentityProviderModule {}

src/modules/identity-provider/identity-provider.service.ts

@@ -0,0 +1,185 @@
+import {
+  BadRequestException,
+  Inject,
+  Injectable,
+  OnModuleInit,
+} from '@nestjs/common';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { lastValueFrom } from 'rxjs';
+import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { Metadata } from '@grpc/grpc-js';
+import { Issuer, generators } from 'openid-client';
+import { SsoSignInDto } from './dto/sso-signin.dto';
+import { CacheService } from 'src/services/cache.service';
+import { CreateIdentityProvider } from './dto/identity-provider.dto';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+@Injectable()
+export class IdentityProviderService implements OnModuleInit {
+  private logger: DadosferaLogger;
+  private identityProviderService: IdentityProviderProtoService;
+  constructor(
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+    private readonly cacheService: CacheService<SsoSignInDto>,
+    @Inject(DadosferaLogger)
+    private dadosferaLoggger: DadosferaLogger
+  ) {
+    this.logger = dadosferaLoggger.logger;
+  }
+
+  onModuleInit() {
+    this.identityProviderService =
+      this.grpcClient.getService<IdentityProviderProtoService>(
+        ProtoServices.IdentityProviderProtoService,
+      );
+  }
+
+  async create(body: IdentityProviderRequest, metadata: Metadata) {
+    this.logger.info("Call IdentityProvider GRPC Create")
+    return await lastValueFrom(
+      this.identityProviderService.Create(body, metadata),
+    );
+  }
+
+  async getList(metadata: Metadata) {
+    this.logger.info("Call IdentityProvider GRPC GetList")
+    return await lastValueFrom(
+      this.identityProviderService.GetList({}, metadata),
+    );
+  }
+
+  async loginIdentityProvider(id: string, callbackUrl: string) {
+    this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
+    const idp = await lastValueFrom(
+      this.identityProviderService.FindIdentityProvider({ id }),
+    );
+
+    this.logger.info("Discovery issueURL: " + idp.issuerUrl)
+    const issuer = await Issuer.discover(idp.issuerUrl);
+    const client = new issuer.Client({
+      client_id: idp.clientId,
+      client_secret: idp.clientSecret,
+      redirect_uris: idp.redirectUrls,
+      response_types: ['code'],
+    });
+
+    this.logger.info("Generate Challenge")
+    const code_verifier: string = generators.codeVerifier();
+    const code_challenge: string = generators.codeChallenge(code_verifier);
+
+    this.logger.info("Generate State")
+    const state = generators.state();
+
+    this.logger.info("Generate Nonce")
+    const nonce = generators.nonce();
+
+    // Using state because it is returned in the callback
+    // and we can use it to retrieve the code_verifier and nonce
+    this.logger.info("Save Login parameters in redis")
+    await this.cacheService.set(state, {
+      codeVerifier: code_verifier,
+      nonce,
+      id: idp.id,
+      state,
+      clientId: idp.clientId,
+      clientSecret: idp.clientSecret,
+      issuerUrl: idp.issuerUrl,
+      redirectUrls: idp.redirectUrls,
+    });
+
+    this.logger.info("Generate Authorization URL")
+    const url = client.authorizationUrl({
+      scope: 'openid email',
+      response_type: 'code',
+      code_challenge,
+      code_challenge_method: 'S256',
+      state,
+      nonce,
+      redirect_uri: callbackUrl,
+    });
+    const idpUrl = url + '&identity_provider=' + idp.name;
+    this.logger.info(idpUrl)
+    return idpUrl;
+  }
+
+  async getTokenByIdp(code: string, state: string, callbackUrl: string) {
+    this.logger.info("Get login parameters in redis")
+    const ssoSign = await this.cacheService.get(state);
+
+    if (!ssoSign) {
+      this.logger.info("Login Parameters Not Found")
+      throw new BadRequestException('SSO sign-in is expired or not found');
+    }
+
+    this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
+    const issuer = await Issuer.discover(ssoSign.issuerUrl);
+    const client = new issuer.Client({
+      client_id: ssoSign.clientId,
+      client_secret: ssoSign.clientSecret,
+      redirect_uris: ssoSign.redirectUrls,
+    });
+
+    const params = client.callbackParams(
+      `${callbackUrl}?code=${code}&state=${state}`,
+    );
+    try {
+
+      this.logger.info("Get Token Set");
+      const tokenSet = await client.callback(callbackUrl, params, {
+        nonce: ssoSign.nonce,
+        code_verifier: ssoSign.codeVerifier,
+        state: ssoSign.state
+      });
+
+      this.logger.info("Delete parameters in redis");
+      await this.cacheService.delete(ssoSign.state);
+
+      this.logger.info("Call IdentityProvider GRPC SignInUser");
+      return await lastValueFrom(
+        this.identityProviderService.SignInUser({
+          accessToken: tokenSet.access_token,
+          idToken: tokenSet.id_token,
+          refreshToken: tokenSet.refresh_token,
+          id: ssoSign.id,
+        }),
+      );
+    } catch (error) {
+      this.logger.error(error);
+      throw error;
+    }
+  }
+
+  async deleteIdentityProvider(id: string, metadata: Metadata) {
+    this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
+    return await lastValueFrom(
+      this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
+    );
+  }
+
+  async updateIdentityProviders(
+    id: string,
+    body: CreateIdentityProvider,
+    metadata: Metadata,
+  ) {
+    this.logger.info("Call IdentityProvider GRPC Update with: " + id)
+    return await lastValueFrom(
+      this.identityProviderService.UpdateIdentityProvider(
+        {
+          id,
+          ...body,
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async identityProvidersLinksPerDomain(frontDomain: string) {
+    this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
+    return await lastValueFrom(
+      this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
+    );
+  }
+}

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,4 +1,4 @@
-import { Info } from 'protospack/dist/lib/interfaces';
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
 interface Values {
   jdbc_user: string;

src/modules/inputs/inputs-client.config.ts

@@ -6,7 +6,9 @@ import {
   type GrpcOptions,
 } from '@nestjs/microservices';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 export class InputsGrpcClient {
   public readonly name = 'InputsGrpcClient';
   private config: GrpcOptions = {

src/modules/mixpanel/mixpanel.controller.ts

@@ -1,93 +1,46 @@
-import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
-import { init } from 'mixpanel';
-import { Authenticated } from 'src/decorators/authentication.decorator';
+import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
-import { RequestUser, User } from 'src/decorators/user.decorator';
+import { RequestUser } from 'src/decorators/user.decorator';
+import { MixpanelService } from './mixpanel.service';
+import { extractUserFrom } from 'src/authentication/extract-user';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 @ApiInternalOnlyController()
-@Authenticated()
 @Controller('trackEvent')
 export class MixpanelController {
+  logger: DadosferaLogger;
+
   constructor(
-    @Inject('MIXPANEL_TOKEN')
-    private readonly mixpanelToken: string,
-  ) {}
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private mixpanelService: MixpanelService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
   @Post(':id')
-  async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
+  async trackEvent(
+    @Param('id') id,
+    @Body() body,
+    @Req() request
+  ) {
+    this.logger.info(`POST Track Event: ${id}`)
     delete body.info;
-    const mixpanel = init(this.mixpanelToken);
 
-    const separator = user.username.includes('-') ? '-' : '.';
-    const removeValues = [
-      '.dadosferatech.dadosfera',
-      '.demo.dadosfera',
-      '.dadosferademo',
-      '.dadosferarh.dadosfera',
-      '.dadosferatech.dadosfera2',
-      '.dadosferatech.dadosfera',
-      '.dadosfera.fin',
-      '.dadosferafin.dadosfera',
-      '.praxio.dadosfera',
-      '.dadosfera.tech',
-      '.treinamentos@dadosfera.ai',
-      '.dadosfera2',
-      '.treinamentosfera',
-      '.dadosfera',
-    ];
+    const anonymousUser = {
+      username: "anonymous",
+      customer_name: "anonymous"
+    } as RequestUser
 
-    let username = user.username;
+    const hasToken = request.headers['authorization'];
 
-    removeValues.forEach((value) => {
-      username = username.replace(value, '');
-    });
+    const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
 
-    username = username.split('@')?.[0];
-    username = username.split('+')?.[0];
-
-    let firstName = username
-      .substring(0, username.indexOf(separator))
-      .replace('dadosfera', '');
-    let lastName = username
-      .substring(username.lastIndexOf(separator) + 1)
-      .replace('dadosfera', '');
-
-    if (!firstName) {
-      firstName = lastName;
-      lastName = '';
-    }
-
-    firstName = this.capitalize(firstName);
-    lastName = this.capitalize(lastName);
-
-    await mixpanel.people.set(user.username, {
-      $first_name: firstName,
-      $last_name: lastName,
-      $name: this.getFullName(firstName, lastName),
-      $email: user.username.includes('@')
-        ? user.username
-        : user.username + '@dadosfera.ai',
-      customer_name: user.customer_name,
-    });
-
-    await mixpanel.track(id, {
-      distinct_id: user.username,
-      customer: user.customer_name,
-      env: process.env.ENV,
-      ...body,
-    });
+    this.logger.info(`Has user: ${typeof hasToken == "string"}`)
+ 
+    await this.mixpanelService.track(id, user, request, body)
 
+    this.logger.info(`Event successful`)
     return { id, body, user: user.username };
   }
-
-  capitalize(sentence: string): string {
-    if (!sentence) {
-      return '';
-    }
-
-    return sentence[0].toUpperCase() + sentence.substring(1);
-  }
-
-  getFullName(firstName: string, lastName: string) {
-    return `${firstName}${lastName ? ' ' + lastName : ''}`;
-  }
 }

src/modules/mixpanel/mixpanel.module.ts

@@ -1,6 +1,8 @@
 import { Module } from '@nestjs/common';
 import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
 import { MixpanelController } from './mixpanel.controller';
+import { MixpanelService } from './mixpanel.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 @Module({
   controllers: [MixpanelController],
@@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller';
     {
       provide: 'MIXPANEL_TOKEN',
       useValue: getSecretFromSecretsManager(
-        `${process.env.ENV}/root/mixpanel_token`,
+        `prd/root/mixpanel_token`,
       ),
     },
+    MixpanelService,
+    DadosferaLogger
   ],
+  exports: [MixpanelService]
 })
 export class MixpanelModule {}

src/modules/mixpanel/mixpanel.service.ts

@@ -0,0 +1,118 @@
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { Inject } from '@nestjs/common';
+import { Request } from 'express';
+import mixpanel, { init } from 'mixpanel';
+import { RequestUser } from 'src/decorators/user.decorator';
+
+export class MixpanelService {
+    logger: DadosferaLogger;
+
+  constructor(
+    @Inject('MIXPANEL_TOKEN')
+    private readonly mixpanelToken: string,
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  async track(eventName: string, user: RequestUser, request: Request, body: any) {
+    this.logger.info("track: " + JSON.stringify({
+        eventName,
+        ...body
+    }))
+    const mixpanel = init(this.mixpanelToken);
+    await this.setPeople(user, mixpanel);
+
+    await mixpanel.track(eventName, {
+      distinct_id: user.username,
+      customer: user.customer_name,
+      env: process.env.ENV,
+      $ip: request.ip,
+      $os: request.headers['sec-ch-ua-platform'] || '',
+      $browser: request.headers['user-agent'],
+      ...body,
+    });
+    return;
+  }
+
+  async trackShare(request: Request,  body: any) {
+    this.logger.info("trackShare: " + JSON.stringify(body))
+    const mixpanel = init(this.mixpanelToken);
+
+    await mixpanel.track("share_page", {
+      env: process.env.ENV,
+      $ip: request.ip,
+      $os: request.headers['sec-ch-ua-platform'] || '',
+      $browser: request.headers['user-agent'],
+      ...body,
+    });
+    return;
+  }
+
+  private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
+    const separator = user.username.includes('-') ? '-' : '.';
+    const removeValues = [
+      '.dadosferatech.dadosfera',
+      '.demo.dadosfera',
+      '.dadosferademo',
+      '.dadosferarh.dadosfera',
+      '.dadosferatech.dadosfera2',
+      '.dadosferatech.dadosfera',
+      '.dadosfera.fin',
+      '.dadosferafin.dadosfera',
+      '.praxio.dadosfera',
+      '.dadosfera.tech',
+      '.treinamentos@dadosfera.ai',
+      '.dadosfera2',
+      '.treinamentosfera',
+      '.dadosfera',
+    ];
+
+    let username = user.username;
+
+    removeValues.forEach((value) => {
+      username = username.replace(value, '');
+    });
+
+    username = username.split('@')?.[0];
+    username = username.split('+')?.[0];
+
+    let firstName = username
+      .substring(0, username.indexOf(separator))
+      .replace('dadosfera', '');
+    let lastName = username
+      .substring(username.lastIndexOf(separator) + 1)
+      .replace('dadosfera', '');
+
+    if (!firstName) {
+      firstName = lastName;
+      lastName = '';
+    }
+
+    firstName = this.capitalize(firstName);
+    lastName = this.capitalize(lastName);
+
+    await mixpanel.people.set(user.username, {
+      $first_name: firstName,
+      $last_name: lastName,
+      $name: this.getFullName(firstName, lastName),
+      $email: user.username.includes('@')
+        ? user.username
+        : user.username + '@dadosfera.ai',
+      customer_name: user.customer_name,
+    });
+  }
+
+  private capitalize(sentence: string): string {
+    if (!sentence) {
+      return '';
+    }
+
+    return sentence[0].toUpperCase() + sentence.substring(1);
+  }
+
+  private getFullName(firstName: string, lastName: string) {
+    return `${firstName}${lastName ? ' ' + lastName : ''}`;
+  }
+}

src/modules/network-config/network-config-client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { NetworkConfig } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class NetworkConfigGrpcClient {
   public name = 'NetworkConfigGrpcClient';

src/modules/network-policy/dto/network-policy.dto.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class NetworkPoliciesDTO {
+    @ApiProperty()
+    policies: string []
+}

src/modules/network-policy/network-policy.controller.ts

@@ -0,0 +1,64 @@
+import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Post } from '@nestjs/common';
+import { ApiOkResponse } from '@nestjs/swagger';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { NetworkPoliciesDTO } from './dto/network-policy.dto';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { NetworkPolicyService } from './network-policy.service';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+
+@Controller('network-policy')
+export class NetworkPolicyController {
+  constructor(private networkPolicyService: NetworkPolicyService) {}
+
+  @Get()
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  async getNetworks(
+    @User() user: RequestUser,
+  ) {
+    return await this.networkPolicyService.getByCustomer(
+      user.customer_id
+    );
+  }
+
+  @Post()
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @ApiOkResponse()
+  @HttpCode(HttpStatus.CREATED)
+  async applyNetworkPolicies(
+    @User() user: RequestUser,
+    @Body() data: NetworkPoliciesDTO,
+  ) {
+    const metadata = PackTheMetadata(user);
+
+    return await this.networkPolicyService.apply(
+      data.policies,
+      user.customer_id,
+      metadata,
+    );
+  }
+
+  @Delete()
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @ApiOkResponse()
+  @HttpCode(HttpStatus.OK)
+  async removeNetworkPolicies(
+    @User() user: RequestUser,
+    @Body() data: NetworkPoliciesDTO,
+  ) {
+    const metadata = PackTheMetadata(user);
+
+    return await this.networkPolicyService.delete(
+      data.policies,
+      user.customer_id,
+      metadata,
+    );
+  }
+}

src/modules/network-policy/network-policy.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common';
+import { NetworkPolicyController } from './network-policy.controller';
+import { NetworkPolicyService } from './network-policy.service';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+const ducClient = new DucClient();
+
+@Module({
+  imports: [
+    ClientsModule.register([
+      ducClient.providerOptions
+    ]),
+  ],
+  controllers: [NetworkPolicyController],
+  providers: [NetworkPolicyService, DadosferaLogger]
+})
+export class NetworkPolicyModule {}

src/modules/network-policy/network-policy.service.ts

@@ -0,0 +1,74 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { Metadata } from '@grpc/grpc-js';
+import { lastValueFrom } from 'rxjs';
+import { NetworkPoliciesDTO } from './dto/network-policy.dto';
+
+
+@Injectable()
+export class NetworkPolicyService {
+  private customerService: CustomersProtoService;
+  private logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.customerService = this.grpcClient.getService<CustomersProtoService>(
+      ProtoServices.CustomersProtoService,
+    );
+  }
+
+  async getByCustomer(id: string): Promise<NetworkPoliciesDTO> {
+    const {
+      customer
+    } = await lastValueFrom(this.customerService.CustomerFindOneById({
+      id
+    }));
+
+    return {
+      policies: customer.networkPolicies
+    }
+  }
+
+  async apply(
+    networkPolicies: string[],
+    customerId: string,
+    metadata: Metadata,
+  ) {
+    return await lastValueFrom(
+      this.customerService.CustomerCreateNetworkPolicy(
+        {
+          customerId,
+          networkPolicies,
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async delete(
+    networkPolicies: string[],
+    customerId: string,
+    metadata: Metadata,
+  ) {
+    return await lastValueFrom(
+      this.customerService.CustomerRemoveNetworkPolicy(
+        {
+          customerId,
+          networkPolicies,
+        },
+        metadata,
+      ),
+    );
+  }
+}

src/modules/oauth/oauth.controller.ts

@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
 import { ConnectionClientService } from '../connection/client.service';
 import jwt from 'jsonwebtoken';
 import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
-import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 @ApiTags('oauth')
 @Controller('oauth')

src/modules/open-data/dto/wordpres-form.ts

@@ -0,0 +1,48 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class CreateUserOpenDataDTO {
+  @ApiProperty()
+  firstName: string;
+  @ApiProperty()
+  lastName: string;
+  @ApiProperty()
+  email: string;
+  @ApiProperty()
+  organization: string;
+  @ApiProperty()
+  enquiryType: string;
+}
+
+type FormField = {
+  id: string;
+  type: string;
+  title: string;
+  value: string;
+  raw_value: string;
+  required: string;
+};
+
+type MetaData = {
+  title: string;
+  value: string;
+};
+
+export type WordpressForm = {
+  form: {
+    id: string;
+    name: string;
+  };
+  fields: {
+    [key: string]: FormField;
+  };
+  meta: {
+    date: MetaData;
+    time: MetaData;
+    page_url: MetaData;
+    user_agent: MetaData;
+    remote_ip: MetaData;
+    credit: MetaData;
+  };
+};
+
+

src/modules/open-data/open-data.controller.ts

@@ -0,0 +1,101 @@
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { Body, Controller, ForbiddenException, Header, Headers, HttpCode, HttpException, Inject, Param, Post, Query, Req, Res, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
+import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { UsersService } from '../users/users.service';
+import { Language } from 'src/decorators/language.decorator';
+import { OpenDataService } from './open-data.service';
+import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
+import { Metadata } from '@grpc/grpc-js';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { request } from 'http';
+import { Request } from 'express';
+
+@Controller('open-data')
+@ApiInternalOnlyController()
+@ApiTags('OpenData')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
+@UseFilters(GrpcToHttpExceptionFilter)
+export class OpenDataController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private openDataService: OpenDataService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post("/sharing-ocean-data")
+  @HttpCode(200)
+  @Header('content-type', 'application/json')
+  @ApiOkResponse()
+  async createUser(
+    @Body()
+    body: WordpressForm,
+    @Query('language')
+    language: string,
+    @Req()
+    request: Request,
+    @Headers('origin')
+    origin: string
+  ) {
+    this.logger.info('createUser for open data' + JSON.stringify(request.headers));
+
+    // const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
+    // if (!corslist.includes(origin)) {
+    //   this.logger.info('block request by cors list: '+ origin);
+    //   throw new ForbiddenException();
+    // }
+
+    const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
+    const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
+    const roles = [process.env.OPEN_GROUP_ID];
+    const metadata = PackTheMetadata({
+      language: language || 'en-us'
+    }); 
+
+    const data = {}
+
+    try {
+      Object.keys(body.fields)
+        .filter(key => body.fields[key].required === "1")
+        .forEach(key => {
+          const field = body.fields[key]
+          data[field.id] = field.value
+        });
+    } catch (e) {
+      this.logger.error('user data ' + e.message);
+    }
+
+    const user: CreateUserOpenDataDTO = {
+      email: data["email"],
+      enquiryType: data["enquiry_type"],
+      firstName: data["first_name"],
+      lastName: data["last_name"],
+      organization: data["organization"]
+    }
+    this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`);
+
+    try {
+      const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata);
+      this.logger.info('user created with id: '+ id);
+      return {
+        success: true,
+        status: 'success',
+        message: 'user created with succesfull'
+      }
+    } catch (e) {
+      this.logger.error('failed with exception: ' + e.message);
+      return {
+        success: false,
+        status: 'failed',
+        message: e.message
+      };
+    }
+  }
+
+}

src/modules/open-data/open-data.module.ts

@@ -0,0 +1,22 @@
+import { Module } from '@nestjs/common';
+import { OpenDataController } from './open-data.controller';
+import { UsersService } from '../users/users.service';
+import { ClientsModule } from '@nestjs/microservices'
+import { DucClient } from '../duc/client.config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { RolesModule } from '../roles/roles.module';
+import { PermissionsModule } from '../permissions/permissions.module';
+import { OpenDataService } from './open-data.service';
+
+const client = new DucClient();
+
+@Module({
+  controllers: [OpenDataController],
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    RolesModule,
+    // PermissionsModule,
+  ],
+  providers: [DadosferaLogger, UsersService, OpenDataService]
+})
+export class OpenDataModule {}

src/modules/open-data/open-data.service.ts

@@ -0,0 +1,51 @@
+import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
+import { lastValueFrom } from 'rxjs';
+import { CreateUserOpenDataDTO } from './dto/wordpres-form';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { Metadata } from '@grpc/grpc-js';
+
+@Injectable()
+export class OpenDataService implements OnModuleInit {
+  logger: DadosferaLogger;
+
+  private usersClientService: UsersProtoService;
+  constructor(
+    @Inject(DadosferaLogger)
+    private dadosferaLogger: DadosferaLogger,
+    @Inject(DucClient.name)
+    private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.usersClientService = this.grpcClient.getService(
+      ProtoServices.UsersProtoService,
+    );
+
+  }
+
+  async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) {
+    const body = {
+      email: data.email,
+      name: data.firstName + " " + data.lastName,
+      department: data.organization,
+      jobTitle: data.enquiryType,
+      customerId: customerId,
+      roleIds: roleIds
+    }
+
+    try {
+      const { user } = await lastValueFrom(
+        this.usersClientService.SimpleUserCreate(body, metadata),
+      );
+      return user.id;
+    } catch(err) {
+      return err;
+    }
+  }
+}