Merge pull request #219 from dadosfera/feat/pipeline-monitoring-dashboard

Feat: pipeline monitoring dashboard

.ebextensions/01_setup_swap.config

@@ -0,0 +1,3 @@
+container_commands:
+    01setup_swap:
+        command: "bash .ebextensions/setup_swap.sh"

.ebextensions/setup_swap.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+SWAPFILE=/var/swapfile
+SWAP_MEGABYTES=1024
+
+if [ -f $SWAPFILE ]; then
+    echo "Swapfile $SWAPFILE found, assuming already setup"
+    exit;
+fi
+
+/bin/dd if=/dev/zero of=$SWAPFILE bs=1M count=$SWAP_MEGABYTES
+/bin/chmod 600 $SWAPFILE
+/sbin/mkswap $SWAPFILE
+/sbin/swapon $SWAPFILE

.eslintrc.js

@@ -6,8 +6,9 @@ module.exports = {
   },
   plugins: ['@typescript-eslint/eslint-plugin'],
   extends: [
+    'eslint:recommended',
     'plugin:@typescript-eslint/recommended',
-    'plugin:prettier/recommended',
+    'prettier',
   ],
   root: true,
   env: {

.github/workflows/deploy-manually.yml

@@ -0,0 +1,228 @@
+name: Deploy
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Deployment environment"
+        required: true
+        type: choice
+        options:
+          - stg
+          - stg2
+          - prd
+      push_to_dockerhub:
+        description: "Push image to Dockerhub?"
+        required: true
+        type: boolean
+        default: false
+
+jobs:
+  extract_environment:
+    runs-on: ubuntu-latest
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    env:
+      EVENT: ${{ github.event_name }}
+      DEPLOY_ENV: ${{ github.event.inputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ ${EVENT} == "workflow_dispatch" ]; then
+            echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
+          elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          fi
+        id: extract_environment
+
+  semantic_release:
+    runs-on: ubuntu-latest
+    outputs:
+      new_release_published: ${{ steps.semantic.outputs.new_release_published }}
+      new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - if: github.event_name != 'workflow_dispatch'
+        name: Semantic Release
+        uses: cycjimmy/semantic-release-action@v4
+        id: semantic
+        with:
+          extra_plugins: |
+            conventional-changelog-eslint@4.0.0
+          branches: |
+            [
+              'main'
+            ]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  deploy-info:
+    if: ${{ github.event_name == 'workflow_dispatch'}}
+    needs: [extract_environment, semantic_release]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Create summary
+        env:
+          EVENT: ${{ github.event_name }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY
+
+  deploy:
+    if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
+    needs: [extract_environment, semantic_release]
+    runs-on:
+      [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update Pip
+        run: |
+          python3 -m pip install --upgrade pip
+
+      - name: Install Docker Compose
+        run: |
+          python3 -m pip install docker-compose --upgrade
+
+      - name: Install AWS CLI
+        run: |
+          python3 -m pip install awscli --upgrade
+
+      - name: Install AWS Elastic Beanstalk CLI
+        run: |
+          python3 -m pip install awsebcli --upgrade
+
+      - name: Configure AWS Region
+        uses: aws-actions/configure-aws-credentials@v4
+        id: aws
+        with:
+          aws-region: us-east-1
+
+      - name: Login to AWS ECR
+        id: login_ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, Tag, and Push Image to AWS ECR
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+        run: |
+          docker-compose -f build.docker-compose.yml build
+          docker-compose -f build.docker-compose.yml push
+
+      - name: Login to Docker Hub
+        if: ${{inputs.push_to_dockerhub}}
+        uses: docker/login-action@v2
+        with:
+          username: dadosfera
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build, Tag, and Push Image to Dockerhub
+        if: ${{inputs.push_to_dockerhub}}
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+        run: |
+          docker-compose -f build.docker-compose.dockerhub.yml build
+          docker-compose -f build.docker-compose.dockerhub.yml push
+
+      - name: Create ZIP file to Deploy AWS Beanstalk
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+          NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
+        run: |
+          sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
+          sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+          sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
+          sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
+          zip deploy.zip docker-compose.yml  -r .ebextensions
+
+      - name: Deploy AWS Beanstalk
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          AWS_REGION: us-east-1
+          APP_NAME: ${{ github.event.repository.name }}
+        run: |
+          eb use $APP_NAME-$ENV
+          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
+          eb deploy
+
+      - name: Remove Docker's Trash
+        if: always()
+        run: |
+          docker system prune --volumes -a -f
+          docker system df
+  api_docs:
+    needs: [extract_environment, semantic_release, deploy]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract Docs BlockId and PageId
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
+          STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
+          STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
+          STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
+          PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
+          PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
+        shell: bash
+        run: |
+          if [ $ENV == "stg2" ]; then
+            echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
+          elif [ $ENV == "stg" ]; then
+            echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
+          elif [ $ENV == "prd" ]; then
+            echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
+          fi
+        id: extract_docs_info
+
+      - name: Generate API docs
+        env:
+          DOCS_URL: ${{ secrets.DOCS_URL }}
+          DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
+          DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
+          DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
+          EVENT: ${{ github.event_name }}
+          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
+        run: |
+          if [ ${EVENT} != "workflow_dispatch" ]; then
+            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
+          fi
+          sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
+          sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
+          sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
+          curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
+
+      - name: Generate External API docs
+        env:
+          DOCS_URL: ${{ secrets.DOCS_URL }}
+          DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
+          DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}}
+          DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}}
+          EVENT: ${{ github.event_name }}
+          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
+        run: |
+          if [ ${EVENT} != "workflow_dispatch" ]; then
+            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json
+          fi
+          sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json
+          sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json
+          sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json
+          curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL

.github/workflows/deploy.yml

@@ -1,150 +0,0 @@
-name: Deploy
-on:
-  push:
-    branches:
-      - alpha
-      - beta
-      - main
-
-jobs:
-  extract_environment:
-    runs-on: ubuntu-latest
-    outputs:
-      environment: ${{ steps.extract_environment.outputs.environment }}
-    steps:
-      - name: Extract Environment
-        run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=environment;]$(echo "prd")"
-          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
-            echo "##[set-output name=environment;]$(echo "dev")"
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "##[set-output name=environment;]$(echo "stg")"
-          fi
-        id: extract_environment
-
-  semantic_release:
-    runs-on: ubuntu-latest
-    outputs:
-      new_release_published: ${{ steps.semantic.outputs.new_release_published }}
-      new_release_version: ${{ steps.semantic.outputs.new_release_version }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
-        id: semantic
-        with:
-          semantic_version: 16
-          extra_plugins: |
-            @saithodev/semantic-release-backmerge
-            conventional-changelog-eslint
-          branches: |
-            [
-              'main',
-              {
-                name: 'alpha',
-                prerelease: true
-              },
-              {
-                name: 'beta',
-                prerelease: true
-              }
-            ]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-
-  deploy:
-    needs: [extract_environment, semantic_release]
-    runs-on: [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Update Pip
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        run: |
-          python3 -m pip install --upgrade pip
-
-      - name: Install Docker Compose
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        run: |
-          python3 -m pip install docker-compose --upgrade
-
-      - name: Install AWS CLI
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        run: |
-          python3 -m pip install awscli --upgrade
-
-      - name: Install AWS Elastic Beanstalk CLI
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        run: |
-          python3 -m pip install awsebcli --upgrade
-
-      - name: Configure AWS Region
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        uses: aws-actions/configure-aws-credentials@v1
-        id: aws
-        with:
-          aws-region: us-east-1
-
-      - name: Extract Environment
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        shell: bash
-        run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=result;]$(echo "prd")"
-          elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
-            echo "##[set-output name=result;]$(echo "dev")"
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "##[set-output name=result;]$(echo "stg")"
-          fi
-        id: extract_environment
-
-      - name: Login to AWS ECR
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        id: login_ecr
-        uses: aws-actions/amazon-ecr-login@v1
-
-      - name: Build, Tag, and Push Image to AWS ECR
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
-          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
-          docker-compose -f build.docker-compose.yml push
-
-      - name: Create ZIP file to Deploy AWS Beanstalk
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
-          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
-        run: |
-          sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
-          sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
-          sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
-          zip deploy.zip docker-compose.yml  -r .ebextensions
-
-      - name: Deploy AWS Beanstalk
-        if: needs.semantic_release.outputs.new_release_published == 'true'
-        env:
-          ENV: ${{ steps.extract_environment.outputs.result }}
-          AWS_REGION: us-east-1
-          APP_NAME: maestro
-        run: |
-          eb use $APP_NAME-$ENV
-          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
-          eb deploy
-
-      - name: Remove Docker's Trash
-        continue-on-error: true
-        run: |
-          docker system prune
-          docker rmi -f $(docker images -aq)

.github/workflows/test.yml

@@ -2,8 +2,6 @@ name: Test
 on:
   pull_request:
     branches:
-      - alpha
-      - beta
       - main
 
 jobs:
@@ -11,36 +9,23 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Install Docker Compose
-        run: |
-          python3 -m pip install docker-compose --upgrade
+        uses: actions/checkout@v3
 
       - name: Build
         env:
-          ENV: test
-          IMAGE_TAG: test
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          APP_NAME: ${{ github.event.repository.name }}
         run: |
-          export ENV=test
-          export IMAGE_TAG=test
-          export ACCOUNT_ID=test
-          docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
+          docker build -t $APP_NAME --target test .
 
       - name: Run Test
         env:
           ENV: test
-          IMAGE_TAG: test
-          ACCOUNT_ID: test
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          APP_NAME: ${{ github.event.repository.name }}
         run: |
-          export ENV=test
-          export IMAGE_TAG=test
-          docker-compose -f build.docker-compose.yml run -e NPM_TOKEN=${NPM_TOKEN} --rm --entrypoint="npm run test" maestro
+          docker run --rm --entrypoint="npm" $APP_NAME run test
 
       - name: Remove Docker's Trash
-        continue-on-error: true
+        if: always()
         run: |
-          docker system prune
-          docker rmi -f $(docker images -aq)
+          docker system prune --volumes -a -f
+          docker system df

.gitignore

@@ -29,7 +29,7 @@ lerna-debug.log*
 
 # IDE - VSCode
 .vscode/*
-!.vscode/settings.json
+#!.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
@@ -37,4 +37,4 @@ lerna-debug.log*
 # Ignore .env files
 *.env
 
-.env
+.env

.npmrc

@@ -1 +0,0 @@
-//registry.npmjs.org/:_authToken=${NPM_TOKEN}

.nvmrc

@@ -0,0 +1 @@
+18.10.0

.releaserc.json

@@ -5,18 +5,18 @@
       {
         "preset": "eslint",
         "releaseRules": [
-          { "tag": "FIX", "release": "patch" },
+          { "tag": "BREAKING", "release": "major" },
           { "tag": "FEAT", "release": "minor" },
           { "tag": "UPDATE", "release": "minor" },
           { "tag": "UPGRADE", "release": "minor" },
-          { "tag": "BREAKING", "release": "major" },
-          { "tag": "DOCS", "release": "patch" },
           { "tag": "CHORE", "release": "minor" },
-          { "tag": "BUILD", "release": "patch" },
           { "tag": "REFACTOR", "release": "minor" },
+          { "tag": "PERF", "release": "minor" },
+          { "tag": "FIX", "release": "patch" },
+          { "tag": "DOCS", "release": "patch" },
+          { "tag": "BUILD", "release": "patch" },
           { "tag": "TEST", "release": "patch" },
-          { "tag": "CI", "release": "patch" },
-          { "tag": "PERF", "release": "minor" }
+          { "tag": "CI", "release": "patch" }
         ]
       }
     ],
@@ -26,19 +26,6 @@
         "preset": "eslint"
       }
     ],
-    [
-      "@saithodev/semantic-release-backmerge",
-      {
-        "branches": [
-          { "from": "main", "to": "alpha" },
-          { "from": "main", "to": "beta" }
-        ],
-        "backmergeStrategy": "merge",
-        "clearWorkspace": true,
-        "restoreWorkspace": true
-      }
-    ],
-    "@semantic-release/npm",
     "@semantic-release/github"
   ]
 }

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug NestJs",
+      "request": "launch",
+      "runtimeArgs": [
+        "run",
+        "start:debug"
+      ],
+      "runtimeExecutable": "npm",
+      "runtimeVersion": "18.10.0",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "type": "node",
+      "console": "integratedTerminal"
+    }
+  ]
+}

Dockerfile

@@ -1,16 +1,21 @@
-FROM node:14.15.4-alpine3.12 as packages
-WORKDIR /packages
-ARG NPM_TOKEN
-COPY package.json .
-COPY .npmrc .
-RUN npm install
-RUN rm -f ./.npmrc 
+# install all dependencies
+FROM node:18.10-alpine as packages
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN apk add --no-cache aws-cli \
+ && aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
+ && npm install
 
-FROM node:14.15.4-alpine3.12
+# unit test specific build
+FROM node:18.10-alpine as test
+WORKDIR /app
+COPY --from=packages /app/node_modules ./node_modules
+COPY . ./
+
+FROM node:18.10-alpine
 WORKDIR /app
 COPY . /app/
-ARG NPM_TOKEN
-COPY --from=packages /packages/node_modules /app/node_modules
+COPY --from=packages /app/node_modules ./node_modules
 RUN npm run build
 EXPOSE 3333
-ENTRYPOINT npm run start
+ENTRYPOINT npm run start

LICENSE

@@ -0,0 +1,10 @@
+LICENSE
+Copyright (C) 2023 Dadosfera
+
+All rights reserved.
+
+This software and related documentation are provided under a license agreement containing restrictions on use and disclosure and are protected by intellectual property laws. Except as expressly permitted in your license agreement or allowed by law, you may not use, copy, reproduce, translate, broadcast, modify, license, transmit, distribute, exhibit, perform, publish, or display any part, in any form, or by any means.
+
+Reverse engineering, disassembly, or decompilation of this software, unless required by law for interoperability, is prohibited.
+
+If you have any questions about this, please contact Dadosfera.ai at legal@dadosfera.ai

README.md

@@ -1,4 +1,350 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-  <h1 align="center">Maestro</h1> 
-</p>
+</p>
+
+# Maestro
+
+Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
+
+## 🚀 Starting
+
+These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
+
+### 📋 Requirements
+
+- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
+- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
+  - Create your Access Key on the "Security credentials" menu
+  - Set the Access Key on your local development machine
+- Request access to the dev, stg and prd VPNs
+
+### 🔧 Installation<a id="installation"></a>
+
+- Clone the repository
+
+  - SSH
+
+    ```
+    git clone git@github.com:dadosfera/maestro.git
+    ```
+
+    or
+
+  - HTTPS
+    ```
+    git clone https://github.com/dadosfera/maestro.git
+    ```
+
+- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
+
+  ```sh
+  nvm use
+  ```
+
+- Install the project dependencies:
+
+  ```sh
+  npm i
+  ```
+
+- Setup the following enviroment variables:
+
+  ```
+  ENV=
+  DUC_URL=
+  INFACTORY_URL=
+  OTFACTORY_URL=
+  PIFACTORY_URL=
+  SM_OAUTH_PATH=
+  ```
+
+- Start the server:
+
+  ```sh
+  # dev mode
+  npm run start:dev
+
+  # or in debug mode
+  npm run start:debug
+  ```
+
+  The service should start successfully.
+
+## 📄 Documentation
+
+NestJs makes it easy to document each route using decorators on all requests and responses properties. It automatically generates a swagger for given information and provides a route to access it http://localhost:3333/api.
+For more info check the [official documentation](https://docs.nestjs.com/openapi/introduction).
+
+### - Multiple documentations
+
+We are currently generating **2 different** documentations: Internal and External.
+
+All routes that have the decorator `@ApiInternalOnly()` will not be visible on the **External** API swagger.
+
+- When you start the application with `npm run start` it will serve and generate the swagger JSON of the **External** API
+- When you start the application with `npm run start:internal` it will serve and generate the swagger JSON of the **Internal** API
+- When you run `npm run docs` it will generate the swagger JSON of both **Internal** and **External** API, and save them to `docsfera.json` and `docsfera.external.json` respectively;
+
+It is important to run `npm run docs` before every deploy so we can always have the most updated docs published.
+
+## Authentication decorators
+
+Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
+
+### `@Authenticated`
+
+If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
+
+```ts
+import { Authenticated } from '../../authentication/authentication.decorator';
+
+@Controller('foo')
+@Authenticated()
+class FooController {
+  /* ... */
+
+  @Get('bar')
+  async getBar() {
+    this.logger.info('user is authenticated!');
+
+    return { authenticated: true };
+  }
+}
+```
+
+```ts
+import { Authenticated } from '../../authentication/authentication.decorator';
+
+@Controller('foo')
+class FooController {
+  /* ... */
+
+  @Get('bar')
+  @Authenticated()
+  async getBar() {
+    this.logger.info('user is authenticated!');
+
+    return { authenticated: true };
+  }
+
+  @Post('bar')
+  async postBar() {
+    this.logger.info('user is NOT authenticated!');
+
+    return { authenticated: false };
+  }
+}
+```
+
+### `@RequireAllPermissions`
+
+This decorator requires that **all permissions** listed are granted to the requesting user.
+
+```ts
+import { RequireAllPermissions } from '../../authentication/authentication.decorator';
+import { PERMISSIONS } from '../../authentication/permissions.enum';
+
+@Controller('foo')
+class FooController {
+  /* ... */
+
+  @Get('bar')
+  @RequireAllPermissions(Permissions.BAR.MANAGE, Permissions.BAR.CREATE)
+  async getBar() {
+    /* ... */
+  }
+}
+```
+
+### `@RequireSomePermission`
+
+In the following case, the user is required to have **at least one** listed permission.
+
+```ts
+import { RequireSomePermission } from '../../authentication/authentication.decorator';
+import { PERMISSIONS } from '../../authentication/permissions.enum';
+
+@Controller('foo')
+@RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.CREATE)
+class FooController {
+  /* ... */
+
+  @Get('bar')
+  async getBar() {
+    /* ... */
+  }
+}
+```
+
+### `@AuthenticateCondition`
+
+If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
+
+In the following example:
+
+- all routes on the `FooController` controller can only be requested from localhost
+- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
+
+```ts
+import { AuthenticateCondition } from '../../authentication/authentication.decorator';
+
+@Controller('foo')
+// allow requests only from localhost
+@AuthenticateCondition((request: Request) => request.ip === '::ffff:127.0.0.1')
+class FooController {
+  /* ... */
+
+  @Get('bar')
+  async getBar() {
+    /* ... */
+  }
+
+  @Post('bar')
+  // allow requests only from a specific customer
+  @AuthenticateCondition(
+    (request: Request, user: RequestUser) =>
+      user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
+  )
+  async postBar() {
+    /* ... */
+  }
+}
+```
+
+### Note on authentication decorators
+
+- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
+
+- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (_and wouldn't like to..._) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
+
+  ```ts
+  import {
+    RequireAllPermissions,
+    RequireSomePermission,
+    AuthenticateCondition,
+  } from '../../authentication/authentication.decorator';
+  import { PERMISSIONS } from '../../authentication/permissions.enum';
+  import { Waa, Baz } from './authenticationFunctions';
+
+  @Controller('foo')
+  @RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
+  @RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.ADMIN)
+  @AuthenticateCondition(Waa)
+  @AuthenticateCondition(Baz)
+  class FooController {
+    /* ... */
+
+    @Get('bar')
+    async getBar() {
+      /* ... */
+    }
+
+    @Post('bar')
+    @RequireAllPermissions(
+      Permissions.BAR.MANAGE,
+      Permissions.BAR.USE,
+      Permissions.BAR.REQUEST,
+    )
+    async postBar() {
+      /* ... */
+    }
+  }
+  ```
+
+- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
+
+  ```ts
+  // same behavior
+  @RequireAllPermissions(Permissions.BAR.MANAGE)
+  @RequireSomePermission(Permissions.BAR.MANAGE)
+  ```
+
+## `@User` parameter decorator
+
+The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
+
+```ts
+import { User, RequestUser } from '../../authentication/user.decorator';
+
+@Controller('foo')
+class FooController {
+  /* ... */
+
+  @Get('bar')
+  async getBar(@User() user: RequestUser) {
+    this.logger.info(user);
+
+    return { user };
+  }
+}
+```
+
+If the user is required to be logged in, set `required` to `true`, as you would want in the `change-password` operation:
+
+```ts
+import { User, RequestUser } from '../../authentication/user.decorator';
+
+@Controller('user')
+class UserController {
+  /* ... */
+
+  @Post('change-password')
+  @HttpCode(HttpStatus.OK)
+  async changePassword(
+    @User({ required: true }) user: RequestUser,
+    @Body() body: AuthChangePasswordRequest,
+  ) {
+    const { oldPassword, newPassword } = body;
+
+    return this.authClient.changePassword({
+      userId: user.user_id,
+      oldPassword,
+      newPassword,
+    });
+  }
+}
+```
+
+## 📦 Development
+
+### ⌨️ Coding Style
+
+By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
+
+**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
+
+### Commits pattern
+
+Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
+
+Format: `<type>[optional scope]: <description>`
+
+Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
+
+### Branching naming convention
+
+- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
+
+  **It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (\_) could be used to separate the ID and description.**
+
+- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
+
+- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
+
+- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
+
+### Making a Pull Request
+
+1. Commit your changes
+2. Open the Pull Request on GitHub
+3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
+
+## 🛠️ Built with
+
+Some technologies used in this project:
+
+- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
+
+## ⚙️ Back-end Architecture
+
+The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).

build.docker-compose.dockerhub.yml

@@ -0,0 +1,5 @@
+version: "3.8"
+services:
+  maestro:
+    build: .
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

docker-compose.dockerhub.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+services:
+  maestro:
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+    container_name: maestro
+    restart: always
+    ports:
+      - 3333:3333
+    env_file:
+      - .env
+
+  node_exporter:
+    image: ${NODE_EXPORTER_URL}
+    container_name: node_exporter
+    restart: always
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - .env
 
   node_exporter:
-    image: prom/node-exporter:v1.3.1
+    image: ${NODE_EXPORTER_URL}
     container_name: node_exporter
     restart: always
     volumes:

environment.d.ts

@@ -0,0 +1,18 @@
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      NODE_ENV: 'test';
+      ENV: 'local' | 'stg' | 'prd' | 'test';
+      LOCAL_ENV: 'stg' | 'prd';
+
+      DUC_URL: string;
+      INFACTORY_URL: string;
+      PIFACTORY_URL: string;
+
+      INTERNAL_SWAGGER: 'true' | 'false';
+
+      AWS_REGION: string;
+    }
+  }
+}
+export {};

jest.config.ts

@@ -0,0 +1,18 @@
+import type { Config } from '@jest/types';
+const config: Config.InitialOptions = {
+  moduleFileExtensions: ['ts', 'js', 'json'],
+  modulePaths: ['<rootDir>'],
+  roots: ['<rootDir>/src/', '<rootDir>/test/'],
+  testRegex: '.*\\.(test|spec)\\.[jt]s$',
+  transform: { '\\.[jt]s$': 'ts-jest' },
+  collectCoverageFrom: ['**/*.[jt]s'],
+  coverageDirectory: 'coverage',
+  coveragePathIgnorePatterns: [
+    '<rootDir>/node_modules/',
+    '.*\\.module\\.[jt]s$',
+  ],
+  // moduleDirectories: ['node_modules'], // default is already 'node_modules'
+  // rootDir: '.', //No need
+  // testEnvironment: 'node', //Defaults to 'node'
+};
+export default config;

nest-cli.json

@@ -2,7 +2,16 @@
   "collection": "@nestjs/schematics",
   "sourceRoot": "src",
   "compilerOptions": {
-    "assets": ["**/*.proto"],
+    "assets": [
+      "**/*.proto",
+      {
+        "include": "i18n/**/*",
+        "watchAssets": true
+      }
+    ],
+    "plugins": [
+      "@nestjs/swagger"
+    ],
     "watchAssets": true
   }
-}
+}

package.json

@@ -1,18 +1,24 @@
 {
-  "name": "grpc-server-and-client-nestjs-micro-pattern",
+  "name": "maestro",
   "version": "0.0.0",
   "description": "",
   "author": "",
   "private": true,
   "license": "UNLICENSED",
+  "engines": {
+    "node": "18.10.0"
+  },
   "scripts": {
+    "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
+    "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
     "prebuild": "rimraf dist",
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "nest start",
-    "start:dev": "nest start --watch",
-    "start:debug": "nest start --debug --watch",
+    "start:dev": "export INTERNAL_SWAGGER=true && nest start --watch",
+    "start:debug": "npm run start:dev",
     "start:prod": "node dist/main",
+    "docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "jest",
     "test:watch": "jest --watch",
@@ -21,66 +27,72 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
-    "@grpc/grpc-js": "^1.5.10",
-    "@grpc/proto-loader": "^0.6.9",
-    "@nestjs/common": "^8.4.3",
-    "@nestjs/config": "^1.2.1",
-    "@nestjs/core": "^8.4.3",
-    "@nestjs/mapped-types": "*",
-    "@nestjs/microservices": "^8.4.3",
-    "@nestjs/platform-express": "^8.4.3",
-    "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.5.5",
-    "axios": "^0.25.0",
-    "dotenv": "^14.2.0",
-    "helmet": "^5.0.2",
-    "jsonwebtoken": "^8.5.1",
+    "@aws-sdk/client-secrets-manager": "^3.112.0",
+    "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
+    "@dadosfera/protospack-v2": "3.33.0",
+    "@grpc/grpc-js": "^1.6.7",
+    "@grpc/proto-loader": "^0.6.13",
+    "@nestjs/cli": "^9.4.2",
+    "@nestjs/common": "^9.4.0",
+    "@nestjs/config": "^2.3.1",
+    "@nestjs/core": "^9.4.0",
+    "@nestjs/mapped-types": "^1.2.2",
+    "@nestjs/microservices": "^9.4.0",
+    "@nestjs/passport": "^9.0.3",
+    "@nestjs/platform-express": "^9.4.0",
+    "@nestjs/schematics": "^9.1.0",
+    "@nestjs/swagger": "^6.3.0",
+    "@nestjs/testing": "^9.4.0",
+    "axios": "^0.27.2",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.0",
+    "cron-parser": "^4.4.0",
+    "dotenv": "^14.3.2",
+    "elastic-apm-node": "^3.36.0",
+    "helmet": "^5.1.0",
+    "jsonwebtoken": "^9.0.0",
     "jwk-to-pem": "^2.0.5",
+    "mixpanel": "^0.17.0",
+    "ms": "^3.0.0-canary.1",
+    "passport": "^0.6.0",
+    "passport-facebook": "^3.0.0",
+    "passport-forcedotcom": "^0.2.0",
+    "passport-google-oauth20": "^2.0.0",
+    "passport-hubspot-oauth2": "^1.0.3",
+    "passport-mailchimp": "^1.1.0",
+    "protospack": "2.5.2",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",
-    "swagger-ui-express": "^4.3.0"
+    "swagger-ui-express": "^4.4.0"
+  },
+  "overrides": {
+    "multer": "1.4.5-lts.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^8.2.4",
-    "@nestjs/schematics": "^8.0.8",
-    "@nestjs/testing": "^8.4.3",
     "@types/express": "^4.17.13",
     "@types/jest": "27.0.2",
     "@types/jsonwebtoken": "^8.5.8",
     "@types/jwk-to-pem": "^2.0.1",
-    "@types/node": "^16.11.26",
+    "@types/multer": "^1.4.7",
+    "@types/node": "^16.11.41",
+    "@types/passport-facebook": "^2.1.11",
+    "@types/passport-google-oauth20": "^2.0.11",
+    "@types/passport-oauth2": "^1.4.11",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.17.0",
-    "@typescript-eslint/parser": "^5.17.0",
-    "eslint": "^8.12.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "eslint": "^8.18.0",
     "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-prettier": "^4.0.0",
     "jest": "^27.5.1",
-    "prettier": "^2.6.1",
+    "nock": "^13.2.7",
+    "prettier": "^2.7.1",
     "source-map-support": "^0.5.20",
-    "supertest": "^6.1.3",
-    "ts-jest": "^27.1.4",
-    "ts-loader": "^9.2.7",
-    "ts-node": "^10.7.0",
+    "supertest": "^6.2.3",
+    "ts-jest": "^27.1.5",
+    "ts-loader": "^9.3.1",
+    "ts-node": "^10.8.1",
     "tsconfig-paths": "^3.14.1",
     "typescript": "^4.6.3"
-  },
-  "jest": {
-    "moduleFileExtensions": [
-      "js",
-      "json",
-      "ts"
-    ],
-    "rootDir": "src",
-    "testRegex": ".*\\.spec\\.ts$",
-    "transform": {
-      "^.+\\.(t|j)s$": "ts-jest"
-    },
-    "collectCoverageFrom": [
-      "**/*.(t|j)s"
-    ],
-    "coverageDirectory": "../coverage",
-    "testEnvironment": "node"
   }
 }

src/app.module.ts

@@ -1,107 +1,65 @@
-import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
-import { ClientsModule } from '@nestjs/microservices';
+import { Module } from '@nestjs/common';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { AuthenticationGuard } from './authentication/authentication.guard';
+
+import { APP_GUARD } from '@nestjs/core';
+
+import { UsersModule } from './modules/users/users.module';
+import { RolesModule } from './modules/roles/roles.module';
+import { ConnectorModule } from './modules/connector/connector.module';
+import { AuthModule } from './modules/auth/auth.module';
+import { PermissionsModule } from './modules/permissions/permissions.module';
+import { TermsOfUseModule } from './modules/termsOfUse/termsOfUse.module';
+import { ConnectionModule } from './modules/connection/connection.module';
+import { ConnectionTestModule } from './modules/connection-test/connection-test.module';
+import { NetworkConfigModule } from './modules/network-config/network-config.module';
+import { InputsModule } from './modules/inputs/inputs.module';
+import { OauthModule } from './modules/oauth/oauth.module';
+import { PipelinesModule } from './modules/pipelines/pipelines.module';
+import { TransformationsModule } from './modules/transformations/transformations.module';
+import { HealthModule } from './modules/health/health.module';
+import { CatalogModule } from './modules/catalog/catalog.module';
 import { ConfigModule } from '@nestjs/config';
-
-import { LoggerMiddleware } from './middlewares/authentication';
-
-import { InputsController } from './modules/inputs/inputs.controller';
-import { TransformationsController } from './modules/transformations/transformations.controller';
-import { OutputsController } from './modules/outputs/outputs.controllers';
-import { PipelinesController } from './modules/pipelines/pipelines.controller';
-import { AuthController } from './modules/auth/auth.controller';
-import { HealthController } from './modules/health/health.controller';
-
-import { InputsService } from './modules/inputs/inputs.service';
-import { TransformationsService } from './modules/transformations/transformations.service';
-import { OutputsService } from './modules/outputs/outputs.service';
-import { PipelinesService } from './modules/pipelines/pipelines.service';
-// import { AuthService } from './modules/auth/auth.service';
-import { HealthService } from './modules/health/health.service';
-
-import { InputsClientService } from './clients/inputs/client.service';
-import { TransformationsClientService } from './clients/transformations/client.service';
-import { OutputsClientService } from './clients/outputs/client.service';
-import { PipelinesClientService } from './clients/pipelines/client.service';
-import { AuthClientService } from './clients/auth/client.service';
-
-import { OutputsClientConfiguration } from './clients/outputs/client.config';
-import { TransformationsClientConfiguration } from './clients/transformations/client.config';
-import { AuthClient } from './clients/auth/client.config';
-import { InputsClientConfiguration } from './clients/inputs/client.config';
-import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
-import { CatalogController } from './modules/catalog/catalog.controller';
-import { CatalogService } from './modules/catalog/catalog.service';
-
-const authClient = new AuthClient();
-const inputClient = new InputsClientConfiguration();
-const outputClient = new OutputsClientConfiguration();
-const pipelineClient = new PipelinesClientConfiguration();
-const transformationClient = new TransformationsClientConfiguration();
+import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
+import { ProductboardModule } from './modules/productboard/productboard.module';
+import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
+import { CustomersModule } from './modules/customers/customers.module';
 
 @Module({
-  controllers: [
-    InputsController,
-    TransformationsController,
-    OutputsController,
-    PipelinesController,
-    AuthController,
-    HealthController,
-    CatalogController,
-  ],
+  controllers: [],
   providers: [
-    InputsService,
-    TransformationsService,
-    OutputsService,
-    PipelinesService,
-    // AuthService,
-    HealthService,
-    InputsClientService,
-    TransformationsClientService,
-    OutputsClientService,
-    PipelinesClientService,
-    AuthClientService,
-    CatalogService,
+    DadosferaLogger,
+    {
+      provide: APP_GUARD,
+      useClass: AuthenticationGuard,
+    },
   ],
   imports: [
     ConfigModule.forRoot({
       isGlobal: true,
     }),
-
-    ClientsModule.register([
-      {
-        name: 'INPUTS_PACKAGE',
-        ...inputClient.config(),
-      },
-
-      {
-        name: 'TRANSFORMATIONS_PACKAGE',
-        ...transformationClient.config(),
-      },
-      {
-        name: 'OUTPUTS_PACKAGE',
-        ...outputClient.config(),
-      },
-      {
-        name: 'AUTH_PACKAGE',
-        ...authClient.config(),
-      },
-      {
-        name: 'PIPELINES_PACKAGE',
-        ...pipelineClient.config(),
-      },
-    ]),
+    AuthModule,
+    ConnectionModule,
+    NetworkConfigModule,
+    InputsModule,
+    PipelinesV2Module,
+    CatalogModule,
+    ConnectorModule,
+    PermissionsModule,
+    TermsOfUseModule,
+    ConnectionTestModule,
+    PipelinesModule,
+    TransformationsModule,
+    UsersModule,
+    RolesModule,
+    OauthModule,
+    ProductboardModule,
+    MixpanelModule,
+    CustomersModule,
+    //Always leave HealthModule last, so it is on the bottom of swagger
+    HealthModule,
   ],
 })
-export class AppModule implements NestModule {
-  configure(consumer: MiddlewareConsumer) {
-    consumer
-      .apply(LoggerMiddleware)
-      .forRoutes(
-        InputsController,
-        TransformationsController,
-        OutputsController,
-        PipelinesController,
-        CatalogController,
-      );
-  }
-}
+export class AppModule {}

src/authentication/authentication.guard.spec.ts

@@ -0,0 +1,457 @@
+import request from 'supertest';
+import { Test } from '@nestjs/testing';
+import { HttpStatus, INestApplication } from '@nestjs/common';
+import { APP_GUARD } from '@nestjs/core';
+import jwt from 'jsonwebtoken';
+
+import { Body, Controller, Get } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import {
+  AuthenticateCondition,
+  Authenticated,
+  RequireAllPermissions,
+  RequireSomePermission,
+} from '../decorators/authentication.decorator';
+import { AuthenticationGuard } from './authentication.guard';
+import { PERMISSIONS_GROUPS } from './permissions.enum';
+import { AuthClientService } from '../modules/auth/auth.service';
+
+import ErrorCodes from '../utils/errorCodes';
+
+const logger = {
+  info: (...args) => args,
+  error: (...args) => args,
+};
+@Controller('no-class-auth')
+class NoClassAuthController {
+  @Get('body')
+  async getBody(@Body() body) {
+    return { body };
+  }
+
+  @Get('authenticated')
+  @Authenticated()
+  async mustBeAuthenticated(@Body() body) {
+    return { body };
+  }
+
+  @Get('required-permission')
+  @RequireSomePermission(PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN)
+  async requiredPermission(@Body() body) {
+    return { body };
+  }
+
+  @Get('has-all-permissions')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
+  )
+  async hasAllPermissions(@Body() body) {
+    return { body };
+  }
+
+  @Get('has-some-permission')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
+  )
+  async hasSomePermission(@Body() body) {
+    return { body };
+  }
+}
+
+@Controller('class-auth-condition')
+@AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
+@RequireSomePermission(PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE)
+class ClassAuthConditionController {
+  @Get('body')
+  async getBody(@Body() body) {
+    return { body };
+  }
+
+  @Get('authenticated')
+  @Authenticated()
+  async mustBeAuthenticated(@Body() body) {
+    return { body };
+  }
+
+  @Get('required-permission')
+  @AuthenticateCondition((req) => req.get('x-on-route') === 'ok')
+  @RequireSomePermission(PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN)
+  async requiredPermission(@Body() body) {
+    return { body };
+  }
+}
+
+describe('authentication.guard', () => {
+  let app: INestApplication;
+  const jwtSecretA = {
+    kid: 'token-a-testing-shared-key',
+    pem: '$tr0ng-SH4Red-secr3t!!!~gl0ba1~]',
+  };
+  const jwtSecretB = {
+    kid: 'token-b',
+    pem: 'another-shared-token',
+  };
+  const fakeUserPayload = {
+    user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
+    username: 'super.admin',
+    customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
+    customer_name: 'dadosfera',
+    customer_tier: 'BASIC',
+  };
+
+  beforeAll(async () => {
+    const moduleRef = await Test.createTestingModule({
+      providers: [
+        {
+          provide: AuthClientService,
+          useValue: {
+            getPublicKeys: async () => ({
+              keys: [jwtSecretA, jwtSecretB],
+            }),
+          },
+        },
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        {
+          provide: APP_GUARD,
+          useClass: AuthenticationGuard,
+        },
+      ],
+      controllers: [NoClassAuthController, ClassAuthConditionController],
+    }).compile();
+
+    app = moduleRef.createNestApplication();
+    await app.init();
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  function AssertBodyNoAuth(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.OK);
+    expect(res.body).toStrictEqual({ body: {} });
+  }
+
+  function AssertBodyWithAuth(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.OK);
+    expect(res.body).toStrictEqual({
+      body: {
+        info: {
+          customer: 'dadosfera',
+          customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
+          customer_tier: 'BASIC',
+          user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
+        },
+      },
+    });
+  }
+
+  function AssertUnauthorized(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.UNAUTHORIZED);
+    expect(res.body).toStrictEqual({
+      code: ErrorCodes.AUTH.UNAUTHORIZED,
+      error: 'Não autenticado',
+      message: 'É necessário estar logado para realizar essa operação',
+      statusCode: HttpStatus.UNAUTHORIZED,
+    });
+  }
+
+  function AssertForbidden(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.FORBIDDEN);
+    expect(res.body).toStrictEqual({
+      code: ErrorCodes.AUTH.FORBIDDEN,
+      error: 'Não autorizado',
+      message:
+        'Você não tem permissões suficientes para realizar essa operação',
+      statusCode: HttpStatus.FORBIDDEN,
+    });
+  }
+
+  function NoClassAuthTest(accessToken: string, tokenName: string[]) {
+    const route = '/no-class-auth';
+
+    describe(`${route}, ${tokenName?.length ? tokenName : 'none'} auth`, () => {
+      it('should GET /body with user data (if available)', async () => {
+        const res = await request(app.getHttpServer())
+          .get(`${route}/body`)
+          .set({ Authorization: accessToken });
+
+        if (tokenName?.length) {
+          AssertBodyWithAuth(res);
+        } else {
+          AssertBodyNoAuth(res);
+        }
+      });
+
+      it('should GET /authenticated if authenticated', async () => {
+        const res = await request(app.getHttpServer())
+          .get(`${route}/authenticated`)
+          .set({ Authorization: accessToken });
+
+        if (tokenName?.length) {
+          AssertBodyWithAuth(res);
+        } else {
+          AssertUnauthorized(res);
+        }
+      });
+
+      it('should GET /required-permission if authenticated', async () => {
+        const res = await request(app.getHttpServer())
+          .get(`${route}/required-permission`)
+          .set({ Authorization: accessToken });
+
+        // zendesk required
+        if (tokenName?.includes('zendesk')) {
+          AssertBodyWithAuth(res);
+        } else if (tokenName?.length) {
+          AssertForbidden(res);
+        } else {
+          AssertUnauthorized(res);
+        }
+      });
+
+      it('should GET /has-all-permissions if authenticated', async () => {
+        const res = await request(app.getHttpServer())
+          .get(`${route}/has-all-permissions`)
+          .set({ Authorization: accessToken });
+
+        // zendesk and metabase
+        if (tokenName?.includes('zendesk') && tokenName?.includes('metabase')) {
+          AssertBodyWithAuth(res);
+        } else if (tokenName?.length) {
+          AssertForbidden(res);
+        } else {
+          AssertUnauthorized(res);
+        }
+      });
+
+      it('should GET /has-some-permission if authenticated', async () => {
+        const res = await request(app.getHttpServer())
+          .get(`${route}/has-some-permission`)
+          .set({ Authorization: accessToken });
+
+        // zendesk or metabase
+        if (tokenName?.includes('zendesk') || tokenName?.includes('metabase')) {
+          AssertBodyWithAuth(res);
+        } else if (tokenName?.length) {
+          AssertForbidden(res);
+        } else {
+          AssertUnauthorized(res);
+        }
+      });
+    });
+  }
+
+  function ClassAuthConditionTest(accessToken: string, tokenName: string[]) {
+    const route = '/class-auth-condition';
+
+    describe(`${route}, ${tokenName?.length ? tokenName : 'none'} auth`, () => {
+      describe('GET /body', () => {
+        it('should GET /body if authenticated and with correct headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/body`)
+            .set({
+              'x-on-class': 'ok',
+              Authorization: accessToken,
+            });
+
+          // metabase required
+          if (tokenName?.includes('metabase')) {
+            AssertBodyWithAuth(res);
+          } else if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+
+        it('should not GET /body without correct headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/body`)
+            .set({ Authorization: accessToken });
+
+          if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+      });
+
+      describe('GET /authenticated', () => {
+        it('should GET /authenticated if authenticated and with correct headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/authenticated`)
+            .set({
+              'x-on-class': 'ok',
+              Authorization: accessToken,
+            });
+
+          // metabase required
+          if (tokenName?.includes('metabase')) {
+            AssertBodyWithAuth(res);
+          } else if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+
+        it('should not GET /authenticated without correct class headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/authenticated`)
+            .set({ Authorization: accessToken });
+
+          if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+      });
+
+      describe('GET /required-permission', () => {
+        it('should GET /required-permission if authenticated and with correct headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/required-permission`)
+            .set({
+              'x-on-class': 'ok',
+              'x-on-route': 'ok',
+              Authorization: accessToken,
+            });
+
+          // zendesk and metabase
+          if (
+            tokenName?.includes('zendesk') &&
+            tokenName?.includes('metabase')
+          ) {
+            AssertBodyWithAuth(res);
+          } else if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+
+        it('should not GET /required-permission without correct route headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/required-permission`)
+            .set({
+              'x-on-class': 'ok',
+              Authorization: accessToken,
+            });
+
+          if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+
+        it('should not GET /required-permission without correct class headers', async () => {
+          const res = await request(app.getHttpServer())
+            .get(`${route}/required-permission`)
+            .set({
+              'x-on-route': 'ok',
+              Authorization: accessToken,
+            });
+
+          if (tokenName?.length) {
+            AssertForbidden(res);
+          } else {
+            AssertUnauthorized(res);
+          }
+        });
+      });
+    });
+  }
+
+  describe('token validation', () => {
+    it('should accept token kid A', async () => {
+      const Authorization = CreateToken([], {
+        jwt: {
+          kid: jwtSecretA.kid,
+          pem: jwtSecretA.pem,
+        },
+      });
+
+      const res = await request(app.getHttpServer())
+        .get('/no-class-auth/authenticated')
+        .set({
+          Authorization,
+        });
+
+      AssertBodyWithAuth(res);
+    });
+
+    it('should accept token kid B', async () => {
+      const Authorization = CreateToken([], {
+        jwt: {
+          kid: jwtSecretB.kid,
+          pem: jwtSecretB.pem,
+        },
+      });
+
+      const res = await request(app.getHttpServer())
+        .get('/no-class-auth/authenticated')
+        .set({
+          Authorization,
+        });
+
+      AssertBodyWithAuth(res);
+    });
+
+    it('should not accept unknown kid', async () => {
+      const Authorization = CreateToken([], {
+        jwt: {
+          kid: 'unknown-key-id',
+          pem: jwtSecretA.pem,
+        },
+      });
+
+      const res = await request(app.getHttpServer())
+        .get('/no-class-auth/authenticated')
+        .set({
+          Authorization,
+        });
+
+      AssertUnauthorized(res);
+    });
+  });
+
+  function CreateToken(permissions, overrides?: { user?: any; jwt?: any }) {
+    const tokenPayload = {
+      ...fakeUserPayload,
+      permissions: permissions.map(({ seqid }) => seqid),
+      token_use: 'access',
+      ...overrides?.user,
+    };
+
+    return jwt.sign(tokenPayload, overrides?.jwt?.pem ?? jwtSecretA.pem, {
+      keyid: overrides?.jwt?.kid ?? jwtSecretA.kid,
+    });
+  }
+
+  NoClassAuthTest(null, null);
+  ClassAuthConditionTest(null, null);
+
+  const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
+  NoClassAuthTest(tokenZ, ['zendesk']);
+  ClassAuthConditionTest(tokenZ, ['zendesk']);
+
+  const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
+  NoClassAuthTest(tokenM, ['metabase']);
+  ClassAuthConditionTest(tokenM, ['metabase']);
+
+  const tokenZM = CreateToken([
+    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
+  ]);
+  NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
+  ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
+});

src/authentication/authentication.guard.ts

@@ -0,0 +1,136 @@
+import {
+  Injectable,
+  CanActivate,
+  OnApplicationBootstrap,
+  ExecutionContext,
+  Inject,
+} from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
+import assert from 'assert';
+import jwt from 'jsonwebtoken';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { AuthClientService } from '../modules/auth/auth.service';
+import {
+  AuthenticationFunction,
+  AUTH_FUNCTION_KEY,
+} from '../decorators/authentication.decorator';
+import { RequestUser } from '../decorators/user.decorator';
+import ErrorBuilder from '../utils/ErrorBuilder';
+import ErrorCodes from '../utils/errorCodes';
+
+@Injectable()
+export class AuthenticationGuard
+  implements CanActivate, OnApplicationBootstrap
+{
+  logger: DadosferaLogger;
+
+  pems: Map<string, string>;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private reflector: Reflector,
+    private authClient: AuthClientService,
+  ) {
+    this.pems = new Map();
+    this.logger = dadosferaLogger.logger;
+  }
+
+  async onApplicationBootstrap() {
+    return this.loadDucJWKS();
+  }
+
+  private async loadDucJWKS() {
+    const { keys } = await this.authClient.getPublicKeys();
+
+    keys.forEach((key) => {
+      this.pems.set(key.kid, key.pem);
+    });
+  }
+
+  canActivate(ctx: ExecutionContext): boolean {
+    const authFunctions = this.reflector.getAllAndMerge<
+      AuthenticationFunction[]
+    >(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
+    const mustBeAuthenticated = authFunctions.length > 0;
+
+    const request = ctx.switchToHttp().getRequest();
+    const accessToken = this.validateToken(request, mustBeAuthenticated);
+
+    if (!mustBeAuthenticated) {
+      // no need to be authenticated
+      return true;
+    }
+
+    if (!accessToken) {
+      // couldn't load valid token
+      throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
+    }
+
+    // every authentication function must return true to authenticate
+    if (!authFunctions.every((func) => func(request, accessToken))) {
+      throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
+    }
+
+    return true;
+  }
+
+  private validateToken(
+    request,
+    mustBeAuthenticated: boolean,
+  ): RequestUser | false {
+    const accessToken = request.get('Authorization');
+    let accessTokenPayload: RequestUser;
+
+    // If the user isn't authenticated, an error will occurr anywhere here.
+    // Fancy error avoidance isn't performed by purpose, such as avoiding to access null values.
+    try {
+      const accessTokenDecoded: any = jwt.decode(accessToken, {
+        complete: true,
+      });
+
+      /*assert(
+        accessTokenDecoded.payload.iss === 'duc',
+        'token should be issued by DUC',
+      );*/
+
+      assert(
+        accessTokenDecoded?.payload?.token_use === 'access',
+        'should be an access token',
+      );
+
+      const { kid } = accessTokenDecoded.header;
+      const pemValue: string = this.pems.get(kid);
+
+      jwt.verify(accessToken, pemValue);
+      accessTokenPayload = accessTokenDecoded.payload;
+    } catch (err) {
+      // log errors if authentication is required
+      if (mustBeAuthenticated) {
+        this.logger.error('Unable to verify duc access token\n' + err?.stack);
+      }
+
+      return false;
+    }
+
+    request.accessTokenPayload = accessTokenPayload;
+    request.user = {
+      user_id: accessTokenPayload.user_id,
+      username: accessTokenPayload.username,
+      permissions: accessTokenPayload.permissions,
+      customer_id: accessTokenPayload.customer_id,
+      customer_name: accessTokenPayload.customer_name,
+      customer_tier: accessTokenPayload.customer_tier,
+      access_token: accessToken,
+    };
+    // TODO: for backwards compatibility. remove in the future
+    request.body.info = {
+      user_id: accessTokenPayload.user_id,
+      customer_id: accessTokenPayload.customer_id,
+      customer: accessTokenPayload.customer_name,
+      customer_tier: accessTokenPayload.customer_tier,
+    };
+
+    return accessTokenPayload;
+  }
+}

src/authentication/permissions.enum.ts

@@ -0,0 +1,651 @@
+import { Logger } from '@nestjs/common';
+import assert from 'assert';
+import { LanguageEnum } from 'src/utils/languages.enum';
+
+const logger = new Logger();
+
+export enum PermissionUsages {
+  PUBLIC = 'public',
+  INTERNAL = 'internal',
+}
+
+export interface PermissionsObjectPermission {
+  seqid: number;
+  claim: string;
+  usage: PermissionUsages;
+  name?: Record<LanguageEnum, string>;
+}
+export interface PermissionsObjectPermissionTranslated {
+  seqid: number;
+  claim: string;
+  usage: PermissionUsages;
+  name?: string;
+}
+export interface PermissionsObjectGroup {
+  title: {
+    'pt-br': string;
+    'en-us': string;
+  };
+  permissions: Record<string, PermissionsObjectPermission>;
+}
+
+/*
+- like field numbers in gRPC, avoid to change (or reuse previously used) seqids once its deployed to DUC
+- it is possible to update claim and usage, not seqid
+*/
+
+/*export const Permissions: {
+  [P in keyof any]: { [Q in keyof any]: Permission };
+} = {*/
+export const PERMISSIONS_GROUPS = {
+  AUTH: {
+    title: {
+      'pt-br': 'Autenticação',
+      'en-us': 'Authentication',
+      'es-es': 'Authentication',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 18,
+        claim: 'POST /auth',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': '',
+          'en-us': '',
+          'es-es': '',
+        },
+      },
+      GENERATE_TOKEN: {
+        seqid: 46,
+        claim: 'customer:generate-token',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Gerar Token de Acesso',
+          'en-us': 'Generate Acess Token',
+          'es-es': 'Gerar Token de Acceso',
+        },
+      },
+    },
+  },
+
+  PIPELINE: {
+    title: {
+      'pt-br': 'Coletar | Pipelines',
+      'en-us': 'Collect | Pipelines',
+      'es-es': 'Colecta | Pipelines',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 23,
+        claim: 'POST /pipelines',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Criar pipelines',
+          'en-us': 'Create pipelines',
+          'es-es': 'Crear pipelines',
+        },
+      },
+      GET: {
+        seqid: 13,
+        claim: 'GET /pipelines',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Buscar pipelines',
+          'en-us': 'Get pipelines',
+          'es-es': 'Obtener pipelines',
+        },
+      },
+      UPDATE: {
+        seqid: 29,
+        claim: 'PUT /pipelines',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Atualizar pipeline',
+          'en-us': 'Update pipeline',
+          'es-es': 'Actualizar pipeline',
+        },
+      },
+      DELETE: {
+        seqid: 5,
+        claim: 'DELETE /pipelines',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Excluir pipeline',
+          'en-us': 'Delete pipeline',
+          'es-es': 'Eliminar pipeline',
+        },
+      },
+    },
+  },
+
+  CONNECTION: {
+    title: {
+      'pt-br': 'Coletar | Fontes de dados',
+      'en-us': 'Collect | Sources',
+      'es-es': 'Colecta | Sources',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 37,
+        claim: 'connection:create',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Cadastrar Fonte',
+          'en-us': 'Create Source',
+          'es-es': 'Crear Source',
+        },
+      },
+      DELETE: {
+        seqid: 38,
+        claim: 'connection:delete',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Excluir Fonte',
+          'en-us': 'Remove Source',
+          'es-es': 'Eliminar Source',
+        },
+      },
+      UPDATE: {
+        seqid: 39,
+        claim: 'connection:update',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Editar Fonte',
+          'en-us': 'Edit Source',
+          'es-es': 'Editar Source',
+        },
+      },
+    },
+  },
+
+  NETWORK_CONFIG: {
+    title: {
+      'pt-br': 'Coletar | Redes',
+      'en-us': 'Collect | Network',
+      'es-es': 'Colecta | Network',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 35,
+        claim: 'network_config:create',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Cadastrar VPN',
+          'en-us': 'Create VPN',
+          'es-es': 'Crear VPN',
+        },
+      },
+      // DELETE: {
+      //   seqid: 36,
+      //   claim: 'network_config:delete',
+      //   usage: PermissionUsages.PUBLIC,
+      //   name: {
+      //     'pt-br': 'Excluir VPN',
+      //     'en-us': 'Remove VPN',
+      //     'es-es': 'Eliminar VPN',
+      //   },
+      // },
+    },
+  },
+
+  OUTPUT: {
+    title: {
+      'pt-br': 'Output',
+      'en-us': 'Output',
+      'es-es': 'Output',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 22,
+        claim: 'POST /outputs',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Criar outputs',
+          'en-us': 'Create outputs',
+          'es-es': 'Crear outputs',
+        },
+      },
+      GET: {
+        seqid: 12,
+        claim: 'GET /outputs',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Buscar outputs',
+          'en-us': 'Get outputs',
+          'es-es': 'Obtener outputs',
+        },
+      },
+      UPDATE: {
+        seqid: 28,
+        claim: 'PUT /outputs',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Editar outputs',
+          'en-us': 'Edit outputs',
+          'es-es': 'Editar outputs',
+        },
+      },
+      DELETE: {
+        seqid: 4,
+        claim: 'DELETE /outputs',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Excluir outputs',
+          'en-us': 'Remove outputs',
+          'es-es': 'Eliminar outputs',
+        },
+      },
+    },
+  },
+
+  TRANSFORMATIONS: {
+    title: {
+      'pt-br': 'Micro-transformações',
+      'en-us': 'Micro-transformations',
+      'es-es': 'Micro-transformaciones',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 24,
+        claim: 'POST /transformations',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Criar micro-transformações',
+          'en-us': 'Create micro-transformations',
+          'es-es': 'Crear micro-transformaciones',
+        },
+      },
+      GET: {
+        seqid: 15,
+        claim: 'GET /transformations',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Buscar micro-transformações',
+          'en-us': 'Get micro-transformations',
+          'es-es': 'Obtener micro-transformaciones',
+        },
+      },
+      UPDATE: {
+        seqid: 30,
+        claim: 'PUT /transformations',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Editar micro-transformações',
+          'en-us': 'Edit micro-transformations',
+          'es-es': 'Editar micro-transformaciones',
+        },
+      },
+      DELETE: {
+        seqid: 6,
+        claim: 'DELETE /transformations',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Excluir micro-transformações',
+          'en-us': 'Remove micro-transformations',
+          'es-es': 'Eliminar micro-transformaciones',
+        },
+      },
+    },
+  },
+
+  CATALOG: {
+    title: {
+      'pt-br': 'Explorar | Catálogo',
+      'en-us': 'Explore | Catalog',
+      'es-es': 'Explorar | Catalogar',
+    },
+    permissions: {
+      GET: {
+        seqid: 7,
+        claim: 'GET /catalog',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar lista de Ativos e visualizar atributos',
+          'en-us': 'Access Assets list and its attributes',
+          'es-es': 'Acceda a la lista de Activos y vea los atributos',
+        },
+      },
+      CREATE: {
+        seqid: 19,
+        claim: 'catalog:create',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Criar Ativos',
+          'en-us': 'Create Assets',
+          'es-es': 'Crear Activos',
+        },
+      },
+      UPDATE: {
+        seqid: 25,
+        claim: 'catalog:edit',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Criar e editar atributos no catálogo',
+          'en-us': 'Create and edit attributes on the catalog',
+          'es-es': 'Crear y editar atributos en el catálogo',
+        },
+      },
+      DELETE: {
+        seqid: 1,
+        claim: 'catalog:delete',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Excluir Ativos do catálogo',
+          'en-us': 'Remove Assets from the catalog',
+          'es-es': 'Eliminar Activos del catálogo',
+        },
+      },
+      DATA_MANAGER: {
+        seqid: 42,
+        claim: 'catalog:data_manager',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br':
+            'Gerente do catálogo. Consegue vizualizar e editar todos os ativos.',
+          'en-us': 'Catalog manager. Able to view and edit all assets.',
+          'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
+        },
+      },
+      EMBED_ANALYTICS: {
+        seqid: 44,
+        claim: 'catalog:embed',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Acessar Módulo de Incorporação de Ativos',
+          'en-us': 'Access Embedding analytics Module',
+          'es-es': 'Acceder al Módulo de Incorporación de Activos',
+        },
+      },
+      TRIGGER_CATALOG_TASK: {
+        seqid: 45,
+        claim: 'catalog:trigger-task',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Executar a catalogação de um ativo',
+          'en-us': 'Trigger an asset cataloging',
+          'es-es': 'Realizar el listado de un activo',
+        },
+      },
+    },
+  },
+
+  CONNECTORS: {
+    title: {
+      'pt-br': 'Conectores',
+      'en-us': 'Connectors',
+      'es-es': 'Conectores',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 20,
+        claim: 'POST /connectors',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Criar conectores',
+          'en-us': 'Create connectors',
+          'es-es': 'Crear conectores',
+        },
+      },
+      GET: {
+        seqid: 8,
+        claim: 'GET /connectors',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Buscar conectores',
+          'en-us': 'Get connectors',
+          'es-es': 'Obtener conectores',
+        },
+      },
+      UPDATE: {
+        seqid: 26,
+        claim: 'PUT /connectors',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Editar conectores',
+          'en-us': 'Edit connectors',
+          'es-es': 'Editar conectores',
+        },
+      },
+      DELETE: {
+        seqid: 2,
+        claim: 'DELETE /connectors',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Excluir conectores',
+          'en-us': 'Remove connectors',
+          'es-es': 'Eliminar conectores',
+        },
+      },
+    },
+  },
+
+  SNOWFLAKE: {
+    title: {
+      'pt-br': 'Explorar | Consolidar',
+      'en-us': 'Explore | Data consolidation',
+      'es-es': 'Explorar | Consolidación de datos',
+    },
+    permissions: {
+      OPEN: {
+        seqid: 14,
+        claim: 'GET /snowflake',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Snowflake',
+          'en-us': 'Access Snowflake',
+          'es-es': 'Acceso Snowflake',
+        },
+      },
+    },
+  },
+
+  ZENDESK: {
+    title: {
+      'pt-br': 'Zendesk',
+      'en-us': 'Zendesk',
+      'es-es': 'Zendesk',
+    },
+    permissions: {
+      OPEN: {
+        seqid: 16,
+        claim: 'support-chat:access',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Acessar chat de suporte',
+          'en-us': 'Access support chat',
+          'es-es': 'Acceder al chat de soporte',
+        },
+      },
+    },
+  },
+
+  DATAVIZ: {
+    title: {
+      'pt-br': 'Analisar | Visualização',
+      'en-us': 'Analyze | Visualization',
+      'es-es': 'Analizar | Visualización',
+    },
+    permissions: {
+      METABASE: {
+        seqid: 11,
+        claim: 'GET /metabase',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Metabase',
+          'en-us': 'Access Metabase',
+          'es-es': 'Acceso Metabase',
+        },
+      },
+    },
+  },
+  INTELLIGENCE: {
+    title: {
+      'pt-br': 'Analisar | Inteligência',
+      'en-us': 'Analyze | Intelligence',
+      'es-es': 'Analizar | Inteligencia',
+    },
+    permissions: {
+      INTELLIGENCE: {
+        seqid: 31,
+        claim: 'intelligence:open',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Módulo de Inteligência',
+          'en-us': 'Access Intelligence Module',
+          'es-s': 'Acceder Módulo de Inteligencia',
+        },
+      },
+    },
+  },
+
+  MODULES: {
+    title: {
+      'pt-br': 'Módulos da Dadosfera',
+      'en-us': 'Dadosfera Modules',
+      'es-es': 'Módulos de la Dadosfera',
+    },
+    permissions: {
+      APP_BUILDER: {
+        seqid: 32,
+        claim: 'app-builder:open',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Acessar App Builder',
+          'en-us': 'Access App Builder',
+          'es-es': 'Acceder App Builder',
+        },
+      },
+      MACHINE_LEARNING: {
+        seqid: 33,
+        claim: 'machine-learning:open',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Acessar Machine Learning',
+          'en-us': 'Access Machine Learning',
+          'es-es': 'Acceder Machine Learning',
+        },
+      },
+    },
+  },
+  USERS: {
+    title: {
+      'pt-br': 'Usuários',
+      'en-us': 'Users',
+      'es-es': 'Usuarios',
+    },
+    permissions: {
+      ADMIN: {
+        seqid: 34,
+        claim: 'users:admin',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Gestão de usuários e grupos',
+          'en-us': 'Manage user and roles',
+          'es-es': 'Administrar usuarios y roles',
+        },
+      },
+    },
+  },
+  PROCESS: {
+    title: {
+      'pt-br': 'Processar',
+      'en-us': 'Process',
+      'es-es': 'Procesar',
+    },
+    permissions: {
+      TRANSFORMATION: {
+        seqid: 43,
+        claim: 'process:open',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Módulo de Transformação',
+          'en-us': 'Access Transformation Module',
+          'es-es': 'Acceder Módulo de transformación',
+        },
+      },
+    },
+  },
+  DADOSFERA: {
+    title: {
+      'pt-br': 'Usuários',
+      'en-us': 'Users',
+      'es-es': 'Usuarios',
+    },
+    permissions: {
+      CONNECTIONS_MIGRATION: {
+        seqid: 40,
+        claim: 'dadosfera:connections-migration',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'connections-migration',
+          'en-us': 'connections-migration',
+          'es-es': 'connections-migration',
+        },
+      },
+      SYNC_USER: {
+        seqid: 41,
+        claim: 'dadosfera:sync-user',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'sync-user',
+          'en-us': 'sync-user',
+          'es-es': 'sync-user',
+        },
+      },
+    },
+  },
+};
+export interface DadosferaModule {
+  name: string;
+  description: string;
+  key: string;
+  permissionSeqId: number;
+}
+export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+  {
+    name: 'Intelligence Module',
+    description: 'Orchest Module',
+    key: 'intelligence',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid,
+  },
+  {
+    name: 'Proccessing Module',
+    description: 'Proccessing Module',
+    key: 'process',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid,
+  },
+  {
+    name: 'Embedded Analytics',
+    description: 'Embedded Analytics Module',
+    key: 'embedded-analytics',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid,
+  },
+];
+// traverses the object searching for duplicate seqids or claims (executes at runtime)
+let nextAvailableSeqid = 0;
+const seqids = Object.values(PERMISSIONS_GROUPS).flatMap((namespace) =>
+  Object.values(namespace.permissions).map(({ seqid }) => seqid),
+);
+const claims = Object.values(PERMISSIONS_GROUPS).flatMap((namespace) =>
+  Object.values(namespace.permissions).map(({ claim }) => claim),
+);
+Object.values(PERMISSIONS_GROUPS).map((namespace) =>
+  Object.values(namespace.permissions).map(({ seqid, claim }) => {
+    const seqidsCount = seqids.filter((x) => x === seqid).length;
+    assert(seqidsCount === 1, `seqid ${seqid} count is not 1`);
+
+    const claimsCount = claims.filter((x) => x === claim).length;
+    assert(claimsCount === 1, `claim '${claim}' count is not 1`);
+    nextAvailableSeqid = Math.max(nextAvailableSeqid, seqid);
+  }),
+);
+
+DADOSFERA_MODULES.map((m) => m.key).forEach((m, i, arr) => {
+  if (arr.indexOf(m) !== i)
+    throw new Error(`DADOSFERA_MODULES[${i}] does not have a unique key`);
+});
+
+logger.log(`next available seqid ${nextAvailableSeqid + 1}`);

src/clients/auth/client.config.ts

@@ -1,23 +0,0 @@
-import { credentials } from '@grpc/grpc-js';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-
-import { DucProtoFilePath, DucPackages } from '@victorradael/protospack';
-
-export class AuthClient {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.DUC_URL,
-        package: DucPackages,
-        credentials: credentials.createSsl(),
-        protoPath: DucProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/auth/client.service.ts

@@ -1,46 +0,0 @@
-import { OnModuleInit, Inject } from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-
-import {
-  AuthServiceInterface,
-  DucServicesNames,
-} from '@victorradael/protospack';
-
-import { ILogin } from './interfaces';
-
-export class AuthClientService implements OnModuleInit {
-  private authService: AuthServiceInterface;
-  constructor(
-    @Inject('AUTH_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {}
-
-  onModuleInit() {
-    this.authService = this.grpcClient.getService<AuthServiceInterface>(
-      DucServicesNames.AuthProtoService,
-    );
-  }
-
-  async signIn({ username, password }: ILogin): Promise<any> {
-    console.log('AuthClientService', 'SignIn');
-
-    const tokens = await new Promise((resolve, reject) => {
-      this.authService.signIn({ username, password }).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-    return tokens;
-  }
-}

src/clients/auth/interfaces.d.ts

@@ -1,4 +0,0 @@
-export interface ILogin {
-  username: string;
-  password: string;
-}

src/clients/inputs/client.config.ts

@@ -1,23 +0,0 @@
-import { credentials } from '@grpc/grpc-js';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-
-import { InputPackages, InputProtoFilePath } from '@victorradael/protospack';
-
-export class InputsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.INFACTORY_URL,
-        package: InputPackages,
-        credentials: credentials.createSsl(),
-        protoPath: InputProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/inputs/client.service.ts

@@ -1,199 +0,0 @@
-import { OnModuleInit, Inject } from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import {
-  InputNewCreateRequest,
-  InputService,
-  TestConnectionGetColumnsRequest,
-  TestConnectionRequest,
-} from '@victorradael/protospack';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import { IIdRequest, UpdateInputRequest } from './interfaces';
-
-export class InputsClientService implements OnModuleInit {
-  private inputService: InputService;
-  constructor(
-    @Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {}
-
-  onModuleInit() {
-    this.inputService =
-      this.grpcClient.getService<InputService>('InputService');
-  }
-
-  async create(createInputDto: InputNewCreateRequest) {
-    console.log('InputClientService', 'Create');
-
-    const createInputResponse = await new Promise((resolve, reject) => {
-      this.inputService
-        .NewCreate(objectSnakeToCamel(createInputDto))
-        .subscribe({
-          next(x) {
-            resolve(objectCamelToSnake(x));
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return createInputResponse;
-  }
-
-  async findOne(data: IIdRequest) {
-    console.log('InputClientService', 'FindOne');
-
-    const findOneInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return findOneInputResponse;
-  }
-
-  async findAll(data) {
-    console.log('InputClientService', 'FindAll');
-
-    const findAllInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return findAllInputResponse;
-  }
-
-  async update(updateInputDTO: UpdateInputRequest) {
-    console.log('InputClientService', 'Update');
-    const updateInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Update(objectSnakeToCamel(updateInputDTO)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return updateInputResponse;
-  }
-
-  async remove(idRequest: IIdRequest) {
-    console.log('InputClientService', 'Remove');
-
-    const removeInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Remove(objectSnakeToCamel(idRequest)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return removeInputResponse;
-  }
-
-  async testConnection(data: TestConnectionRequest) {
-    console.log('InputClientService', 'TestConnection');
-    const testConnectionResponse = await new Promise((resolve, reject) => {
-      this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return testConnectionResponse;
-  }
-
-  async getColumns(data: TestConnectionGetColumnsRequest) {
-    console.log('InputClientService', 'TestConnection/Get-Columns');
-    const getColumnsResponse = await new Promise((resolve, reject) => {
-      this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return getColumnsResponse;
-  }
-}

src/clients/outputs/client.config.ts

@@ -1,22 +0,0 @@
-import { OutputPackages, OutputProtoFilePath } from '@victorradael/protospack';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import { credentials } from '@grpc/grpc-js';
-
-export class OutputsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.OTFACTORY_URL,
-        package: OutputPackages,
-        credentials: credentials.createSsl(),
-        protoPath: OutputProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/outputs/client.service.ts

@@ -1,160 +0,0 @@
-import { Controller, Inject, OnModuleInit } from '@nestjs/common';
-import { ClientGrpc, Payload } from '@nestjs/microservices';
-import {
-  OutputServicesNames,
-  OutputsServiceInterface,
-} from '@victorradael/protospack';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import {
-  ICreateOutputRequest,
-  IIdRequest,
-  IOutputUpdateRequest,
-} from './interfaces';
-
-@Controller('output')
-export class OutputsClientService implements OnModuleInit {
-  private outputService: OutputsServiceInterface;
-  constructor(
-    @Inject('OUTPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {}
-
-  onModuleInit() {
-    this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
-      OutputServicesNames.OutputService,
-    );
-  }
-
-  async create(@Payload() createOutputDto: ICreateOutputRequest) {
-    console.log('OutputClientService', 'Create');
-
-    const createOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Create(objectSnakeToCamel(createOutputDto)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    const convertedItem = objectCamelToSnake(createOutputResponse['output']);
-    createOutputResponse['output'] = convertedItem;
-    return createOutputResponse;
-  }
-
-  async findAll(data) {
-    console.log('OutputClientService', 'FindAll');
-
-    const findAllOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    const convertedOutputs = findAllOutputResponse['outputs'].map((ot) => {
-      return objectCamelToSnake(ot);
-    });
-
-    return { outputs: convertedOutputs };
-  }
-
-  async findOne(data: IIdRequest) {
-    console.log('OutputClientService', 'FindOne');
-
-    const findOneOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(findOneOutputResponse);
-  }
-
-  async update(updateOutPutDTO: IOutputUpdateRequest) {
-    console.log('OutputClientService', 'Update');
-
-    const updateOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Update(objectSnakeToCamel(updateOutPutDTO)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(updateOutputResponse);
-  }
-
-  async remove(data: IIdRequest) {
-    console.log('OutputClientService', 'Remove');
-
-    const removeOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Remove(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return removeOutputResponse;
-  }
-}

src/clients/outputs/interfaces.d.ts

@@ -1,29 +0,0 @@
-export interface ICreateOutputRequest {
-  operation: string;
-  name: string;
-  plugin: string;
-  values: Values;
-}
-
-export interface Values {
-  prefix: string;
-}
-
-interface Info {
-  user_id: string;
-  customer_id: string;
-}
-
-export interface IIdRequest {
-  id: string;
-  info: Info;
-}
-
-export interface IOutputUpdateRequest {
-  id: string;
-  operation: string;
-  values: Values;
-  plugin: string;
-  name: string;
-  info: Info;
-}

src/clients/pipelines/client.config.ts

@@ -1,25 +0,0 @@
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import {
-  PipelinePackages,
-  PipelineProtoFilePath,
-} from '@victorradael/protospack';
-import { credentials } from '@grpc/grpc-js';
-
-export class PipelinesClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.PIFACTORY_URL,
-        package: PipelinePackages,
-        credentials: credentials.createSsl(),
-        protoPath: PipelineProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/pipelines/client.service.ts

@@ -1,235 +0,0 @@
-import { Inject, OnModuleInit } from '@nestjs/common';
-import { ClientGrpc, Payload } from '@nestjs/microservices';
-import {
-  PipelineServicesNames,
-  PipelinesServiceInterface,
-} from '@victorradael/protospack';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import {
-  ICreatePipelineDto,
-  IGetPipelineLogsRequest,
-  IIdRequest,
-  IUpdatePipelineRequest,
-} from './interfaces';
-
-export class PipelinesClientService implements OnModuleInit {
-  private pipelineService: PipelinesServiceInterface;
-  constructor(
-    @Inject('PIPELINES_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {}
-
-  onModuleInit() {
-    this.pipelineService =
-      this.grpcClient.getService<PipelinesServiceInterface>(
-        PipelineServicesNames.PipelineService,
-      );
-  }
-
-  async create(@Payload() createPipelineDto: ICreatePipelineDto) {
-    console.log('PipelinesClientService', 'Create');
-
-    const createPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService
-        .Create(objectSnakeToCamel(createPipelineDto))
-        .subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return createPipelineResponse;
-  }
-
-  async findAll(data: IIdRequest) {
-    console.log('PipelinesClientService', 'FindAll');
-
-    const findAllPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return findAllPipelineResponse;
-  }
-
-  async findOne(data: IIdRequest) {
-    console.log('PipelinesClientService', 'FindOne');
-
-    const findOnePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return findOnePipelineResponse;
-  }
-
-  async update(UpdatePipelineRequest: IUpdatePipelineRequest) {
-    console.log('PipelinesClientService', 'Update');
-
-    const updatePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService
-        .Update(objectSnakeToCamel(UpdatePipelineRequest))
-        .subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return updatePipelineResponse;
-  }
-
-  async remove(data: IIdRequest) {
-    console.log('PipelinesClientService', 'Remove');
-
-    const removePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.remove(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return removePipelineResponse;
-  }
-
-  async getPipelineLogsMessages(data: IGetPipelineLogsRequest) {
-    console.log('PipelinesClientService', 'GetPipelineLogsMessages');
-
-    const logsPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.getPipelineLogs(data).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return logsPipelineResponse;
-  }
-
-  async getPipelineStatus(data: IIdRequest) {
-    console.log('PipelinesClientService', 'GetPipelineStatus');
-
-    const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.getPipelineStatus(data).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          console.log('Observable Error');
-          reject(err);
-        },
-        complete() {
-          console.log('done');
-        },
-      });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return statusPipelineResponse;
-  }
-
-  async runPipeline({ id, info }: IIdRequest) {
-    console.log('PipelinesClientService', 'RunPipeline');
-    const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService
-        .triggerPipeline(objectSnakeToCamel({ id, info }))
-        .subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-    })
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return statusPipelineResponse;
-  }
-}

src/clients/transformations/client.config.ts

@@ -1,25 +0,0 @@
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import {
-  TransformationPackages,
-  TransformationProtoFilePath,
-} from '@victorradael/protospack';
-import { credentials } from '@grpc/grpc-js';
-
-export class TransformationsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.TRFACTORY_URL,
-        package: TransformationPackages,
-        credentials: credentials.createSsl(),
-        protoPath: TransformationProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/transformations/client.service.ts

@@ -1,173 +0,0 @@
-import { Controller, Inject, OnModuleInit, Post } from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import { TransformationsServiceInterface } from '@victorradael/protospack';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import { ICreateTransformationsRequest, IIdRequest } from './interfaces';
-
-@Controller('transformation')
-export class TransformationsClientService implements OnModuleInit {
-  private transformationService: TransformationsServiceInterface;
-  constructor(
-    @Inject('TRANSFORMATIONS_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {}
-
-  onModuleInit() {
-    this.transformationService =
-      this.grpcClient.getService<TransformationsServiceInterface>(
-        'TransformationService',
-      );
-  }
-
-  @Post()
-  async create(createTransformationsDto: ICreateTransformationsRequest) {
-    console.log('TransformationClientService', 'Create');
-
-    const createTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService
-          .Create(objectSnakeToCamel(createTransformationsDto))
-          .subscribe({
-            next(x) {
-              resolve(x);
-            },
-            error(err) {
-              console.log('Observable Error');
-              reject(err);
-            },
-            complete() {
-              console.log('done');
-            },
-          });
-      },
-    )
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    const convertedTransforms = createTransformationResponse[
-      'transformations'
-    ].map((tr) => {
-      return objectCamelToSnake(tr);
-    });
-
-    return { transformations: convertedTransforms };
-  }
-
-  async findAll(data) {
-    console.log('TransformationClientService', 'FindAll');
-
-    const findAllTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService.FindAll(objectSnakeToCamel(data)).subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-      },
-    )
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-    const convertedTransforms = findAllTransformationResponse[
-      'transformations'
-    ].map((tr) => {
-      return objectCamelToSnake(tr);
-    });
-    return { transformations: convertedTransforms };
-  }
-
-  async findOne(data: IIdRequest) {
-    console.log('TransformationClientService', 'FindOne');
-
-    const findOneTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService.FindOne(objectSnakeToCamel(data)).subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-      },
-    )
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(findOneTransformationResponse);
-  }
-
-  async update(updateTransformationDTO) {
-    console.log('TransformationClientService', 'Update');
-
-    const updateTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService
-          .Update(objectSnakeToCamel(updateTransformationDTO))
-          .subscribe({
-            next(x) {
-              resolve(x);
-            },
-            error(err) {
-              console.log('Observable Error');
-              reject(err);
-            },
-            complete() {
-              console.log('done');
-            },
-          });
-      },
-    )
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(updateTransformationResponse);
-  }
-
-  async remove(data: IIdRequest) {
-    console.log('TransformationClientService', 'Remove');
-
-    const removeTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService.Remove(objectSnakeToCamel(data)).subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            console.log('Observable Error');
-            reject(err);
-          },
-          complete() {
-            console.log('done');
-          },
-        });
-      },
-    )
-      .then((res) => res)
-      .catch((err) => {
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(removeTransformationResponse);
-  }
-}

src/decorators/authentication.decorator.ts

@@ -0,0 +1,41 @@
+import { applyDecorators } from '@nestjs/common';
+import { ApiSecurity } from '@nestjs/swagger';
+import { type Request } from 'express';
+import { type PermissionsObjectPermission } from '../authentication/permissions.enum';
+import { type RequestUser } from './user.decorator';
+import { SetMultipleMetadata } from './shared';
+
+export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
+
+export type AuthenticationFunction = (req: Request, user: any) => boolean;
+
+export function RequireAllPermissions(
+  ...permissions: PermissionsObjectPermission[]
+) {
+  return createAuthenticatedDecorator((req, user: RequestUser) =>
+    permissions.every(({ seqid }) => user.permissions.includes(seqid)),
+  );
+}
+
+export function RequireSomePermission(
+  ...permissions: PermissionsObjectPermission[]
+) {
+  return createAuthenticatedDecorator((req, user: RequestUser) =>
+    permissions.some(({ seqid }) => user.permissions.includes(seqid)),
+  );
+}
+
+export function AuthenticateCondition(func: AuthenticationFunction) {
+  return createAuthenticatedDecorator(func);
+}
+
+export function Authenticated() {
+  return createAuthenticatedDecorator(() => true);
+}
+
+function createAuthenticatedDecorator(metadataValue: AuthenticationFunction) {
+  return applyDecorators(
+    ApiSecurity('access-token'),
+    SetMultipleMetadata(AUTH_FUNCTION_KEY, metadataValue),
+  );
+}

src/decorators/language.decorator.ts

@@ -0,0 +1,25 @@
+import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
+import { LanguageEnum } from 'src/utils/languages.enum';
+
+export const Language: () => ParameterDecorator = createParamDecorator(
+  (options: any, ctx: ExecutionContext): LanguageEnum => {
+    const headers = ctx.switchToHttp()?.getRequest()?.headers;
+
+    if (!headers) return LanguageEnum.ptbr;
+
+    let language: LanguageEnum =
+      headers['dadosfera-lang'] ?? headers['accept-language'];
+    language = language?.toLowerCase().trim() as LanguageEnum;
+
+    if (!language) return LanguageEnum.ptbr;
+
+    Object.values(LanguageEnum).some((l) => {
+      if (language === l || l.includes(language)) {
+        language = l;
+        return true;
+      }
+      language = LanguageEnum.enus;
+    });
+    return language;
+  },
+);

src/decorators/shared.ts

@@ -0,0 +1,28 @@
+import { type CustomDecorator } from '@nestjs/common';
+
+// implementation copied from SetMetadata, but tweaked to get existing values
+// of the metadataKey and accumulate it with the new metadataValue
+export function SetMultipleMetadata(
+  metadataKey,
+  metadataValue,
+): CustomDecorator {
+  const decoratorFactory: CustomDecorator = (target, key?, descriptor?) => {
+    // .start: tweak
+    // descriptor?.value = function decorator; target = class decorator
+    const accumulatedVal =
+      Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
+    accumulatedVal.push(metadataValue);
+    // .end: tweak
+
+    if (descriptor) {
+      Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
+      return descriptor;
+    }
+
+    Reflect.defineMetadata(metadataKey, accumulatedVal, target);
+    return target;
+  };
+
+  decoratorFactory.KEY = metadataKey;
+  return decoratorFactory;
+}

src/decorators/swagger.decorator.ts

@@ -0,0 +1,13 @@
+import { applyDecorators } from '@nestjs/common';
+import { ApiExcludeController, ApiExcludeEndpoint } from '@nestjs/swagger';
+
+export function ApiInternalOnlyEndpoint() {
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    return applyDecorators(ApiExcludeEndpoint());
+  return () => null;
+}
+export function ApiInternalOnlyController() {
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    return applyDecorators(ApiExcludeController());
+  return () => null;
+}

src/decorators/user.decorator.spec.ts

@@ -0,0 +1,179 @@
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { Controller, Get, HttpStatus, INestApplication } from '@nestjs/common';
+import { APP_GUARD } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+import jwt from 'jsonwebtoken';
+import request from 'supertest';
+import { AuthenticationGuard } from '../authentication/authentication.guard';
+import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
+import { AuthClientService } from '../modules/auth/auth.service';
+import ErrorCodes from '../utils/errorCodes';
+import { User } from './user.decorator';
+
+const logger = {
+  info: (...args) => args,
+  error: (...args) => args,
+};
+@Controller('user')
+class UserController {
+  @Get()
+  public getUser(@User() user) {
+    return { user };
+  }
+
+  @Get('options')
+  public getUserWithOptions(@User({}) user) {
+    return { user };
+  }
+
+  @Get('not-required')
+  public getUserNotRequired(@User({ required: false }) user) {
+    return { user };
+  }
+
+  @Get('required')
+  public getUserRequired(@User({ required: true }) user) {
+    return { user };
+  }
+}
+
+describe('user.decorator', () => {
+  let app: INestApplication;
+  const jwt_secret = {
+    kid: 'token-testing-shared-key',
+    pem: '$tr0ng-SH4Red-secr3t!!!~gl0ba1~]',
+  };
+  const fakeUserPayload = {
+    user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
+    username: 'super.admin',
+    permissions: [PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE].map(
+      ({ seqid }) => seqid,
+    ),
+    customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
+    customer_name: 'dadosfera',
+    customer_tier: 'BASIC',
+    access_token: '',
+  };
+
+  beforeAll(async () => {
+    const moduleRef = await Test.createTestingModule({
+      providers: [
+        {
+          provide: AuthClientService,
+          useValue: {
+            getPublicKeys: async () => ({
+              keys: [jwt_secret],
+            }),
+          },
+        },
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        {
+          provide: APP_GUARD,
+          useClass: AuthenticationGuard,
+        },
+      ],
+      controllers: [UserController],
+    }).compile();
+
+    app = moduleRef.createNestApplication();
+    await app.init();
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  function AssertNoAuth(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.OK);
+    expect(res.body).toStrictEqual({});
+  }
+
+  function AssertWithAuth(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.OK);
+    expect(res.body).toStrictEqual({ user: fakeUserPayload });
+  }
+
+  function AssertRequiredNoAuth(res: request.Response) {
+    expect(res.statusCode).toBe(HttpStatus.UNAUTHORIZED);
+    expect(res.body).toStrictEqual({
+      code: ErrorCodes.AUTH.UNAUTHORIZED,
+      error: 'Não autenticado',
+      message: 'É necessário estar logado para realizar essa operação',
+      statusCode: HttpStatus.UNAUTHORIZED,
+    });
+  }
+
+  function UserTest(accessToken: string) {
+    describe(`with${accessToken ? '' : 'out'} token`, () => {
+      it('should GET /user', async () => {
+        const res = await request(app.getHttpServer())
+          .get('/user')
+          .set({ Authorization: accessToken });
+
+        if (accessToken) {
+          AssertWithAuth(res);
+        } else {
+          AssertNoAuth(res);
+        }
+      });
+
+      it('should GET /user/options', async () => {
+        const res = await request(app.getHttpServer())
+          .get('/user/options')
+          .set({ Authorization: accessToken });
+
+        if (accessToken) {
+          AssertWithAuth(res);
+        } else {
+          AssertNoAuth(res);
+        }
+      });
+
+      it('should GET /user/not-required', async () => {
+        const res = await request(app.getHttpServer())
+          .get('/user/not-required')
+          .set({ Authorization: accessToken });
+
+        if (accessToken) {
+          AssertWithAuth(res);
+        } else {
+          AssertNoAuth(res);
+        }
+      });
+
+      it('should GET /user/required if logged', async () => {
+        const res = await request(app.getHttpServer())
+          .get('/user/required')
+          .set({ Authorization: accessToken });
+
+        if (accessToken) {
+          AssertWithAuth(res);
+        } else {
+          AssertRequiredNoAuth(res);
+        }
+      });
+    });
+  }
+
+  function CreateToken(overrides?) {
+    const tokenPayload = {
+      ...fakeUserPayload,
+      token_use: 'access',
+      ...overrides?.user,
+    };
+
+    return jwt.sign(tokenPayload, jwt_secret.pem, {
+      keyid: jwt_secret.kid,
+      ...overrides?.jwt,
+    });
+  }
+
+  UserTest(null);
+
+  const token = CreateToken();
+  fakeUserPayload.access_token = token;
+  UserTest(token);
+});

src/decorators/user.decorator.ts

@@ -0,0 +1,24 @@
+import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
+
+import ErrorBuilder from '../utils/ErrorBuilder';
+import ErrorCodes from '../utils/errorCodes';
+
+export interface RequestUser {
+  user_id: string;
+  username: string;
+  permissions: number[];
+  customer_id: string;
+  customer_name: string;
+  customer_tier: string;
+  access_token: string;
+}
+
+export const User: (options?: { required?: boolean }) => ParameterDecorator =
+  createParamDecorator((options: any, ctx: ExecutionContext) => {
+    const request = ctx.switchToHttp().getRequest();
+
+    if (!request.user && options?.required) {
+      throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
+    }
+    return request.user;
+  });

src/error/grpc-to-http-exception.filter.ts

@@ -0,0 +1,37 @@
+import {
+  ExceptionFilter,
+  Catch,
+  ArgumentsHost,
+  HttpException,
+} from '@nestjs/common';
+import { Response } from 'express';
+
+import ErrorBuilder from '../utils/ErrorBuilder';
+
+/**
+ * ExceptionFilter to handle RpcException thrown by our microservices
+ *
+ * If it detects that the exception is already an **HttpException** or it is generated by our **ErrorBuilder**,
+ * it will return the exception without any treatment.
+ */
+@Catch(Error)
+export class GrpcToHttpExceptionFilter implements ExceptionFilter {
+  catch(exception: any, host: ArgumentsHost) {
+    const ctx = host.switchToHttp();
+    const response = ctx.getResponse<Response>();
+
+    if (exception instanceof HttpException) {
+      return response
+        .status(exception.getStatus())
+        .json(exception.getResponse());
+    }
+
+    const err =
+      exception.constructor.name === ErrorBuilder.name
+        ? exception
+        : new ErrorBuilder(exception.details);
+
+    const { statusCode } = err.response;
+    return response.status(statusCode).json(err.response);
+  }
+}

src/error/http-exception.filter.spec.ts

@@ -1,7 +0,0 @@
-import { HttpExceptionFilter } from './http-exception.filter';
-
-describe('HttpExceptionFilter', () => {
-  it('should be defined', () => {
-    expect(new HttpExceptionFilter()).toBeDefined();
-  });
-});

src/main.ts

@@ -1,12 +1,22 @@
+if (process.env.ENV !== 'local') require('elastic-apm-node/start');
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
-import { writeFileSync } from 'fs';
 import helmet from 'helmet';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import { AppModule } from './app.module';
-
+import { writeFileSync } from 'fs';
+import { execSync } from 'child_process';
+import { INestApplication } from '@nestjs/common';
 async function bootstrap() {
+  DadosferaLogger.setupLogger({
+    serviceName: 'maestro',
+    serviceEnvironment: process.env.ENV,
+  });
+  const logger = new DadosferaLogger();
+
   const app = await NestFactory.create(AppModule, {
+    logger,
     cors: {
       origin: '*',
       methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
@@ -14,20 +24,60 @@ async function bootstrap() {
       optionsSuccessStatus: 204,
     },
   });
-
   app.use(helmet());
+  configureSwagger(app);
+  await app.listen(3333);
+  if (process.env.KILL_AFTER_START) await app.close();
+}
 
-  const config = new DocumentBuilder()
-    .setTitle('Maestro Grpc Documentation')
+function configureSwagger(app: INestApplication) {
+  let branchName = '';
+  try {
+    branchName = execSync('git rev-parse --abbrev-ref HEAD')
+      .toString()
+      .replace('\n', '')
+      .trim();
+  } catch (error) {
+    console.warn("Can't get branch name", error);
+  }
+  const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
+
+  const swaggerConfig = new DocumentBuilder()
+    .setTitle(swaggerTitle)
     .setDescription('Documentation for Maestro gateway')
-    .setVersion('1.0')
-    .addBearerAuth()
-    .build();
+    .addSecurity('access-token', {
+      type: 'apiKey',
+      name: 'Authorization',
+      in: 'header',
+    })
+    .setDescription('This is the Maestro API');
+
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
+
+  const config = swaggerConfig.build();
 
   const document = SwaggerModule.createDocument(app, config);
-  writeFileSync('./swagger.json', JSON.stringify(document));
-  SwaggerModule.setup('api', app, document);
 
-  await app.listen(3333);
+  const swaggerJsonFile =
+    process.env.INTERNAL_SWAGGER === 'true'
+      ? 'docsfera.json'
+      : 'docsfera.external.json';
+
+  if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
+
+  writeFileSync(
+    swaggerJsonFile,
+    JSON.stringify(
+      {
+        service: '${DOCS_API_TOKEN}',
+        pageId: '${DOCS_PAGE_ID}',
+        blockId: '${DOCS_BLOCK_ID}',
+        schema: document,
+      },
+      null,
+      2,
+    ),
+  );
 }
 bootstrap();

src/middlewares/authentication.ts

@@ -1,127 +0,0 @@
-import { Request, Response, NextFunction } from 'express';
-import axios from 'axios';
-import {
-  ForbiddenException,
-  InternalServerErrorException,
-  NestMiddleware,
-  UnauthorizedException,
-  UseFilters,
-} from '@nestjs/common';
-import jwkToPem from 'jwk-to-pem';
-import { decode, verify } from 'jsonwebtoken';
-import { HttpExceptionFilter } from 'src/error/http-exception.filter';
-
-@UseFilters(new HttpExceptionFilter())
-export class LoggerMiddleware implements NestMiddleware {
-  use = async (request: Request, response: Response, next: NextFunction) => {
-    const idToken = request.get('Dadosfera-User');
-    const accessToken = request.get('Authorization');
-    const privateKey = process.env.JWT_PRIVATE_KEY;
-
-    const requiredMethod = request.method.trim();
-    const requiredRoute = request.route.path.split('/')[1].trim();
-
-    verify(idToken, privateKey, (err) => {
-      if (err) {
-        throw new UnauthorizedException();
-      }
-    });
-    const jwtDecoded: any = decode(idToken);
-
-    const permissions = jwtDecoded.user.permissions;
-    const clienId = jwtDecoded.user.customerId;
-    const customer = jwtDecoded.user.customer;
-    const userId = jwtDecoded.user.id;
-
-    await verifyToken(accessToken);
-
-    let havePermission = false;
-    permissions.forEach((permission) => {
-      permission = permission.split('/');
-      const method = permission[0].trim();
-      const route = permission[1].trim();
-
-      if (method === requiredMethod && route === requiredRoute) {
-        havePermission = true;
-      }
-    });
-
-    if (havePermission) {
-      request.body.info = {
-        customer_id: clienId,
-        user_id: userId,
-        customer,
-      };
-
-      next();
-    } else {
-      throw new ForbiddenException();
-    }
-  };
-}
-
-let pems: { [key: string]: Record<string, unknown> }[];
-
-const setUp = async (region: string, id: string) => {
-  const URL = `https://cognito-idp.${region}.amazonaws.com/${id}/.well-known/jwks.json`;
-
-  try {
-    const response = await axios.get(URL);
-
-    if (response.status !== 200) {
-      throw new InternalServerErrorException();
-    }
-    const data = await response.data;
-    const { keys } = data;
-
-    pems = keys.map((key: any) => {
-      const modulus = key.n;
-      const exponent = key.e;
-      const keyType = key.kty;
-      const jwk = { kty: keyType, n: modulus, e: exponent };
-      const pem = jwkToPem(jwk);
-      const keyId = key.kid;
-
-      return { [keyId]: pem };
-    });
-  } catch (error) {
-    // console.log(error);
-    // console.log('Error! Unable to download JWKs');
-  }
-};
-
-const verifyToken = async (accessToken: string) => {
-  const awsRegion = process.env.AWS_REGION;
-  const awsPoolId = process.env.AWS_IDENTITY_POOL_ID;
-
-  try {
-    await setUp(awsRegion, awsPoolId);
-    if (!accessToken) {
-      throw new UnauthorizedException();
-    }
-
-    const user: any = decode(accessToken, { complete: true });
-
-    if (user === null) {
-      throw new UnauthorizedException();
-    }
-
-    const { kid } = user.header;
-    const pem = pems.filter((item: any) => item[kid]);
-    const pemValue: any = pem[0][kid];
-
-    if (!pem) {
-      throw new UnauthorizedException();
-    }
-
-    verify(accessToken, pemValue, (err: any) => {
-      if (err) {
-        throw new UnauthorizedException();
-      }
-
-      return;
-    });
-  } catch (error) {
-    throw new UnauthorizedException();
-  }
-};

src/modules/auth/README.md

@@ -0,0 +1,38 @@
+# Auth
+
+## SSO/oAuth
+
+### Strategy
+We are using [PassportJs](https://www.passportjs.org/) to handle oAuth authentications.
+
+When the client (front end) makes a `GET /auth/oauth/{strategy}` Passport automatically redirects the user to the `strategy` login page. To do that we must configure and use a **Passport Strategy**. We must also have a callback route, conventionally `GET /auth/oauth/{strategy}/callback`, so the oAuth app can report the status of the user's login.
+
+- If the oAuth is successfull we call DUC's `AuthOauthSignIn` request that gets the tokens from Cognito and saves them on cache temporarily under a key we call `session`. Duc returns that `session` to maestro which then redirects the user to our app login page with that `session` as a query param.
+
+- If the oAuth login is not successfull for some reason or the user **does not** exist on DUC's database we redirect the user to our login page with an `error` and `error_description` as query params.
+
+### Routes
+
+So in order to have an SSO login, besides configuring the Strategy, we must have two routes for each Strategy, like in the example below:
+
+```ts
+  @Get('oauth/google')
+  @UseGuards(AuthGuard('google-login'))
+  googleOauth() {
+    this.logger.info('/oauth/google');
+    return true;
+  }
+
+  @Get('oauth/google/callback')
+  @UseGuards(AuthGuard('google-login'))
+  @Redirect()
+  async googleOauthCallback(@Req() req) {
+    const { url, email, token, language = 'pt-br' } = await this.callback(req);
+    if (url.searchParams.get('error')) {
+      this.logger.error('/oauth/google - ERROR');
+      return { url: url.href };
+    }
+    // ... Rest of the logic
+    return { url: url.href };
+  }
+```

src/modules/auth/auth.controller.ts

@@ -1,27 +1,347 @@
-import { Body, Controller, Post, UnauthorizedException } from '@nestjs/common';
+import {
+  Body,
+  Controller,
+  Headers,
+  Post,
+  HttpCode,
+  HttpStatus,
+  Inject,
+  UseFilters,
+  Get,
+  UseGuards,
+  Redirect,
+  Req,
+  Param,
+} from '@nestjs/common';
+import {
+  ApiHeaders,
+  ApiOkResponse,
+  ApiSecurity,
+  ApiTags,
+} from '@nestjs/swagger';
+import {
+  AuthChangePasswordRequest,
+  AuthResetPasswordRequest,
+  AuthVerifyResetPasswordCodeRequest,
+  AuthConfirmResetPasswordRequest,
+  AuthEnableTotpMfaRequest,
+  AuthDisableTotpMfaRequest,
+  AuthVerifyTotpMfaRequest,
+} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
-import { AuthClientService } from 'src/clients/auth/client.service';
-
-interface ISignIn {
-  username: string;
-  password: string;
-}
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { AuthClientService } from './auth.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import {
+  AuthRefreshAccessTokenReq,
+  AuthRefreshAccessTokenRes,
+  AuthSignInReq,
+  AuthSignInRes,
+} from './dtos/login';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { AuthGuard } from '@nestjs/passport';
+import { Request } from 'express';
+import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
+@ApiTags('Auth')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
+@UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('auth')
 export class AuthController {
-  constructor(private authClient: AuthClientService) {}
+  logger: DadosferaLogger;
+  redirectUrl: string;
 
-  @Post()
-  async signIn(@Body() { username, password }: ISignIn) {
-    console.log(`/auth`, 'SignIn');
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private authClient: AuthClientService,
+  ) {
+    this.logger = dadosferaLogger.logger;
 
-    const tokens = await this.authClient
-      .signIn({ username, password })
-      .then((result) => result)
+    switch (process.env.ENV) {
+      case 'stg':
+        this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/auth/login`;
+        break;
+      case 'prd':
+        this.redirectUrl = `https://app.dadosfera.ai/auth/login`;
+        break;
+      default:
+        this.redirectUrl = `http://localhost:4200/auth/login`;
+    }
+  }
+
+  @Post('sign-in')
+  @HttpCode(HttpStatus.OK)
+  async signIn(
+    @Body() { username, password, totp }: AuthSignInReq,
+    @Language() language: LanguageEnum,
+  ): Promise<AuthSignInRes> {
+    this.logger.info('/auth - SignIn');
+    const metadata = PackTheMetadata({ language });
+    return this.authClient.signIn({ username, password, totp }, metadata);
+  }
+
+  @Post('refresh-access-token')
+  @HttpCode(HttpStatus.OK)
+  @ApiOkResponse({ type: AuthRefreshAccessTokenRes })
+  async refreshAccessToken(
+    @Body() body: AuthRefreshAccessTokenReq,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('/auth - RefreshAccessToken');
+    const { refreshToken, customerName: customer_name } = body;
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      language,
+    });
+
+    return this.authClient.refreshAccessToken({ refreshToken }, metadata);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('/sso/snowflake')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.SNOWFLAKE.permissions.OPEN)
+  @HttpCode(HttpStatus.OK)
+  async snowflakeSignIn(
+    @User() user: RequestUser,
+    @Body('RelayState') relayState: string,
+  ) {
+    this.logger.info('/auth - snowflakeSignIn');
+
+    return this.authClient.snowflakeSignIn({
+      userId: user.user_id,
+      relayState,
+    });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('change-password')
+  @HttpCode(HttpStatus.OK)
+  async changePassword(
+    @Body() body: AuthChangePasswordRequest,
+    @Headers() headers,
+  ) {
+    this.logger.info('/auth - change-password');
+
+    const { oldPassword, newPassword } = body;
+    const { authorization: accessToken } = headers;
+
+    return this.authClient.changePassword({
+      accessToken,
+      oldPassword,
+      newPassword,
+    });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('reset-password')
+  @HttpCode(HttpStatus.OK)
+  async resetPassword(
+    @Body() body: AuthResetPasswordRequest,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('/auth - reset-password');
+    const metadata = PackTheMetadata({
+      language: language,
+    });
+
+    const { username } = body;
+
+    return this.authClient.resetPassword({ username }, metadata);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('verify-reset-password-code')
+  @HttpCode(HttpStatus.OK)
+  async verifyResetPasswordCode(
+    @Body() body: AuthVerifyResetPasswordCodeRequest,
+  ) {
+    this.logger.info('/auth - verify-reset-password-code');
+
+    const { username, code } = body;
+
+    return this.authClient.verifyResetPasswordCode({ username, code });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('confirm-reset-password')
+  @HttpCode(HttpStatus.OK)
+  async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
+    this.logger.info('/auth - confirm-reset-password');
+
+    const { username, code, newPassword } = body;
+
+    return this.authClient.confirmResetPassword({
+      username,
+      code,
+      newPassword,
+    });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('enable-totp')
+  @HttpCode(HttpStatus.OK)
+  async enableTotpMFA(
+    @Body() body: AuthEnableTotpMfaRequest,
+    @Headers() headers,
+  ) {
+    this.logger.info('/auth - enable-totp');
+
+    const { password } = body;
+    const { authorization: accessToken } = headers;
+
+    return this.authClient.enableTotpMFA({ accessToken, password });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('disable-totp')
+  @HttpCode(HttpStatus.OK)
+  async disableTotpMFA(
+    @Body() body: AuthDisableTotpMfaRequest,
+    @Headers() headers,
+  ) {
+    this.logger.info('/auth - disable-totp');
+
+    const { password } = body;
+    const { authorization: accessToken } = headers;
+
+    return this.authClient.disableTotpMFA({ accessToken, password });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('dismiss-totp')
+  @HttpCode(HttpStatus.OK)
+  async dismissTotpMFA(@Headers() headers) {
+    this.logger.info('/auth - disable-totp');
+
+    const { authorization: accessToken } = headers;
+
+    return this.authClient.dismissTotpMFA({ accessToken });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('verify-totp')
+  @HttpCode(HttpStatus.OK)
+  async verifyTotp(@Body() body: AuthVerifyTotpMfaRequest, @Headers() headers) {
+    this.logger.info('/auth - enable-totp');
+
+    const { totp } = body;
+    const { authorization: accessToken } = headers;
+
+    return this.authClient.verifyTotp({ accessToken, totp });
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Authenticated()
+  @ApiSecurity('access-token')
+  @Get('verify-access-token')
+  verifyAccessToken() {
+    return { access_token_status: 'valid' };
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('session/:session')
+  getSession(@Param('session') session: string) {
+    return this.authClient.getSession(session);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('oauth/google')
+  @UseGuards(AuthGuard('google-login'))
+  googleOauth() {
+    this.logger.info('/oauth/google');
+    return true;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('oauth/google/callback')
+  @UseGuards(AuthGuard('google-login'))
+  @Redirect()
+  async googleOauthCallback(@Req() req) {
+    const { url, email, token, language = 'pt-br' } = await this.callback(req);
+    if (url.searchParams.get('error')) {
+      this.logger.error('/oauth/google - ERROR');
+      return { url: url.href };
+    }
+
+    await this.authClient
+      .oauthSignIn({
+        username: email,
+        token,
+      })
+      .then(({ session }) => {
+        this.logger.info('/oauth/google - SUCESS');
+        url.searchParams.set('session', session);
+      })
       .catch((err) => {
-        throw new UnauthorizedException(err.message);
+        this.logger.error('/oauth/google - ERROR');
+        let error = OauthErrors.INVALID_CREDENTIALS[language].error;
+        let error_description =
+          OauthErrors.INVALID_CREDENTIALS[language].error_description;
+        switch (err.details) {
+          case ErrorCodes.USER.NOT_FOUND:
+            error = OauthErrors.USER_NOT_FOUND[language].error;
+            error_description =
+              OauthErrors.USER_NOT_FOUND[language].error_description(email);
+            break;
+          case ErrorCodes.AUTH.UNAUTHORIZED:
+            error = OauthErrors.INVALID_SESSION[language].error;
+            error_description =
+              OauthErrors.INVALID_SESSION[language].error_description;
+            break;
+        }
+        url.searchParams.set('error', error);
+        url.searchParams.set('error_description', error_description);
+        return null;
       });
+    return { url: url.href };
+  }
 
-    return tokens;
+  async callback(req: Request) {
+    const { error, state } = req.query;
+    const { authInfo } = req;
+    const url = new URL(this.redirectUrl);
+    let email, token, error_title, error_description;
+    let language: 'pt-br' | 'en-us' = 'pt-br';
+
+    const stateObject = jwt.verify(
+      state as string,
+      process.env.JWT_PRIVATE_KEY,
+    );
+    if (typeof stateObject != 'string') language = stateObject.language;
+
+    if (error || !authInfo) {
+      this.logger.error(error);
+      if (!authInfo) this.logger.error('No authInfo', { request: req });
+
+      error_title = OauthErrors.INVALID_CREDENTIALS[language].error;
+      error_description =
+        OauthErrors.INVALID_CREDENTIALS[language].error_description;
+      if (error) error_description += ` - [${error}]`;
+    } else {
+      const { accessToken } = authInfo as any;
+      const { _json: userInfo } = req.user as any;
+
+      email = userInfo.email;
+      token = accessToken;
+    }
+
+    if (error_title) {
+      url.searchParams.set('error', error_title);
+      url.searchParams.set('error_description', error_description);
+    }
+
+    return { token, email, url, language };
   }
 }

src/modules/auth/auth.module.ts

@@ -0,0 +1,24 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { AuthController } from './auth.controller';
+import { AuthClientService } from './auth.service';
+
+import { DucClient } from '../duc/client.config';
+import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
+import { getOauthSecrets } from 'src/utils/OauthSecrets';
+const client = new DucClient();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [AuthController],
+  providers: [
+    AuthClientService,
+    DadosferaLogger,
+    GoogleLoginStrategy,
+    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
+  ],
+  exports: [AuthClientService],
+})
+export class AuthModule {}

src/modules/auth/auth.service.spec.ts

@@ -1,5 +0,0 @@
-describe('PipelinesGrpcServerService', () => {
-  it('should be defined', () => {
-    expect(2 + 2).toBe(4);
-  });
-});

src/modules/auth/auth.service.ts

@@ -0,0 +1,171 @@
+import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { lastValueFrom } from 'rxjs';
+
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import {
+  AuthSnowflakeSignInRequest,
+  AuthSignInRequest,
+  AuthRefreshAccessTokenRequest,
+  AuthEnableTotpMfaRequest,
+  AuthDisableTotpMfaRequest,
+  AuthDismissTotpMfaRequest,
+  AuthVerifyTotpMfaRequest,
+  AuthChangePasswordRequest,
+  AuthResetPasswordRequest,
+  AuthVerifyResetPasswordCodeRequest,
+  AuthConfirmResetPasswordRequest,
+} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { DucClient } from '../duc/client.config';
+import { Metadata } from '@grpc/grpc-js';
+
+@Injectable()
+export class AuthClientService implements OnModuleInit {
+  logger: DadosferaLogger;
+
+  private authService: AuthServiceInterface;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.authService = this.grpcClient.getService<AuthServiceInterface>(
+      ProtoServices.AuthProtoService,
+    );
+  }
+
+  async getPublicKeys() {
+    this.logger.info('GetPublicKeys');
+
+    return lastValueFrom(this.authService.AuthGetPublicKeys({}));
+  }
+
+  async snowflakeSignIn(input: AuthSnowflakeSignInRequest) {
+    this.logger.info('snowflakeSignIn');
+
+    return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
+  }
+
+  async signIn(
+    { username, password, totp }: AuthSignInRequest,
+    metadata: Metadata,
+  ) {
+    this.logger.info('SignIn');
+
+    return lastValueFrom(
+      this.authService.AuthSignIn({ username, password, totp }, metadata),
+    );
+  }
+
+  async refreshAccessToken(
+    { refreshToken }: AuthRefreshAccessTokenRequest,
+    metadata: Metadata,
+  ) {
+    this.logger.info('RefreshAccessToken');
+
+    return lastValueFrom(
+      this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
+    );
+  }
+
+  async changePassword({
+    accessToken,
+    oldPassword,
+    newPassword,
+  }: AuthChangePasswordRequest) {
+    this.logger.info('ChangePassword');
+
+    return lastValueFrom(
+      this.authService.AuthChangePassword({
+        accessToken,
+        oldPassword,
+        newPassword,
+      }),
+    );
+  }
+
+  async resetPassword(
+    { username }: AuthResetPasswordRequest,
+    metadata: Metadata,
+  ) {
+    this.logger.info('resetPassword');
+
+    return lastValueFrom(
+      this.authService.AuthResetPassword({ username }, metadata),
+    );
+  }
+
+  async verifyResetPasswordCode({
+    username,
+    code,
+  }: AuthVerifyResetPasswordCodeRequest) {
+    this.logger.info('verifyResetPasswordCode');
+
+    return lastValueFrom(
+      this.authService.AuthVerifyResetPasswordCode({ username, code }),
+    );
+  }
+
+  async confirmResetPassword({
+    username,
+    code,
+    newPassword,
+  }: AuthConfirmResetPasswordRequest) {
+    this.logger.info('confirmResetPassword');
+
+    return lastValueFrom(
+      this.authService.AuthConfirmResetPassword({
+        username,
+        code,
+        newPassword,
+      }),
+    );
+  }
+
+  async enableTotpMFA({ accessToken, password }: AuthEnableTotpMfaRequest) {
+    this.logger.info('enableTotpMFA');
+
+    return lastValueFrom(
+      this.authService.AuthEnableTotpMfa({ accessToken, password }),
+    );
+  }
+
+  async disableTotpMFA({ accessToken, password }: AuthDisableTotpMfaRequest) {
+    this.logger.info('disableTotpMFA');
+
+    return lastValueFrom(
+      this.authService.AuthDisableTotpMfa({ accessToken, password }),
+    );
+  }
+
+  async dismissTotpMFA({ accessToken }: AuthDismissTotpMfaRequest) {
+    this.logger.info('dismissTotpMFA');
+
+    return lastValueFrom(this.authService.AuthDismissTotpMfa({ accessToken }));
+  }
+
+  async verifyTotp({ accessToken, totp }: AuthVerifyTotpMfaRequest) {
+    this.logger.info('disableTotpMFA');
+
+    return lastValueFrom(
+      this.authService.AuthVerifyTotpMfa({ accessToken, totp }),
+    );
+  }
+
+  async getSession(session: string) {
+    return lastValueFrom(this.authService.AuthGetSession({ session }));
+  }
+
+  async oauthSignIn(data: { username: string; token: string }) {
+    const { username, token } = data;
+    return lastValueFrom(
+      this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
+    );
+  }
+}

src/modules/auth/dtos/login.ts

@@ -0,0 +1,120 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import {
+  AuthSignInRequest,
+  AuthSignInResponse,
+} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+
+export class LoginDto {
+  @ApiProperty()
+  username: string;
+
+  @ApiProperty()
+  password: string;
+
+  @ApiProperty({ required: false })
+  totp: string;
+}
+export class AuthTokens {
+  @ApiProperty()
+  accessToken: string;
+  @ApiProperty()
+  refreshToken: string;
+  @ApiPropertyOptional()
+  termsOfUseToken: string;
+  @ApiProperty({ deprecated: true })
+  idToken: string;
+}
+export enum MfaStatuses {
+  PENDING = 'pending',
+  NONE = 'none',
+  TOTP = 'totp',
+}
+
+export enum TouStatuses {
+  PENDING = 'pending',
+  REQUIRED = 'required',
+  OK = 'ok',
+}
+export class TermsOfUse {
+  @ApiProperty()
+  version: number;
+  @ApiProperty()
+  publicUrl: string;
+  @ApiProperty()
+  enforceDate: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class TermsOfUseStatus {
+  @ApiProperty({ enum: TouStatuses })
+  status: string;
+  @ApiPropertyOptional()
+  lastSigned?: TermsOfUse;
+  @ApiPropertyOptional()
+  next?: TermsOfUse;
+}
+export class AuthUser {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class AuthCustomer {
+  @ApiProperty()
+  modules: string[];
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  tier: string;
+  @ApiProperty()
+  scheduleLimit: string;
+  @ApiProperty()
+  links: Link[];
+}
+
+export class AuthSignInReq implements AuthSignInRequest {
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  password: string;
+  @ApiPropertyOptional({
+    description:
+      'TOTP code used to login when Multi-Factor Authentication is enabled',
+  })
+  totp: string;
+}
+
+export class AuthSignInRes implements AuthSignInResponse {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty({ enum: MfaStatuses })
+  mfaStatus: string;
+  @ApiPropertyOptional()
+  customer?: AuthCustomer;
+  @ApiPropertyOptional()
+  user?: AuthUser;
+  @ApiPropertyOptional()
+  tokens?: AuthTokens;
+  @ApiPropertyOptional()
+  termsOfUse?: TermsOfUseStatus;
+}
+
+export class AuthRefreshAccessTokenReq {
+  @ApiProperty()
+  refreshToken: string;
+  @ApiProperty()
+  customerName: string;
+}
+export class AuthRefreshAccessTokenRes {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty()
+  accessToken: string;
+}

src/modules/auth/passport-strategies/google-strategy.ts

@@ -0,0 +1,46 @@
+import {
+  AuthenticateOptionsGoogle,
+  Profile,
+  Strategy,
+  StrategyOptions,
+} from 'passport-google-oauth20';
+import { PassportStrategy } from '@nestjs/passport';
+import { Inject, Injectable } from '@nestjs/common';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+import { Request } from 'express';
+import jwt from 'jsonwebtoken';
+
+@Injectable()
+export class GoogleLoginStrategy extends PassportStrategy(
+  Strategy,
+  'google-login',
+) {
+  redirect_uri: string;
+  constructor(
+    @Inject('OAUTH_SECRETS')
+    private readonly oauthSecrets: OauthSecrets,
+  ) {
+    const options: StrategyOptions = {
+      clientID: oauthSecrets['google-login'].client_id,
+      clientSecret: oauthSecrets['google-login'].client_secret,
+      callbackURL: oauthSecrets['google-login'].redirect_uri,
+      scope: ['email', 'profile', 'openid'],
+    };
+    const verify = (
+      accessToken: string,
+      refreshToken: string,
+      profile: Profile,
+      done,
+    ) => {
+      return done(null, profile, { accessToken, refreshToken });
+    };
+
+    super(options, verify);
+  }
+
+  authenticate(req: Request, options: AuthenticateOptionsGoogle) {
+    const language = req.headers['dadosfera-lang'] || req.query.language;
+    options.state = jwt.sign({ language }, process.env.JWT_PRIVATE_KEY);
+    super.authenticate(req, options);
+  }
+}

src/modules/catalog/catalog-client.ts

@@ -0,0 +1,37 @@
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { credentials } from '@grpc/grpc-js';
+import { Catalog } from '@dadosfera/protospack-v2';
+
+const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+
+export class CatalogClientConfiguration {
+  public name = 'CatalogClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.PIFACTORY_URL,
+      package: [
+        Catalog.ProtoPackages.ReadPackage,
+        Catalog.ProtoPackages.WritePackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        Catalog.ProtoPaths.ReadFilePath,
+        Catalog.ProtoPaths.WriteFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        defaults: false,
+      },
+    },
+  };
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/catalog/catalog.controller.ts

@@ -1,322 +1,636 @@
 import {
+  BadRequestException,
   Body,
   Controller,
   Delete,
+  ForbiddenException,
   Get,
+  Headers,
+  Inject,
+  NotFoundException,
   Param,
   Post,
+  Put,
   Query,
+  UseFilters,
 } from '@nestjs/common';
+import {
+  ApiCreatedResponse,
+  ApiHeaders,
+  ApiOkResponse,
+  ApiTags,
+} from '@nestjs/swagger';
+import {
+  Authenticated,
+  RequireAllPermissions,
+  RequireSomePermission,
+} from '../../decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import {
+  GetDatasetCatalogTaskRes,
+  ICatalogAllRequest,
+  ICatalogAllResponse,
+  IColumnsMetadataResponse,
+  ICreateDataAsset,
+  IDeleteComment,
+  IDocsResponse,
+  IMakeAComment,
+  IOneDataAsset,
+  IPreviewResponse,
+  IUpdateDataRequest,
+  TriggerCatalogReq,
+  TriggerCatalogRes,
+} from './dtos';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
+@ApiTags('Catalog')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @Controller('catalog')
+@UseFilters(new GrpcToHttpExceptionFilter())
+@Authenticated()
 export class CatalogController {
-  constructor(private catalogService: CatalogService) {}
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private catalogService: CatalogService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
 
-  @Get('all')
-  async catalogAll(@Body() body) {
-    console.log(`/catalog`, 'ON CATALOG ALL ROUTE');
+  @Get()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async searchCatalog(
+    @User() user: RequestUser,
+    @Query() query: ICatalogAllRequest,
+  ): Promise<ICatalogAllResponse> {
+    const { user_id, customer_name, customer_id, username, permissions } = user;
+    this.logger.info(`/catalog - searchCatalog`, {
+      user_id,
+      customer_name,
+    });
 
-    const catalogService = new CatalogService();
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
 
-    const res = await catalogService.catalogAll(body);
+    const roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+      roles,
+      is_data_manager,
+    });
+
+    const res = await this.catalogService.searchDataAssets(
+      query,
+      metadata,
+      customer_id,
+    );
 
     return res;
   }
 
-  @Get('data_apps')
-  async dataAppsAll(@Body() body) {
-    console.log(`/catalog`, 'ON FIND ALL DATA APPS ROUTE');
+  @ApiInternalOnlyEndpoint()
+  @Get('data-asset')
+  async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+    const { pipeline, object } = query;
+    this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
+      username,
+      customer_name,
+    });
 
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.dataAppsAll(body);
-
-    return res;
-  }
-
-  @Delete('data_apps/:id')
-  async dataAppsOne(@Param() params, @Body() body) {
-    console.log(`/catalog`, 'ON FIND ONE DATA APP ROUTE');
-    const { id } = params;
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.dataAppsOne(id, body);
-
-    return res;
-  }
-
-  @Get('dashboard-metabase')
-  async getAllDashboardMetabase(@Body() body) {
-    console.log(`/catalog`, 'ON GET ALL DASHBOARDS METABASE ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getAllDashboardMetabase(body);
-
-    return res;
-  }
-
-  @Get('dashboard-metabase/:id')
-  async getOneDashboardMetabase(@Body() body, @Param() params) {
-    const { id } = params;
-    console.log(`/catalog`, 'ON GET ONE DASHBOARD METABASE ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getOneDashboardMetabase(id, body);
-
-    return res;
-  }
-
-  @Get('table-metadata')
-  async getAllTableMetadata(@Body() body, @Query() query) {
-    console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE');
-
-    const catalogService = new CatalogService();
-    if (!query) {
-      const res = await catalogService.getAllTableMetadata(body);
-      return res;
-    } else {
-      const res = await catalogService.getOneTableMetadata(body, query);
-      return res;
+    if (!pipeline || !object) {
+      throw new BadRequestException('Query params not provided');
     }
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id: undefined,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } =
+      await this.catalogService.getOneDataAssetByPipelineAndObject({
+        pipeline: query.pipeline,
+        object: query.object,
+        customer_id,
+        metadata,
+      });
+
+    if (data_asset?.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (data_asset.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (data_asset.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      return { data_asset };
+    }
+
+    throw new ForbiddenException(
+      'You do not have permission to access this data asset.',
+    );
   }
 
-  @Delete('table-metadata/:id')
-  async deleteOneTableMetadata(@Param() params, @Body() body) {
-    const { id } = params;
-    console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.deleteOneTableMetadata(id, body);
-
-    return res;
-  }
-
-  @Get('column-metadata')
-  async getOneColumnMetadata(@Body() body) {
-    console.log(`/catalog`, 'ON GET ONE COLUMN METADATA ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getOneColumnMetadata(body);
-
-    return res;
-  }
-
-  @Delete('column-metadata/:id')
-  async deleteOneColumnMetadata(@Param() params, @Body() body) {
-    console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE');
-    const { id } = params;
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.deleteOneColumnMetadata(id, body);
-
-    return res;
-  }
-
-  @Get('data-preview')
-  async getOneDataPreview(@Body() body) {
-    console.log(`/catalog`, 'ON GET ONE DATAPREVIEW ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getOneDataPreview(body);
-
-    return res;
-  }
-
-  @Get('data-status')
-  async getDataStatus(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA STATUS ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getDataStatus(body);
-
-    return res;
-  }
-
-  @Get('data-description')
-  async getDataDescription(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA DESCRIPTION ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getDataDescription(body);
-
-    return res;
-  }
-
-  @Get('data-docs')
-  async getDataDocs(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA DOCS ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getDataDocs(body);
-
-    return res;
-  }
-
-  @Get('data-rating')
-  async getDataRating(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getDataRating(body);
-
-    return res;
-  }
-
-  @Post('data-rating')
-  async createDataRating(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.createDataRating(body);
-
-    return res;
-  }
-
-  @Get('summary-rating/:id')
-  async getSummaryRating(@Param() params, @Body() body) {
-    console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE');
-    const { id } = params;
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getSummaryRating(id, body);
-
-    return res;
-  }
-
-  @Get('data-comment')
-  async getDataComment(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getDataComment(body);
-
-    return res;
-  }
-
-  @Post('data-comment')
-  async createDataComment(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.createDataComment(body);
-
-    return res;
-  }
-
-  @Get('data-review')
-  async getDataReview(@Body() body) {
-    console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getDataReview(body);
-
-    return res;
-  }
-
-  @Post('tags')
-  async createTags(@Body() body) {
-    console.log(`/catalog`, 'ON CREATE TAG ROUTE');
-
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.createTags(body);
-
-    return res;
-  }
-
+  @ApiInternalOnlyEndpoint()
   @Get('tags')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async findAllTags(@Body() body) {
-    console.log(`/catalog`, 'ON FIND ALL TAGS ROUTE');
+    this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
+      user: body.info.user_id,
+      customer: body.info.customer,
+    });
 
-    const catalogService = new CatalogService();
+    const { user_id, customer, customer_id } = body.info;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name: customer,
+    });
 
-    const res = await catalogService.findAllTags(body);
+    const res = await this.catalogService.findAllTags(body, metadata);
 
     return res;
   }
 
-  @Delete('tags/:id')
-  async deleteTags(@Param() params, @Body() body) {
-    console.log(`/catalog`, 'ON DELETE TAG ROUTE');
-    const { id } = params;
+  @Get('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getDataAsset(
+    @User() user: RequestUser,
+    @Param('id') id: string,
+    @Query('shared') shared?: 'true',
+  ) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
 
-    const catalogService = new CatalogService();
+    this.logger.info(`GET /data-asset/${id}`, {
+      username,
+      customer_name,
+    });
 
-    const res = await catalogService.deleteTags(id, body);
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } = await this.catalogService.getOneDataAsset({
+      customer_id,
+      id,
+      metadata,
+    });
+    has_permission =
+      is_data_manager ||
+      data_asset?.owner === username ||
+      (user_roles as Array<any>).some((r) => data_asset.p_roles.includes(r)) ||
+      data_asset.p_users.includes(user_id);
+    if (
+      shared === 'true' &&
+      (data_asset.share_type === undefined || data_asset.share_type === 'none')
+    )
+      throw new NotFoundException();
+    if (!has_permission)
+      throw new ForbiddenException(
+        'You do not have permission to access this data asset.',
+      );
+    delete data_asset.p_roles;
+    delete data_asset.p_users;
+
+    return { data_asset };
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('data-asset/rls/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getDataAssetRls(
+    @User() user: RequestUser,
+    @Headers() headers,
+    @Param('id') id: string,
+  ) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+
+    this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
+      username,
+      customer_name,
+    });
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } = await this.catalogService.getOneDataAsset({
+      customer_id,
+      id,
+      metadata,
+    });
+
+    if (data_asset?.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (data_asset.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (data_asset.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      return { data_asset };
+    }
+
+    throw new ForbiddenException(
+      'You do not have permission to access this data asset.',
+    );
+  }
+
+  @Get('data-asset/:id/columns-metadata')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getDataAssetColumnsMetadata(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+  ): Promise<IColumnsMetadataResponse> {
+    const { customer_name, customer_id, user_id, username } = user;
+
+    this.logger.info(`/catalog - columns-metadata`, {
+      user_id,
+      customer_name,
+    });
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      customer_id,
+      user_id,
+      username,
+      language,
+    });
+
+    const columns_metadata =
+      await this.catalogService.getDatasetColumnsMetadata(id, metadata);
+
+    return { columns_metadata };
+  }
+
+  @Get('data-asset/:id/preview')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getDataAssetPreview(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+  ): Promise<IPreviewResponse> {
+    const { customer_name, customer_id, user_id, username } = user;
+
+    this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
+      user_id,
+      customer_name,
+    });
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      customer_id,
+      user_id,
+      username,
+      language,
+    });
+
+    const preview = await this.catalogService.getDatasetPreview(id, metadata);
+
+    return { preview };
+  }
+
+  @Get('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getDataAssetDocs(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+  ): Promise<IDocsResponse> {
+    const { customer_name, customer_id, user_id, username } = user;
+
+    this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
+      user_id,
+      customer_name,
+    });
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      customer_id,
+      user_id,
+      username,
+      language,
+    });
+
+    const docs = await this.catalogService.getDataDocs(id, metadata);
+
+    return { docs };
+  }
+
+  @Put('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async updateDataAsset(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') data_asset_id: string,
+    @Body() body: IUpdateDataRequest,
+  ): Promise<IOneDataAsset> {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const result = await this.catalogService.updateOneDataAsset({
+      body,
+      data_asset_id,
+      customer_id,
+      metadata,
+    });
+
+    delete result.data_asset.p_roles;
+    delete result.data_asset.p_users;
+
+    return result;
+  }
+
+  @Post('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async manageDataAssetDocs(
+    @User() user: RequestUser,
+    @Headers() headers,
+    @Param('id') table_id: string,
+    @Body('docs') docs: string,
+  ) {
+    const { user_id, customer_name } = user;
+
+    this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
+      user_id,
+      customer_name,
+    });
+
+    const res = await this.catalogService.createDataDocs({
+      table_id,
+      docs,
+      info: {
+        customer: customer_name,
+      },
+    });
 
     return res;
   }
 
-  @Post('table-tags')
-  async createTableTags(@Body() body) {
-    console.log(`/catalog`, 'ON CREATE TABLE TAG ROUTE');
+  @ApiInternalOnlyEndpoint()
+  @Put('data-asset/:id/manage-permissions')
+  async manageDataAssetPermissions(
+    @Param('id') id: string,
+    @User() user: RequestUser,
+    @Body() body,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.managePermissions(
+      { ...body, id },
+      metadata,
+    );
 
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.createTableTags(body);
-
-    return res;
+    return { data_asset: JSON.parse(response.data_asset) };
   }
 
-  @Get('table-tags')
-  async getAllTableTags(@Body() body) {
-    console.log(`/catalog`, 'ON GET ALL TABLE TAGS ROUTE');
+  @ApiInternalOnlyEndpoint()
+  @Put('data-asset/:id/revoke-permissions')
+  async revokeDataAssetPermissions(
+    @Param('id') id: string,
+    @User() user: RequestUser,
+    @Body() body,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.revokePermissions(
+      { ...body, id },
+      metadata,
+    );
 
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.findAllTableTags(body);
-
-    return res;
+    return { message: response.message };
   }
 
-  @Delete('table-tags/:id')
-  async deleteTableTags(@Param() params, @Body() body) {
-    console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');
-    const { id } = params;
+  @Post()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async createDataAsset(
+    @User() user: RequestUser,
+    @Body() body: ICreateDataAsset,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const { data_asset } = await this.catalogService.createDataAsset(
+      body,
+      metadata,
+    );
 
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.deleteTableTags(id, body);
-
-    return res;
+    return { data_asset: JSON.parse(data_asset) };
   }
 
-  @Get('table-rules')
-  async getTableRules(@Body() body) {
-    console.log(`/catalog`, 'ON GET TABLE RULES ROUTE');
+  @Post('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async commentOnDataAsset(
+    @Param('id') id: string,
+    @User() user: RequestUser,
+    @Body() body: IMakeAComment,
+  ) {
+    const { message } = body;
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.commentOnDataAsset(
+      { data_asset_id: id, username, message },
+      metadata,
+    );
 
-    const catalogService = new CatalogService();
-
-    const res = await catalogService.getTableRules(body);
-
-    return res;
+    return response;
   }
 
-  @Delete('table-rules/:id')
-  async deleteTableRules(@Param() params, @Body() body) {
-    console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE');
-    const { id } = params;
+  @Delete('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.deleteDataAsset(
+      { id, type: undefined },
+      metadata,
+    );
 
-    const catalogService = new CatalogService();
+    return response;
+  }
 
-    const res = await catalogService.deleteTableRules(id, body);
+  @Delete('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async deleteComment(
+    @Param('id') id: string,
+    @User() user: RequestUser,
+    @Body() body: IDeleteComment,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.deleteComment(
+      {
+        data_asset_id: id,
+        comment: { ...body.comment, created_at: undefined, message: undefined },
+      },
+      metadata,
+    );
 
-    return res;
+    return response;
+  }
+
+  @Post('dataset-catalog-task')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiCreatedResponse({ type: TriggerCatalogRes })
+  async triggerCatalog(
+    @User() user: RequestUser,
+    @Body() body: TriggerCatalogReq,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const session = await this.catalogService.triggerCatalog(body, metadata);
+
+    return { session };
+  }
+
+  @Get('dataset-catalog-task/:session')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiOkResponse({ type: GetDatasetCatalogTaskRes })
+  async getCatalogTask(
+    @User() user: RequestUser,
+    @Param('session') session: string,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.getDatasetCatalogTask(
+      session,
+      metadata,
+    );
+
+    return response;
   }
 }

src/modules/catalog/catalog.module.ts

@@ -0,0 +1,26 @@
+import { Module } from '@nestjs/common';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { CatalogController } from './catalog.controller';
+import { CatalogService } from './catalog.service';
+import { CatalogClientConfiguration } from './catalog-client';
+import { ClientsModule } from '@nestjs/microservices';
+import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
+import { UsersModule } from '../users/users.module';
+import { RolesModule } from '../roles/roles.module';
+
+const client = new CatalogClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    OldPipelineModule,
+    UsersModule,
+    RolesModule,
+  ],
+  controllers: [CatalogController],
+  providers: [CatalogService, DadosferaLogger],
+  exports: [CatalogService],
+})
+export class CatalogModule {}

src/modules/catalog/catalog.service.ts

@@ -1,349 +1,349 @@
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+import {
+  WriteService,
+  ReadService,
+  ProtoServices,
+  Messages,
+} from '@dadosfera/protospack-v2/dist/lib/Catalog';
+import {
+  HttpException,
+  HttpStatus,
+  Inject,
+  OnModuleInit,
+} from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
 import axios from 'axios';
+import { lastValueFrom } from 'rxjs';
+import { CatalogClientConfiguration } from './catalog-client';
+import { UsersService } from '../users/users.service';
+import { RolesService } from '../roles/roles.service';
+import { Metadata } from '@grpc/grpc-js';
+import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
 
-class CatalogService {
-  async catalogAll(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/all/`,
-    );
-    console.log(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/all/`,
-    );
-    return data;
+class CatalogService implements OnModuleInit {
+  catalogReadService: ReadService.CatalogReadServices;
+  catalogWriteService: WriteService.CatalogWriteServices;
+  logger: any;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject(CatalogClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+    private readonly userService: UsersService,
+    private readonly roleService: RolesService,
+  ) {
+    this.logger = dadosferaLogger.logger;
   }
 
-  async dataAppsAll(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data_apps/`,
-    );
-    return data;
+  onModuleInit() {
+    this.catalogReadService =
+      this.grpcClient.getService<ReadService.CatalogReadServices>(
+        ProtoServices.CatalogReadServices,
+      );
+    this.catalogWriteService =
+      this.grpcClient.getService<WriteService.CatalogWriteServices>(
+        ProtoServices.CatalogWriteServices,
+      );
   }
 
-  async dataAppsOne(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  _getNimbusUrl(body) {
+    const customer = body.info.customer.toLowerCase();
 
-    const { data } = await axios.delete(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data_apps/${id}/`,
-    );
-    return data;
+    if (process.env.ENV === 'prd') {
+      return `https://nimbus-${customer}.dadosfera.ai`;
+    }
+
+    return `https://nimbus-${customer}.${process.env.ENV.replace(
+      'local',
+      'stg',
+    )}.dadosfera.ai`;
   }
 
-  async getAllDashboardMetabase(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
+    if (!data.embed) data.embed = undefined;
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/dashboard-metabase/`,
+    return lastValueFrom(
+      this.catalogWriteService.CreateDataAsset(data, metadata),
     );
-    return data;
   }
 
-  async getOneDashboardMetabase(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/dashboard-metabase/${id}`,
-    );
-    return data;
+    return lastValueFrom(
+      this.catalogWriteService.ManagePermission(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
   }
 
-  async getAllTableMetadata(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/`,
-    );
-    return data;
+    return lastValueFrom(
+      this.catalogWriteService.RevokePermission(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
   }
 
-  async getOneTableMetadata(body, params) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/`,
-      { params: params },
-    );
-    return data;
+    return lastValueFrom(
+      this.catalogWriteService.MakeAComment(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
   }
 
-  async deleteOneTableMetadata(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
 
-    const { data } = await axios.delete(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/${id}`,
-    );
-    return data;
+    return lastValueFrom(
+      this.catalogWriteService.UpdateAComment(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
   }
 
-  async getOneColumnMetadata(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/column-metadata/`,
-    );
-    return data;
+    return lastValueFrom(
+      this.catalogWriteService.DeleteDataAsset(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
   }
 
-  async deleteOneColumnMetadata(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async getUserRolesIds(userId: string) {
+    const result = await this.userService.findOneById(userId).catch(() => null);
 
-    const { data } = await axios.delete(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/column-metadata/${id}/`,
-    );
-    return data;
+    const roles_ids = result.user.roles.map((role) => role.id);
+
+    return roles_ids;
   }
 
-  async getOneDataPreview(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async searchDataAssets(
+    query: Record<string, any>,
+    metadata: Metadata,
+    customer_id: string,
+  ) {
+    this.logger.info('CatalogService - searchDataAssets');
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-preview/`,
+    const { search, page, size, sort_by, order, ...filters } = query;
+
+    const { data_assets, total } = await lastValueFrom(
+      this.catalogReadService.GetAllDataAssets(
+        {
+          search,
+          page,
+          size,
+          sort_by,
+          order,
+          filters: JSON.stringify(filters),
+        },
+        metadata,
+      ),
     );
-    return data;
+
+    const result = JSON.parse(data_assets);
+
+    const response = await this.getAssetsUsersAndRoles(
+      result.data_assets,
+      customer_id,
+    );
+
+    return { data_assets: response, total };
   }
 
-  async getDataStatus(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-status/`,
+  async getOneDataAsset(data: {
+    id: string;
+    customer_id: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, id, metadata } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAsset(
+        { id, type: undefined },
+        metadata,
+      ),
     );
-    return data;
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
   }
 
-  async getDataDescription(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-description/`,
+  async getOneDataAssetByPipelineAndObject(data: {
+    customer_id: string;
+    pipeline: string;
+    object: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, metadata, pipeline, object } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAssetByPipelineAndObject(
+        { pipeline, object },
+        metadata,
+      ),
     );
-    return data;
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
   }
 
-  async getDataDocs(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async updateOneDataAsset(data: {
+    data_asset_id: string;
+    customer_id: string;
+    body: IUpdateDataRequest;
+    metadata: Metadata;
+  }) {
+    const { body, customer_id, data_asset_id, metadata } = data;
 
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-docs/`,
+    const { data_asset } = await lastValueFrom(
+      this.catalogWriteService.UpdateDataAsset(
+        { id: data_asset_id, changes: JSON.stringify(body) },
+        metadata,
+      ),
     );
-    return data;
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
   }
 
-  async getDataRating(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-rating/`,
+  async getDataDocs(id: string, metadata: Metadata) {
+    const { documentation } = await lastValueFrom(
+      this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
     );
-    return data;
+    console.log(documentation);
+    const docs = JSON.parse(documentation);
+    return docs;
   }
 
-  async createDataRating(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
+  async getDatasetPreview(id: string, metadata: Metadata) {
+    const { preview } = await lastValueFrom(
+      this.catalogReadService.GetDatasetPreview(
+        { id, type: undefined },
+        metadata,
+      ),
+    );
+    const result = JSON.parse(preview);
+    return result;
+  }
 
+  async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
+    const { columns_metadata } = await lastValueFrom(
+      this.catalogReadService.GetDatasetColumnsMetadata(
+        { id, type: undefined },
+        metadata,
+      ),
+    );
+    const result = JSON.parse(columns_metadata);
+    return result;
+  }
+
+  async createDataDocs(body) {
+    const nimbusUrl = this._getNimbusUrl(body);
     const { data } = await axios.post(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-rating/`,
+      `${nimbusUrl}/api/catalog/data-docs/`,
       body,
     );
     return data;
   }
 
-  async getSummaryRating(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/summary-rating/${id}`,
-    );
-    return data;
+  async findAllTags(data, metadata) {
+    this.logger.info('CatalogService - findAllCustomerTags');
+
+    const response = await lastValueFrom(
+      this.catalogReadService.GetCustomerTags(data, metadata),
+    )
+      .then((res) => {
+        this.logger.info('Done');
+        return res;
+      })
+      .catch((err) => {
+        this.logger.error(err.message);
+        throw new Error(err);
+      });
+
+    return response;
   }
-
-  async getDataComment(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-comment/`,
+  async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
+    const { users: customer_users } =
+      await this.userService.findAllUsersByCustomerId(customer_id);
+    const { roles: customer_roles } = await this.roleService.roleSearch(
+      {},
+      { customer_id },
     );
-    return data;
+    return data_assets.map((data_asset) => {
+      const owner = customer_users.find(
+        (u) => u.id === data_asset.owner,
+      )?.username;
+
+      const roles = [];
+      const users = [];
+      for (const role_id of data_asset.roles) {
+        const role = customer_roles.find((r) => r.id === role_id);
+        if (role) roles.push({ id: role.id, name: role.name });
+      }
+      for (const user_id of data_asset.users) {
+        const user = customer_users.find((r) => r.id === user_id);
+        if (user) users.push({ id: user.id, username: user.username });
+      }
+      return {
+        ...data_asset,
+        roles,
+        users,
+        owner,
+      } as typeof data_asset;
+    });
   }
-
-  async createDataComment(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.post(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-comment/`,
-      body,
+  async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
+    const { session } = await lastValueFrom(
+      this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
     );
-    return data;
+    return session;
   }
-
-  async getDataReview(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-review/`,
+  async getDatasetCatalogTask(session: string, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
     );
-    return data;
-  }
-
-  async createTags(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.post(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/tags/`,
-      body,
-    );
-    return data;
-  }
-
-  async findAllTags(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/tags/`,
-      body,
-    );
-    return data;
-  }
-
-  async findAllTableTags(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-tags/`,
-      body,
-    );
-    return data;
-  }
-
-  async deleteTags(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.delete(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/tags/${id}`,
-    );
-    return data;
-  }
-
-  async createTableTags(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.post(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-tags/`,
-      body,
-    );
-    return data;
-  }
-
-  async deleteTableTags(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.delete(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-tags/${id}`,
-    );
-    return data;
-  }
-
-  async getTableRules(body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.get(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-rules/`,
-      body,
-    );
-    return data;
-  }
-
-  async deleteTableRules(id, body) {
-    const customer =
-      body.info.customer.toLowerCase() === 'dadosfera'
-        ? ``
-        : `-${body.info.customer.toLowerCase()}`;
-
-    const { data } = await axios.delete(
-      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-rules/${id}`,
-    );
-    return data;
+    return res;
   }
 }
 

src/modules/catalog/dtos/index.ts

@@ -0,0 +1,314 @@
+import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
+import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+
+export enum DataAssetShareType {
+  none = 'none',
+  public = 'public',
+  private = 'private',
+}
+export enum OrderEnum {
+  asc = 'asc',
+  desc = 'desc',
+}
+export class EmbedObject {
+  @ApiProperty()
+  url: string;
+}
+export class IRole {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  id: string;
+}
+
+export class IUser {
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  id: string;
+}
+
+export class IComment {
+  @ApiProperty()
+  id: number;
+  @ApiProperty()
+  message: string;
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  username: string;
+}
+
+export class IDataAsset {
+  @ApiProperty()
+  id: number;
+  @ApiPropertyOptional()
+  database_name?: string;
+  @ApiPropertyOptional()
+  table_name?: string;
+  @ApiPropertyOptional()
+  num_columns?: number;
+  @ApiPropertyOptional()
+  location?: string;
+  @ApiPropertyOptional()
+  created_time?: string;
+  @ApiPropertyOptional()
+  table_type?: string;
+  @ApiPropertyOptional()
+  size_in_bytes?: number;
+  @ApiPropertyOptional()
+  num_files?: number;
+  @ApiPropertyOptional()
+  data_encrypted?: string;
+  @ApiPropertyOptional()
+  num_rows?: number;
+  @ApiProperty()
+  display_name: string;
+  @ApiProperty()
+  pipeline_id: string;
+  @ApiProperty()
+  user_id: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  data_asset_type: string;
+  @ApiProperty()
+  tags: string[];
+  @ApiProperty()
+  owner: string;
+  @ApiProperty({ type: [IRole] })
+  roles: IRole[];
+  @ApiProperty({ type: [IUser] })
+  users: IUser[];
+  @ApiPropertyOptional()
+  dashboard_id?: number;
+  @ApiPropertyOptional()
+  name?: string;
+  @ApiPropertyOptional()
+  parameters?: string[];
+  @ApiPropertyOptional()
+  embedding_params?: object;
+  @ApiProperty()
+  created_at: string;
+  @ApiPropertyOptional()
+  updated_at?: string;
+  @ApiProperty({ type: [IComment] })
+  comments: IComment[];
+  @ApiPropertyOptional()
+  embed?: EmbedObject;
+  @ApiPropertyOptional({ enum: DataAssetShareType })
+  share_type?: DataAssetShareType;
+}
+
+export class IOneDataAsset {
+  @ApiProperty()
+  data_asset: IDataAsset;
+}
+
+export class IMakeAComment {
+  @ApiProperty()
+  message: string;
+}
+
+export class CommentToDelete extends PickType(IComment, ['id', 'username']) {}
+export class IDeleteComment {
+  @ApiProperty()
+  comment: CommentToDelete;
+}
+
+export class ICatalogAllRequest {
+  @ApiPropertyOptional({
+    description: 'Texto usado para filtrar os ativos no catálogo.',
+  })
+  search;
+
+  @ApiPropertyOptional({
+    description: 'Número de registros a ser retornado',
+    maximum: 10000,
+    minimum: 1,
+  })
+  size: number;
+
+  @ApiPropertyOptional({
+    description: 'Usado para paginação, em conjunto com `size`',
+    minimum: 1,
+  })
+  page: number;
+
+  @ApiPropertyOptional({
+    description:
+      'Campo do data asset para usar na ordenação. Padrão: `created_at` ',
+  })
+  sort_by: string;
+
+  @ApiPropertyOptional({
+    enum: OrderEnum,
+    default: OrderEnum.asc,
+    description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
+  })
+  order?: OrderEnum;
+}
+
+export class ICatalogAllResponse {
+  @ApiProperty({ type: [IDataAsset] })
+  data_assets: IDataAsset[];
+  @ApiProperty()
+  total: number;
+}
+
+export class IData {
+  @ApiProperty()
+  var: number;
+  @ApiProperty()
+  date: string;
+  @ApiProperty()
+  last: number;
+  @ApiProperty()
+  maximum: number;
+  @ApiProperty()
+  minimum: number;
+  @ApiProperty()
+  day_opening: number;
+}
+
+export class IUpdateDataRequest {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  tags: string[];
+  @ApiPropertyOptional()
+  embed: EmbedObject;
+  @ApiPropertyOptional({ enum: DataAssetShareType })
+  share_type?: DataAssetShareType;
+}
+export class ICreateDataAsset implements CreateDataAssetRequest {
+  @ApiProperty()
+  display_name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  data_asset_type: string;
+  @ApiProperty()
+  external_url: string;
+  @ApiProperty()
+  location: string;
+  @ApiPropertyOptional()
+  embed: EmbedObject;
+}
+
+export class IPreview {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  database_name: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty({ type: [IData] })
+  data: IData[];
+}
+
+export class IDocs {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  docs: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  table_id: string;
+}
+
+export class IColumnMetadata {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  column_name: string;
+  @ApiProperty()
+  data_type: string;
+  @ApiProperty()
+  missing_count: string;
+  @ApiProperty()
+  min: string;
+  @ApiProperty()
+  max: string;
+  @ApiProperty()
+  count: string;
+  @ApiProperty()
+  histogram: string;
+  @ApiProperty()
+  mean: string;
+  @ApiProperty()
+  median: string;
+  @ApiProperty()
+  missing_rate: string;
+  @ApiProperty()
+  stddev: string;
+  @ApiProperty()
+  unique_count: string;
+  @ApiProperty()
+  unique_rate: string;
+  @ApiProperty()
+  analyzer_rules: string[];
+  @ApiProperty()
+  version: string;
+}
+
+export class IPreviewResponse {
+  @ApiProperty()
+  preview: IPreview;
+}
+
+export class IDocsResponse {
+  @ApiProperty()
+  docs: IDocs;
+}
+
+export class IColumnsMetadataResponse {
+  @ApiProperty({ type: [IColumnMetadata] })
+  columns_metadata: IColumnMetadata[];
+}
+
+export class TriggerCatalogReq {
+  @ApiProperty({
+    description: 'Name of the table on Snowflake',
+    example: 'TB__4DXSD4_TEST',
+  })
+  table_name: string;
+
+  @ApiPropertyOptional({
+    description:
+      'Schema where the asset is located. If not set defaults to "PUBLIC"',
+  })
+  table_schema?: string;
+
+  @ApiPropertyOptional({
+    description: 'Id of the pipeline that generated the asset.',
+  })
+  pipeline_id: string;
+}
+
+export class TriggerCatalogRes {
+  @ApiProperty()
+  session: string;
+}
+
+export class GetDatasetCatalogTaskRes {
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  customer_name: string;
+  @ApiProperty()
+  session_id: string;
+  @ApiProperty()
+  status: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  updated_by: string;
+}

src/modules/connection-test/connection-test-client.config.ts

@@ -0,0 +1,31 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { ConnectionTest } from '@dadosfera/protospack-v2';
+
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
+export class ConnectionTestClientConfiguration {
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [ConnectionTest.ProtoPackages.ReadPackage],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [ConnectionTest.ProtoPaths.ReadFilePath],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: 'ConnectionTestGrpcClient',
+    ...this.config,
+  };
+}

src/modules/connection-test/connection-test.controller.ts

@@ -0,0 +1,124 @@
+import {
+  Body,
+  Controller,
+  HttpCode,
+  HttpStatus,
+  Inject,
+  Post,
+  UseFilters,
+  ValidationPipe,
+} from '@nestjs/common';
+import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import { User, RequestUser } from 'src/decorators/user.decorator';
+import { ConnectionTestService } from './connection-test.service';
+import {
+  ConnectionTestPingRes,
+  ConnectionTestPingReq,
+  ConnectionTestCredentialsReq,
+  ConnectionTestCredentialsRes,
+  ConnectionTestListSchemasReq,
+  ConnectionTestListSchemasRes,
+  ConnectionTestListTablesReq,
+  ConnectionTestListTablesRes,
+  GetTableMetadataRes,
+  GetTableMetadataReq,
+} from './dto/connection-test';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+
+@ApiInternalOnlyController()
+@ApiTags('Connection Test')
+@Controller('connection-test')
+@UseFilters(new GrpcToHttpExceptionFilter())
+@Authenticated()
+export class ConnectionTestController {
+  logger: any;
+  constructor(
+    private connectionTestService: ConnectionTestService,
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post('ping')
+  @ApiOkResponse({ type: ConnectionTestPingRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestPing(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: ConnectionTestPingReq,
+  ) {
+    this.logger.info('/connection-test/ping', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestPing(body, user);
+  }
+
+  @Post('credentials')
+  @ApiOkResponse({ type: ConnectionTestCredentialsRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestCredentials(
+    @User() user: RequestUser,
+    @Body() body: ConnectionTestCredentialsReq,
+  ) {
+    this.logger.info('/connection-test/credentials', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestCredentials(body, user);
+  }
+
+  @Post('schemas')
+  @ApiOkResponse({ type: ConnectionTestListSchemasRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestListSchemas(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: ConnectionTestListSchemasReq,
+  ) {
+    this.logger.info('/connection-test/schemas', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestListSchemas(
+      body,
+      user.customer_name,
+    );
+  }
+
+  @Post('tables')
+  @ApiOkResponse({ type: ConnectionTestListTablesRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestListTables(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: ConnectionTestListTablesReq,
+  ) {
+    this.logger.info('/connection-test/tables', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestListTables(
+      body,
+      user.customer_name,
+    );
+  }
+
+  @Post('table-metadata')
+  @ApiOkResponse({ type: GetTableMetadataRes })
+  @HttpCode(HttpStatus.OK)
+  async getTableMetadata(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: GetTableMetadataReq,
+  ) {
+    this.logger.info('/connection-test/table-metadata', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.getTableMetadata(
+      body,
+      user.customer_name,
+    );
+  }
+}

src/modules/connection-test/connection-test.module.ts

@@ -0,0 +1,14 @@
+import { Module } from '@nestjs/common';
+import { ConnectionTestController } from './connection-test.controller';
+import { ConnectionTestService } from './connection-test.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { ConnectionTestClientConfiguration } from './connection-test-client.config';
+import { ConnectionModule } from '../connection/connection.module';
+const client = new ConnectionTestClientConfiguration();
+@Module({
+  controllers: [ConnectionTestController],
+  providers: [ConnectionTestService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
+})
+export class ConnectionTestModule {}

src/modules/connection-test/connection-test.service.ts

@@ -0,0 +1,191 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ConnectionTest } from '@dadosfera/protospack-v2';
+import { lastValueFrom } from 'rxjs';
+import {
+  ConnectionTestCredentialsReq,
+  ConnectionTestCredentialsRes,
+  ConnectionTestListSchemasReq,
+  ConnectionTestListSchemasRes,
+  ConnectionTestListTablesReq,
+  ConnectionTestListTablesRes,
+  ConnectionTestPingReq,
+  ConnectionTestPingRes,
+  GetTableMetadataReq,
+  GetTableMetadataRes,
+} from './dto/connection-test';
+import { ConnectionClientService } from '../connection/client.service';
+import {
+  CreateConnectionDto,
+  DatabaseConnectionPropertiesDto,
+} from '../connection/dtos/connection';
+import { RequestUser } from 'src/decorators/user.decorator';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+
+@Injectable()
+export class ConnectionTestService {
+  private connectionTestReadClient: ConnectionTest.ReadService.ConnectionTestReadServices;
+  constructor(
+    @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
+    private connectionsService: ConnectionClientService,
+  ) {
+    this.connectionTestReadClient =
+      grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
+        ConnectionTest.ProtoServices.ConnectionTestReadServices,
+      );
+  }
+
+  async connectionTestPing(
+    body: ConnectionTestPingReq,
+    user: RequestUser,
+  ): Promise<ConnectionTestPingRes> {
+    const {
+      host: endpoint,
+      plugin,
+      port,
+      network_config_id,
+      connection_id,
+    } = body;
+    const { user_id, customer_id, customer_name } = user;
+    const metadata = PackTheMetadata({
+      customer_name,
+      customer_id,
+      user_id,
+    });
+
+    if (connection_id) {
+      const con = await this.connectionsService.getConnectionDetails({
+        body,
+        metadata,
+      });
+      const conPayload = JSON.parse(con.connection.properties);
+      const { plugin, network_config_id } = con.connection;
+      const { host, port, endpoint } = conPayload;
+
+      return lastValueFrom(
+        this.connectionTestReadClient.ConnectionTestPing({
+          customer_name,
+          endpoint: endpoint || host,
+          network_config_id,
+          plugin,
+          port: port ? port.toString() : undefined,
+          connection_id,
+        }),
+      );
+    } else {
+      return lastValueFrom(
+        this.connectionTestReadClient.ConnectionTestPing({
+          customer_name,
+          endpoint,
+          network_config_id,
+          plugin,
+          port: port ? port.toString() : undefined,
+          connection_id,
+        }),
+      );
+    }
+  }
+  async connectionTestCredentials(
+    body: ConnectionTestCredentialsReq,
+    user: RequestUser,
+  ): Promise<ConnectionTestCredentialsRes> {
+    const { plugin, ...connectionBody } = body;
+    let { connection_id } = body;
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+
+    if (body.network_config_id == '') {
+      body.network_config_id = undefined;
+    }
+
+    /********* CREATE CONNECTION *********/
+    if (!connection_id) {
+      const properties: DatabaseConnectionPropertiesDto = {
+        ...connectionBody.properties,
+        plugin,
+        credentials_type: 'basic_auth',
+      };
+      const createConnectionBody: CreateConnectionDto = {
+        ...body,
+        properties,
+      };
+      const { connection } = await this.connectionsService.createConnection(
+        createConnectionBody,
+        metadata,
+      );
+      connection_id = connection.id;
+    }
+    /************************************/
+    const { operation_result } = await lastValueFrom(
+      this.connectionTestReadClient.ConnectionTestCredentials({
+        connection_id,
+        customer_name,
+        plugin,
+      }),
+    ).catch(async (e) => {
+      if (!body.connection_id) {
+        await this.connectionsService.deleteConnection({
+          body: { id: connection_id },
+          metadata,
+        });
+      }
+      throw e;
+    });
+    if (!operation_result && !body.connection_id) {
+      await this.connectionsService.deleteConnection({
+        body: { id: connection_id },
+        metadata,
+      });
+      connection_id = null;
+    }
+    return { operation_result, connection_id };
+  }
+  async connectionTestListSchemas(
+    body: ConnectionTestListSchemasReq,
+    customer_name: string,
+  ): Promise<ConnectionTestListSchemasRes> {
+    const { connection_id, plugin } = body;
+    return lastValueFrom(
+      this.connectionTestReadClient.ListSchemas({
+        connection_id,
+        customer_name,
+        plugin,
+      }),
+    );
+  }
+  async connectionTestListTables(
+    body: ConnectionTestListTablesReq,
+    customer_name: string,
+  ): Promise<ConnectionTestListTablesRes> {
+    const { connection_id, plugin, schema } = body;
+    return lastValueFrom(
+      this.connectionTestReadClient.ListTables({
+        connection_id,
+        customer_name,
+        plugin,
+        schema,
+      }),
+    );
+  }
+
+  async getTableMetadata(
+    body: GetTableMetadataReq,
+    customer_name: string,
+  ): Promise<GetTableMetadataRes> {
+    const { schema, plugin, table_list, connection_id } = body;
+    return lastValueFrom(
+      this.connectionTestReadClient.GetTableMetadata({
+        connection_id,
+        customer_name,
+        plugin,
+        schema,
+        table_list,
+      }),
+    );
+  }
+}

src/modules/connection-test/dto/connection-test.ts

@@ -0,0 +1,133 @@
+import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
+import { IsString, IsOptional } from 'class-validator';
+import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
+import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
+export class ColumnDto {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  type: string;
+}
+export class TableMetadataDto {
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty({ type: [ColumnDto] })
+  columns: ColumnDto[];
+  @ApiProperty({ type: [ColumnDto] })
+  references: ColumnDto[];
+}
+
+export class ConnectionTestPingReq {
+  @ApiPropertyOptional()
+  @IsString()
+  @IsOptional()
+  plugin: string;
+
+  @ApiPropertyOptional()
+  @IsString()
+  @IsOptional()
+  host: string;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  port: number;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  @IsString()
+  network_config_id?: string;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  @IsString()
+  connection_id: string;
+}
+
+export class ConnectionTestPingRes {
+  @ApiProperty()
+  operation_result: boolean;
+}
+
+export class ConnectionTestCredentialsProperties extends OmitType(
+  DatabaseConnectionPropertiesDto,
+  ['plugin', 'credentials_type'],
+) {}
+export class ConnectionTestCredentialsReq extends OmitType(
+  CreateConnectionDto,
+  ['properties'],
+) {
+  @ApiProperty()
+  properties: ConnectionTestCredentialsProperties;
+  @ApiPropertyOptional()
+  connection_id?: string;
+}
+
+export class ConnectionTestCredentialsRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty()
+  connection_id: string;
+}
+
+export class ConnectionTestListSchemasReq {
+  @ApiProperty()
+  @IsString()
+  plugin: string;
+
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+}
+
+export class ConnectionTestListSchemasRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty()
+  schema_list: string[];
+}
+
+export class ConnectionTestListTablesReq {
+  @ApiProperty()
+  @IsString()
+  plugin: string;
+
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+
+  @ApiProperty()
+  @IsString()
+  schema: string;
+}
+
+export class ConnectionTestListTablesRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty()
+  table_list: string[];
+}
+
+export class GetTableMetadataReq {
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+
+  @ApiProperty()
+  @IsString()
+  plugin: string;
+
+  @ApiProperty()
+  @IsString()
+  schema: string;
+
+  @ApiProperty()
+  @IsString({ each: true })
+  table_list: string[];
+}
+
+export class GetTableMetadataRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty({ type: [TableMetadataDto] })
+  tables_metadata: TableMetadataDto[];
+}

src/modules/connection/client.config.ts

@@ -0,0 +1,39 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { ConnectionManager } from '@dadosfera/protospack-v2';
+
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
+export class ConnectionClientConfiguration {
+  public name = 'ConnectionClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [
+        ConnectionManager.ProtoPackages.WritePackage,
+        ConnectionManager.ProtoPackages.ReadPackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        ConnectionManager.ProtoPaths.WriteFilePath,
+        ConnectionManager.ProtoPaths.ReadFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/connection/client.service.ts

@@ -0,0 +1,340 @@
+import { OnModuleInit, Inject } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ConnectionManager } from '@dadosfera/protospack-v2';
+import { lastValueFrom } from 'rxjs';
+import {
+  ConnectionApiConnection,
+  ConnectionRes,
+  ConnectionsRes,
+  ConnectionToCatalogDto,
+  CreateConnectionDto,
+  UpdateConnectionDto,
+} from './dtos/connection';
+import { ConnectionClientConfiguration } from './client.config';
+import {
+  GetConnectionDetailsResponse,
+  INTERNAL_GetAllConnectionsResponse,
+} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
+import { UsersService } from '../users/users.service';
+import { ConnectorClientService } from '../connector/client.service';
+import {
+  ConnectionsApiNetworkConfig,
+  NetworkConfigDto,
+} from '../network-config/dto/network-config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+
+export class ConnectionClientService implements OnModuleInit {
+  private connectionServiceRead: ConnectionManager.ReadService.ConnectionManagerReadServices;
+  private connectionServiceWrite: ConnectionManager.WriteService.ConnectionManagerWriteServices;
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(ConnectionClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+    private usersService: UsersService,
+    private connectorService: ConnectorClientService,
+    private dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.connectionServiceWrite =
+      this.grpcClient.getService<ConnectionManager.WriteService.ConnectionManagerWriteServices>(
+        ConnectionManager.ProtoServices.ConnectionManagerWriteServices,
+      );
+
+    this.connectionServiceRead =
+      this.grpcClient.getService<ConnectionManager.ReadService.ConnectionManagerReadServices>(
+        ConnectionManager.ProtoServices.ConnectionManagerReadServices,
+      );
+  }
+
+  async createConnection(
+    body: CreateConnectionDto,
+    metadata,
+  ): Promise<ConnectionRes> {
+    const {
+      connector_id,
+      description,
+      image_url,
+      name,
+      network_config_id,
+      properties,
+      type,
+      connector_name,
+      plugin,
+    } = body;
+
+    return lastValueFrom(
+      this.connectionServiceWrite.CreateConnection(
+        {
+          connector_id,
+          connector_name,
+          description,
+          image_url,
+          name,
+          network_config_id,
+          properties: JSON.stringify(properties),
+          type,
+          plugin,
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async updateConnection(
+    id: string,
+    body: UpdateConnectionDto,
+    metadata,
+  ): Promise<ConnectionRes> {
+    const { description, name, network_config_id, properties } = body;
+
+    return lastValueFrom(
+      this.connectionServiceWrite.UpdateConnection(
+        {
+          id,
+          description,
+          name,
+          network_config_id,
+          properties: JSON.stringify(properties),
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async deleteConnection({ body, metadata }): Promise<ConnectionRes> {
+    const { id } = body;
+
+    return lastValueFrom(
+      this.connectionServiceWrite.DeleteConnection(
+        {
+          id,
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async getAllConnections({ body, metadata }): Promise<ConnectionsRes> {
+    const { search, filters, size, page } = body;
+
+    return lastValueFrom(
+      this.connectionServiceRead.GetAllConnection(
+        { search, filters: JSON.stringify(filters), size, page },
+        metadata,
+      ),
+    );
+  }
+
+  async getConnectionDetails({
+    body,
+    metadata,
+  }): Promise<GetConnectionDetailsResponse> {
+    const id = body.connection_id || body.id;
+
+    return lastValueFrom(
+      this.connectionServiceRead.GetConnectionDetails({ id }, metadata),
+    );
+  }
+
+  async INTERNAL_getAllConnections({
+    customer_name,
+  }: {
+    customer_name: string;
+  }): Promise<INTERNAL_GetAllConnectionsResponse> {
+    return lastValueFrom(
+      this.connectionServiceRead.INTERNAL_GetAllConnections({ customer_name }),
+    );
+  }
+
+  async catalogExistingConnection(data: {
+    connection?: ConnectionToCatalogDto;
+    networkConfig?: NetworkConfigDto;
+  }) {
+    return lastValueFrom(
+      this.connectionServiceRead.INTERNAL_CatalogExistingConnection({
+        data_str: JSON.stringify(data),
+      }),
+    );
+  }
+
+  async synchronizeConnections(apply: boolean) {
+    const connectors = await this.connectorService
+      .getAllConnectors({
+        search: undefined,
+        filters: undefined,
+        language: 'pt-br',
+        page: 1,
+        size: 500,
+      })
+      .then(({ connectors: connectorsStr }) =>
+        JSON.parse(connectorsStr).map((c) => c),
+      );
+    const { customers } = await this.usersService.findAllCustomers();
+
+    const customer_connections: {
+      customer_name: string;
+      customer_id: string;
+      productConnections: Record<string, any>[];
+      platformConnections: ConnectionApiConnection[];
+      productNetworkConfigs: Record<string, any>[];
+      platformNetworkConfigs: ConnectionsApiNetworkConfig[];
+    }[] = [];
+    for (const customer of customers) {
+      const { name: customer_name, id: customer_id } = customer;
+
+      const customer_connection = await this.INTERNAL_getAllConnections({
+        customer_name,
+      }).then((res) => ({
+        customer_name,
+        customer_id,
+        platformConnections: res.platformConnections
+          ? JSON.parse(res.platformConnections)
+          : [],
+        productConnections: res.productConnections
+          ? JSON.parse(res.productConnections)
+          : [],
+        productNetworkConfigs: res.productNetworkConfigs
+          ? JSON.parse(res.productNetworkConfigs)
+          : [],
+        platformNetworkConfigs: res.platformNetworkConfigs
+          ? JSON.parse(res.platformNetworkConfigs)
+          : [],
+      }));
+      customer_connections.push(customer_connection);
+    }
+    this.logger.info(
+      'DEBUG_ConnectionsMigrations - ' + JSON.stringify(customer_connections),
+    );
+
+    const migrations: {
+      customer_name: string;
+      status: any[];
+      productConnectionsMissing: ConnectionApiConnection[];
+      productNetworkConfigsMissing: ConnectionsApiNetworkConfig[];
+    }[] = [];
+    for (const customer_connection of customer_connections) {
+      const productConnectionsMissing: ConnectionApiConnection[] = [];
+      const productNetworkConfigsMissing: ConnectionsApiNetworkConfig[] = [];
+      const status: any[] = [];
+      let updatePromises: Promise<any>[] = [];
+
+      this.logger.info(
+        'DEBUG_ConnectionsMigrations - customer_connections ' +
+          JSON.stringify(customer_connection),
+      );
+      const {
+        customer_name,
+        productConnections,
+        platformConnections,
+        productNetworkConfigs,
+        platformNetworkConfigs,
+        customer_id,
+      } = customer_connection;
+
+      platformConnections?.forEach((plc) => {
+        const productConnection = productConnections.find(
+          (prc) => prc.id === plc.config_id,
+        );
+        if (!productConnection) productConnectionsMissing.push(plc);
+      });
+      platformNetworkConfigs?.forEach((platNetConfig) => {
+        const productNetworkConfig = productNetworkConfigs.find(
+          (prodNetConfig) => prodNetConfig.id === platNetConfig.config_id,
+        );
+        if (!productNetworkConfig)
+          productNetworkConfigsMissing.push(platNetConfig);
+      });
+      if (apply) {
+        for (const network_config of productNetworkConfigsMissing) {
+          const networkConfigToCatalog = {
+            id: network_config.config_id,
+            created_at: network_config.created_at,
+            updated_at: network_config.updated_at,
+            customer_name: customer_name,
+            customer_id: customer_id,
+            user_id: '',
+            network_type_id: '',
+            network_type_name: network_config.type,
+            type: network_config.type,
+            plugin: network_config.plugin,
+            name: network_config.name,
+            description: network_config.description,
+            properties: undefined,
+            keywords: undefined,
+            tags: undefined,
+          };
+
+          updatePromises.push(
+            this.catalogExistingConnection({
+              networkConfig: networkConfigToCatalog,
+            }),
+          );
+        }
+
+        this.logger.info(
+          'DEBUG_ConnectionsMigrations - Creating missing network configs for customer' +
+            customer_name +
+            ' - ' +
+            JSON.stringify(customer_connection),
+        );
+        const updateStatuses = await Promise.all(updatePromises);
+        status.push(...updateStatuses);
+        updatePromises = [];
+        for (const connection of productConnectionsMissing) {
+          const connector = connectors.find(
+            (c) => c._source.plugin === connection.plugin,
+          );
+          const connectionToCatalog = {
+            id: connection.config_id,
+            created_at: connection.created_at,
+            updated_at: new Date().toISOString(),
+            customer_id: customer_id,
+            user_id: '',
+            plugin: connection.plugin,
+            name: connection.name,
+            description: connection.description,
+            image_url: connector._source.image,
+            network_config_id: connection.network_config_id,
+            type: connection.type,
+            connector_id: connector._id,
+            connector_name: connector._source.name,
+            in_use: 1,
+            username: '',
+            customer_name,
+          };
+
+          updatePromises.push(
+            this.catalogExistingConnection({ connection: connectionToCatalog }),
+          );
+        }
+        this.logger.info(
+          'DEBUG_ConnectionsMigrations - Creating missing connections for customer' +
+            ' - ' +
+            customer_name +
+            JSON.stringify(customer_connection),
+        );
+        const updateStatuses_2 = await Promise.all(updatePromises);
+        status.push(...updateStatuses_2);
+      }
+
+      this.logger.info(
+        'DEBUG_ConnectionsMigrations - migrations for customer ' +
+          customer_name +
+          ' - ' +
+          JSON.stringify(customer_connection),
+      );
+      migrations.push({
+        customer_name,
+        status,
+        productConnectionsMissing,
+        productNetworkConfigsMissing,
+      });
+    }
+
+    return { migrations };
+  }
+}

src/modules/connection/connection.controller.ts

@@ -0,0 +1,212 @@
+import {
+  Body,
+  Controller,
+  Post,
+  Inject,
+  Put,
+  Param,
+  Delete,
+  Get,
+  Query,
+  UseFilters,
+} from '@nestjs/common';
+import { ApiExcludeEndpoint, ApiTags } from '@nestjs/swagger';
+import { ConnectionClientService } from './client.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { ValidationPipe } from '../../pipes/object-validation.pipe';
+import {
+  ConnectionDetailsRes,
+  ConnectionRes,
+  ConnectionsRes,
+  UpdateConnectionDto,
+} from './dtos/connection';
+import { CreateConnectionDto } from './dtos/connection';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+
+const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
+@UseFilters(new GrpcToHttpExceptionFilter())
+@ApiTags('connections')
+@Authenticated()
+@Controller('connections')
+export class ConnectionController {
+  logger: any;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private clientService: ConnectionClientService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('connections-migration')
+  @ApiExcludeEndpoint()
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.DADOSFERA.permissions.CONNECTIONS_MIGRATION,
+  )
+  async synchronizeConnections(@Query('apply') apply: string) {
+    return this.clientService.synchronizeConnections(apply === 'true');
+  }
+
+  @Post()
+  @RequireAllPermissions(connectionPermissions.CREATE)
+  async createConnection(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: CreateConnectionDto,
+  ): Promise<ConnectionRes> {
+    this.logger.info('/connections - Create Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+    });
+
+    const response: any = await this.clientService.createConnection(
+      body,
+      metadata,
+    );
+
+    return response;
+  }
+
+  @Put('/:id')
+  @RequireAllPermissions(connectionPermissions.CREATE)
+  async updateConnection(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @Body(new ValidationPipe()) body: UpdateConnectionDto,
+  ): Promise<ConnectionRes> {
+    this.logger.info('/connections - Update Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      language,
+      username,
+    });
+
+    const response: any = await this.clientService.updateConnection(
+      id,
+      body,
+      metadata,
+    );
+
+    return response;
+  }
+
+  @Delete('/:id')
+  @RequireAllPermissions(connectionPermissions.DELETE)
+  async deleteConnection(
+    @User() user: RequestUser,
+    @Param('id') id: string,
+  ): Promise<ConnectionRes> {
+    this.logger.info('/connections - Delete Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+    });
+
+    const response: any = await this.clientService.deleteConnection({
+      body: { id },
+      metadata,
+    });
+
+    return response;
+  }
+
+  @Get()
+  async getAllConnections(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Query() queries,
+  ): Promise<ConnectionsRes> {
+    this.logger.info('/connections - Get All Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const { search, size, page, ...filters } = queries;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      language,
+      username,
+    });
+
+    const response = await this.clientService.getAllConnections({
+      body: { search, size, page, filters },
+      metadata,
+    });
+
+    return response;
+  }
+
+  @Get('/:id')
+  async getConnectionDetails(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Param('id') id: string,
+    @Query() queries,
+  ): Promise<ConnectionDetailsRes> {
+    this.logger.info('/connections - Get Connection Details');
+
+    console.log(queries);
+
+    const { details, sensitive } = queries;
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+      details,
+      language,
+      sensitive,
+    });
+
+    const response: any = await this.clientService.getConnectionDetails({
+      body: { id },
+      metadata,
+    });
+
+    if (details === 'true') {
+      const properties = JSON.parse(response.connection.properties);
+
+      Object.assign(response.connection, {
+        pipelines: JSON.parse(response.connection.pipelines),
+        connection_controls: properties.connection_controls,
+        config_controls: properties.config_controls,
+        connection_steps: properties.connection_steps,
+      });
+
+      delete response.connection.properties;
+    } else {
+      Object.assign(response.connection, {
+        properties: JSON.parse(response.connection.properties),
+        pipelines: JSON.parse(response.connection.pipelines),
+      });
+    }
+
+    return response;
+  }
+}

src/modules/connection/connection.module.ts

@@ -0,0 +1,22 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { ConnectionClientService } from './client.service';
+import { ConnectionClientConfiguration } from './client.config';
+import { ConnectionController } from './connection.controller';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { UsersModule } from '../users/users.module';
+import { ConnectorModule } from '../connector/connector.module';
+
+const client = new ConnectionClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    UsersModule,
+    ConnectorModule,
+  ],
+  controllers: [ConnectionController],
+  providers: [ConnectionClientService, DadosferaLogger],
+  exports: [ConnectionClientService],
+})
+export class ConnectionModule {}

src/modules/connection/dtos/connection.ts

@@ -0,0 +1,189 @@
+import {
+  ApiProperty,
+  ApiPropertyOptional,
+  OmitType,
+  PickType,
+} from '@nestjs/swagger';
+import {
+  Connection,
+  ConnectionToCatalog,
+} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
+import {
+  GetAllConnectionResponse,
+  GetConnectionDetailsResponse,
+  GetConnectionResponse,
+} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
+export type ConnectionTypes = 'database' | 'file' | 'application';
+export type ConnectionCredentialsType =
+  | 'basic_auth'
+  | 'iam_user'
+  | 'role_name'
+  | 'oauth'
+  | 'api_key'
+  | 'service_account'
+  | 'headers_authF';
+export interface ConnectionApiConnection {
+  config_id: string;
+  plugin: string;
+  secret_id: string;
+  created_at: string;
+  network_config_id: null;
+  type: ConnectionTypes;
+  credentials_type: ConnectionCredentialsType;
+  credentials: Record<string, any>;
+  sensitive_fields: string[];
+  name: string;
+  customer_name: string;
+  description: string;
+  updated_at: string;
+}
+export class ConnectionDto implements Connection {
+  // ---Automatically generated information will not be sent by the frontend--- //
+  @ApiProperty()
+  id: string;
+
+  @ApiProperty()
+  customer_id: string;
+
+  @ApiProperty()
+  customer_name: string;
+
+  @ApiProperty()
+  user_id: string;
+
+  @ApiProperty()
+  updated_at: string;
+
+  @ApiProperty()
+  created_at: string;
+
+  @ApiProperty()
+  pipelines: string;
+
+  // ---Information sent by the frontend--- //
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  type: string;
+
+  @ApiPropertyOptional()
+  network_config_id: string | null;
+
+  @ApiProperty()
+  properties: DatabaseConnectionPropertiesDto | any;
+
+  // ---Information sent by the frontend (Heirs of Connector)--- //
+
+  @ApiProperty()
+  connector_id: string;
+
+  @ApiProperty()
+  connector_name: string;
+
+  @ApiProperty()
+  plugin: string;
+
+  @ApiProperty()
+  image_url: string;
+}
+
+export class ConnectionToCatalogDto implements ConnectionToCatalog {
+  // ---Automatically generated information - will not be sent by the frontend--- //
+  id: string;
+  customer_id: string;
+  updated_at: string;
+  created_at: string;
+  // ---Information extracted from token--- //
+  user_id: string;
+  username: string;
+  customer_name: string;
+
+  // ---Information sent by the frontend--- //
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  type: string;
+
+  @ApiPropertyOptional()
+  network_config_id: string | null;
+
+  // ---Information sent by the frontend (Heirs of Connector)--- //
+  @ApiProperty()
+  connector_id: string;
+
+  @ApiProperty()
+  connector_name: string;
+
+  @ApiProperty()
+  plugin: string;
+
+  @ApiProperty()
+  image_url: string;
+}
+
+export class CreateConnectionDto extends OmitType(ConnectionDto, [
+  'id',
+  'customer_id',
+  'customer_name',
+  'user_id',
+  'created_at',
+  'updated_at',
+  'pipelines',
+]) {}
+
+export class UpdateConnectionDto extends PickType(ConnectionDto, [
+  'name',
+  'description',
+  'network_config_id',
+  'properties',
+]) {}
+
+export class ConnectionsRes implements GetAllConnectionResponse {
+  @ApiProperty({ type: [ConnectionToCatalogDto] })
+  connections: ConnectionToCatalogDto[];
+
+  @ApiProperty()
+  message: string;
+}
+
+export class ConnectionRes implements GetConnectionResponse {
+  @ApiProperty()
+  connection?: ConnectionToCatalogDto;
+
+  @ApiProperty()
+  message: string;
+}
+
+export class ConnectionDetailsRes implements GetConnectionDetailsResponse {
+  @ApiProperty()
+  connection?: ConnectionDto;
+
+  @ApiProperty()
+  message: string;
+}
+
+export class DatabaseConnectionPropertiesDto {
+  @ApiProperty()
+  credentials_type: 'basic_auth';
+  @ApiProperty()
+  plugin: string;
+  @ApiProperty()
+  host: string;
+  @ApiProperty()
+  port: string;
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  password: string;
+  @ApiProperty()
+  database: string;
+}

src/modules/connector/client.config.ts

@@ -0,0 +1,38 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { ConnectorManager } from '@dadosfera/protospack-v2';
+
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
+export class ConnectorClientConfiguration {
+  public name = 'ConnectorClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [
+        ConnectorManager.ProtoPackages.WritePackage,
+        ConnectorManager.ProtoPackages.ReadPackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        ConnectorManager.ProtoPaths.WriteFilePath,
+        ConnectorManager.ProtoPaths.ReadFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/connector/client.service.ts

@@ -0,0 +1,204 @@
+import { Metadata } from '@grpc/grpc-js';
+import {
+  OnModuleInit,
+  Inject,
+  HttpException,
+  HttpStatus,
+} from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ConnectorManager } from '@dadosfera/protospack-v2';
+import { lastValueFrom } from 'rxjs';
+import { ConnectorClientConfiguration } from './client.config';
+
+export class ConnectorClientService implements OnModuleInit {
+  private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices;
+  private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices;
+  constructor(
+    @Inject(ConnectorClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.connectorServiceWrite =
+      this.grpcClient.getService<ConnectorManager.WriteService.ConnectorManagerWriteServices>(
+        ConnectorManager.ProtoServices.ConnectorManagerWriteServices,
+      );
+
+    this.connectorServiceRead =
+      this.grpcClient.getService<ConnectorManager.ReadService.ConnectorManagerReadServices>(
+        ConnectorManager.ProtoServices.ConnectorManagerReadServices,
+      );
+  }
+
+  async uploadConnector(uploadConnector) {
+    const connector: ConnectorManager.Entities.ConnectorCreateRequest = {
+      file: {
+        buffer: uploadConnector.file.buffer,
+        mimetypes: uploadConnector.file.mimetype,
+      },
+      connector: JSON.stringify(uploadConnector.connector),
+    };
+    const serviceBody: ConnectorManager.Messages.RegisterConnectorRequest = {
+      connector,
+    };
+
+    return lastValueFrom(
+      this.connectorServiceWrite.RegisterConnector(serviceBody),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+
+  async uploadFile(uploadFile) {
+    const body: ConnectorManager.Messages.UploadFileRequest = {
+      file: {
+        buffer: uploadFile.file.buffer,
+        mimetypes: uploadFile.file.mimetype,
+      },
+      name: uploadFile.name,
+    };
+
+    return lastValueFrom(this.connectorServiceWrite.UploadFile(body)).catch(
+      (err) => {
+        throw new HttpException(
+          err.details,
+          err.code === 6 ? HttpStatus.CONFLICT : 400,
+        );
+      },
+    );
+  }
+
+  async uploadConnectorsFile(uploadFile: Express.Multer.File) {
+    const res = await lastValueFrom(
+      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage({
+        connectors: uploadFile.buffer,
+      }),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+    const responseParsed = JSON.parse(res.message);
+    const connectorsResponse = responseParsed.items.filter(
+      (i) => i.update._index === process.env.CONNECTORS_INDEX,
+    );
+    const connectorErrors = connectorsResponse
+      .filter((i) => i.update.error)
+      .map((i) => ({
+        id: i.update?._id,
+        reason: i.update?.error,
+        status: i.error?.status,
+      }));
+    const connectorCreated = connectorsResponse
+      .filter((i) => i.update.result === 'created')
+      .map((i) => i.update._id);
+    const connectorUpdated = connectorsResponse
+      .filter((i) => i.update.result === 'updated')
+      .map((i) => i.update._id);
+    const connectorNoop = connectorsResponse
+      .filter((i) => i.update.result === 'noop')
+      .map((i) => i.update._id);
+    return {
+      errors: connectorErrors,
+      created: connectorCreated,
+      updated: connectorUpdated,
+      noop: connectorNoop,
+    };
+  }
+
+  async getAllConnectors(body) {
+    const meta = new Metadata();
+    meta.add('Language', body.language);
+    return lastValueFrom(
+      this.connectorServiceRead.GetAllConnectors(
+        {
+          search: body.search,
+          filters: JSON.stringify(body.filters),
+          language: body.language,
+          page: body.page,
+          size: body.size,
+        },
+        meta,
+      ),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+
+  async getConnectorDetails(connectorId: string, language: string) {
+    const meta = new Metadata();
+    meta.add('Language', language);
+    return lastValueFrom(
+      this.connectorServiceRead.GetConnectorDetails(
+        {
+          plugin: connectorId,
+        },
+        meta,
+      ),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+
+  async getConnector(plugin: string, language: string) {
+    const meta = new Metadata();
+    meta.add('Language', language);
+    return lastValueFrom(
+      this.connectorServiceRead.GetConnector(
+        {
+          plugin,
+        },
+        meta,
+      ),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+
+  async getConnectorsTags() {
+    return lastValueFrom(this.connectorServiceRead.GetConnectorsTags({})).catch(
+      (err) => {
+        throw new HttpException(
+          err.details,
+          err.code === 6 ? HttpStatus.CONFLICT : 400,
+        );
+      },
+    );
+  }
+
+  async deleteConnector(plugin: string, version: string) {
+    return lastValueFrom(
+      this.connectorServiceWrite.DeleteConnector({
+        version,
+        plugin,
+      }),
+    );
+  }
+
+  async updateConnector({ plugin, changes }) {
+    return lastValueFrom(
+      this.connectorServiceWrite.UpdateConnector({
+        plugin,
+        changes,
+      }),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+}

src/modules/connector/connector.controller.ts

@@ -0,0 +1,285 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpException,
+  Param,
+  Put,
+  Post,
+  Query,
+  UploadedFile,
+  UseInterceptors,
+  Inject,
+  UploadedFiles,
+} from '@nestjs/common';
+import { FileInterceptor, FilesInterceptor } from '@nestjs/platform-express';
+import { ApiConsumes, ApiTags } from '@nestjs/swagger';
+import { ConnectorClientService } from './client.service';
+import { AddTagDto } from './dtos/add-tag';
+import { CreateConnectorDto } from './dtos/create-connector';
+import { GetAllDto } from './dtos/get-all';
+import { RemoveTagDto } from './dtos/remove-tag';
+import { UpdateDto } from './dtos/update';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import {
+  Authenticated,
+  RequireAllPermissions,
+  RequireSomePermission,
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+
+@ApiInternalOnlyController()
+@ApiTags('connectors')
+@Authenticated()
+@Controller('connectors')
+export class ConnectorController {
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private connectorClientService: ConnectorClientService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('file'))
+  async uploadConnector(
+    @UploadedFile() file,
+    @Body() body: CreateConnectorDto,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+
+    const response = await this.connectorClientService.uploadConnector({
+      file,
+      connector: JSON.parse(body.connector),
+    });
+
+    return response;
+  }
+
+  @Post('/uploads')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FilesInterceptor('files'))
+  async uploadFile(@UploadedFiles() files) {
+    this.logger.info('/upload - Upload Connector Route');
+    const promises = files.map(async (file) => {
+      return await this.connectorClientService.uploadFile({
+        file,
+        name: file.originalname.split('.')[0],
+      });
+    });
+
+    const response = await Promise.all(promises);
+    return response;
+  }
+
+  @Post('/multiple-connectors-upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('file'))
+  async uploadConnectorsFile(@UploadedFile() file: Express.Multer.File) {
+    this.logger.info(`/upload`, 'Upload Connector Route');
+
+    return await this.connectorClientService.uploadConnectorsFile(file);
+  }
+
+  @Get()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
+  async getAllConnectors(
+    @Language() language: LanguageEnum,
+    @Query() queries: GetAllDto,
+  ) {
+    this.logger.info('/GET - getAllConnectors Route');
+
+    const { search, size, page, ...filters } = queries;
+
+    const response: any = await this.connectorClientService.getAllConnectors({
+      filters: filters || {},
+      search: search || '',
+      language: language || 'pt-br',
+      size: size || 10,
+      page: page || 1,
+    });
+
+    const connectors = JSON.parse(response.connectors);
+
+    return {
+      message: response.message,
+      connectors,
+    };
+  }
+
+  @Get('/tags')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
+  async getConnectorsTags() {
+    return await this.connectorClientService.getConnectorsTags();
+  }
+
+  @Get('/:plugin')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
+  async getConnector(
+    @Language() language: LanguageEnum,
+    @Param('plugin') plugin: string,
+    @Query('version') version: string,
+  ) {
+    this.logger.info('/GET/:plugin - getConnector Route');
+
+    const pluginId = `${plugin}-${version}`;
+
+    const response: any = await this.connectorClientService.getConnector(
+      pluginId,
+      language,
+    );
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Get('/:plugin/details')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
+  async getConnectorDetails(
+    @Language() language: LanguageEnum,
+    @Param('plugin') plugin: string,
+    @Query('version') version: string,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+    const pluginId = `${plugin}-${version}`;
+
+    const response: any = await this.connectorClientService.getConnectorDetails(
+      pluginId,
+      language || 'pt-br',
+    );
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Put('/:plugin')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
+  @ApiConsumes('multipart/form-data')
+  async updateConnector(
+    @Param('plugin') plugin: string,
+    @Body() body: UpdateDto,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+
+    const changes = body;
+
+    const response: any = await this.connectorClientService.updateConnector({
+      plugin,
+      changes: JSON.stringify(changes),
+    });
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Put('/:plugin/add-tag')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
+  async addTagOnConnector(
+    @Param('plugin') plugin: string,
+    @Body() body: AddTagDto,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+
+    const { tags } = body;
+
+    if (!tags || !(typeof tags === 'object') || !tags.length) {
+      throw new HttpException('Not found tags attributes', 400);
+    }
+
+    const changes = { plugin, tags };
+
+    const response: any = await this.connectorClientService.updateConnector({
+      plugin,
+      changes: JSON.stringify(changes),
+    });
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Put('/:plugin/remove-tag')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
+  async removeTagOnConnector(
+    @Param('plugin') plugin: string,
+    @Body() body: RemoveTagDto,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+
+    const { tags } = body;
+    const remove = tags;
+
+    if (!remove || !(typeof remove === 'object') || !remove.length) {
+      throw new HttpException('Not found remove attribute', 400);
+    }
+
+    const changes = { plugin, remove };
+
+    const response: any = await this.connectorClientService.updateConnector({
+      plugin,
+      changes: JSON.stringify(changes),
+    });
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Delete('/:plugin')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
+  async deleteConnector(
+    @Param('plugin') plugin: string,
+    @Query('version') version: string,
+  ) {
+    this.logger.info('DELETE /:plugin - Delete Connector Route');
+
+    const response = await this.connectorClientService.deleteConnector(
+      plugin,
+      version,
+    );
+
+    return { message: response.message, connector: response.connector };
+  }
+}

src/modules/connector/connector.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { ConnectorClientService } from './client.service';
+import { ConnectorClientConfiguration } from './client.config';
+import { ConnectorController } from './connector.controller';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+const client = new ConnectorClientConfiguration();
+
+@Module({
+  controllers: [ConnectorController],
+  providers: [ConnectorClientService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions])],
+  exports: [ConnectorClientService],
+})
+export class ConnectorModule {}

src/modules/connector/dtos/add-tag.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class AddTagDto {
+  @ApiProperty()
+  tags: string[];
+}

src/modules/connector/dtos/create-connector.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class CreateConnectorDto {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+
+  @ApiProperty()
+  connector: string;
+}

src/modules/connector/dtos/create-multiple-connector.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class UploadConnectorsFileDto {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+}

src/modules/connector/dtos/delete-connector.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class DeleteConnectorDto {
+  @ApiProperty()
+  version: string;
+}

src/modules/connector/dtos/get-all.ts

@@ -0,0 +1,18 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class GetAllDto {
+  @ApiProperty()
+  search: string;
+
+  @ApiProperty()
+  language: string;
+
+  @ApiProperty()
+  size: string;
+
+  @ApiProperty()
+  page: string;
+
+  @ApiProperty()
+  filters: any;
+}

src/modules/connector/dtos/remove-tag.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class RemoveTagDto {
+  @ApiProperty()
+  tags: string[];
+}

src/modules/connector/dtos/update.ts

@@ -0,0 +1,42 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class UpdateDto {
+  @ApiProperty({ format: 'binary', required: false })
+  file?: string;
+
+  @ApiProperty()
+  version: string;
+
+  @ApiProperty({ required: false })
+  category?: string;
+
+  @ApiProperty({ required: false })
+  plugin?: string;
+
+  @ApiProperty({ required: false })
+  image?: string;
+
+  @ApiProperty({ required: false })
+  docs?: string;
+
+  @ApiProperty({ required: false })
+  name?: string;
+
+  @ApiProperty({
+    required: false,
+    description: 'skip_select_entities: boolean skip_select_columns: boolean;',
+  })
+  options?: {
+    skip_select_entities: boolean;
+    skip_select_columns: boolean;
+  };
+
+  @ApiProperty({
+    required: false,
+    type: 'array',
+    items: {
+      properties: {},
+    },
+  })
+  connection_controls?: object[];
+}

src/modules/connector/dtos/upload-file.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class UploadFileDto {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+
+  @ApiProperty()
+  name: string;
+}

src/modules/customers/customers.controller.ts

@@ -0,0 +1,74 @@
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import {
+  Body,
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Put,
+  Query,
+  UseFilters,
+} from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { CustomersService } from './customers.service';
+import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import type { StringValue } from 'ms';
+
+@ApiInternalOnlyController()
+@ApiTags('Customers')
+@Controller('customers')
+@Authenticated()
+@UseFilters(GrpcToHttpExceptionFilter)
+export class CustomersController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private customersService: CustomersService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get(':id/links')
+  async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
+    this.logger.info('getCustomerLinks', { id });
+    const links = await this.customersService.getLinks(id);
+    return { links };
+  }
+
+  @Put(':id/links')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  setCustomerLinks(
+    @Body() body: CustomerDto,
+    @Param('id') id,
+  ): Promise<IdResponse> {
+    const { links } = body;
+    this.logger.info('setCustomerLinks', { id, links });
+    return this.customersService.setLinks(id, links);
+  }
+
+  @Get('token')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
+  async getCustomerToken(
+    @Query('exp') exp: StringValue,
+    @User() user: RequestUser,
+  ): Promise<string> {
+    this.logger.info('getCustomerToken', { exp, user });
+    const data = {
+      customerId: user.customer_id,
+      userId: user.user_id,
+      customerName: user.customer_name,
+    };
+    return this.customersService.generateToken(exp, data);
+  }
+}

src/modules/customers/customers.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import { CustomersController } from './customers.controller';
+import { CustomersService } from './customers.service';
+
+const client = new DucClient();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [CustomersController],
+  providers: [CustomersService, DadosferaLogger],
+  exports: [CustomersService],
+})
+export class CustomersModule {}

src/modules/customers/customers.service.ts

@@ -0,0 +1,123 @@
+import {
+  OnModuleInit,
+  Inject,
+  Injectable,
+  HttpException,
+  HttpStatus,
+  InternalServerErrorException,
+} from '@nestjs/common';
+
+import { firstValueFrom, lastValueFrom } from 'rxjs';
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import ErrorCodes from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import {
+  GetSecretValueCommand,
+  SecretsManagerClient,
+} from '@aws-sdk/client-secrets-manager';
+import getEnv from 'src/utils/getEnv';
+import { logger } from 'elastic-apm-node';
+
+// This function will accept any string, which may result in a bug.
+@Injectable()
+export class CustomersService implements OnModuleInit {
+  private customerService: CustomersProtoService;
+
+  constructor(
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.customerService = this.grpcClient.getService<CustomersProtoService>(
+      ProtoServices.CustomersProtoService,
+    );
+  }
+
+  async getLinks(customerId: string) {
+    try {
+      const result = await lastValueFrom(
+        this.customerService.CustomerFindOneById({ id: customerId }),
+      );
+      return result.customer?.links || [];
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      throw err;
+    }
+  }
+
+  async setLinks(customerId: string, links: Link[]) {
+    if (!customerId || !links) {
+      throw new HttpException(null, HttpStatus.BAD_REQUEST);
+    }
+
+    try {
+      return await firstValueFrom(
+        this.customerService.CustomerUpdate({
+          id: customerId,
+          links,
+        } as CustomerUpdateRequest),
+      );
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      else throw err;
+    }
+  }
+
+  async generateToken(
+    expiresIn = '30m',
+    data: { customerId: string; userId: string; customerName: string },
+  ) {
+    const { customerId, userId: generatedById, customerName } = data;
+
+    const secretsManagerClient = new SecretsManagerClient({});
+
+    const getSecretComand = new GetSecretValueCommand({
+      SecretId: `${getEnv()}/${customerName}/jwt-signing-key`,
+    });
+    let private_key_pem;
+    await secretsManagerClient
+      .send(getSecretComand)
+      .then((res) => {
+        const secret = JSON.parse(res.SecretString);
+
+        private_key_pem = secret.private_key_pem;
+      })
+      .catch((err) => {
+        logger.error(err.stack);
+        throw new InternalServerErrorException(
+          'Error finding keys for customer',
+        );
+      });
+    let jwt_token;
+    try {
+      jwt_token = jwt.sign(
+        {
+          customerId,
+          generatedById,
+        },
+        private_key_pem,
+        {
+          algorithm: 'RS256',
+          expiresIn,
+          issuer: 'maestro',
+        },
+      );
+    } catch (err) {
+      logger.error(err.stack);
+      throw new InternalServerErrorException('Error generating token');
+    }
+    // const verify = jwt.verify(jwt_token, public_key_pem, {
+    //   algorithms: ['RS256'],
+    //   issuer: 'maestro',
+    // });
+    // const decoded = jwt.decode(jwt_token, { complete: true });
+    return jwt_token;
+  }
+}

src/modules/customers/dtos/customers.ts

@@ -0,0 +1,12 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { ApiProperty } from '@nestjs/swagger';
+
+export class CustomerDto {
+  @ApiProperty()
+  links: Link[];
+}
+
+export class CustomerLinksResponse {
+  @ApiProperty()
+  links: Link[];
+}

src/modules/duc/client.config.ts

@@ -0,0 +1,37 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import {
+  ProtoPackages,
+  ProtoPaths,
+} from '@dadosfera/protospack-v2/dist/lib/Duc';
+
+const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0');
+
+export class DucClient {
+  public name = 'DucClient';
+
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.DUC_URL,
+      package: [ProtoPackages.WritePackage, ProtoPackages.ReadPackage],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [ProtoPaths.WriteFilePath, ProtoPaths.ReadFilePath],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/health/health.controller.ts

@@ -1,13 +1,22 @@
-import { Controller, Get } from '@nestjs/common';
+import { Controller, Get, Inject } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
 import { HealthService } from './health.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
+@ApiTags('Health')
 @Controller('health')
 export class HealthController {
-  constructor(private readonly healthService: HealthService) {}
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private readonly healthService: HealthService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
 
   @Get()
   check() {
-    console.log(`/health`, 'check');
     return this.healthService.check();
   }
 }

src/modules/health/health.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { HealthController } from './health.controller';
+import { HealthService } from './health.service';
+
+@Module({
+  controllers: [HealthController],
+  providers: [HealthService, DadosferaLogger],
+})
+export class HealthModule {}

src/modules/health/health.service.spec.ts

@@ -1,5 +0,0 @@
-describe('PipelinesGrpcServerService', () => {
-  it('should be defined', () => {
-    expect(2 + 2).toBe(4);
-  });
-});

src/modules/health/health.service.ts

@@ -1,9 +1,16 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 @Injectable()
 export class HealthService {
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
   check() {
-    console.log('HEALTH');
     return { message: 'Ok' };
   }
 }

src/modules/inputs/dtos/input.model.ts

@@ -0,0 +1,58 @@
+import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
+
+export class Column {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  type: string;
+}
+export class TableColumns {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  columns: string[];
+  @ApiProperty()
+  references: Column[];
+  @ApiProperty()
+  type: string;
+}
+export class AvailableEntity {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  replication_methods: string[];
+}
+
+export class Input {
+  @ApiProperty()
+  id: string;
+  @ApiProperty({ type: [TableColumns] })
+  tables: TableColumns[];
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  plugin: string;
+  @ApiPropertyOptional()
+  type?: string;
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  updated_at: string;
+}
+export class GetAvailableEntitiesReq {
+  @ApiProperty()
+  plugin: string;
+}
+
+export class GetAvailableEntitiesRes {
+  @ApiProperty({ type: [AvailableEntity] })
+  entities: AvailableEntity[];
+}
+
+export class CreateInputReq extends OmitType(Input, [
+  'id',
+  'created_at',
+  'updated_at',
+]) {}

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,3 +1,5 @@
+import { Info } from 'protospack/dist/lib/interfaces';
+
 interface Values {
   jdbc_user: string;
   jdbc_password: string;
@@ -9,30 +11,6 @@ interface Values {
   schema: string;
 }
 
-interface Cron {
-  hours: string[];
-  hour_interval: boolean;
-  hour_resourse: boolean;
-  hour_resourse_value: number;
-  month_day: string[];
-  month_day_interval: boolean;
-  month_day_resourse: boolean;
-  month_day_resourse_value: number;
-  month: string[];
-  month_interval: boolean;
-  month_resourse: boolean;
-  month_resourse_value: number;
-  week_days: string[];
-  week_days_interval: boolean;
-  week_days_resourse: boolean;
-  week_days_resourse_value: number;
-}
-
-export interface Info {
-  costumer_id: string;
-  user_id: string;
-}
-
 export interface ICreateInputRequest {
   cron: string;
   name: string;
@@ -54,6 +32,7 @@ export interface UpdateInputRequest {
   values: Values;
   operation: string;
   info: Info;
+  tables: any[];
 }
 
 export interface ITestConnectionRequest {

src/modules/inputs/inputs-client.config.ts

@@ -0,0 +1,36 @@
+import { Input } from '@dadosfera/protospack-v2';
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  Transport,
+  type GrpcOptions,
+} from '@nestjs/microservices';
+
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+export class InputsGrpcClient {
+  public readonly name = 'InputsGrpcClient';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [
+        Input.ProtoPackages.WritePackage,
+        Input.ProtoPackages.ReadPackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        Input.ProtoPaths.WriteFilePath,
+        Input.ProtoPaths.ReadFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        defaults: false,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/inputs/inputs.controller.ts

@@ -3,97 +3,164 @@ import {
   Controller,
   Delete,
   Get,
+  Inject,
   Param,
+  Patch,
   Post,
-  Put,
+  UseFilters,
 } from '@nestjs/common';
 import { InputsService } from './inputs.service';
-import { InputsClientService } from 'src/clients/inputs/client.service';
-import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
-import { InputNewCreateRequest } from '@victorradael/protospack';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
+import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import {
+  CreateInputReq,
+  GetAvailableEntitiesReq,
+  GetAvailableEntitiesRes,
+  Input,
+} from './dtos/input.model';
+import { UpdateInputRequest } from './dtos/old_interfaces';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
+@ApiTags('Inputs')
 @Controller('inputs')
+@AuthenticateCondition((req, user) => {
+  switch (req.method) {
+    case 'POST':
+    case 'PUT':
+    case 'PATCH':
+      return (
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE.seqid,
+        ) ||
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
+        )
+      );
+
+    default:
+      return (
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE.seqid,
+        ) ||
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
+        ) ||
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.GET.seqid,
+        )
+      );
+  }
+})
+@UseFilters(new GrpcToHttpExceptionFilter())
 export class InputsController {
-  constructor(private inputsClientService: InputsClientService) {}
-
-  @Post('/test-connection')
-  async testConnection(@Body() data) {
-    console.log(`/test-connection`, 'ON TEST CONNECTION ROUTE');
-
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.testConnection(data);
-
-    return response;
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private inputService: InputsService,
+  ) {
+    this.logger = dadosferaLogger.logger;
   }
 
-  @Post('/test-connection/get-columns')
-  async getColumns(@Body() data) {
-    console.log(
-      `/test-connection/get-columns`,
-      'ON TEST CONNECTION GET COLUMNS ROUTE',
-    );
-
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.getColumns(data);
-
-    return response;
+  @ApiInternalOnlyEndpoint()
+  @Get('available-entities/:plugin')
+  @ApiOkResponse({ type: GetAvailableEntitiesRes })
+  async getAvailableEntities(
+    @Param() params: GetAvailableEntitiesReq,
+    @User() user: RequestUser,
+  ) {
+    const { plugin } = params;
+    const availableEntities = await this.inputService.getAvailableEntities({
+      info: {
+        customer: user.customer_name,
+        customer_id: user.customer_id,
+        user_id: user.user_id,
+      },
+      plugin,
+    });
+    return availableEntities;
   }
 
   @Post()
-  async create(@Body() createInputDto: InputNewCreateRequest) {
-    console.log(`/input`, 'ON CREATE ROUTE');
+  @ApiOkResponse({ type: Input })
+  async create(@Body() body: CreateInputReq, @User() user: RequestUser) {
+    const info: Info = {
+      user_id: user.user_id,
+      customer: user.customer_name,
+      customer_id: user.customer_id,
+    };
+    this.logger.info(`/input - ON CREATE ROUTE`, {
+      user: info.user_id,
+      customer: info.customer,
+    });
 
-    const inputService = new InputsService(this.inputsClientService);
-    const response = await inputService.create(createInputDto);
+    const response = await this.inputService.create({ body, info });
 
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get()
-  async findAll(@Body() body) {
-    console.log(`/input`, 'ON FIND ALL ROUTE');
-
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.findAll(body);
+  async findAll(@User() user: RequestUser) {
+    const { user_id, customer_id, customer_name: customer } = user;
+    this.logger.info(`/input - ON FIND ALL ROUTE`, {
+      user: user.user_id,
+      customer,
+    });
+    const info: Info = {
+      customer,
+      customer_id,
+      user_id,
+    };
+    const response = await this.inputService.findAll(info);
 
     return response;
   }
 
   @Get('/:id')
-  async findOne(@Body() body, @Param() params) {
-    const { id } = params;
-    console.log(`/input/${id}`, 'ON FIND ONE ROUTE');
-    const inputService = new InputsService(this.inputsClientService);
+  async findOne(@Body() body, @Param('id') id: string) {
+    this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
+      user: body.info.user_id,
+      customer: body.info.customer,
+    });
 
-    const response = await inputService.findOne({ id, ...body });
+    const response = await this.inputService.findOne({ id, ...body });
 
     return response;
   }
 
-  @Put(':id')
+  @ApiInternalOnlyEndpoint()
+  @Patch(':id')
   async update(@Body() updateInputDto: UpdateInputRequest, @Param() params) {
     const { id } = params;
     const { info } = updateInputDto;
+    this.logger.info(`/input/${id} - ON UPDATE ROUTE`, {
+      user: info.user_id,
+      customer: info.customer_id,
+    });
     delete updateInputDto.info;
 
-    console.log(`/input/${id}`, 'ON UPDATE ROUTE');
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.update(id, updateInputDto, info);
+    const response = await this.inputService.update(id, updateInputDto, info);
 
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Delete(':id')
   async delete(@Body() data, @Param() params) {
     const { id } = params;
 
-    console.log(`/input/${id}`, 'ON DELETE ROUTE');
+    this.logger.info(`/input/${id} - ON DELETE ROUTE`, {
+      user: data.info.user_id,
+      customer: data.info.customer,
+    });
 
-    const inputService = new InputsService(this.inputsClientService);
-
-    const response = await inputService.remove({ id, ...data });
+    const response = await this.inputService.remove({ id, ...data });
 
     return response;
   }

src/modules/inputs/inputs.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { InputsGrpcClient } from './inputs-client.config';
+import { InputsController } from './inputs.controller';
+import { InputsService } from './inputs.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+const client = new InputsGrpcClient();
+
+@Module({
+  controllers: [InputsController],
+  providers: [InputsService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions])],
+  exports: [InputsService],
+})
+export class InputsModule {}

src/modules/inputs/inputs.service.spec.ts

@@ -1,5 +0,0 @@
-describe('PipelinesGrpcServerService', () => {
-  it('should be defined', () => {
-    expect(2 + 2).toBe(4);
-  });
-});

src/modules/inputs/inputs.service.ts

@@ -1,49 +1,216 @@
-import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
-import { InputNewCreateRequest } from '@victorradael/protospack';
-import { InputsClientService } from 'src/clients/inputs/client.service';
-import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
+import {
+  ForbiddenException,
+  HttpException,
+  HttpStatus,
+  Inject,
+  Injectable,
+} from '@nestjs/common';
+import CronParser, { CronExpression } from 'cron-parser';
+import { ClientGrpc } from '@nestjs/microservices';
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+import { lastValueFrom } from 'rxjs';
+import { objectCamelToSnake } from 'src/utils/CaseConverter';
+import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
+import { Input } from '@dadosfera/protospack-v2';
+import {
+  GetAvailableEntitiesRequest,
+  InputCreateGenericRequest,
+  InputCreateS3Request,
+  InputNewCreateRequest,
+  TestConnectionRequest,
+} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { CreateInputReq } from './dtos/input.model';
 
 @Injectable()
+
+//TODO refactor this class to remove OLD_inputClient
 export class InputsService {
-  constructor(private inputClient: InputsClientService) {}
+  logger: DadosferaLogger;
+  inputWriteService: Input.WriteService.InputWriteService;
+  inputReadService: Input.ReadService.InputReadService;
 
-  async create(@Body() data: InputNewCreateRequest) {
-    try {
-      const createInputResponse = await this.inputClient.create(data);
-
-      return createInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject('InputsGrpcClient') private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
   }
 
-  async findAll(body) {
-    try {
-      const findAllInputResponse = await this.inputClient.findAll(body);
+  async onModuleInit() {
+    this.inputWriteService =
+      this.grpcClient.getService<Input.WriteService.InputWriteService>(
+        Input.ProtoServices.InputWriteService,
+      );
+    this.inputReadService =
+      this.grpcClient.getService<Input.ReadService.InputReadService>(
+        Input.ProtoServices.InputReadService,
+      );
+  }
 
-      return findAllInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+  OLD_inputClient = {
+    newCreate: async (createInputDto: InputNewCreateRequest) => {
+      this.logger.info('InputClientService - Create');
+
+      const input = await lastValueFrom(
+        this.inputWriteService.InputNewCreate(createInputDto),
+      );
+      objectCamelToSnake(input);
+      return input;
+    },
+    createS3Inputs: async (createInputDto: InputCreateS3Request) => {
+      this.logger.info('InputClientService - Create');
+      createInputDto;
+      const createInputResponse = await lastValueFrom(
+        this.inputWriteService.InputCreateS3(createInputDto),
+      ).catch((e) => {
+        throw new HttpException(e.details, 500);
+      });
+      this.logger.info('Done');
+      objectCamelToSnake(createInputResponse);
+      return createInputResponse;
+    },
+    update: async (updateInputDTO: UpdateInputRequest) => {
+      this.logger.info('InputClientService - Update');
+      const updateInputResponse = await lastValueFrom(
+        this.inputWriteService.InputUpdate(updateInputDTO),
+      );
+
+      return updateInputResponse;
+    },
+    testConnection: async (data: TestConnectionRequest) => {
+      this.logger.info('InputClientService - TestConnection');
+      const testConnectionResponse = await lastValueFrom(
+        this.inputReadService.NewTestConnection(data),
+      );
+
+      return testConnectionResponse;
+    },
+    getAvailableEntities: async (data) => {
+      this.logger.info('InputClientService - GetAvailableEntities');
+      data;
+      const response = await lastValueFrom(
+        this.inputReadService.GetAvailableEntities(data),
+      ).catch((err) => {
+        this.logger.info(err.details);
+        if (err.details && err.details.includes('400'))
+          throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
+        throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
+      });
+      objectCamelToSnake(response);
+
+      return response;
+    },
+  };
+
+  secondsInADay = 60 * 60 * 24;
+  secondsInAnHour = 60 * 60;
+
+  adjustInputPayload(payload) {
+    if (payload) {
+      if (payload.input_generic) {
+        const { credentials } = payload.input_generic;
+        return {
+          ...payload.input_generic,
+          credentials: credentials && JSON.parse(credentials),
+        };
+      }
+      if (payload.input_s3) return payload.input_s3;
+      if (payload.input_jdbc) return payload.input_jdbc;
+
+      const { credentials } = payload;
+      return {
+        ...payload,
+        credentials: credentials && JSON.parse(credentials),
+      };
     }
+    return payload;
+  }
+  getDifferenceInSeconds(date1: Date, date2: Date) {
+    const diffInMs = Math.abs(date2.getTime() - date1.getTime());
+    return diffInMs / 1000;
+  }
+  validateCron(data) {
+    const { info, cron } = data;
+    const { customer_tier } = info;
+    if (!cron) return;
+    let interval: CronExpression;
+    try {
+      interval = CronParser.parseExpression(cron);
+    } catch (error) {
+      throw new HttpException(
+        'Intervalo de tempo inválido',
+        HttpStatus.BAD_REQUEST,
+      );
+    }
+    const nextDate = interval.next().toDate();
+    const afterNextDate = interval.next().toDate();
+    const secondsApart = this.getDifferenceInSeconds(nextDate, afterNextDate);
+    if (customer_tier === 'BASIC' && secondsApart < this.secondsInADay) {
+      throw new ForbiddenException(
+        'Intervalo de tempo não pode ser inferior a um dia.',
+      );
+    } else if (secondsApart < this.secondsInAnHour) {
+      throw new ForbiddenException(
+        'Intervalo de tempo não pode ser inferior a uma hora.',
+      );
+    }
+  }
+  async create(data: { body: CreateInputReq; info: Info }) {
+    // this.validateCron(data);
+    const { body, info } = data;
+
+    const inputCreateGenericRequest: InputCreateGenericRequest = {
+      input: {
+        ...body,
+      },
+      info,
+    };
+
+    const { input } = await lastValueFrom(
+      this.inputWriteService.InputCreate(inputCreateGenericRequest),
+    );
+    const adjustedInput = this.adjustInputPayload(input);
+    return { input: adjustedInput };
+  }
+
+  async getAvailableEntities(data: GetAvailableEntitiesRequest) {
+    return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
+  }
+
+  async findAll(info: Info) {
+    const { inputs } = await lastValueFrom(
+      this.inputReadService.InputFindAll({ info }),
+    );
+    if (!inputs) return { inputs: [] };
+
+    const inputsReturn = inputs.map((i) => this.adjustInputPayload(i));
+    return { inputs: inputsReturn };
   }
 
   async findOne(idRequest: IIdRequest) {
-    try {
-      const findOneInputResponse = await this.inputClient.findOne(idRequest);
-      return findOneInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
+    const findOneInputResponse: any = await lastValueFrom(
+      this.inputReadService.InputFindOne(idRequest),
+    );
+    findOneInputResponse.input = this.adjustInputPayload(
+      findOneInputResponse.input,
+    );
+    return findOneInputResponse;
   }
 
   async update(id: string, data, info: Info) {
+    this.validateCron({ ...data, info });
     try {
-      const updateInputResponse = await this.inputClient.update({
+      const updateInputResponse: any = await this.OLD_inputClient.update({
         id,
         info,
         ...data,
       });
 
+      updateInputResponse.input = this.adjustInputPayload(
+        updateInputResponse?.input,
+      );
       return updateInputResponse;
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -51,20 +218,13 @@ export class InputsService {
   }
 
   async remove(idRequest: IIdRequest) {
-    try {
-      const removeInputResponse = await this.inputClient.remove(idRequest);
-
-      return removeInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
+    return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
   }
 
   async testConnection(data) {
     try {
-      const testConnectionInputResponse = await this.inputClient.testConnection(
-        data,
-      );
+      const testConnectionInputResponse =
+        await this.OLD_inputClient.testConnection(data);
 
       return testConnectionInputResponse;
     } catch (err) {
@@ -72,14 +232,30 @@ export class InputsService {
     }
   }
 
-  async getColumns(data) {
-    try {
-      const testConnectionGetColumnsResponse =
-        await this.inputClient.getColumns(data);
+  generateInputS3Payload(payload) {
+    const { credentials, plugin, cron } = payload;
+    if (!credentials) return payload;
 
-      return testConnectionGetColumnsResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
+    const {
+      client_aws_access_key_id,
+      client_aws_secret_access_key,
+      file_format_params,
+      format_file_params,
+      client_bucket,
+      file_to_extract,
+    } = credentials;
+
+    const formatedPayload = {
+      plugin,
+      cron,
+      source_bucket: client_bucket,
+      source_prefix: file_to_extract,
+      auth_parameters: {
+        aws_access_key_id: client_aws_access_key_id,
+        aws_secret_access_key: client_aws_secret_access_key,
+      },
+      file_format_params: file_format_params || format_file_params,
+    };
+    return formatedPayload;
   }
 }