UPDATE: add toleration for spot instances

FIX: env fixed and add logger for the process.env

.eslintrc.js

@@ -8,7 +8,7 @@ module.exports = {
   extends: [
     'eslint:recommended',
     'plugin:@typescript-eslint/recommended',
-    'plugin:prettier/recommended',
+    'prettier',
   ],
   root: true,
   env: {

.github/workflows/deploy-manually.yml

@@ -3,16 +3,21 @@ on:
   push:
     branches:
       - main
+      - beta
   workflow_dispatch:
     inputs:
       environment:
-        description: 'Deployment environment'
+        description: "Deployment environment"
         required: true
         type: choice
         options:
-          - dev
           - stg
           - prd
+      push_to_dockerhub:
+        description: "Push image to Dockerhub?"
+        required: true
+        type: boolean
+        default: false
 
 jobs:
   extract_environment:
@@ -26,9 +31,11 @@ jobs:
       - name: Extract Environment
         run: |
           if [ ${EVENT} == "workflow_dispatch" ]; then
-            echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
+            echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
           elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=environment;]$(echo "prd")"
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
           fi
         id: extract_environment
 
@@ -39,28 +46,35 @@ jobs:
       new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - if: github.event_name != 'workflow_dispatch'
         name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
+        uses: cycjimmy/semantic-release-action@v3
         id: semantic
         with:
-          semantic_version: 16
           extra_plugins: |
-            conventional-changelog-eslint
+            conventional-changelog-eslint@4.0.0
           branches: |
             [
-              'main'
+              'main',
+              {
+                name: 'alpha',
+                prerelease: true
+              },
+              {
+                name: 'beta',
+                prerelease: true
+              }
             ]
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  deploy:
+  build_ecr_image:
     if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
     needs: [extract_environment, semantic_release]
     runs-on:
-      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+      [self-hosted, "prd"]
 
     steps:
       - name: Printing stats
@@ -71,16 +85,12 @@ jobs:
         run: echo ${GITHUB_REF#refs/heads/}
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Update Pip
         run: |
           python3 -m pip install --upgrade pip
 
-      - name: Install Docker Compose
-        run: |
-          python3 -m pip install docker-compose --upgrade
-
       - name: Install AWS CLI
         run: |
           python3 -m pip install awscli --upgrade
@@ -90,14 +100,14 @@ jobs:
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4
         id: aws
         with:
           aws-region: us-east-1
 
       - name: Login to AWS ECR
         id: login_ecr
-        uses: aws-actions/amazon-ecr-login@v1
+        uses: aws-actions/amazon-ecr-login@v2
 
       - name: Build, Tag, and Push Image to AWS ECR
         env:
@@ -105,32 +115,103 @@ jobs:
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
-          docker-compose -f build.docker-compose.yml build
+          docker compose -f build.docker-compose.yml build
-          docker-compose -f build.docker-compose.yml push
+          docker compose -f build.docker-compose.yml push
 
-      - name: Create ZIP file to Deploy AWS Beanstalk
+      - name: Login to Docker Hub
+        if: ${{inputs.push_to_dockerhub}}
+        uses: docker/login-action@v2
+        with:
+          username: dadosfera
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build, Tag, and Push Image to Dockerhub
+        if: ${{inputs.push_to_dockerhub}}
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
-          sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
+          docker compose -f build.docker-compose.dockerhub.yml build
-          sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+          docker compose -f build.docker-compose.dockerhub.yml push
-          sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
-          zip deploy.zip docker-compose.yml  -r .ebextensions
 
-      - name: Deploy AWS Beanstalk
+      # - name: Create ZIP file to Deploy AWS Beanstalk
-        env:
+      #   env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
+      #     ENV: ${{ needs.extract_environment.outputs.environment }}
-          AWS_REGION: us-east-1
+      #     IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
-          APP_NAME: ${{ github.event.repository.name }}
+      #     ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
-        run: |
+      #     NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
-          eb use $APP_NAME-$ENV
+      #   run: |
-          echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
+      #     sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
-          eb deploy
+      #     sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+      #     sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
+      #     sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
+      #     zip deploy.zip docker-compose.yml  -r .ebextensions
+
+      # - name: Deploy AWS Beanstalk
+      #   env:
+      #     ENV: ${{ needs.extract_environment.outputs.environment }}
+      #     AWS_REGION: us-east-1
+      #     APP_NAME: ${{ github.event.repository.name }}
+      #   run: |
+      #     eb use $APP_NAME-$ENV
+      #     echo -e "deploy:\n  artifact: deploy.zip" >> .elasticbeanstalk/config.yml
+      #     eb deploy
 
       - name: Remove Docker's Trash
         if: always()
         run: |
           docker system prune --volumes -a -f
           docker system df
+
+  helmfile-deploy:
+    needs: [extract_environment, semantic_release, build_ecr_image]
+    runs-on: [self-hosted, "prd-azure"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install Azure ClI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - uses: azure/login@v2
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+
+      - name: Run Helmfile Apply
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+        run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG

.github/workflows/test.yml

@@ -6,23 +6,18 @@ on:
 
 jobs:
   test:
-    runs-on: self-hosted
+    runs-on: [self-hosted, prd]
+    env:
+      APP_NAME: ${{ github.event.repository.name }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build
-        env:
+        run: docker build -t ${APP_NAME}_teste --target test .
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          docker build -t $APP_NAME --target test .
 
       - name: Run Test
-        env:
+        run: docker run ${APP_NAME}_teste
-          ENV: test
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          docker run --rm --entrypoint="npm" $APP_NAME run test
 
       - name: Remove Docker's Trash
         if: always()

.github/workflows/validate-k8s.yml

@@ -0,0 +1,79 @@
+name: Validate K8S Modifications
+
+on:
+  pull_request:
+    branches:
+      - main
+      - stg
+
+jobs:
+  extract_environment:
+    runs-on: ubuntu-22.04
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
+          fi
+        id: extract_environment
+
+  helmfile-deploy:
+    needs: [extract_environment]
+    runs-on: [self-hosted, "prd-azure"]
+
+    steps:
+      - name: Summary
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: |
+          echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\`  para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install Azure ClI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - uses: azure/login@v2
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+
+      - name: Run Helmfile Diff
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: helmfile -f helmfiles/${ENV}.yaml diff

.nvmrc

@@ -1 +1 @@
-18.3.0
+18.17

.vscode/launch.json

@@ -12,6 +12,7 @@
         "start:debug"
       ],
       "runtimeExecutable": "npm",
+      "runtimeVersion": "18.17",
       "skipFiles": [
         "<node_internals>/**"
       ],

Dockerfile

@@ -1,21 +1,43 @@
-# install all dependencies
+FROM node:18.17-alpine AS base_image
-FROM node:18.3.0-alpine3.15 as packages
+
+FROM base_image AS build_base
 WORKDIR /app
-COPY package.json package-lock.json ./
+RUN apk update
-RUN apk add --no-cache aws-cli \
+# needed packages to build dependencies from source
- && aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
+RUN apk add --no-cache aws-cli
- && npm install
+COPY package*.json ./
+
+
+# run aws cli without mounting secret, because CI already has AWS credentials
+FROM build_base AS ci_image
+RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+RUN npm ci
+COPY . .
+
 
 # unit test specific build
-FROM node:18.3.0-alpine3.15 as test
+FROM ci_image AS test
-WORKDIR /app
+ENV DUC_URL=0.0.0.0:50051
-COPY --from=packages /app/node_modules ./node_modules
+ENTRYPOINT ["npm", "run", "test"]
-COPY . ./
 
-FROM node:18.3.0-alpine3.15
+
-WORKDIR /app
+# dev build
-COPY . /app/
+FROM build_base AS dev
-COPY --from=packages /app/node_modules ./node_modules
+RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
+  aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+# flag --build-from-source is required to force-build sqlite3
+RUN npm ci
+COPY . .
+ENTRYPOINT npm run start:dev
+
+
+FROM ci_image AS prod_build
 RUN npm run build
-EXPOSE 3333
+
-ENTRYPOINT npm run start
+
+FROM base_image
+WORKDIR /app
+COPY --from=prod_build /app/dist ./dist
+COPY --from=prod_build /app/node_modules ./node_modules
+COPY --from=prod_build /app/package*.json ./
+ENTRYPOINT npm run start:prod

LICENSE

@@ -0,0 +1,10 @@
+LICENSE
+Copyright (C) 2023 Dadosfera
+
+All rights reserved.
+
+This software and related documentation are provided under a license agreement containing restrictions on use and disclosure and are protected by intellectual property laws. Except as expressly permitted in your license agreement or allowed by law, you may not use, copy, reproduce, translate, broadcast, modify, license, transmit, distribute, exhibit, perform, publish, or display any part, in any form, or by any means.
+
+Reverse engineering, disassembly, or decompilation of this software, unless required by law for interoperability, is prohibited.
+
+If you have any questions about this, please contact Dadosfera.ai at legal@dadosfera.ai

README.md

@@ -2,43 +2,58 @@
   <image src="./assets/maestro.svg" style="width:10rem">
 </p>
 
+
 # Maestro
-Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
 
-## 🚀 Starting
+Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
-These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
 
-### 📋 Requirements
+```mermaid
-- [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
+graph TD;
-- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
+  Frontend<-->Maestro;
-    - Create your Access Key on the "Security credentials" menu
+  Maestro<-->duc;
-    - Set the Access Key on your local development machine
+  Maestro<-->pi-factory;
-- Request access to the dev, stg and prd VPNs
+  Maestro<-->in-factory;
-
-### 🔧 Installation<a id="installation"></a>
-- Clone the repository
-    - SSH
 ```
+
+É uma API REST, desenvolvida em NodeJs utilizando o Framework [NestJs](https://docs.nestjs.com/).
+
+## 🚀 Iniciando
+
+Estas instruções permitirão que você obtenha uma cópia funcional do projeto em sua máquina local para desenvolvimento e testes.
+
+### 📋 Requisitos
+
+- [NodeJS v18.17 / NPM v9.6.7](https://nodejs.org/pt-br/download/)
+
+> Dica: Utilize [NVM](https://github.com/nvm-sh/nvm) para gerenciar facilmente as versões do node
+
+- Solicite acesso à conta de desenvolvimento do AWS Console para **todos os serviços necessários**
+  - Crie sua Access Key no menu "Security credentials"
+  - Defina a Access Key em sua máquina de desenvolvimento local
+- Solicite acesso às VPNs de stg e prd
+
+### 🔧 Instalação<a id="installation"></a>
+
+- Clone o repositório
+
+  ```sh
   git clone git@github.com:dadosfera/maestro.git
   ```
-        or
 
-    - HTTPS
+- Selecione a versão correta do node (opcional, apenas se estiver usando o [NVM](https://github.com/nvm-sh/nvm)):
-        ```
-        git clone https://github.com/dadosfera/maestro.git
-        ```
 
-- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
   ```sh
   nvm use
   ```
 
-- Install the project dependencies:
+- Instale as dependências do projeto:
+
   ```sh
   npm i
   ```
 
-- Setup the following enviroment variables:
+- Configure as seguintes variáveis de ambiente:
+
   ```
   ENV=
   DUC_URL=
@@ -48,21 +63,41 @@ These instructions will allow you to get a working copy of the project on your l
   SM_OAUTH_PATH=
   ```
 
-- Start the server:
+- Inicie o servidor:
+
   ```sh
-    # dev mode
   npm run start:dev
-
-    # or in debug mode
-    npm run start:debug
   ```
-    The service should start successfully.
 
-## Authentication decorators
+  > Se preferir, utilize o debbuger do VSCode apertando F5
-Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
+
+  O serviço deve iniciar com sucesso.
+
+## 📄 Documentação
+
+O NestJs facilita a documentação de cada rota usando decoradores em todas as propriedades de solicitações e respostas. Ele gera automaticamente um swagger para as informações fornecidas e fornece uma rota para acessá-lo em http://localhost:3333/api. Para mais informações, consulte a [documentação oficial](https://docs.nestjs.com/openapi/introduction).
+
+### - Documentações múltiplas
+
+Atualmente, estamos gerando **2 documentações diferentes**: Interna e Externa.
+
+Todas as rotas que têm o decorador `@ApiInternalOnly()` não serão visíveis no swagger da API **Externa**.
+
+- Quando você inicia a aplicação com `npm run start:dev`, ela servirá e gerará o JSON do swagger da API **Interna**
+
+- Quando você executa `npm run docs`, ele gerará o JSON do swagger de ambas as APIs **Interna** e **Externa** e os salvará em `docsfera.json` e `docsfera.external.json`, respectivamente;
+
+> Link para documentação interna: https://dadosfera.github.io/docsfera
+
+É importante executar `npm run docs` antes de cada implantação para que sempre tenhamos as documentações mais atualizadas publicadas.
+
+## Decoradores de autenticação
+
+O Maestro possui utilitários para facilitar a autenticação do usuário em todos os controladores e rotas. Os seguintes decoradores estão disponíveis:
 
 ### `@Authenticated`
-If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
+
+Se o usuário precisar estar autenticado para fazer uma solicitação, podemos usar o decorador `@Authenticated` no controlador ou rota, conforme necessário.
 
 ```ts
 import { Authenticated } from '../../authentication/authentication.decorator';
@@ -74,7 +109,7 @@ class FooController {
 
   @Get('bar')
   async getBar() {
-    this.logger.info('user is authenticated!');
+    this.logger.info('usuário está autenticado!');
 
     return { authenticated: true };
   }
@@ -91,14 +126,14 @@ class FooController {
   @Get('bar')
   @Authenticated()
   async getBar() {
-    this.logger.info('user is authenticated!');
+    this.logger.info('usuário está autenticado!');
 
     return { authenticated: true };
   }
 
   @Post('bar')
   async postBar() {
-    this.logger.info('user is NOT authenticated!');
+    this.logger.info('usuário NÃO está autenticado!');
 
     return { authenticated: false };
   }
@@ -106,11 +141,12 @@ class FooController {
 ```
 
 ### `@RequireAllPermissions`
-This decorator requires that **all permissions** listed are granted to the requesting user.
+
+Este decorador exige que **todas as permissões** listadas sejam concedidas ao usuário solicitante.
 
 ```ts
 import { RequireAllPermissions } from '../../authentication/authentication.decorator';
-import { Permissions } from '../../authentication/permissions.enum';
+import { PERMISSIONS } from '../../authentication/permissions.enum';
 
 @Controller('foo')
 class FooController {
@@ -125,11 +161,12 @@ class FooController {
 ```
 
 ### `@RequireSomePermission`
-In the following case, the user is required to have **at least one** listed permission.
+
+No caso seguinte, é necessário que o usuário tenha **pelo menos uma** das permissões listadas.
 
 ```ts
 import { RequireSomePermission } from '../../authentication/authentication.decorator';
-import { Permissions } from '../../authentication/permissions.enum';
+import { PERMISSIONS } from '../../authentication/permissions.enum';
 
 @Controller('foo')
 @RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.CREATE)
@@ -144,17 +181,19 @@ class FooController {
 ```
 
 ### `@AuthenticateCondition`
-If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
 
-In the following example:
+Se for necessária uma verificação de autenticação mais complicada, podemos usar o decorador `@AuthenticateCondition` para defini-la. A função personalizada deve retornar `true` para autenticar a solicitação.
-- all routes on the `FooController` controller can only be requested from localhost
+
-- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
+No exemplo a seguir:
+
+- todas as rotas no controlador `FooController` só podem ser solicitadas a partir do localhost
+- `POST /foo/bar` só pode ser solicitado a partir do localhost **e** por usuários do cliente com id `111...eef`
 
 ```ts
 import { AuthenticateCondition } from '../../authentication/authentication.decorator';
 
 @Controller('foo')
-// allow requests only from localhost
+// permitir solicitações apenas do localhost
 @AuthenticateCondition((request: Request) => request.ip === '::ffff:127.0.0.1')
 class FooController {
   /* ... */
@@ -165,18 +204,22 @@ class FooController {
   }
 
   @Post('bar')
-  // allow requests only from a specific customer
+  // permitir solicitações apenas de um cliente específico
-  @AuthenticateCondition((request: Request, user: RequestUser) => user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef')
+  @AuthenticateCondition(
+    (request: Request, user: RequestUser) =>
+      user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
+  )
   async postBar() {
     /* ... */
   }
 }
 ```
 
-### Note on authentication decorators
+### Nota sobre decoradores de autenticação
-- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
 
-- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (*and wouldn't like to...*) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
+- O método antigo de autenticação colocava os dados do usuário no campo `request.body.info`, o que impõe certos problemas em relação ao corpo da solicitação, pois esses dados devem vir do frontend sem qualquer modificação pelo Maestro. Agora, esse uso está ⚠️ **DESCONTINUADO** ⚠️. Estamos trabalhando para migrar para o decorador de parâmetro `@User`. O método antigo está funcionando enquanto a migração está em andamento.
+
+- Os decoradores de autenticação podem ser usados juntos e todos eles **devem** passar para que a solicitação seja autenticada, mas, no caso geral, você não precisa (_e não gostaria de..._) usar todos eles juntos, pois você pode codificar toda a lógica de autenticação no decorador `@AuthenticateCondition`.
 
   ```ts
   import {
@@ -184,8 +227,8 @@ class FooController {
     RequireSomePermission,
     AuthenticateCondition,
   } from '../../authentication/authentication.decorator';
-    import { Permissions } from '../../authentication/permissions.enum';
+  import { PERMISSIONS } from '../../authentication/permissions.enum';
-    import { Waa, Baz } from './authenticationFunctions'
+  import { Waa, Baz } from './authenticationFunctions';
 
   @Controller('foo')
   @RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
@@ -201,25 +244,28 @@ class FooController {
     }
 
     @Post('bar')
-      @RequireAllPermissions(Permissions.BAR.MANAGE, Permissions.BAR.USE, Permissions.BAR.REQUEST)
+    @RequireAllPermissions(
+      Permissions.BAR.MANAGE,
+      Permissions.BAR.USE,
+      Permissions.BAR.REQUEST,
+    )
     async postBar() {
       /* ... */
     }
   }
   ```
 
-- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
+- Se `@RequireAllPermissions` e `@RequireSomePermission` forem usados com **apenas uma única permissão**, eles apresentam o **exatamente mesmo comportamento**.
 
   ```ts
-    // same behavior
+  // mesmo comportamento
   @RequireAllPermissions(Permissions.BAR.MANAGE)
   @RequireSomePermission(Permissions.BAR.MANAGE)
   ```
 
+## Decorador de parâmetro `@User`
 
-
+Os dados do usuário solicitante podem ser obtidos usando o decorador de parâmetro @User, como no exemplo a seguir:
-## `@User` parameter decorator
-The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
 
 ```ts
 import { User, RequestUser } from '../../authentication/user.decorator';
@@ -237,7 +283,7 @@ class FooController {
 }
 ```
 
-If the user is required to be logged in, set `required` to `true`, as you would want in the `change-password` operation:
+Se o usuário precisar estar logado, defina `required` como `true`, como por exemplo:
 
 ```ts
 import { User, RequestUser } from '../../authentication/user.decorator';
@@ -263,39 +309,46 @@ class UserController {
 }
 ```
 
-## 📦 Development
+## 📦 Desenvolvimento
-### ⌨️ Coding Style
-By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
 
-**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
+### ⌨️ Estilo de Codificação
 
-### Commits pattern
+Por padrão, usamos [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) com configurações padrão.
-Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
 
-Format: `<type>[optional scope]: <description>`
+**Recomendamos usar o Visual Studio Code e instalar as extensões recomendadas para facilitar o processo de desenvolvimento.**
 
-Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
+### Padrão de commits
 
-### Branching naming convention
+Nosso pipeline de fluxo de trabalho segue as especificações de commits convencionais ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) para liberar versões conforme necessário.
-- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
 
-  **It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (_) could be used to separate the ID and description.**
+Formato: `<type>[optional scope]: <description>`
 
-- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`,  `bugfix/JIRA-1444_gray-on-blur-fix`.
+Exemplo: `FIX: ensure Range headers adhere more closely to RFC 2616`
 
-- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
+### Convenção de nomenclatura de branchs
 
-- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
+- **Feature**: Quaisquer alterações de código para um novo módulo ou caso de uso devem ser feitas em uma branch de feature. Esta branch é criada com base na branch `main`. Quando todas as alterações estiverem concluídas, será necessário um Pull Request/Merge Request para colocar todas essas alterações de volta na branch `main`. Exemplos: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
 
-### Making a Pull Request
+  **Recomenda-se usar todas as letras em minúsculas e hífen (-) para separar palavras, a menos que seja um nome ou ID de item específico. O sublinhado (\_) pode ser usado para separar o ID e a descrição.**
-1. Commit your changes
-2. Open the Pull Request on GitHub
-3. Send Pull Request link in microsfera Google Chat Group for review and possible approval
 
-## 🛠️ Built with
+- **Bug Fix**: Se as alterações de código feitas na branch de feature foram rejeitadas após um lançamento, sprint ou demo, quaisquer correções necessárias após isso devem ser feitas na branch de correção de bug. Exemplos: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
-Some technologies used in this project:
 
-- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
+- **Hot Fix**: Se houver necessidade de corrigir um bloqueador, fazer um patch temporário, aplicar uma mudança crítica de framework ou configuração que deva ser tratada imediatamente, ela deve ser criada como um Hotfix. Exemplos: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
 
-## ⚙️ Back-end Architecture
+- **Experimental**: Uma branch para experimentar. Qualquer nova feature ou ideia que não faça parte de um lançamento ou sprint. Exemplo: `experimental/dark-theme-support`.
-The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
+
+### Fazendo um Pull Request
+
+1. Comite suas alterações
+2. Abra o Pull Request no GitHub
+3. Envie o link do Pull Request no grupo de chat do Google da Microsfera para revisão e possível aprovação
+
+## 🛠️ Construído com
+
+Algumas tecnologias usadas neste projeto:
+
+- [NestJS](https://docs.nestjs.com) - Framework para construir aplicações NodeJS eficientes e escaláveis no lado do servidor
+
+## ⚙️ Arquitetura de Back-end
+
+A arquitetura pode ser encontrada neste [link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).

build.docker-compose.dockerhub.yml

@@ -0,0 +1,4 @@
+services:
+  maestro:
+    build: .
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

build.docker-compose.yml

@@ -1,5 +1,4 @@
-version: "3.8"
 services:
   maestro:
     build: .
-    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+    image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd:${IMAGE_TAG}

docker-compose.dockerhub.yml

@@ -0,0 +1,28 @@
+services:
+  maestro:
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+    container_name: maestro
+    restart: always
+    ports:
+      - 3333:3333
+    env_file:
+      - .env
+
+  node_exporter:
+    image: ${NODE_EXPORTER_URL}
+    container_name: node_exporter
+    restart: always
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   maestro:
     image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
@@ -10,7 +9,7 @@ services:
       - .env
 
   node_exporter:
-    image: prom/node-exporter:v1.3.1
+    image: ${NODE_EXPORTER_URL}
     container_name: node_exporter
     restart: always
     volumes:

environment.d.ts

@@ -0,0 +1,20 @@
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      NODE_ENV: 'test';
+      ENV: 'local' | 'stg' | 'prd' | 'test';
+      LOCAL_ENV: 'stg' | 'prd';
+
+      DUC_URL: string;
+      INFACTORY_URL: string;
+      PIFACTORY_URL: string;
+
+      INTERNAL_SWAGGER: 'true' | 'false';
+
+      AWS_REGION: string;
+      OPEN_GROUP_ID: string;
+      OPEN_CUSTOMER_ID: string;
+    }
+  }
+}
+export {};

helmfiles/prd.yaml

@@ -0,0 +1,20 @@
+charts:
+  - name: maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: maestro.duc_url
+        value: duc.dadosfera.ai
+      - name: hostname
+        value: maestro.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
+      - name: maestro.open_group_id
+        value: 401573bb-334f-44b2-b30e-88d4cea31ae9

helmfiles/stg.yaml

@@ -0,0 +1,22 @@
+charts:
+  - name: maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: maestro.duc_url
+        value: duc.stg.dadosfera.ai
+      - name: hostname
+        value: maestro.stg.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.open_customer_id
+        value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
+      - name: maestro.open_group_id
+        value: e3f98a2f-7748-4981-8505-7695c8ca8218
+      - name: replicaCount
+        value: 1

jest.config.ts

@@ -0,0 +1,18 @@
+import type { Config } from '@jest/types';
+const config: Config.InitialOptions = {
+  moduleFileExtensions: ['ts', 'js', 'json'],
+  modulePaths: ['<rootDir>'],
+  roots: ['<rootDir>/src/', '<rootDir>/test/'],
+  testRegex: '.*\\.(test|spec)\\.[jt]s$',
+  transform: { '\\.[jt]s$': 'ts-jest' },
+  collectCoverageFrom: ['**/*.[jt]s'],
+  coverageDirectory: 'coverage',
+  coveragePathIgnorePatterns: [
+    '<rootDir>/node_modules/',
+    '.*\\.module\\.[jt]s$',
+  ],
+  // moduleDirectories: ['node_modules'], // default is already 'node_modules'
+  // rootDir: '.', //No need
+  // testEnvironment: 'node', //Defaults to 'node'
+};
+export default config;

maestro/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

maestro/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: maestro
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

maestro/templates/deployment.yaml

@@ -0,0 +1,117 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: maestro
+  namespace: applications
+  labels:
+    app: maestro
+
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: maestro
+  
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: maestro
+    
+    spec:
+      imagePullSecrets:
+        - name: {{ .Values.imagePullSecrets }}
+      nodeSelector:
+        "beta.kubernetes.io/os": linux
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: application
+                    operator: In
+                    values:
+                      - backend
+      tolerations:
+      - key: "kubernetes.azure.com/scalesetpriority"
+        operator: "Equal"
+        value: "spot"
+        effect: "NoSchedule"
+
+      containers:
+        - name: maestro
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+          ports:
+            - containerPort: {{ .Values.containerPort }}
+          resources:
+            requests:
+              cpu: {{ .Values.resources.requests.cpu }}
+              memory: {{ .Values.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.resources.limits.cpu }}
+              memory: {{ .Values.resources.limits.memory }}
+          env:
+            - name: AWS_IDENTITY_POOL_ID
+              value: {{ .Values.maestro.aws_identity_pool_id }}
+            - name: AWS_REGION
+              value: "us-east-1"
+            - name: BASE_HOST
+              value: "maestro_prd"
+            - name: BUCKET_CUSTOMER_CSV_ASSETS
+              value: {{ .Values.maestro.bucket_customer_csv_assets }}
+            - name: CONNECTORS_INDEX
+              value: "connectors"
+            - name: DUC_URL
+              value: {{ .Values.maestro.duc_url }}
+            - name: ELASTIC_APM_ENVIRONMENT
+              value: {{ .Values.maestro.env }}
+            - name: ELASTIC_APM_SERVER_URL
+              value: "https://apm-server.dadosfera.ai"
+            - name: ELASTIC_APM_SERVICE_NAME
+              value: {{ .Values.maestro.apm_service }}
+            - name: ENV
+              value: {{ .Values.maestro.env }}
+            - name: INFACTORY_URL
+              value: {{ .Values.maestro.in_factory_url }}
+            - name: LOGGER_GELF_HOST
+              value: "logstash-pipelines.dadosfera.ai"
+            - name: LOGGER_GELF_PORT
+              value: "{{ .Values.maestro.logger_gelf_port }}"
+            - name: NIMBUS_BASE_URL
+              value: "http://nimbus-api"
+            - name: NPM_TOKEN
+              value: {{ .Values.maestro.npm_token }}
+            - name: PB_TOKEN_PATH
+              value: {{ .Values.maestro.pb_token_path }}
+            - name: PIFACTORY_URL
+              value: {{ .Values.maestro.pi_factory_url }}
+            - name: SM_OAUTH_PATH
+              value: {{ .Values.maestro.sm_oauth_path }}
+            - name: TRFACTORY_URL
+              value: {{ .Values.maestro.tr_factory_url }}
+            - name: UPLOAD_FILE_AGENT_CONNECTION
+              value: {{ .Values.maestro.upload_file_agent_connection }}
+            - name: JWT_PRIVATE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: prd-duc
+                  key: jwt_token
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: prd-maestro
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: prd-maestro
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: AWS_DEFAULT_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: prd-maestro
+                  key: AWS_DEFAULT_REGION

maestro/templates/ingress.yaml

@@ -0,0 +1,27 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/server-snippet: |
+      underscores_in_headers on;
+      ignore_invalid_headers on;
+      
+  generation: 1
+  labels:
+    app: maestro
+  name: maestro
+  namespace: applications
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{ .Values.hostname }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: maestro
+            port:
+              number: {{ .Values.ingress.port }}
+        path: /
+        pathType: Prefix

maestro/templates/secret.yaml

@@ -0,0 +1,40 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: prd-maestro
+  namespace: applications
+  labels:
+    app: maestro
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: secretsmanager-prd
+    kind: SecretStore
+  target:
+    name: prd-maestro
+    creationPolicy: Owner
+  data:
+  - secretKey: AWS_ACCESS_KEY_ID
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_ACCESS_KEY_ID
+
+  - secretKey: AWS_SECRET_ACCESS_KEY
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_SECRET_ACCESS_KEY
+
+  - secretKey: AWS_DEFAULT_REGION
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_DEFAULT_REGION
+
+  - secretKey: jwt_token
+    remoteRef:
+      key: {{ .Values.maestro.jwt_token_secret_id }}
+      version: "AWSCURRENT"
+      property: token
+

maestro/templates/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: maestro
+  namespace: applications
+  labels:
+    app: maestro
+
+spec:
+  type: ClusterIP
+  ports:
+  - name: maestro
+    protocol: TCP
+    port: {{ .Values.service.port }}
+    targetPort: {{ .Values.service.targetPort }}
+  selector:
+    app: maestro
+

maestro/values.yaml

@@ -0,0 +1,50 @@
+# Default values for metabase.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 3
+hostname: maestro-temp.dadosfera.ai
+image:
+  repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: 1.56.0
+containerPort: 3333
+imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
+service:
+  type: ClusterIP
+  port: 3333
+  targetPort: 3333
+ingress:
+  enabled: false
+  port: 3333
+resources:
+  requests:
+    cpu: 100m
+    memory: 1500Mi
+  limits:
+    cpu: 2000m
+    memory: 2Gi
+maestro:
+  aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
+  duc_url: duc.dadosfera.ai
+  in_factory_url: in-factory.dadosfera.ai
+  bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153"
+  env: prd
+  apm_service: maestro
+  jwt_token_secret_id: prd/root/jwt_token
+  logger_gelf_port: "1026"
+  npm_token: npm_Xp17h3daMkORcZ55NY95Ez4gRfdzsQ3UvINP
+  pb_token_path: prd/root/productboard_token
+  pi_factory_url: pi-factory.dadosfera.ai
+  sm_oauth_path: prd/root/oauth_applications
+  tr_factory_url: in-factory.dadosfera.ai
+  upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
+  open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
+  open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  targetMemoryUtilizationPercentage: 80

package.json

@@ -6,111 +6,94 @@
   "private": true,
   "license": "UNLICENSED",
   "engines": {
-    "node": "18.3.0"
+    "node": "18.17"
   },
   "scripts": {
+    "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
+    "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
     "prebuild": "rimraf dist",
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "nest start",
-    "start:dev": "nest start --watch",
+    "start:dev": "export INTERNAL_SWAGGER=true && nest start --watch",
-    "start:debug": "nest start --debug --watch",
+    "start:debug": "npm run start:dev",
     "start:prod": "node dist/main",
+    "docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
-    "test": "jest",
+    "test": "jest --detectOpenHandles --forceExit",
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
-    "@aws-sdk/client-secrets-manager": "^3.112.0",
+    "@aws-sdk/client-secrets-manager": "^3.414.0",
-    "@dadosfera/dadosfera-logs": "1.0.0-beta.3",
+    "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.11.0",
+    "@dadosfera/protospack": "2.5.3",
-    "@grpc/grpc-js": "^1.6.7",
+    "@dadosfera/protospack-v2": "3.37.0-beta.3",
-    "@grpc/proto-loader": "^0.6.13",
+    "@grpc/grpc-js": "^1.9.3",
-    "@nestjs/common": "^8.4.7",
+    "@grpc/proto-loader": "^0.7.9",
-    "@nestjs/config": "^1.2.1",
+    "@nestjs/cli": "^9.5.0",
-    "@nestjs/core": "^8.4.7",
+    "@nestjs/common": "^9.4.3",
-    "@nestjs/mapped-types": "*",
+    "@nestjs/config": "^2.3.4",
-    "@nestjs/microservices": "^8.4.7",
+    "@nestjs/core": "^9.4.3",
-    "@nestjs/passport": "^8.2.2",
+    "@nestjs/mapped-types": "^1.2.2",
-    "@nestjs/platform-express": "^8.4.7",
+    "@nestjs/microservices": "^9.4.3",
-    "@nestjs/schedule": "^1.1.0",
+    "@nestjs/passport": "^9.0.3",
-    "@nestjs/swagger": "^5.2.1",
+    "@nestjs/platform-express": "^9.4.3",
-    "axios": "^0.25.0",
+    "@nestjs/schematics": "^9.2.0",
-    "cron-parser": "^4.4.0",
+    "@nestjs/swagger": "^6.3.0",
+    "@nestjs/testing": "^9.4.3",
+    "axios": "^0.27.2",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.0",
+    "cron-parser": "^4.9.0",
     "dotenv": "^14.3.2",
-    "elastic-apm-node": "^3.36.0",
+    "elastic-apm-node": "^3.50.0",
-    "helmet": "^5.1.0",
+    "helmet": "^5.1.1",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "^9.0.2",
     "jwk-to-pem": "^2.0.5",
+    "mixpanel": "^0.17.0",
+    "ms": "^3.0.0-canary.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
-    "passport-forcedotcom": "^0.1.5",
+    "passport-forcedotcom": "^0.2.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-hubspot-oauth2": "^1.0.3",
     "passport-mailchimp": "^1.1.0",
-    "protospack": "2.5.2",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.5",
-    "swagger-ui-express": "^4.4.0"
+    "swagger-ui-express": "^4.6.3"
   },
   "overrides": {
     "multer": "1.4.5-lts.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^8.2.8",
+    "@types/express": "^4.17.17",
-    "@nestjs/schematics": "^8.0.11",
-    "@nestjs/testing": "^8.4.7",
-    "@types/express": "^4.17.13",
     "@types/jest": "27.0.2",
-    "@types/jsonwebtoken": "^8.5.8",
+    "@types/jsonwebtoken": "^8.5.9",
     "@types/jwk-to-pem": "^2.0.1",
-    "@types/multer": "^1.4.7",
+    "@types/multer": "^1.4.12",
-    "@types/node": "^16.11.41",
+    "@types/node": "^16.18.52",
     "@types/passport-facebook": "^2.1.11",
     "@types/passport-google-oauth20": "^2.0.11",
-    "@types/passport-oauth2": "^1.4.11",
+    "@types/passport-oauth2": "^1.4.12",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.29.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
-    "@typescript-eslint/parser": "^5.29.0",
+    "@typescript-eslint/parser": "^5.62.0",
-    "eslint": "^8.18.0",
+    "eslint": "^8.49.0",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^8.10.0",
-    "eslint-plugin-prettier": "^4.0.0",
+    "eslint-plugin-prettier": "^4.2.1",
     "jest": "^27.5.1",
-    "nock": "^13.2.7",
+    "nock": "^13.3.3",
-    "prettier": "^2.7.1",
+    "prettier": "^2.8.8",
     "source-map-support": "^0.5.20",
-    "supertest": "^6.2.3",
+    "supertest": "^6.3.3",
     "ts-jest": "^27.1.5",
-    "ts-loader": "^9.3.1",
+    "ts-loader": "^9.4.4",
-    "ts-node": "^10.8.1",
+    "ts-node": "^10.9.1",
-    "tsconfig-paths": "^3.14.1",
+    "tsconfig-paths": "^3.14.2",
-    "typescript": "^4.6.3"
+    "typescript": "^4.9.5"
-  },
-  "jest": {
-    "moduleFileExtensions": [
-      "js",
-      "json",
-      "ts"
-    ],
-    "rootDir": ".",
-    "modulePaths": [
-      "<rootDir>"
-    ],
-    "moduleDirectories": [
-      "node_modules"
-    ],
-    "testRegex": ".*\\.spec\\.ts$",
-    "transform": {
-      "^.+\\.(t|j)s$": "ts-jest"
-    },
-    "collectCoverageFrom": [
-      "**/*.(t|j)s"
-    ],
-    "coverageDirectory": "../coverage",
-    "testEnvironment": "node"
   }
 }

src/app.module.ts

@@ -1,81 +1,37 @@
 import { Module } from '@nestjs/common';
-import { ClientsModule } from '@nestjs/microservices';
-import { ConfigModule } from '@nestjs/config';
-import { InputsController } from './modules/inputs/inputs.controller';
-import { TransformationsController } from './modules/transformations/transformations.controller';
-import { OutputsController } from './modules/outputs/outputs.controllers';
-import { PipelinesController } from './modules/pipelines/pipelines.controller';
-import { HealthController } from './modules/health/health.controller';
 
-import { InputsService } from './modules/inputs/inputs.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { TransformationsService } from './modules/transformations/transformations.service';
-import { OutputsService } from './modules/outputs/outputs.service';
-import { PipelinesService } from './modules/pipelines/pipelines.service';
-import { HealthService } from './modules/health/health.service';
 
-import { InputsClientService } from './clients/inputs/client.service';
-import { TransformationsClientService } from './clients/transformations/client.service';
-import { OutputsClientService } from './clients/outputs/client.service';
-import { PipelinesClientService } from './clients/pipelines/client.service';
-
-import { OutputsClientConfiguration } from './clients/outputs/client.config';
-import { TransformationsClientConfiguration } from './clients/transformations/client.config';
-import { InputsClientConfiguration } from './clients/inputs/client.config';
-import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
-import { CatalogController } from './modules/catalog/catalog.controller';
-import { CatalogService } from './modules/catalog/catalog.service';
-import { OauthController } from './modules/oauth/oauth.controller';
-import { getOauthSecrets } from './utils/OauthSecrets';
-import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot-strategy';
-import { FacebookStrategy } from './modules/oauth/passport-strategies/facebook-strategy';
 import { AuthenticationGuard } from './authentication/authentication.guard';
+
 import { APP_GUARD } from '@nestjs/core';
-import { GoogleStrategy } from './modules/oauth/passport-strategies/google-strategy';
+
-import { MailChimpStrategy } from './modules/oauth/passport-strategies/mailchimp-strategy';
-import { SalesforceStrategy } from './modules/oauth/passport-strategies/salesforce-strategy';
 import { UsersModule } from './modules/users/users.module';
 import { RolesModule } from './modules/roles/roles.module';
-
 import { ConnectorModule } from './modules/connector/connector.module';
 import { AuthModule } from './modules/auth/auth.module';
 import { PermissionsModule } from './modules/permissions/permissions.module';
-
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { TermsOfUseModule } from './modules/termsOfUse/termsOfUse.module';
-
+import { ConnectionModule } from './modules/connection/connection.module';
-const inputClient = new InputsClientConfiguration();
+import { ConnectionTestModule } from './modules/connection-test/connection-test.module';
-const outputClient = new OutputsClientConfiguration();
+import { NetworkConfigModule } from './modules/network-config/network-config.module';
-const pipelineClient = new PipelinesClientConfiguration();
+import { InputsModule } from './modules/inputs/inputs.module';
-const transformationClient = new TransformationsClientConfiguration();
+import { OauthModule } from './modules/oauth/oauth.module';
+import { PipelinesModule } from './modules/pipelines/pipelines.module';
+import { TransformationsModule } from './modules/transformations/transformations.module';
+import { HealthModule } from './modules/health/health.module';
+import { CatalogModule } from './modules/catalog/catalog.module';
+import { ConfigModule } from '@nestjs/config';
+import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
+import { ProductboardModule } from './modules/productboard/productboard.module';
+import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
+import { CustomersModule } from './modules/customers/customers.module';
+import { OpenDataModule } from './modules/open-data/open-data.module';
+import { ThemeModule } from './modules/theme/theme.module';
 
 @Module({
-  controllers: [
-    InputsController,
-    TransformationsController,
-    OutputsController,
-    PipelinesController,
-    HealthController,
-    CatalogController,
-    OauthController,
-  ],
   providers: [
     DadosferaLogger,
-    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
-    InputsService,
-    TransformationsService,
-    OutputsService,
-    PipelinesService,
-    HealthService,
-    InputsClientService,
-    TransformationsClientService,
-    OutputsClientService,
-    PipelinesClientService,
-    CatalogService,
-    HubspotStrategy,
-    FacebookStrategy,
-    GoogleStrategy,
-    MailChimpStrategy,
-    SalesforceStrategy,
     {
       provide: APP_GUARD,
       useClass: AuthenticationGuard,
@@ -85,33 +41,28 @@ const transformationClient = new TransformationsClientConfiguration();
     ConfigModule.forRoot({
       isGlobal: true,
     }),
-    ConnectorModule,
     AuthModule,
+    ConnectionModule,
+    NetworkConfigModule,
+    InputsModule,
+    PipelinesV2Module,
+    CatalogModule,
+    ConnectorModule,
     PermissionsModule,
     TermsOfUseModule,
-
+    ConnectionTestModule,
-    ClientsModule.register([
+    PipelinesModule,
-      {
+    TransformationsModule,
-        name: 'INPUTS_PACKAGE',
-        ...inputClient.config(),
-      },
-      {
-        name: 'TRANSFORMATIONS_PACKAGE',
-        ...transformationClient.config(),
-      },
-      {
-        name: 'OUTPUTS_PACKAGE',
-        ...outputClient.config(),
-      },
-      {
-        name: 'PIPELINES_PACKAGE',
-        ...pipelineClient.config(),
-      },
-    ]),
-
     UsersModule,
-
     RolesModule,
+    OauthModule,
+    ProductboardModule,
+    MixpanelModule,
+    CustomersModule,
+    OpenDataModule,
+    ThemeModule,
+    //Always leave HealthModule last, so it is on the bottom of swagger
+    HealthModule
   ],
 })
 export class AppModule {}

src/authentication/authentication.decorator.ts

@@ -1,56 +0,0 @@
-import { Request } from 'express';
-import { CustomDecorator } from '@nestjs/common';
-import { PermissionsObjectPermission } from './permissions.enum';
-import { RequestUser } from './user.decorator';
-
-export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
-
-export type AuthenticationFunction = (req: Request, user: any) => boolean;
-
-// implementation copied from SetMetadata, but tweaked to get existing values
-// of the metadataKey and accumulate it with the new metadataValue
-function SetMultipleMetadata(metadataKey, metadataValue): CustomDecorator {
-  const decoratorFactory = (target, key, descriptor) => {
-    // .start: tweak
-    // descriptor?.value = function decorator; target = class decorator
-    const accumulatedVal =
-      Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
-    accumulatedVal.push(metadataValue);
-    // .end: tweak
-
-    if (descriptor) {
-      Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
-      return descriptor;
-    }
-
-    Reflect.defineMetadata(metadataKey, accumulatedVal, target);
-    return target;
-  };
-
-  decoratorFactory.KEY = metadataKey;
-  return decoratorFactory as any;
-}
-
-export function RequireAllPermissions(
-  ...permissions: PermissionsObjectPermission[]
-) {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
-    permissions.every(({ seqid }) => user.permissions.includes(seqid)),
-  );
-}
-
-export function RequireSomePermission(
-  ...permissions: PermissionsObjectPermission[]
-) {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
-    permissions.some(({ seqid }) => user.permissions.includes(seqid)),
-  );
-}
-
-export function AuthenticateCondition(func: AuthenticationFunction) {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, func);
-}
-
-export function Authenticated() {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, () => true);
-}

src/authentication/authentication.guard.spec.ts

@@ -11,9 +11,9 @@ import {
   Authenticated,
   RequireAllPermissions,
   RequireSomePermission,
-} from './authentication.decorator';
+} from '../decorators/authentication.decorator';
 import { AuthenticationGuard } from './authentication.guard';
-import { PERMISSIONS } from './permissions.enum';
+import { PERMISSIONS_GROUPS } from './permissions.enum';
 import { AuthClientService } from '../modules/auth/auth.service';
 
 import ErrorCodes from '../utils/errorCodes';
@@ -36,15 +36,15 @@ class NoClassAuthController {
   }
 
   @Get('required-permission')
-  @RequireSomePermission(PERMISSIONS.ZENDESK.permissions.OPEN)
+  @RequireSomePermission(PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN)
   async requiredPermission(@Body() body) {
     return { body };
   }
 
   @Get('has-all-permissions')
   @RequireAllPermissions(
-    PERMISSIONS.ZENDESK.permissions.OPEN,
+    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   )
   async hasAllPermissions(@Body() body) {
     return { body };
@@ -52,8 +52,8 @@ class NoClassAuthController {
 
   @Get('has-some-permission')
   @RequireSomePermission(
-    PERMISSIONS.ZENDESK.permissions.OPEN,
+    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   )
   async hasSomePermission(@Body() body) {
     return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
 
 @Controller('class-auth-condition')
 @AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
-@RequireSomePermission(PERMISSIONS.METABASE.permissions.OPEN)
+@RequireSomePermission(PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE)
 class ClassAuthConditionController {
   @Get('body')
   async getBody(@Body() body) {
@@ -77,7 +77,7 @@ class ClassAuthConditionController {
 
   @Get('required-permission')
   @AuthenticateCondition((req) => req.get('x-on-route') === 'ok')
-  @RequireSomePermission(PERMISSIONS.ZENDESK.permissions.OPEN)
+  @RequireSomePermission(PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN)
   async requiredPermission(@Body() body) {
     return { body };
   }
@@ -440,17 +440,17 @@ describe('authentication.guard', () => {
   NoClassAuthTest(null, null);
   ClassAuthConditionTest(null, null);
 
-  const tokenZ = CreateToken([PERMISSIONS.ZENDESK.permissions.OPEN]);
+  const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
   NoClassAuthTest(tokenZ, ['zendesk']);
   ClassAuthConditionTest(tokenZ, ['zendesk']);
 
-  const tokenM = CreateToken([PERMISSIONS.METABASE.permissions.OPEN]);
+  const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
   NoClassAuthTest(tokenM, ['metabase']);
   ClassAuthConditionTest(tokenM, ['metabase']);
 
   const tokenZM = CreateToken([
-    PERMISSIONS.ZENDESK.permissions.OPEN,
+    PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   ]);
   NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
   ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);

src/authentication/authentication.guard.ts

@@ -13,8 +13,8 @@ import { AuthClientService } from '../modules/auth/auth.service';
 import {
   AuthenticationFunction,
   AUTH_FUNCTION_KEY,
-} from './authentication.decorator';
+} from '../decorators/authentication.decorator';
-import { RequestUser } from './user.decorator';
+import { RequestUser } from '../decorators/user.decorator';
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';
 

src/authentication/permissions.enum.ts

@@ -37,11 +37,12 @@ export interface PermissionsObjectGroup {
 /*export const Permissions: {
   [P in keyof any]: { [Q in keyof any]: Permission };
 } = {*/
-export const PERMISSIONS = {
+export const PERMISSIONS_GROUPS = {
   AUTH: {
     title: {
       'pt-br': 'Autenticação',
       'en-us': 'Authentication',
+      'es-es': 'Authentication',
     },
     permissions: {
       CREATE: {
@@ -51,15 +52,26 @@ export const PERMISSIONS = {
         name: {
           'pt-br': '',
           'en-us': '',
+          'es-es': '',
+        },
+      },
+      GENERATE_TOKEN: {
+        seqid: 46,
+        claim: 'customer:generate-token',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Gerar Token de Acesso',
+          'en-us': 'Generate Acess Token',
+          'es-es': 'Gerar Token de Acceso',
         },
       },
     },
   },
-
   PIPELINE: {
     title: {
       'pt-br': 'Coletar | Pipelines',
       'en-us': 'Collect | Pipelines',
+      'es-es': 'Colecta | Pipelines',
     },
     permissions: {
       CREATE: {
@@ -69,6 +81,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Criar pipelines',
           'en-us': 'Create pipelines',
+          'es-es': 'Crear pipelines',
         },
       },
       GET: {
@@ -78,6 +91,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Buscar pipelines',
           'en-us': 'Get pipelines',
+          'es-es': 'Obtener pipelines',
         },
       },
       UPDATE: {
@@ -87,6 +101,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Atualizar pipeline',
           'en-us': 'Update pipeline',
+          'es-es': 'Actualizar pipeline',
         },
       },
       DELETE: {
@@ -96,69 +111,84 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Excluir pipeline',
           'en-us': 'Delete pipeline',
+          'es-es': 'Eliminar pipeline',
         },
       },
     },
   },
-
+  CONNECTION: {
-  INPUT: {
     title: {
-      'pt-br': 'Coletar | Conexões',
+      'pt-br': 'Coletar | Fontes de dados',
-      'en-us': 'Collect | Connections',
+      'en-us': 'Collect | Sources',
+      'es-es': 'Colecta | Sources',
     },
     permissions: {
       CREATE: {
-        seqid: 21,
+        seqid: 37,
-        claim: 'POST /inputs',
+        claim: 'connection:create',
-        usage: PermissionUsages.INTERNAL,
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Criar conexões',
+          'pt-br': 'Cadastrar Fonte',
-          'en-us': 'Create connections',
+          'en-us': 'Create Source',
-        },
+          'es-es': 'Crear Source',
-      },
-      GET: {
-        seqid: 9,
-        claim: 'GET /inputs',
-        usage: PermissionUsages.INTERNAL,
-        name: {
-          'pt-br': 'Buscar conexões',
-          'en-us': 'Get connections',
-        },
-      },
-      UPDATE: {
-        seqid: 27,
-        claim: 'PUT /inputs',
-        usage: PermissionUsages.INTERNAL,
-        name: {
-          'pt-br': 'Atualizar conexões',
-          'en-us': 'Create connections',
         },
       },
       DELETE: {
-        seqid: 3,
+        seqid: 38,
-        claim: 'DELETE /inputs',
+        claim: 'connection:delete',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Excluir Fonte',
+          'en-us': 'Remove Source',
+          'es-es': 'Eliminar Source',
+        },
+      },
+      UPDATE: {
+        seqid: 39,
+        claim: 'connection:update',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Excluir conexões',
+          'pt-br': 'Editar Fonte',
-          'en-us': 'Delete connections',
+          'en-us': 'Edit Source',
+          'es-es': 'Editar Source',
         },
       },
-      GET_ENTITIES: {
+    },
-        seqid: 10,
+  },
-        claim: 'GET /inputs/available-entities',
+  NETWORK_CONFIG: {
-        usage: PermissionUsages.INTERNAL,
+    title: {
+      'pt-br': 'Coletar | Redes',
+      'en-us': 'Collect | Network',
+      'es-es': 'Colecta | Network',
+    },
+    permissions: {
+      CREATE: {
+        seqid: 35,
+        claim: 'network_config:create',
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Excluir conexões',
+          'pt-br': 'Cadastrar VPN',
-          'en-us': 'Delete connections',
+          'en-us': 'Create VPN',
+          'es-es': 'Crear VPN',
         },
       },
+      // DELETE: {
+      //   seqid: 36,
+      //   claim: 'network_config:delete',
+      //   usage: PermissionUsages.PUBLIC,
+      //   name: {
+      //     'pt-br': 'Excluir VPN',
+      //     'en-us': 'Remove VPN',
+      //     'es-es': 'Eliminar VPN',
+      //   },
+      // },
     },
   },
-
   OUTPUT: {
     title: {
       'pt-br': 'Output',
       'en-us': 'Output',
+      'es-es': 'Output',
     },
     permissions: {
       CREATE: {
@@ -168,6 +198,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Criar outputs',
           'en-us': 'Create outputs',
+          'es-es': 'Crear outputs',
         },
       },
       GET: {
@@ -177,6 +208,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Buscar outputs',
           'en-us': 'Get outputs',
+          'es-es': 'Obtener outputs',
         },
       },
       UPDATE: {
@@ -186,6 +218,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Editar outputs',
           'en-us': 'Edit outputs',
+          'es-es': 'Editar outputs',
         },
       },
       DELETE: {
@@ -195,15 +228,16 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Excluir outputs',
           'en-us': 'Remove outputs',
+          'es-es': 'Eliminar outputs',
         },
       },
     },
   },
-
   TRANSFORMATIONS: {
     title: {
       'pt-br': 'Micro-transformações',
       'en-us': 'Micro-transformations',
+      'es-es': 'Micro-transformaciones',
     },
     permissions: {
       CREATE: {
@@ -213,6 +247,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Criar micro-transformações',
           'en-us': 'Create micro-transformations',
+          'es-es': 'Crear micro-transformaciones',
         },
       },
       GET: {
@@ -222,6 +257,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Buscar micro-transformações',
           'en-us': 'Get micro-transformations',
+          'es-es': 'Obtener micro-transformaciones',
         },
       },
       UPDATE: {
@@ -231,6 +267,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Editar micro-transformações',
           'en-us': 'Edit micro-transformations',
+          'es-es': 'Editar micro-transformaciones',
         },
       },
       DELETE: {
@@ -240,15 +277,16 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Excluir micro-transformações',
           'en-us': 'Remove micro-transformations',
+          'es-es': 'Eliminar micro-transformaciones',
         },
       },
     },
   },
-
   CATALOG: {
     title: {
       'pt-br': 'Explorar | Catálogo',
       'en-us': 'Explore | Catalog',
+      'es-es': 'Explorar | Catalogar',
     },
     permissions: {
       GET: {
@@ -256,44 +294,79 @@ export const PERMISSIONS = {
         claim: 'GET /catalog',
         usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Buscar data assets',
+          'pt-br': 'Acessar lista de Ativos e visualizar atributos',
-          'en-us': 'Search for data assets',
+          'en-us': 'Access Assets list and its attributes',
+          'es-es': 'Acceda a la lista de Activos y vea los atributos',
         },
       },
       CREATE: {
         seqid: 19,
-        claim: 'POST /catalog',
+        claim: 'catalog:create',
         usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Criar atributos no catálogo',
+          'pt-br': 'Criar Ativos',
-          'en-us': 'Create attributes',
+          'en-us': 'Create Assets',
+          'es-es': 'Crear Activos',
         },
       },
       UPDATE: {
         seqid: 25,
-        claim: 'PUT /catalog',
+        claim: 'catalog:edit',
-        usage: PermissionUsages.INTERNAL,
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Editar atributos no catálogo',
+          'pt-br': 'Criar e editar atributos no catálogo',
-          'en-us': 'Edit attributes',
+          'en-us': 'Create and edit attributes on the catalog',
+          'es-es': 'Crear y editar atributos en el catálogo',
         },
       },
       DELETE: {
         seqid: 1,
-        claim: 'DELETE /catalog',
+        claim: 'catalog:delete',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Excluir Ativos do catálogo',
+          'en-us': 'Remove Assets from the catalog',
+          'es-es': 'Eliminar Activos del catálogo',
+        },
+      },
+      DATA_MANAGER: {
+        seqid: 42,
+        claim: 'catalog:data_manager',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Excluir atributos no catálogo',
+          'pt-br':
-          'en-us': 'Remove attributes',
+            'Gerente do catálogo. Consegue vizualizar e editar todos os ativos.',
+          'en-us': 'Catalog manager. Able to view and edit all assets.',
+          'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
+        },
+      },
+      EMBED_ANALYTICS: {
+        seqid: 44,
+        claim: 'catalog:embed',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Acessar Módulo de Incorporação de Ativos',
+          'en-us': 'Access Embedding analytics Module',
+          'es-es': 'Acceder al Módulo de Incorporación de Activos',
+        },
+      },
+      TRIGGER_CATALOG_TASK: {
+        seqid: 45,
+        claim: 'catalog:trigger-task',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Executar a catalogação de um ativo',
+          'en-us': 'Trigger an asset cataloging',
+          'es-es': 'Realizar el listado de un activo',
         },
       },
     },
   },
-
   CONNECTORS: {
     title: {
       'pt-br': 'Conectores',
       'en-us': 'Connectors',
+      'es-es': 'Conectores',
     },
     permissions: {
       CREATE: {
@@ -303,6 +376,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Criar conectores',
           'en-us': 'Create connectors',
+          'es-es': 'Crear conectores',
         },
       },
       GET: {
@@ -312,6 +386,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Buscar conectores',
           'en-us': 'Get connectors',
+          'es-es': 'Obtener conectores',
         },
       },
       UPDATE: {
@@ -321,6 +396,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Editar conectores',
           'en-us': 'Edit connectors',
+          'es-es': 'Editar conectores',
         },
       },
       DELETE: {
@@ -330,15 +406,16 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Excluir conectores',
           'en-us': 'Remove connectors',
+          'es-es': 'Eliminar conectores',
         },
       },
     },
   },
-
   SNOWFLAKE: {
     title: {
       'pt-br': 'Explorar | Consolidar',
       'en-us': 'Explore | Data consolidation',
+      'es-es': 'Explorar | Consolidación de datos',
     },
     permissions: {
       OPEN: {
@@ -348,62 +425,75 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Acessar Snowflake',
           'en-us': 'Access Snowflake',
+          'es-es': 'Acceso Snowflake',
         },
       },
     },
   },
-
   ZENDESK: {
     title: {
       'pt-br': 'Zendesk',
       'en-us': 'Zendesk',
+      'es-es': 'Zendesk',
     },
     permissions: {
       OPEN: {
         seqid: 16,
-        claim: 'GET /zendesk',
+        claim: 'support-chat:access',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Acessar Zendesk',
+          'pt-br': 'Acessar chat de suporte',
-          'en-us': 'Access Zendesk',
+          'en-us': 'Access support chat',
+          'es-es': 'Acceder al chat de soporte',
         },
       },
     },
   },
-
+  DATAVIZ: {
-  METABASE: {
     title: {
       'pt-br': 'Analisar | Visualização',
       'en-us': 'Analyze | Visualization',
+      'es-es': 'Analizar | Visualización',
     },
     permissions: {
-      OPEN: {
+      METABASE: {
         seqid: 11,
         claim: 'GET /metabase',
         usage: PermissionUsages.PUBLIC,
         name: {
           'pt-br': 'Acessar Metabase',
           'en-us': 'Access Metabase',
+          'es-es': 'Acceso Metabase',
         },
       },
     },
   },
-
+  INTELLIGENCE: {
-  ORCHEST: {
     title: {
       'pt-br': 'Analisar | Inteligência',
       'en-us': 'Analyze | Intelligence',
+      'es-es': 'Analizar | Inteligencia',
     },
     permissions: {
       INTELLIGENCE: {
         seqid: 31,
         claim: 'intelligence:open',
-        usage: PermissionUsages.INTERNAL,
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Acessar Orchest',
+          'pt-br': 'Acessar Módulo de Inteligência',
-          'en-us': 'Access Orchest',
+          'en-us': 'Access Intelligence Module',
+          'es-s': 'Acceder Módulo de Inteligencia',
         },
       },
+    },
+  },
+  MODULES: {
+    title: {
+      'pt-br': 'Módulos da Dadosfera',
+      'en-us': 'Dadosfera Modules',
+      'es-es': 'Módulos de la Dadosfera',
+    },
+    permissions: {
       APP_BUILDER: {
         seqid: 32,
         claim: 'app-builder:open',
@@ -411,6 +501,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Acessar App Builder',
           'en-us': 'Access App Builder',
+          'es-es': 'Acceder App Builder',
         },
       },
       MACHINE_LEARNING: {
@@ -420,6 +511,7 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Acessar Machine Learning',
           'en-us': 'Access Machine Learning',
+          'es-es': 'Acceder Machine Learning',
         },
       },
     },
@@ -428,6 +520,7 @@ export const PERMISSIONS = {
     title: {
       'pt-br': 'Usuários',
       'en-us': 'Users',
+      'es-es': 'Usuarios',
     },
     permissions: {
       ADMIN: {
@@ -437,21 +530,117 @@ export const PERMISSIONS = {
         name: {
           'pt-br': 'Gestão de usuários e grupos',
           'en-us': 'Manage user and roles',
+          'es-es': 'Administrar usuarios y roles',
+        },
+      },
+    },
+  },
+  PROCESS: {
+    title: {
+      'pt-br': 'Processar',
+      'en-us': 'Process',
+      'es-es': 'Procesar',
+    },
+    permissions: {
+      TRANSFORMATION: {
+        seqid: 43,
+        claim: 'process:open',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Módulo de Transformação',
+          'en-us': 'Access Transformation Module',
+          'es-es': 'Acceder Módulo de transformación',
+        },
+      },
+    },
+  },
+  DADOSFERA: {
+    title: {
+      'pt-br': 'Usuários',
+      'en-us': 'Users',
+      'es-es': 'Usuarios',
+    },
+    permissions: {
+      CONNECTIONS_MIGRATION: {
+        seqid: 40,
+        claim: 'dadosfera:connections-migration',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'connections-migration',
+          'en-us': 'connections-migration',
+          'es-es': 'connections-migration',
+        },
+      },
+      SYNC_USER: {
+        seqid: 41,
+        claim: 'dadosfera:sync-user',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'sync-user',
+          'en-us': 'sync-user',
+          'es-es': 'sync-user',
+        },
+      },
+    },
+  },
+  CUSTOMER: {
+    title: {
+      'pt-br': 'Organização',
+      'en-us': 'Organization',
+      'es-es': 'Organización',
+    },
+    permissions: {
+      MONITORING_DASHBOARD: {
+        seqid: 47,
+        claim: 'customer:monitoring-dashboard',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Ver o dashboard de monitoramento',
+          'en-us': 'View the monitoring dashboard',
+          'es-es': 'Ver el dashboard de monitoreo',
         },
       },
     },
   },
 };
-
+export interface DadosferaModule {
+  name: string;
+  description: string;
+  key: string;
+  permissionSeqId: number;
+}
+export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+  {
+    name: 'Intelligence Module',
+    description: 'Orchest Module',
+    key: 'intelligence',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid,
+  },
+  {
+    name: 'Proccessing Module',
+    description: 'Proccessing Module',
+    key: 'process',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid,
+  },
+  {
+    name: 'Embedded Analytics',
+    description: 'Embedded Analytics Module',
+    key: 'embedded-analytics',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid,
+  },
+];
 // traverses the object searching for duplicate seqids or claims (executes at runtime)
 let nextAvailableSeqid = 0;
-const seqids = Object.values(PERMISSIONS).flatMap((namespace) =>
+const seqids = Object.values(PERMISSIONS_GROUPS).flatMap((namespace) =>
   Object.values(namespace.permissions).map(({ seqid }) => seqid),
 );
-const claims = Object.values(PERMISSIONS).flatMap((namespace) =>
+const claims = Object.values(PERMISSIONS_GROUPS).flatMap((namespace) =>
   Object.values(namespace.permissions).map(({ claim }) => claim),
 );
-Object.values(PERMISSIONS).map((namespace) =>
+Object.values(PERMISSIONS_GROUPS).map((namespace) =>
   Object.values(namespace.permissions).map(({ seqid, claim }) => {
     const seqidsCount = seqids.filter((x) => x === seqid).length;
     assert(seqidsCount === 1, `seqid ${seqid} count is not 1`);
@@ -462,4 +651,9 @@ Object.values(PERMISSIONS).map((namespace) =>
   }),
 );
 
+DADOSFERA_MODULES.map((m) => m.key).forEach((m, i, arr) => {
+  if (arr.indexOf(m) !== i)
+    throw new Error(`DADOSFERA_MODULES[${i}] does not have a unique key`);
+});
+
 logger.log(`next available seqid ${nextAvailableSeqid + 1}`);

src/clients/duc/client.config.ts

@@ -1,26 +0,0 @@
-import { credentials } from '@grpc/grpc-js';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import { ProtoPackages, ProtoPaths } from '@dadosfera/protospack-v2/dist/lib/Duc';
-
-export class DucClient {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.DUC_URL,
-        package: ProtoPackages.WritePackage,
-        // credentials: credentials.createSsl(),
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
-        protoPath: ProtoPaths.WriteFilePath,
-        loader: {
-          keepCase: true,
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/inputs/client.config.ts

@@ -1,25 +0,0 @@
-import { credentials } from '@grpc/grpc-js';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-
-import { InputPackages, InputProtoFilePath } from 'protospack';
-
-export class InputsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.INFACTORY_URL,
-        package: InputPackages,
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
-        protoPath: InputProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/inputs/client.service.ts

@@ -1,310 +0,0 @@
-import {
-  OnModuleInit,
-  Inject,
-  HttpException,
-  HttpStatus,
-  NotFoundException,
-  InternalServerErrorException,
-} from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import {
-  DecodeGrpcStruct,
-  EncodeJsonToGrpcStruct,
-  InputCreateS3Request,
-  InputNewCreateRequest,
-  InputService,
-  TestConnectionGetColumnsRequest,
-  TestConnectionRequest,
-} from 'protospack';
-import axios, { AxiosRequestConfig } from 'axios';
-import { lastValueFrom } from 'rxjs';
-import { InputModel } from 'src/modules/inputs/dtos/input.model';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import { mustache } from 'src/utils/mustache';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { OauthSecrets } from 'src/utils/OauthSecrets';
-import { URLSearchParams } from 'url';
-import { IIdRequest, UpdateInputRequest } from './interfaces';
-
-export class InputsClientService implements OnModuleInit {
-  private inputService: InputService;
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-    @Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  async onModuleInit() {
-    this.inputService =
-      this.grpcClient.getService<InputService>('InputService');
-  }
-
-  async newCreate(createInputDto: InputNewCreateRequest) {
-    this.logger.info('InputClientService - Create');
-
-    const input = await lastValueFrom(
-      this.inputService.NewCreate(objectSnakeToCamel(createInputDto)),
-    );
-    objectCamelToSnake(input);
-    return input;
-  }
-
-  async createGeneric(createInputGeneric) {
-    this.logger.info('InputClientService - Create');
-    const { info, ...data } = createInputGeneric;
-    // data.credentials = await this.getInputTokens(data);
-    if (data.options) delete data.options;
-    if (data.credentials.oauth_code) delete data.credentials.oauth_code;
-    const grpcPayload = {
-      input: EncodeJsonToGrpcStruct(data),
-      info,
-    };
-    objectSnakeToCamel(grpcPayload);
-
-    const structReturn = await lastValueFrom(
-      this.inputService.Create(grpcPayload),
-    ).catch((err: { details: string }) => {
-      if (err.details === 'Item Not found!') {
-        throw new NotFoundException('Input not found');
-      }
-      throw new InternalServerErrorException(err.details);
-    });
-    objectCamelToSnake(structReturn);
-    const inputCreated = DecodeGrpcStruct(structReturn.input);
-    return { input: inputCreated };
-  }
-
-  async createS3Inputs(createInputDto: InputCreateS3Request) {
-    this.logger.info('InputClientService - Create');
-    objectSnakeToCamel(createInputDto);
-    const createInputResponse = await lastValueFrom(
-      this.inputService.CreateS3(createInputDto),
-    ).catch((e) => {
-      throw new HttpException(e.details, 500);
-    });
-    this.logger.info('Done');
-    objectCamelToSnake(createInputResponse);
-    return createInputResponse;
-  }
-
-  async findOne(data: IIdRequest) {
-    this.logger.info('InputClientService - FindOne');
-    const findOneInputResponse = await lastValueFrom(
-      this.inputService.FindOne(objectSnakeToCamel(data)),
-    ).catch((e) => {
-      this.logger.error(e.details);
-      throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
-    });
-    objectCamelToSnake(findOneInputResponse);
-    return findOneInputResponse;
-  }
-
-  async findAll(data) {
-    this.logger.info('InputClientService - FindAll');
-    const findAllInputResponse = await lastValueFrom(
-      this.inputService.FindAll(objectSnakeToCamel(data)),
-    ).catch((e) => {
-      this.logger.error(e.details);
-      throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
-    });
-    findAllInputResponse.inputs.forEach((input) => objectCamelToSnake(input));
-
-    return findAllInputResponse;
-  }
-
-  async update(updateInputDTO: UpdateInputRequest) {
-    this.logger.info('InputClientService - Update');
-    const updateInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Update(objectSnakeToCamel(updateInputDTO)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return updateInputResponse;
-  }
-
-  async remove(idRequest: IIdRequest) {
-    this.logger.info('InputClientService - Remove');
-
-    const removeInputResponse = await new Promise((resolve, reject) => {
-      this.inputService.Remove(objectSnakeToCamel(idRequest)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return removeInputResponse;
-  }
-
-  async testConnection(data: TestConnectionRequest) {
-    this.logger.info('InputClientService - TestConnection');
-    const testConnectionResponse = await new Promise((resolve, reject) => {
-      this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return testConnectionResponse;
-  }
-
-  async getColumns(data: TestConnectionGetColumnsRequest) {
-    this.logger.info('InputClientService - TestConnection/Get-Columns');
-    const getColumnsResponse = await new Promise((resolve, reject) => {
-      this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return getColumnsResponse;
-  }
-  async getAuthSecrets(plugin) {
-    switch (plugin) {
-      case 'hubspot':
-        return {
-          client_id: this.oauthSecrets.hubspot.client_id || '',
-          client_secret: this.oauthSecrets.hubspot.client_secret || '',
-        };
-    }
-    return {};
-  }
-  async getInputTokens(input: InputModel) {
-    const { credentials } = input;
-    let credentialsTokens = {};
-    switch (credentials.connection_type) {
-      case 'oauth':
-        credentialsTokens = await this.getOauthTokens(input);
-        break;
-    }
-    return { ...credentials, ...credentialsTokens };
-  }
-  async getOauthTokens(input: InputModel) {
-    const tokens: Record<string, any> = {};
-    const { options, credentials } = input;
-    const { oauth } = options;
-    const { get_tokens_url_params, get_tokens_set_response } = oauth;
-    const secrets = await this.getAuthSecrets(input.plugin);
-    const get_tokens_url_params_string = mustache(get_tokens_url_params, {
-      ...secrets,
-      ...credentials,
-    });
-    let get_tokens_url_params_obj: Record<string, any> = {};
-    try {
-      get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
-    } catch (error) {
-      throw new HttpException('Erro transformando get_tokens_url_params', 400);
-    }
-    let params = {};
-    switch (oauth.content_type) {
-      case 'application/x-www-form-urlencoded':
-        params = new URLSearchParams(get_tokens_url_params_obj);
-        params = params.toString();
-        break;
-      default:
-        break;
-    }
-    const axiosRequestConfig: AxiosRequestConfig = {
-      url: oauth.get_tokens_url,
-      method: oauth.get_tokens_method || 'POST',
-      data: params,
-      headers: { 'content-type': oauth.content_type || 'application/json' },
-    };
-
-    const { data } = await axios(axiosRequestConfig).catch((err) => {
-      this.logger.error(err.response.data);
-      throw new HttpException(err.response.data.message, err.response.status);
-    });
-    for (const key in get_tokens_set_response) {
-      const responseKey = get_tokens_set_response[key];
-      tokens[key] = data[responseKey];
-    }
-    const { redirect_uri } = get_tokens_url_params_obj;
-    tokens.redirect_uri = redirect_uri;
-    return tokens;
-  }
-
-  async getAvailableEntities(data) {
-    this.logger.info('InputClientService - GetAvailableEntities');
-    objectSnakeToCamel(data);
-    const response = await lastValueFrom(
-      this.inputService.GetAvailableEntities(data),
-    ).catch((err) => {
-      this.logger.info(err.details);
-      if (err.details && err.details.includes('400'))
-        throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
-      throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
-    });
-    objectCamelToSnake(response);
-
-    return response;
-  }
-}

src/clients/outputs/client.config.ts

@@ -1,24 +0,0 @@
-import { OutputPackages, OutputProtoFilePath } from 'protospack';
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import { credentials } from '@grpc/grpc-js';
-
-export class OutputsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.OTFACTORY_URL,
-        package: OutputPackages,
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
-        protoPath: OutputProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/outputs/client.service.ts

@@ -1,178 +0,0 @@
-import { Controller, Inject, OnModuleInit } from '@nestjs/common';
-import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { OutputServicesNames, OutputsServiceInterface } from 'protospack';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import {
-  ICreateOutputRequest,
-  IIdRequest,
-  IOutputUpdateRequest,
-} from './interfaces';
-
-@Controller('output')
-export class OutputsClientService implements OnModuleInit {
-  private outputService: OutputsServiceInterface;
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject('OUTPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
-      OutputServicesNames.OutputService,
-    );
-  }
-
-  async create(@Payload() createOutputDto: ICreateOutputRequest) {
-    this.logger.info('OutputClientService - Create');
-
-    const createOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Create(objectSnakeToCamel(createOutputDto)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    const convertedItem = objectCamelToSnake(createOutputResponse['output']);
-    createOutputResponse['output'] = convertedItem;
-    return createOutputResponse;
-  }
-
-  async findAll(data) {
-    this.logger.info('OutputClientService - FindAll');
-
-    const findAllOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    const convertedOutputs = findAllOutputResponse['outputs'].map((ot) => {
-      return objectCamelToSnake(ot);
-    });
-
-    return { outputs: convertedOutputs };
-  }
-
-  async findOne(data: IIdRequest) {
-    this.logger.info('OutputClientService - FindOne');
-
-    const findOneOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(findOneOutputResponse);
-  }
-
-  async update(updateOutPutDTO: IOutputUpdateRequest) {
-    this.logger.info('OutputClientService - Update');
-
-    const updateOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Update(objectSnakeToCamel(updateOutPutDTO)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(updateOutputResponse);
-  }
-
-  async remove(data: IIdRequest) {
-    this.logger.info('OutputClientService - Remove');
-
-    const removeOutputResponse = await new Promise((resolve, reject) => {
-      this.outputService.Remove(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return removeOutputResponse;
-  }
-}

src/clients/outputs/interfaces.d.ts

@@ -1,29 +0,0 @@
-export interface ICreateOutputRequest {
-  operation: string;
-  name: string;
-  plugin: string;
-  values: Values;
-}
-
-export interface Values {
-  prefix: string;
-}
-
-interface Info {
-  user_id: string;
-  customer_id: string;
-}
-
-export interface IIdRequest {
-  id: string;
-  info: Info;
-}
-
-export interface IOutputUpdateRequest {
-  id: string;
-  operation: string;
-  values: Values;
-  plugin: string;
-  name: string;
-  info: Info;
-}

src/clients/pipelines/client.config.ts

@@ -1,24 +0,0 @@
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import { PipelinePackages, PipelineProtoFilePath } from 'protospack';
-import { credentials } from '@grpc/grpc-js';
-
-export class PipelinesClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.PIFACTORY_URL,
-        package: PipelinePackages,
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
-        protoPath: PipelineProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/pipelines/client.service.ts

@@ -1,273 +0,0 @@
-import {
-  BadRequestException,
-  Inject,
-  InternalServerErrorException,
-  OnModuleInit,
-} from '@nestjs/common';
-import { ClientGrpc, Payload } from '@nestjs/microservices';
-import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack';
-import { lastValueFrom } from 'rxjs';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import {
-  ICreatePipelineDto,
-  IGetPipelineLogsRequest,
-  IIdRequest,
-  IUpdatePipelineRequest,
-} from './interfaces';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-export class PipelinesClientService implements OnModuleInit {
-  private pipelineService: PipelinesServiceInterface;
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject('PIPELINES_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.pipelineService =
-      this.grpcClient.getService<PipelinesServiceInterface>(
-        PipelineServicesNames.PipelineService,
-      );
-  }
-
-  async create(@Payload() createPipelineDto: ICreatePipelineDto) {
-    this.logger.info('PipelinesClientService - Create');
-    const createPipelineResponse = await lastValueFrom(
-      this.pipelineService.Create(objectSnakeToCamel(createPipelineDto)),
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((error: { details: string }) => {
-        if (error.details.includes('INVALID_REQUEST')) {
-          // eslint-disable-next-line @typescript-eslint/no-unused-vars
-          const [errorType, message] = error.details.split('|');
-          throw new BadRequestException(message);
-        }
-        throw new InternalServerErrorException(error.details);
-      });
-
-    return createPipelineResponse;
-  }
-
-  async findAll(data: IIdRequest) {
-    this.logger.info('PipelinesClientService - FindAll');
-    const findAllPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x.pipelines);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return findAllPipelineResponse;
-  }
-
-  async findOne(data: IIdRequest) {
-    this.logger.info('PipelinesClientService - FindOne');
-
-    const findOnePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.FindOne(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // console.log('done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return findOnePipelineResponse;
-  }
-
-  async update(UpdatePipelineRequest: IUpdatePipelineRequest) {
-    this.logger.info('PipelinesClientService - Update');
-
-    const updatePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService
-        .Update(objectSnakeToCamel(UpdatePipelineRequest))
-        .subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            //console.log('done');
-          },
-        });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return updatePipelineResponse;
-  }
-
-  async remove(data: IIdRequest) {
-    this.logger.info('PipelinesClientService - Remove');
-
-    const removePipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.remove(objectSnakeToCamel(data)).subscribe({
-        next(x) {
-          resolve(objectCamelToSnake(x));
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // this.logger.info('Done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return removePipelineResponse;
-  }
-
-  async getPipelineLogsMessages(data: IGetPipelineLogsRequest) {
-    this.logger.info('PipelinesClientService - GetPipelineLogsMessages');
-
-    const logsPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService.getPipelineLogs(data).subscribe({
-        next(x) {
-          resolve(x);
-        },
-        error(err) {
-          reject(err);
-        },
-        complete() {
-          // this.logger.info('Done');
-        },
-      });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return logsPipelineResponse;
-  }
-
-  async getPipelineStatus(data) {
-    this.logger.info('PipelinesClientService - GetPipelineStatus');
-
-    const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService
-        .getPipelineStatus(objectSnakeToCamel(data))
-        .subscribe({
-          next(x) {
-            const statusArray = x.status.sort((a, b) => {
-              if (a.id < b.id) {
-                return 1;
-              } else {
-                return -1;
-              }
-            });
-            resolve({ status: statusArray });
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // this.logger.info('Done');
-          },
-        });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return statusPipelineResponse;
-  }
-
-  async runPipeline({ id, info }: IIdRequest) {
-    this.logger.info('PipelinesClientService - RunPipeline');
-    const statusPipelineResponse = await new Promise((resolve, reject) => {
-      this.pipelineService
-        .triggerPipeline(objectSnakeToCamel({ id, info }))
-        .subscribe({
-          next(x) {
-            if (x.status == false) {
-              reject(
-                'This pipeline is not ready yet to execute, Try again later!',
-              );
-            }
-            resolve(x);
-          },
-          error(err) {
-            this.logger.error('Observable Error');
-            reject(err);
-          },
-          complete() {
-            //this.logger.info('Done');
-          },
-        });
-    })
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return statusPipelineResponse;
-  }
-}

src/clients/transformations/client.config.ts

@@ -1,25 +0,0 @@
-import { ClientOptions, Transport } from '@nestjs/microservices';
-import {
-  TransformationPackages,
-  TransformationProtoFilePath,
-} from 'protospack';
-import { credentials } from '@grpc/grpc-js';
-
-export class TransformationsClientConfiguration {
-  config(): ClientOptions {
-    return {
-      transport: Transport.GRPC,
-      options: {
-        url: process.env.INFACTORY_URL,
-        package: TransformationPackages,
-        credentials: credentials.createSsl(),
-        protoPath: TransformationProtoFilePath,
-        loader: {
-          enums: String,
-          objects: true,
-          arrays: true,
-        },
-      },
-    };
-  }
-}

src/clients/transformations/client.service.ts

@@ -1,194 +0,0 @@
-import { Controller, Inject, OnModuleInit, Post } from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import { TransformationsServiceInterface } from 'protospack';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import {
-  objectCamelToSnake,
-  objectSnakeToCamel,
-} from 'src/utils/CaseConverter';
-import { ICreateTransformationsRequest, IIdRequest } from './interfaces';
-
-@Controller('transformation')
-export class TransformationsClientService implements OnModuleInit {
-  private transformationService: TransformationsServiceInterface;
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    @Inject('TRANSFORMATIONS_PACKAGE') private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.transformationService =
-      this.grpcClient.getService<TransformationsServiceInterface>(
-        'TransformationService',
-      );
-  }
-
-  @Post()
-  async create(createTransformationsDto: ICreateTransformationsRequest) {
-    this.logger.info('TransformationClientService - Create');
-
-    const createTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService
-          .Create(objectSnakeToCamel(createTransformationsDto))
-          .subscribe({
-            next(x) {
-              resolve(x);
-            },
-            error(err) {
-              reject(err);
-            },
-            complete() {
-              // console.log('done');
-            },
-          });
-      },
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    const convertedTransforms = createTransformationResponse[
-      'transformations'
-    ].map((tr) => {
-      return objectCamelToSnake(tr);
-    });
-
-    return { transformations: convertedTransforms };
-  }
-
-  async findAll(data) {
-    this.logger.info('TransformationClientService - FindAll');
-
-    const findAllTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService.FindAll(objectSnakeToCamel(data)).subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // console.log('done');
-          },
-        });
-      },
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-    const convertedTransforms = findAllTransformationResponse[
-      'transformations'
-    ].map((tr) => {
-      return objectCamelToSnake(tr);
-    });
-    return { transformations: convertedTransforms };
-  }
-
-  async findOne(data: IIdRequest) {
-    this.logger.info('TransformationClientService - FindOne');
-
-    const findOneTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService.FindOne(objectSnakeToCamel(data)).subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // console.log('done');
-          },
-        });
-      },
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(findOneTransformationResponse);
-  }
-
-  async update(updateTransformationDTO) {
-    this.logger.info('TransformationClientService - Update');
-
-    const updateTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService
-          .Update(objectSnakeToCamel(updateTransformationDTO))
-          .subscribe({
-            next(x) {
-              resolve(x);
-            },
-            error(err) {
-              reject(err);
-            },
-            complete() {
-              // console.log('done');
-            },
-          });
-      },
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(updateTransformationResponse);
-  }
-
-  async remove(data: IIdRequest) {
-    this.logger.info('TransformationClientService - Remove');
-
-    const removeTransformationResponse = await new Promise(
-      (resolve, reject) => {
-        this.transformationService.Remove(objectSnakeToCamel(data)).subscribe({
-          next(x) {
-            resolve(x);
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // console.log('done');
-          },
-        });
-      },
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-
-    return objectCamelToSnake(removeTransformationResponse);
-  }
-}

src/decorators/authentication.decorator.ts

@@ -0,0 +1,41 @@
+import { applyDecorators } from '@nestjs/common';
+import { ApiSecurity } from '@nestjs/swagger';
+import { type Request } from 'express';
+import { type PermissionsObjectPermission } from '../authentication/permissions.enum';
+import { type RequestUser } from './user.decorator';
+import { SetMultipleMetadata } from './shared';
+
+export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
+
+export type AuthenticationFunction = (req: Request, user: any) => boolean;
+
+export function RequireAllPermissions(
+  ...permissions: PermissionsObjectPermission[]
+) {
+  return createAuthenticatedDecorator((req, user: RequestUser) =>
+    permissions.every(({ seqid }) => user.permissions.includes(seqid)),
+  );
+}
+
+export function RequireSomePermission(
+  ...permissions: PermissionsObjectPermission[]
+) {
+  return createAuthenticatedDecorator((req, user: RequestUser) =>
+    permissions.some(({ seqid }) => user.permissions.includes(seqid)),
+  );
+}
+
+export function AuthenticateCondition(func: AuthenticationFunction) {
+  return createAuthenticatedDecorator(func);
+}
+
+export function Authenticated() {
+  return createAuthenticatedDecorator(() => true);
+}
+
+function createAuthenticatedDecorator(metadataValue: AuthenticationFunction) {
+  return applyDecorators(
+    ApiSecurity('access-token'),
+    SetMultipleMetadata(AUTH_FUNCTION_KEY, metadataValue),
+  );
+}

src/decorators/language.decorator.ts

@@ -0,0 +1,25 @@
+import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
+import { LanguageEnum } from 'src/utils/languages.enum';
+
+export const Language: () => ParameterDecorator = createParamDecorator(
+  (options: any, ctx: ExecutionContext): LanguageEnum => {
+    const headers = ctx.switchToHttp()?.getRequest()?.headers;
+
+    if (!headers) return LanguageEnum.ptbr;
+
+    let language: LanguageEnum =
+      headers['dadosfera-lang'] ?? headers['accept-language'];
+    language = language?.toLowerCase().trim() as LanguageEnum;
+
+    if (!language) return LanguageEnum.ptbr;
+
+    Object.values(LanguageEnum).some((l) => {
+      if (language === l || l.includes(language)) {
+        language = l;
+        return true;
+      }
+      language = LanguageEnum.enus;
+    });
+    return language;
+  },
+);

src/decorators/set-origin.decorator.ts

@@ -0,0 +1,32 @@
+import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
+import { SetMetadata } from '@nestjs/common';
+
+export const SetOrigin = (origin: string) => SetMetadata('allowedOrigin', origin);
+
+@Injectable()
+export class CORSGuard implements CanActivate {
+  constructor(private reflector: Reflector) {}
+
+  canActivate(context: ExecutionContext): boolean {
+    const response = context.switchToHttp().getResponse();
+    const request = context.switchToHttp().getRequest();
+    const origin = request.headers.origin;
+
+    // Obter a origem permitida através do decorador
+    const allowedOrigin = this.reflector.get<string>('allowedOrigin', context.getHandler());
+    if (process.env.ENV === "local") {
+        return true;
+    }
+
+    // Verifica se a origem da requisição é permitida   
+    if (allowedOrigin && origin !== allowedOrigin) {
+      throw new ForbiddenException('Acesso não permitido pela política CORS');
+    }
+
+    response.setHeader('Access-Control-Allow-Origin', allowedOrigin);
+
+    return true;
+  }
+}
+

src/decorators/shared.ts

@@ -0,0 +1,28 @@
+import { type CustomDecorator } from '@nestjs/common';
+
+// implementation copied from SetMetadata, but tweaked to get existing values
+// of the metadataKey and accumulate it with the new metadataValue
+export function SetMultipleMetadata(
+  metadataKey,
+  metadataValue,
+): CustomDecorator {
+  const decoratorFactory: CustomDecorator = (target, key?, descriptor?) => {
+    // .start: tweak
+    // descriptor?.value = function decorator; target = class decorator
+    const accumulatedVal =
+      Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
+    accumulatedVal.push(metadataValue);
+    // .end: tweak
+
+    if (descriptor) {
+      Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
+      return descriptor;
+    }
+
+    Reflect.defineMetadata(metadataKey, accumulatedVal, target);
+    return target;
+  };
+
+  decoratorFactory.KEY = metadataKey;
+  return decoratorFactory;
+}

src/decorators/swagger.decorator.ts

@@ -0,0 +1,13 @@
+import { applyDecorators } from '@nestjs/common';
+import { ApiExcludeController, ApiExcludeEndpoint } from '@nestjs/swagger';
+
+export function ApiInternalOnlyEndpoint() {
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    return applyDecorators(ApiExcludeEndpoint());
+  return () => null;
+}
+export function ApiInternalOnlyController() {
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    return applyDecorators(ApiExcludeController());
+  return () => null;
+}

src/decorators/user.decorator.spec.ts

@@ -1,17 +1,14 @@
-import request from 'supertest';
-import { Test } from '@nestjs/testing';
-import { HttpStatus, INestApplication } from '@nestjs/common';
-import { APP_GUARD } from '@nestjs/core';
-import jwt from 'jsonwebtoken';
-
-import { Controller, Get } from '@nestjs/common';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { AuthenticationGuard } from './authentication.guard';
+import { Controller, Get, HttpStatus, INestApplication } from '@nestjs/common';
+import { APP_GUARD } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+import jwt from 'jsonwebtoken';
+import request from 'supertest';
+import { AuthenticationGuard } from '../authentication/authentication.guard';
+import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
 import { AuthClientService } from '../modules/auth/auth.service';
-
 import ErrorCodes from '../utils/errorCodes';
 import { User } from './user.decorator';
-import { PERMISSIONS } from './permissions.enum';
 
 const logger = {
   info: (...args) => args,
@@ -49,7 +46,7 @@ describe('user.decorator', () => {
   const fakeUserPayload = {
     user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
     username: 'super.admin',
-    permissions: [PERMISSIONS.METABASE.permissions.OPEN].map(
+    permissions: [PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE].map(
       ({ seqid }) => seqid,
     ),
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',

src/decorators/user.decorator.ts

@@ -1,4 +1,4 @@
-import { createParamDecorator, ExecutionContext } from '@nestjs/common';
+import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
 
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';
@@ -20,6 +20,5 @@ export const User: (options?: { required?: boolean }) => ParameterDecorator =
     if (!request.user && options?.required) {
       throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
     }
-
     return request.user;
   });

src/error/grpc-to-http-exception.filter.ts

@@ -1,16 +1,31 @@
-import { ExceptionFilter, Catch, ArgumentsHost } from '@nestjs/common';
+import {
+  ExceptionFilter,
+  Catch,
+  ArgumentsHost,
+  HttpException,
+} from '@nestjs/common';
 import { Response } from 'express';
 
 import ErrorBuilder from '../utils/ErrorBuilder';
 
+/**
+ * ExceptionFilter to handle RpcException thrown by our microservices
+ *
+ * If it detects that the exception is already an **HttpException** or it is generated by our **ErrorBuilder**,
+ * it will return the exception without any treatment.
+ */
 @Catch(Error)
 export class GrpcToHttpExceptionFilter implements ExceptionFilter {
   catch(exception: any, host: ArgumentsHost) {
     const ctx = host.switchToHttp();
     const response = ctx.getResponse<Response>();
 
-    // return this exception directly if is already of type ErrorBuilder,
+    if (exception instanceof HttpException) {
-    // else transform it using the ErrorBuilder
+      return response
+        .status(exception.getStatus())
+        .json(exception.getResponse());
+    }
+
     const err =
       exception.constructor.name === ErrorBuilder.name
         ? exception

src/main.ts

@@ -4,9 +4,10 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import helmet from 'helmet';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
-import documentEmpty from '../swagger_empty.json';
-
 import { AppModule } from './app.module';
+import { writeFileSync } from 'fs';
+import { execSync } from 'child_process';
+import { INestApplication } from '@nestjs/common';
 async function bootstrap() {
   DadosferaLogger.setupLogger({
     serviceName: 'maestro',
@@ -23,29 +24,60 @@ async function bootstrap() {
       optionsSuccessStatus: 204,
     },
   });
-
   app.use(helmet());
+  configureSwagger(app);
+  await app.listen(3333);
+  if (process.env.KILL_AFTER_START) await app.close();
+}
 
-  const config = new DocumentBuilder()
+function configureSwagger(app: INestApplication) {
-    .setTitle('Maestro')
+  let branchName = '';
+  try {
+    branchName = execSync('git rev-parse --abbrev-ref HEAD')
+      .toString()
+      .replace('\n', '')
+      .trim();
+  } catch (error) {
+    console.warn("Can't get branch name", error);
+  }
+  const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
+
+  const swaggerConfig = new DocumentBuilder()
+    .setTitle(swaggerTitle)
     .setDescription('Documentation for Maestro gateway')
-    .setVersion('1.0')
+    .addSecurity('access-token', {
-    .addBearerAuth()
+      type: 'apiKey',
-    .addServer(
+      name: 'Authorization',
-      process.env.ENV === 'local'
+      in: 'header',
-        ? 'http://127.0.0.1:3333'
+    })
-        : 'https://maestro.dev.dadosfera.ai',
+    .setDescription('This is the Maestro API');
-    )
+
-    .build();
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
+
+  const config = swaggerConfig.build();
 
   const document = SwaggerModule.createDocument(app, config);
 
-  if (process.env.ENV != 'stg' && process.env.ENV != 'prd')
+  const swaggerJsonFile =
-    SwaggerModule.setup('api', app, document);
+    process.env.INTERNAL_SWAGGER === 'true'
+      ? 'docsfera.json'
+      : 'docsfera.external.json';
 
-  const swaggerDoc = process.env.ENV === 'dev' ? document : documentEmpty;
+  if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
-  SwaggerModule.setup('api', app, swaggerDoc);
 
-  await app.listen(3333);
+  writeFileSync(
+    swaggerJsonFile,
+    JSON.stringify(
+      {
+        service: '${DOCS_API_TOKEN}',
+        pageId: '${DOCS_PAGE_ID}',
+        blockId: '${DOCS_BLOCK_ID}',
+        schema: document,
+      },
+      null,
+      2,
+    ),
+  );
 }
 bootstrap();

src/modules/auth/README.md

@@ -0,0 +1,38 @@
+# Auth
+
+## SSO/oAuth
+
+### Strategy
+We are using [PassportJs](https://www.passportjs.org/) to handle oAuth authentications.
+
+When the client (front end) makes a `GET /auth/oauth/{strategy}` Passport automatically redirects the user to the `strategy` login page. To do that we must configure and use a **Passport Strategy**. We must also have a callback route, conventionally `GET /auth/oauth/{strategy}/callback`, so the oAuth app can report the status of the user's login.
+
+- If the oAuth is successfull we call DUC's `AuthOauthSignIn` request that gets the tokens from Cognito and saves them on cache temporarily under a key we call `session`. Duc returns that `session` to maestro which then redirects the user to our app login page with that `session` as a query param.
+
+- If the oAuth login is not successfull for some reason or the user **does not** exist on DUC's database we redirect the user to our login page with an `error` and `error_description` as query params.
+
+### Routes
+
+So in order to have an SSO login, besides configuring the Strategy, we must have two routes for each Strategy, like in the example below:
+
+```ts
+  @Get('oauth/google')
+  @UseGuards(AuthGuard('google-login'))
+  googleOauth() {
+    this.logger.info('/oauth/google');
+    return true;
+  }
+
+  @Get('oauth/google/callback')
+  @UseGuards(AuthGuard('google-login'))
+  @Redirect()
+  async googleOauthCallback(@Req() req) {
+    const { url, email, token, language = 'pt-br' } = await this.callback(req);
+    if (url.searchParams.get('error')) {
+      this.logger.error('/oauth/google - ERROR');
+      return { url: url.href };
+    }
+    // ... Rest of the logic
+    return { url: url.href };
+  }
+```

src/modules/auth/auth.controller.ts

@@ -7,11 +7,19 @@ import {
   HttpStatus,
   Inject,
   UseFilters,
+  Get,
+  UseGuards,
+  Redirect,
+  Req,
+  Param,
 } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthSignInRequest,
+  ApiHeaders,
-  AuthRefreshAccessTokenRequest,
+  ApiOkResponse,
+  ApiSecurity,
+  ApiTags,
+} from '@nestjs/swagger';
+import {
   AuthChangePasswordRequest,
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
@@ -21,18 +29,37 @@ import {
   AuthVerifyTotpMfaRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
-import { PERMISSIONS } from 'src/authentication/permissions.enum';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
 import { AuthClientService } from './auth.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import {
+  AuthRefreshAccessTokenReq,
+  AuthRefreshAccessTokenRes,
+  AuthSignInReq,
+  AuthSignInRes,
+} from './dtos/login';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { AuthGuard } from '@nestjs/passport';
+import { Request } from 'express';
+import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
 @ApiTags('Auth')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('auth')
 export class AuthController {
   logger: DadosferaLogger;
+  redirectUrl: string;
 
   constructor(
     @Inject(DadosferaLogger)
@@ -40,10 +67,51 @@ export class AuthController {
     private authClient: AuthClientService,
   ) {
     this.logger = dadosferaLogger.logger;
+
+    switch (process.env.ENV) {
+      case 'stg':
+        this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/auth/login`;
+        break;
+      case 'prd':
+        this.redirectUrl = `https://app.dadosfera.ai/auth/login`;
+        break;
+      default:
+        this.redirectUrl = `http://localhost:4200/auth/login`;
+    }
   }
 
+  @Post('sign-in')
+  @HttpCode(HttpStatus.OK)
+  async signIn(
+    @Body() { username, password, totp }: AuthSignInReq,
+    @Language() language: LanguageEnum,
+  ): Promise<AuthSignInRes> {
+    this.logger.info('/auth - SignIn');
+    const metadata = PackTheMetadata({ language });
+    return this.authClient.signIn({ username, password, totp }, metadata);
+  }
+
+  @Post('refresh-access-token')
+  @HttpCode(HttpStatus.OK)
+  @ApiOkResponse({ type: AuthRefreshAccessTokenRes })
+  async refreshAccessToken(
+    @Body() body: AuthRefreshAccessTokenReq,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('/auth - RefreshAccessToken');
+    const { refreshToken, customerName: customer_name } = body;
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      language,
+    });
+
+    return this.authClient.refreshAccessToken({ refreshToken }, metadata);
+  }
+
+  @ApiInternalOnlyEndpoint()
   @Post('/sso/snowflake')
-  @RequireAllPermissions(PERMISSIONS.SNOWFLAKE.permissions.OPEN)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.SNOWFLAKE.permissions.OPEN)
   @HttpCode(HttpStatus.OK)
   async snowflakeSignIn(
     @User() user: RequestUser,
@@ -57,24 +125,7 @@ export class AuthController {
     });
   }
 
-  @Post('sign-in')
+  @ApiInternalOnlyEndpoint()
-  @HttpCode(HttpStatus.OK)
-  async signIn(@Body() { username, password, totp }: AuthSignInRequest) {
-    this.logger.info('/auth - SignIn');
-
-    return this.authClient.signIn({ username, password, totp });
-  }
-
-  @Post('refresh-access-token')
-  @HttpCode(HttpStatus.OK)
-  async refreshAccessToken(
-    @Body() { refreshToken }: AuthRefreshAccessTokenRequest,
-  ) {
-    this.logger.info('/auth - RefreshAccessToken');
-
-    return this.authClient.refreshAccessToken({ refreshToken });
-  }
-
   @Post('change-password')
   @HttpCode(HttpStatus.OK)
   async changePassword(
@@ -93,16 +144,24 @@ export class AuthController {
     });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('reset-password')
   @HttpCode(HttpStatus.OK)
-  async resetPassword(@Body() body: AuthResetPasswordRequest) {
+  async resetPassword(
+    @Body() body: AuthResetPasswordRequest,
+    @Language() language: LanguageEnum,
+  ) {
     this.logger.info('/auth - reset-password');
+    const metadata = PackTheMetadata({
+      language: language,
+    });
 
     const { username } = body;
 
-    return this.authClient.resetPassword({ username });
+    return this.authClient.resetPassword({ username }, metadata);
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('verify-reset-password-code')
   @HttpCode(HttpStatus.OK)
   async verifyResetPasswordCode(
@@ -115,6 +174,7 @@ export class AuthController {
     return this.authClient.verifyResetPasswordCode({ username, code });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('confirm-reset-password')
   @HttpCode(HttpStatus.OK)
   async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
@@ -129,6 +189,7 @@ export class AuthController {
     });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('enable-totp')
   @HttpCode(HttpStatus.OK)
   async enableTotpMFA(
@@ -143,6 +204,7 @@ export class AuthController {
     return this.authClient.enableTotpMFA({ accessToken, password });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('disable-totp')
   @HttpCode(HttpStatus.OK)
   async disableTotpMFA(
@@ -157,6 +219,7 @@ export class AuthController {
     return this.authClient.disableTotpMFA({ accessToken, password });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('dismiss-totp')
   @HttpCode(HttpStatus.OK)
   async dismissTotpMFA(@Headers() headers) {
@@ -167,6 +230,7 @@ export class AuthController {
     return this.authClient.dismissTotpMFA({ accessToken });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('verify-totp')
   @HttpCode(HttpStatus.OK)
   async verifyTotp(@Body() body: AuthVerifyTotpMfaRequest, @Headers() headers) {
@@ -177,4 +241,107 @@ export class AuthController {
 
     return this.authClient.verifyTotp({ accessToken, totp });
   }
+
+  @ApiInternalOnlyEndpoint()
+  @Authenticated()
+  @ApiSecurity('access-token')
+  @Get('verify-access-token')
+  verifyAccessToken() {
+    return { access_token_status: 'valid' };
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('session/:session')
+  getSession(@Param('session') session: string) {
+    return this.authClient.getSession(session);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('oauth/google')
+  @UseGuards(AuthGuard('google-login'))
+  googleOauth() {
+    this.logger.info('/oauth/google');
+    return true;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('oauth/google/callback')
+  @UseGuards(AuthGuard('google-login'))
+  @Redirect()
+  async googleOauthCallback(@Req() req) {
+    const { url, email, token, language = 'pt-br' } = await this.callback(req);
+    if (url.searchParams.get('error')) {
+      this.logger.error('/oauth/google - ERROR');
+      return { url: url.href };
+    }
+
+    await this.authClient
+      .oauthSignIn({
+        username: email,
+        token,
+      })
+      .then(({ session }) => {
+        this.logger.info('/oauth/google - SUCESS');
+        url.searchParams.set('session', session);
+      })
+      .catch((err) => {
+        this.logger.error('/oauth/google - ERROR');
+        let error = OauthErrors.INVALID_CREDENTIALS[language].error;
+        let error_description =
+          OauthErrors.INVALID_CREDENTIALS[language].error_description;
+        switch (err.details) {
+          case ErrorCodes.USER.NOT_FOUND:
+            error = OauthErrors.USER_NOT_FOUND[language].error;
+            error_description =
+              OauthErrors.USER_NOT_FOUND[language].error_description(email);
+            break;
+          case ErrorCodes.AUTH.UNAUTHORIZED:
+            error = OauthErrors.INVALID_SESSION[language].error;
+            error_description =
+              OauthErrors.INVALID_SESSION[language].error_description;
+            break;
+        }
+        url.searchParams.set('error', error);
+        url.searchParams.set('error_description', error_description);
+        return null;
+      });
+    return { url: url.href };
+  }
+
+  async callback(req: Request) {
+    const { error, state } = req.query;
+    const { authInfo } = req;
+    const url = new URL(this.redirectUrl);
+    let email, token, error_title, error_description;
+    let language: 'pt-br' | 'en-us' = 'pt-br';
+
+    const stateObject = jwt.verify(
+      state as string,
+      process.env.JWT_PRIVATE_KEY,
+    );
+    if (typeof stateObject != 'string') language = stateObject.language;
+
+    if (error || !authInfo) {
+      this.logger.error(error);
+      if (!authInfo) this.logger.error('No authInfo', { request: req });
+
+      error_title = OauthErrors.INVALID_CREDENTIALS[language].error;
+      error_description =
+        OauthErrors.INVALID_CREDENTIALS[language].error_description;
+      if (error) error_description += ` - [${error}]`;
+    } else {
+      const { accessToken } = authInfo as any;
+      const { _json: userInfo } = req.user as any;
+
+      email = userInfo.email;
+      token = accessToken;
+    }
+
+    if (error_title) {
+      url.searchParams.set('error', error_title);
+      url.searchParams.set('error_description', error_description);
+    }
+
+    return { token, email, url, language };
+  }
 }

src/modules/auth/auth.module.ts

@@ -5,20 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { AuthController } from './auth.controller';
 import { AuthClientService } from './auth.service';
 
-import { DucClient } from '../../clients/duc/client.config';
+import { DucClient } from '../duc/client.config';
-const ducClient = new DucClient();
+import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
+import { getOauthSecrets } from 'src/utils/OauthSecrets';
+const client = new DucClient();
 
 @Module({
-  imports: [
+  imports: [ClientsModule.register([client.providerOptions])],
-    ClientsModule.register([
-      {
-        name: 'DUC_PACKAGE',
-        ...ducClient.config(),
-      },
-    ]),
-  ],
   controllers: [AuthController],
-  providers: [AuthClientService, DadosferaLogger],
+  providers: [
+    AuthClientService,
+    DadosferaLogger,
+    GoogleLoginStrategy,
+    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
+  ],
   exports: [AuthClientService],
 })
 export class AuthModule {}

src/modules/auth/auth.service.ts

@@ -18,6 +18,8 @@ import {
   AuthVerifyResetPasswordCodeRequest,
   AuthConfirmResetPasswordRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { DucClient } from '../duc/client.config';
+import { Metadata } from '@grpc/grpc-js';
 
 @Injectable()
 export class AuthClientService implements OnModuleInit {
@@ -27,7 +29,7 @@ export class AuthClientService implements OnModuleInit {
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
-    @Inject('DUC_PACKAGE') private readonly grpcClient: ClientGrpc,
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -50,19 +52,25 @@ export class AuthClientService implements OnModuleInit {
     return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
   }
 
-  async signIn({ username, password, totp }: AuthSignInRequest) {
+  async signIn(
+    { username, password, totp }: AuthSignInRequest,
+    metadata: Metadata,
+  ) {
     this.logger.info('SignIn');
 
     return lastValueFrom(
-      this.authService.AuthSignIn({ username, password, totp }),
+      this.authService.AuthSignIn({ username, password, totp }, metadata),
     );
   }
 
-  async refreshAccessToken({ refreshToken }: AuthRefreshAccessTokenRequest) {
+  async refreshAccessToken(
+    { refreshToken }: AuthRefreshAccessTokenRequest,
+    metadata: Metadata,
+  ) {
     this.logger.info('RefreshAccessToken');
 
     return lastValueFrom(
-      this.authService.AuthRefreshAccessToken({ refreshToken }),
+      this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
     );
   }
 
@@ -82,10 +90,15 @@ export class AuthClientService implements OnModuleInit {
     );
   }
 
-  async resetPassword({ username }: AuthResetPasswordRequest) {
+  async resetPassword(
+    { username }: AuthResetPasswordRequest,
+    metadata: Metadata,
+  ) {
     this.logger.info('resetPassword');
 
-    return lastValueFrom(this.authService.AuthResetPassword({ username }));
+    return lastValueFrom(
+      this.authService.AuthResetPassword({ username }, metadata),
+    );
   }
 
   async verifyResetPasswordCode({
@@ -144,4 +157,15 @@ export class AuthClientService implements OnModuleInit {
       this.authService.AuthVerifyTotpMfa({ accessToken, totp }),
     );
   }
+
+  async getSession(session: string) {
+    return lastValueFrom(this.authService.AuthGetSession({ session }));
+  }
+
+  async oauthSignIn(data: { username: string; token: string }) {
+    const { username, token } = data;
+    return lastValueFrom(
+      this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
+    );
+  }
 }

src/modules/auth/dtos/login.ts

@@ -1,4 +1,9 @@
-import { ApiProperty } from '@nestjs/swagger';
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import {
+  AuthSignInRequest,
+  AuthSignInResponse,
+} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
 export class LoginDto {
   @ApiProperty()
@@ -10,3 +15,106 @@ export class LoginDto {
   @ApiProperty({ required: false })
   totp: string;
 }
+export class AuthTokens {
+  @ApiProperty()
+  accessToken: string;
+  @ApiProperty()
+  refreshToken: string;
+  @ApiPropertyOptional()
+  termsOfUseToken: string;
+  @ApiProperty({ deprecated: true })
+  idToken: string;
+}
+export enum MfaStatuses {
+  PENDING = 'pending',
+  NONE = 'none',
+  TOTP = 'totp',
+}
+
+export enum TouStatuses {
+  PENDING = 'pending',
+  REQUIRED = 'required',
+  OK = 'ok',
+}
+export class TermsOfUse {
+  @ApiProperty()
+  version: number;
+  @ApiProperty()
+  publicUrl: string;
+  @ApiProperty()
+  enforceDate: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class TermsOfUseStatus {
+  @ApiProperty({ enum: TouStatuses })
+  status: string;
+  @ApiPropertyOptional()
+  lastSigned?: TermsOfUse;
+  @ApiPropertyOptional()
+  next?: TermsOfUse;
+}
+export class AuthUser {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class AuthCustomer {
+  @ApiProperty()
+  modules: string[];
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  tier: string;
+  @ApiProperty()
+  scheduleLimit: string;
+  @ApiProperty()
+  links: Link[];
+}
+
+export class AuthSignInReq implements AuthSignInRequest {
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  password: string;
+  @ApiPropertyOptional({
+    description:
+      'TOTP code used to login when Multi-Factor Authentication is enabled',
+  })
+  totp: string;
+}
+
+export class AuthSignInRes implements AuthSignInResponse {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty({ enum: MfaStatuses })
+  mfaStatus: string;
+  @ApiPropertyOptional()
+  customer?: AuthCustomer;
+  @ApiPropertyOptional()
+  user?: AuthUser;
+  @ApiPropertyOptional()
+  tokens?: AuthTokens;
+  @ApiPropertyOptional()
+  termsOfUse?: TermsOfUseStatus;
+}
+
+export class AuthRefreshAccessTokenReq {
+  @ApiProperty()
+  refreshToken: string;
+  @ApiProperty()
+  customerName: string;
+}
+export class AuthRefreshAccessTokenRes {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty()
+  accessToken: string;
+}

src/modules/auth/passport-strategies/google-strategy.ts

@@ -0,0 +1,46 @@
+import {
+  AuthenticateOptionsGoogle,
+  Profile,
+  Strategy,
+  StrategyOptions,
+} from 'passport-google-oauth20';
+import { PassportStrategy } from '@nestjs/passport';
+import { Inject, Injectable } from '@nestjs/common';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+import { Request } from 'express';
+import jwt from 'jsonwebtoken';
+
+@Injectable()
+export class GoogleLoginStrategy extends PassportStrategy(
+  Strategy,
+  'google-login',
+) {
+  redirect_uri: string;
+  constructor(
+    @Inject('OAUTH_SECRETS')
+    private readonly oauthSecrets: OauthSecrets,
+  ) {
+    const options: StrategyOptions = {
+      clientID: oauthSecrets['google-login'].client_id,
+      clientSecret: oauthSecrets['google-login'].client_secret,
+      callbackURL: oauthSecrets['google-login'].redirect_uri,
+      scope: ['email', 'profile', 'openid'],
+    };
+    const verify = (
+      accessToken: string,
+      refreshToken: string,
+      profile: Profile,
+      done,
+    ) => {
+      return done(null, profile, { accessToken, refreshToken });
+    };
+
+    super(options, verify);
+  }
+
+  authenticate(req: Request, options: AuthenticateOptionsGoogle) {
+    const language = req.headers['dadosfera-lang'] || req.query.language;
+    options.state = jwt.sign({ language }, process.env.JWT_PRIVATE_KEY);
+    super.authenticate(req, options);
+  }
+}

src/modules/catalog/catalog-client.ts

@@ -0,0 +1,39 @@
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { credentials } from '@grpc/grpc-js';
+import { Catalog } from '@dadosfera/protospack-v2';
+
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  process.env.PIFACTORY_URL.includes('0.0.0.0');
+
+export class CatalogClientConfiguration {
+  public name = 'CatalogClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.PIFACTORY_URL,
+      package: [
+        Catalog.ProtoPackages.ReadPackage,
+        Catalog.ProtoPackages.WritePackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        Catalog.ProtoPaths.ReadFilePath,
+        Catalog.ProtoPaths.WriteFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        defaults: false,
+      },
+    },
+  };
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/catalog/catalog.module.ts

@@ -0,0 +1,26 @@
+import { Module } from '@nestjs/common';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { CatalogController } from './catalog.controller';
+import { CatalogService } from './catalog.service';
+import { CatalogClientConfiguration } from './catalog-client';
+import { ClientsModule } from '@nestjs/microservices';
+import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
+import { UsersModule } from '../users/users.module';
+import { RolesModule } from '../roles/roles.module';
+
+const client = new CatalogClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    OldPipelineModule,
+    UsersModule,
+    RolesModule,
+  ],
+  controllers: [CatalogController],
+  providers: [CatalogService, DadosferaLogger],
+  exports: [CatalogService],
+})
+export class CatalogModule {}

src/modules/catalog/catalog.service.ts

@@ -1,6 +1,61 @@
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+import {
+  WriteService,
+  ReadService,
+  ProtoServices,
+  Messages,
+} from '@dadosfera/protospack-v2/dist/lib/Catalog';
+import {
+  BadRequestException,
+  HttpException,
+  HttpStatus,
+  Inject,
+  OnModuleInit,
+} from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
 import axios from 'axios';
+import { lastValueFrom } from 'rxjs';
+import { CatalogClientConfiguration } from './catalog-client';
+import { UsersService } from '../users/users.service';
+import { RolesService } from '../roles/roles.service';
+import { Metadata } from '@grpc/grpc-js';
+import {
+  BatchRemoveRlsRulesRequest,
+  IUpdateDataRequest,
+  TriggerCatalogReq,
+} from './dtos';
+import {
+  AddRlsRuleRequest,
+  GetNimbusDashboardsRequest,
+  GetRlsRulesRequest,
+} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+
+class CatalogService implements OnModuleInit {
+  catalogReadService: ReadService.CatalogReadServices;
+  catalogWriteService: WriteService.CatalogWriteServices;
+  logger: any;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject(CatalogClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+    private readonly userService: UsersService,
+    private readonly roleService: RolesService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.catalogReadService =
+      this.grpcClient.getService<ReadService.CatalogReadServices>(
+        ProtoServices.CatalogReadServices,
+      );
+    this.catalogWriteService =
+      this.grpcClient.getService<WriteService.CatalogWriteServices>(
+        ProtoServices.CatalogWriteServices,
+      );
+  }
 
-class CatalogService {
   _getNimbusUrl(body) {
     const customer = body.info.customer.toLowerCase();
 
@@ -8,138 +63,228 @@ class CatalogService {
       return `https://nimbus-${customer}.dadosfera.ai`;
     }
 
-    return `https://nimbus-${customer}.${process.env.ENV}.dadosfera.ai`;
+    return `https://nimbus-${customer}.${process.env.ENV.replace(
+      'local',
+      'stg',
+    )}.dadosfera.ai`;
   }
 
-  async catalogAll(body) {
+  async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - Manage Data assets permissions');
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/all/`);
+    if (!data.embed) data.embed = undefined;
 
-    return data;
+    return lastValueFrom(
-  }
+      this.catalogWriteService.CreateDataAsset(data, metadata),
-
-  async dataAppsAll(body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data_apps/`);
-    return data;
-  }
-
-  async dataAppsOne(id, body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/data_apps/${id}/`,
     );
-
-    return data;
   }
 
-  async getAllDashboardMetabase(body) {
+  async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - Manage Data assets permissions');
-    const { data } = await axios.get(
+
-      `${nimbusUrl}/api/catalog/dashboard-metabase/`,
+    return lastValueFrom(
+      this.catalogWriteService.ManagePermission(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
       );
-    return data;
-  }
-
-  async getOneDashboardMetabase(id, body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/dashboard-metabase/${id}`,
-    );
-
-    // const [, path] = data.iframe_url.split('.dadosfera');
-    // const host = `httpss://metabase-${body.info.customer.toLowerCase()}.dadosfera.ai`;
-    // data.iframe_url = host + path;
-
-    return data;
-  }
-
-  async getAllTableMetadata(body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/table-metadata/`,
-    );
-    return data;
-  }
-
-  async getOneTableMetadata(body, params) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/table-metadata/`,
-      { params: params },
-    );
-    return data;
-  }
-
-  async deleteOneTableMetadata(id, body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.delete(
-      `${nimbusUrl}/api/catalog/table-metadata/${id}`,
-    );
-    return data;
-  }
-
-  async getOneColumnMetadata(body, params) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/column-metadata/`,
-      { params: params },
-    );
-    return data;
-  }
-
-  async deleteOneColumnMetadata(id, body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.delete(
-      `${nimbusUrl}/api/catalog/column-metadata/${id}/`,
-    );
-    return data;
-  }
-
-  async getOneDataPreview(body, params) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-preview/`, {
-      params: params,
     });
-    return data;
   }
 
-  async getDataStatus(body) {
+  async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - Manage Data assets permissions');
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-status/`);
-    return data;
-  }
 
-  async createDataStatus(body) {
+    return lastValueFrom(
-    const nimbusUrl = this._getNimbusUrl(body);
+      this.catalogWriteService.RevokePermission(data, metadata),
-    const { data } = await axios.post(
+    ).catch((err) => {
-      `${nimbusUrl}/api/catalog/data-status/`,
+      throw new HttpException(
-      body,
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
       );
-    return data;
+    });
   }
 
-  async getDataDescription(body) {
+  async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - Manage Data assets permissions');
-    const { data } = await axios.get(
+
-      `${nimbusUrl}/api/catalog/data-description/`,
+    return lastValueFrom(
+      this.catalogWriteService.MakeAComment(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
       );
-    return data;
+    });
   }
 
-  async createDataDescription(body) {
+  async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - Manage Data assets permissions');
-    const { data } = await axios.post(
+
-      `${nimbusUrl}/api/catalog/data-description/`,
+    return lastValueFrom(
-      body,
+      this.catalogWriteService.UpdateAComment(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
       );
-    return data;
+    });
   }
 
-  async getDataDocs(body) {
+  async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - Manage Data assets permissions');
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-docs/`);
+
-    return data;
+    return lastValueFrom(
+      this.catalogWriteService.DeleteDataAsset(data, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 404,
+      );
+    });
+  }
+
+  async getUserRolesIds(userId: string) {
+    const result = await this.userService.findOneById(userId).catch(() => null);
+
+    const roles_ids = result.user.roles.map((role) => role.id);
+
+    return roles_ids;
+  }
+
+  async searchDataAssets(
+    query: Record<string, any>,
+    metadata: Metadata,
+    customer_id: string,
+  ) {
+    this.logger.info('CatalogService - searchDataAssets');
+
+    const { search, page, size, sort_by, order, ...filters } = query;
+
+    const { data_assets, total } = await lastValueFrom(
+      this.catalogReadService.GetAllDataAssets(
+        {
+          search,
+          page,
+          size,
+          sort_by,
+          order,
+          filters: JSON.stringify(filters),
+        },
+        metadata,
+      ),
+    );
+
+    const result = JSON.parse(data_assets);
+
+    const response = await this.getAssetsUsersAndRoles(
+      result.data_assets,
+      customer_id,
+    );
+
+    return { data_assets: response, total };
+  }
+
+  async getOneDataAsset(data: {
+    id: string;
+    customer_id: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, id, metadata } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAsset(
+        { id, type: undefined },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
+  async getOneDataAssetByPipelineAndObject(data: {
+    customer_id: string;
+    pipeline: string;
+    object: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, metadata, pipeline, object } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAssetByPipelineAndObject(
+        { pipeline, object },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
+  async updateOneDataAsset(data: {
+    data_asset_id: string;
+    customer_id: string;
+    body: IUpdateDataRequest;
+    metadata: Metadata;
+  }) {
+    const { body, customer_id, data_asset_id, metadata } = data;
+
+    const { data_asset } = await lastValueFrom(
+      this.catalogWriteService.UpdateDataAsset(
+        { id: data_asset_id, changes: JSON.stringify(body) },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
+  async getDataDocs(id: string, metadata: Metadata) {
+    const { documentation } = await lastValueFrom(
+      this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
+    );
+    console.log(documentation);
+    const docs = JSON.parse(documentation);
+    return docs;
+  }
+
+  async getDatasetPreview(id: string, metadata: Metadata) {
+    const { preview } = await lastValueFrom(
+      this.catalogReadService.GetDatasetPreview(
+        { id, type: undefined },
+        metadata,
+      ),
+    );
+    const result = JSON.parse(preview);
+    return result;
+  }
+
+  async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
+    const { columns_metadata } = await lastValueFrom(
+      this.catalogReadService.GetDatasetColumnsMetadata(
+        { id, type: undefined },
+        metadata,
+      ),
+    );
+    const result = JSON.parse(columns_metadata);
+    return result;
   }
 
   async createDataDocs(body) {
@@ -151,111 +296,129 @@ class CatalogService {
     return data;
   }
 
-  async getDataRating(body) {
+  async findAllTags(data, metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info('CatalogService - findAllCustomerTags');
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-rating/`);
-    return data;
-  }
 
-  async createDataRating(body) {
+    const response = await lastValueFrom(
-    const nimbusUrl = this._getNimbusUrl(body);
+      this.catalogReadService.GetCustomerTags(data, metadata),
-    const { data } = await axios.post(
+    )
-      `${nimbusUrl}/api/catalog/data-rating/`,
+      .then((res) => {
-      body,
+        this.logger.info('Done');
-    );
+        return res;
-    return data;
+      })
-  }
+      .catch((err) => {
-
+        this.logger.error(err.message);
-  async getSummaryRating(id, body) {
+        throw new Error(err);
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/summary-rating/${id}`,
-    );
-    return data;
-  }
-
-  async getDataComment(body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-comment/`);
-    return data;
-  }
-
-  async createDataComment(body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.post(
-      `${nimbusUrl}/api/catalog/data-comment/`,
-      body,
-    );
-    return data;
-  }
-
-  async getDataReview(body, params) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-review/`, {
-      params: params,
       });
-    return data;
-  }
 
-  async createTags(body) {
+    return response;
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.post(`${nimbusUrl}/api/catalog/tags/`, body);
-    return data;
   }
-
+  async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
-  async findAllTags(body) {
+    const { users: customer_users } =
-    const nimbusUrl = this._getNimbusUrl(body);
+      await this.userService.findAllUsersByCustomerId(customer_id);
-    const { data } = await axios.get(`${nimbusUrl}/api/catalog/tags/`, body);
+    const { roles: customer_roles } = await this.roleService.roleSearch(
-    return data;
+      {},
-  }
+      { customer_id },
-
-  async findAllTableTags(body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.get(
-      `${nimbusUrl}/api/catalog/table-tags/`,
-      body,
     );
-    return data;
+    return data_assets.map((data_asset) => {
+      const owner = customer_users.find(
+        (u) => u.id === data_asset.owner,
+      )?.username;
+
+      const roles = [];
+      const users = [];
+      for (const role_id of data_asset.roles) {
+        const role = customer_roles.find((r) => r.id === role_id);
+        if (role) roles.push({ id: role.id, name: role.name });
+      }
+      for (const user_id of data_asset.users) {
+        const user = customer_users.find((r) => r.id === user_id);
+        if (user) users.push({ id: user.id, username: user.username });
+      }
+      return {
+        ...data_asset,
+        roles,
+        users,
+        owner,
+      } as typeof data_asset;
+    });
   }
 
-  async deleteTags(id, body) {
+  async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    const { session } = await lastValueFrom(
-    const { data } = await axios.delete(`${nimbusUrl}/api/catalog/tags/${id}`);
+      this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
-    return data;
-  }
-
-  async createTableTags(body) {
-    const nimbusUrl = this._getNimbusUrl(body);
-    const { data } = await axios.post(
-      `${nimbusUrl}/api/catalog/table-tags/`,
-      body,
     );
-    return data;
+    return session;
+  }
+  async getDatasetCatalogTask(session: string, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
+    );
+    return res;
   }
 
-  async deleteTableTags(id, body) {
+  async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    const res = await lastValueFrom(
-    const { data } = await axios.delete(
+      this.catalogWriteService.AddRlsRule(data, metadata),
-      `${nimbusUrl}/api/catalog/table-tags/${id}`,
     );
-    return data;
+    return res;
   }
 
-  async getTableRules(body) {
+  async removeRlsRule(id: number, metadata: Metadata) {
-    const nimbusUrl = this._getNimbusUrl(body);
+    const res = await lastValueFrom(
-    const { data } = await axios.get(
+      this.catalogWriteService.RemoveRlsRule({ id }, metadata),
-      `${nimbusUrl}/api/catalog/table-rules/`,
-      body,
     );
-    return data;
+    return res;
   }
 
-  async deleteTableRules(id, body) {
+  async batchRemoveRlsRule(
-    const nimbusUrl = this._getNimbusUrl(body);
+    query: BatchRemoveRlsRulesRequest,
-    const { data } = await axios.delete(
+    metadata: Metadata,
-      `${nimbusUrl}/api/catalog/table-rules/${id}`,
+  ) {
+    const { id_rls, nimbus_dashboard_id } = query;
+
+    if (id_rls && nimbus_dashboard_id) {
+      throw new BadRequestException(
+        "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
       );
-    return data;
+    }
+    if (id_rls) {
+      await lastValueFrom(
+        this.catalogWriteService.RemoveRlsRulesByRlsId({ id_rls }, metadata),
+      );
+    } else if (nimbus_dashboard_id) {
+      await lastValueFrom(
+        this.catalogWriteService.RemoveRlsRulesByDashboardId(
+          { nimbus_dashboard_id: parseInt(nimbus_dashboard_id) },
+          metadata,
+        ),
+      );
+    }
+    return 'OK';
+  }
+
+  async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetRlsRules(data, metadata),
+    );
+    return res.rls_rules;
+  }
+
+  async getOneRlsRule(id: number, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetOneRlsRule({ id }, metadata),
+    );
+    return res.rls_rule;
+  }
+
+  async getNimbusDashboards(
+    data: GetNimbusDashboardsRequest,
+    metadata: Metadata,
+  ) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetNimbusDashboards(data, metadata),
+    );
+    return res.dashboards;
   }
 }
 

src/modules/catalog/dtos/index.ts

@@ -0,0 +1,322 @@
+import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
+import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
+
+export enum DataAssetShareType {
+  none = 'none',
+  public = 'public',
+  private = 'private',
+}
+export enum OrderEnum {
+  asc = 'asc',
+  desc = 'desc',
+}
+export class EmbedObject {
+  @ApiProperty()
+  url: string;
+}
+export class IRole {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  id: string;
+}
+
+export class IUser {
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  id: string;
+}
+
+export class IComment {
+  @ApiProperty()
+  id: number;
+  @ApiProperty()
+  message: string;
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  username: string;
+}
+
+export class IDataAsset {
+  @ApiProperty()
+  id: number;
+  @ApiPropertyOptional()
+  database_name?: string;
+  @ApiPropertyOptional()
+  table_name?: string;
+  @ApiPropertyOptional()
+  num_columns?: number;
+  @ApiPropertyOptional()
+  location?: string;
+  @ApiPropertyOptional()
+  created_time?: string;
+  @ApiPropertyOptional()
+  table_type?: string;
+  @ApiPropertyOptional()
+  size_in_bytes?: number;
+  @ApiPropertyOptional()
+  num_files?: number;
+  @ApiPropertyOptional()
+  data_encrypted?: string;
+  @ApiPropertyOptional()
+  num_rows?: number;
+  @ApiProperty()
+  display_name: string;
+  @ApiProperty()
+  pipeline_id: string;
+  @ApiProperty()
+  user_id: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  data_asset_type: string;
+  @ApiProperty()
+  tags: string[];
+  @ApiProperty()
+  owner: string;
+  @ApiProperty({ type: [IRole] })
+  roles: IRole[];
+  @ApiProperty({ type: [IUser] })
+  users: IUser[];
+  @ApiPropertyOptional()
+  dashboard_id?: number;
+  @ApiPropertyOptional()
+  name?: string;
+  @ApiPropertyOptional()
+  parameters?: string[];
+  @ApiPropertyOptional()
+  embedding_params?: object;
+  @ApiProperty()
+  created_at: string;
+  @ApiPropertyOptional()
+  updated_at?: string;
+  @ApiProperty({ type: [IComment] })
+  comments: IComment[];
+  @ApiPropertyOptional()
+  embed?: EmbedObject;
+  @ApiPropertyOptional({ enum: DataAssetShareType })
+  share_type?: DataAssetShareType;
+}
+
+export class IOneDataAsset {
+  @ApiProperty()
+  data_asset: IDataAsset;
+}
+
+export class IMakeAComment {
+  @ApiProperty()
+  message: string;
+}
+
+export class CommentToDelete extends PickType(IComment, ['id', 'username']) {}
+export class IDeleteComment {
+  @ApiProperty()
+  comment: CommentToDelete;
+}
+
+export class ICatalogAllRequest {
+  @ApiPropertyOptional({
+    description: 'Texto usado para filtrar os ativos no catálogo.',
+  })
+  search;
+
+  @ApiPropertyOptional({
+    description: 'Número de registros a ser retornado',
+    maximum: 10000,
+    minimum: 1,
+  })
+  size: number;
+
+  @ApiPropertyOptional({
+    description: 'Usado para paginação, em conjunto com `size`',
+    minimum: 1,
+  })
+  page: number;
+
+  @ApiPropertyOptional({
+    description:
+      'Campo do data asset para usar na ordenação. Padrão: `created_at` ',
+  })
+  sort_by: string;
+
+  @ApiPropertyOptional({
+    enum: OrderEnum,
+    default: OrderEnum.asc,
+    description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
+  })
+  order?: OrderEnum;
+}
+
+export class ICatalogAllResponse {
+  @ApiProperty({ type: [IDataAsset] })
+  data_assets: IDataAsset[];
+  @ApiProperty()
+  total: number;
+}
+
+export class IData {
+  @ApiProperty()
+  var: number;
+  @ApiProperty()
+  date: string;
+  @ApiProperty()
+  last: number;
+  @ApiProperty()
+  maximum: number;
+  @ApiProperty()
+  minimum: number;
+  @ApiProperty()
+  day_opening: number;
+}
+
+export class IUpdateDataRequest {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  tags: string[];
+  @ApiPropertyOptional()
+  embed: EmbedObject;
+  @ApiPropertyOptional({ enum: DataAssetShareType })
+  share_type?: DataAssetShareType;
+}
+export class ICreateDataAsset implements CreateDataAssetRequest {
+  @ApiProperty()
+  display_name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  data_asset_type: string;
+  @ApiProperty()
+  external_url: string;
+  @ApiProperty()
+  location: string;
+  @ApiPropertyOptional()
+  embed: EmbedObject;
+}
+
+export class IPreview {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  database_name: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty({ type: [IData] })
+  data: IData[];
+}
+
+export class IDocs {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  docs: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  table_id: string;
+}
+
+export class IColumnMetadata {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  column_name: string;
+  @ApiProperty()
+  data_type: string;
+  @ApiProperty()
+  missing_count: string;
+  @ApiProperty()
+  min: string;
+  @ApiProperty()
+  max: string;
+  @ApiProperty()
+  count: string;
+  @ApiProperty()
+  histogram: string;
+  @ApiProperty()
+  mean: string;
+  @ApiProperty()
+  median: string;
+  @ApiProperty()
+  missing_rate: string;
+  @ApiProperty()
+  stddev: string;
+  @ApiProperty()
+  unique_count: string;
+  @ApiProperty()
+  unique_rate: string;
+  @ApiProperty()
+  analyzer_rules: string[];
+  @ApiProperty()
+  version: string;
+}
+
+export class IPreviewResponse {
+  @ApiProperty()
+  preview: IPreview;
+}
+
+export class IDocsResponse {
+  @ApiProperty()
+  docs: IDocs;
+}
+
+export class IColumnsMetadataResponse {
+  @ApiProperty({ type: [IColumnMetadata] })
+  columns_metadata: IColumnMetadata[];
+}
+
+export class TriggerCatalogReq {
+  @ApiProperty({
+    description: 'Name of the table on Snowflake',
+    example: 'TB__4DXSD4_TEST',
+  })
+  table_name: string;
+
+  @ApiPropertyOptional({
+    description:
+      'Schema where the asset is located. If not set defaults to "PUBLIC"',
+  })
+  table_schema?: string;
+
+  @ApiPropertyOptional({
+    description: 'Id of the pipeline that generated the asset.',
+  })
+  pipeline_id: string;
+}
+
+export class TriggerCatalogRes {
+  @ApiProperty()
+  session: string;
+}
+
+export class GetDatasetCatalogTaskRes {
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  customer_name: string;
+  @ApiProperty()
+  session_id: string;
+  @ApiProperty()
+  status: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  updated_by: string;
+}
+
+export class BatchRemoveRlsRulesRequest {
+  @ApiPropertyOptional()
+  nimbus_dashboard_id?: string;
+
+  @ApiPropertyOptional()
+  id_rls?: string;
+}

src/modules/connection-test/connection-test-client.config.ts

@@ -0,0 +1,33 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { ConnectionTest } from '@dadosfera/protospack-v2';
+
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
+
+export class ConnectionTestClientConfiguration {
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [ConnectionTest.ProtoPackages.ReadPackage],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [ConnectionTest.ProtoPaths.ReadFilePath],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: 'ConnectionTestGrpcClient',
+    ...this.config,
+  };
+}

src/modules/connection-test/connection-test.controller.ts

@@ -0,0 +1,124 @@
+import {
+  Body,
+  Controller,
+  HttpCode,
+  HttpStatus,
+  Inject,
+  Post,
+  UseFilters,
+  ValidationPipe,
+} from '@nestjs/common';
+import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import { User, RequestUser } from 'src/decorators/user.decorator';
+import { ConnectionTestService } from './connection-test.service';
+import {
+  ConnectionTestPingRes,
+  ConnectionTestPingReq,
+  ConnectionTestCredentialsReq,
+  ConnectionTestCredentialsRes,
+  ConnectionTestListSchemasReq,
+  ConnectionTestListSchemasRes,
+  ConnectionTestListTablesReq,
+  ConnectionTestListTablesRes,
+  GetTableMetadataRes,
+  GetTableMetadataReq,
+} from './dto/connection-test';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+
+@ApiInternalOnlyController()
+@ApiTags('Connection Test')
+@Controller('connection-test')
+@UseFilters(new GrpcToHttpExceptionFilter())
+@Authenticated()
+export class ConnectionTestController {
+  logger: any;
+  constructor(
+    private connectionTestService: ConnectionTestService,
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post('ping')
+  @ApiOkResponse({ type: ConnectionTestPingRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestPing(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: ConnectionTestPingReq,
+  ) {
+    this.logger.info('/connection-test/ping', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestPing(body, user);
+  }
+
+  @Post('credentials')
+  @ApiOkResponse({ type: ConnectionTestCredentialsRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestCredentials(
+    @User() user: RequestUser,
+    @Body() body: ConnectionTestCredentialsReq,
+  ) {
+    this.logger.info('/connection-test/credentials', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestCredentials(body, user);
+  }
+
+  @Post('schemas')
+  @ApiOkResponse({ type: ConnectionTestListSchemasRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestListSchemas(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: ConnectionTestListSchemasReq,
+  ) {
+    this.logger.info('/connection-test/schemas', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestListSchemas(
+      body,
+      user.customer_name,
+    );
+  }
+
+  @Post('tables')
+  @ApiOkResponse({ type: ConnectionTestListTablesRes })
+  @HttpCode(HttpStatus.OK)
+  async connectionTestListTables(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: ConnectionTestListTablesReq,
+  ) {
+    this.logger.info('/connection-test/tables', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.connectionTestListTables(
+      body,
+      user.customer_name,
+    );
+  }
+
+  @Post('table-metadata')
+  @ApiOkResponse({ type: GetTableMetadataRes })
+  @HttpCode(HttpStatus.OK)
+  async getTableMetadata(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: GetTableMetadataReq,
+  ) {
+    this.logger.info('/connection-test/table-metadata', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.connectionTestService.getTableMetadata(
+      body,
+      user.customer_name,
+    );
+  }
+}

src/modules/connection-test/connection-test.module.ts

@@ -0,0 +1,14 @@
+import { Module } from '@nestjs/common';
+import { ConnectionTestController } from './connection-test.controller';
+import { ConnectionTestService } from './connection-test.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { ConnectionTestClientConfiguration } from './connection-test-client.config';
+import { ConnectionModule } from '../connection/connection.module';
+const client = new ConnectionTestClientConfiguration();
+@Module({
+  controllers: [ConnectionTestController],
+  providers: [ConnectionTestService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
+})
+export class ConnectionTestModule {}

src/modules/connection-test/connection-test.service.ts

@@ -0,0 +1,191 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ConnectionTest } from '@dadosfera/protospack-v2';
+import { lastValueFrom } from 'rxjs';
+import {
+  ConnectionTestCredentialsReq,
+  ConnectionTestCredentialsRes,
+  ConnectionTestListSchemasReq,
+  ConnectionTestListSchemasRes,
+  ConnectionTestListTablesReq,
+  ConnectionTestListTablesRes,
+  ConnectionTestPingReq,
+  ConnectionTestPingRes,
+  GetTableMetadataReq,
+  GetTableMetadataRes,
+} from './dto/connection-test';
+import { ConnectionClientService } from '../connection/client.service';
+import {
+  CreateConnectionDto,
+  DatabaseConnectionPropertiesDto,
+} from '../connection/dtos/connection';
+import { RequestUser } from 'src/decorators/user.decorator';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+
+@Injectable()
+export class ConnectionTestService {
+  private connectionTestReadClient: ConnectionTest.ReadService.ConnectionTestReadServices;
+  constructor(
+    @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
+    private connectionsService: ConnectionClientService,
+  ) {
+    this.connectionTestReadClient =
+      grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
+        ConnectionTest.ProtoServices.ConnectionTestReadServices,
+      );
+  }
+
+  async connectionTestPing(
+    body: ConnectionTestPingReq,
+    user: RequestUser,
+  ): Promise<ConnectionTestPingRes> {
+    const {
+      host: endpoint,
+      plugin,
+      port,
+      network_config_id,
+      connection_id,
+    } = body;
+    const { user_id, customer_id, customer_name } = user;
+    const metadata = PackTheMetadata({
+      customer_name,
+      customer_id,
+      user_id,
+    });
+
+    if (connection_id) {
+      const con = await this.connectionsService.getConnectionDetails({
+        body,
+        metadata,
+      });
+      const conPayload = JSON.parse(con.connection.properties);
+      const { plugin, network_config_id } = con.connection;
+      const { host, port, endpoint } = conPayload;
+
+      return lastValueFrom(
+        this.connectionTestReadClient.ConnectionTestPing({
+          customer_name,
+          endpoint: endpoint || host,
+          network_config_id,
+          plugin,
+          port: port ? port.toString() : undefined,
+          connection_id,
+        }),
+      );
+    } else {
+      return lastValueFrom(
+        this.connectionTestReadClient.ConnectionTestPing({
+          customer_name,
+          endpoint,
+          network_config_id,
+          plugin,
+          port: port ? port.toString() : undefined,
+          connection_id,
+        }),
+      );
+    }
+  }
+  async connectionTestCredentials(
+    body: ConnectionTestCredentialsReq,
+    user: RequestUser,
+  ): Promise<ConnectionTestCredentialsRes> {
+    const { plugin, ...connectionBody } = body;
+    let { connection_id } = body;
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+
+    if (body.network_config_id == '') {
+      body.network_config_id = undefined;
+    }
+
+    /********* CREATE CONNECTION *********/
+    if (!connection_id) {
+      const properties: DatabaseConnectionPropertiesDto = {
+        ...connectionBody.properties,
+        plugin,
+        credentials_type: 'basic_auth',
+      };
+      const createConnectionBody: CreateConnectionDto = {
+        ...body,
+        properties,
+      };
+      const { connection } = await this.connectionsService.createConnection(
+        createConnectionBody,
+        metadata,
+      );
+      connection_id = connection.id;
+    }
+    /************************************/
+    const { operation_result } = await lastValueFrom(
+      this.connectionTestReadClient.ConnectionTestCredentials({
+        connection_id,
+        customer_name,
+        plugin,
+      }),
+    ).catch(async (e) => {
+      if (!body.connection_id) {
+        await this.connectionsService.deleteConnection({
+          body: { id: connection_id },
+          metadata,
+        });
+      }
+      throw e;
+    });
+    if (!operation_result && !body.connection_id) {
+      await this.connectionsService.deleteConnection({
+        body: { id: connection_id },
+        metadata,
+      });
+      connection_id = null;
+    }
+    return { operation_result, connection_id };
+  }
+  async connectionTestListSchemas(
+    body: ConnectionTestListSchemasReq,
+    customer_name: string,
+  ): Promise<ConnectionTestListSchemasRes> {
+    const { connection_id, plugin } = body;
+    return lastValueFrom(
+      this.connectionTestReadClient.ListSchemas({
+        connection_id,
+        customer_name,
+        plugin,
+      }),
+    );
+  }
+  async connectionTestListTables(
+    body: ConnectionTestListTablesReq,
+    customer_name: string,
+  ): Promise<ConnectionTestListTablesRes> {
+    const { connection_id, plugin, schema } = body;
+    return lastValueFrom(
+      this.connectionTestReadClient.ListTables({
+        connection_id,
+        customer_name,
+        plugin,
+        schema,
+      }),
+    );
+  }
+
+  async getTableMetadata(
+    body: GetTableMetadataReq,
+    customer_name: string,
+  ): Promise<GetTableMetadataRes> {
+    const { schema, plugin, table_list, connection_id } = body;
+    return lastValueFrom(
+      this.connectionTestReadClient.GetTableMetadata({
+        connection_id,
+        customer_name,
+        plugin,
+        schema,
+        table_list,
+      }),
+    );
+  }
+}

src/modules/connection-test/dto/connection-test.ts

@@ -0,0 +1,133 @@
+import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
+import { IsString, IsOptional } from 'class-validator';
+import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
+import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
+export class ColumnDto {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  type: string;
+}
+export class TableMetadataDto {
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty({ type: [ColumnDto] })
+  columns: ColumnDto[];
+  @ApiProperty({ type: [ColumnDto] })
+  references: ColumnDto[];
+}
+
+export class ConnectionTestPingReq {
+  @ApiPropertyOptional()
+  @IsString()
+  @IsOptional()
+  plugin: string;
+
+  @ApiPropertyOptional()
+  @IsString()
+  @IsOptional()
+  host: string;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  port: number;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  @IsString()
+  network_config_id?: string;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  @IsString()
+  connection_id: string;
+}
+
+export class ConnectionTestPingRes {
+  @ApiProperty()
+  operation_result: boolean;
+}
+
+export class ConnectionTestCredentialsProperties extends OmitType(
+  DatabaseConnectionPropertiesDto,
+  ['plugin', 'credentials_type'],
+) {}
+export class ConnectionTestCredentialsReq extends OmitType(
+  CreateConnectionDto,
+  ['properties'],
+) {
+  @ApiProperty()
+  properties: ConnectionTestCredentialsProperties;
+  @ApiPropertyOptional()
+  connection_id?: string;
+}
+
+export class ConnectionTestCredentialsRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty()
+  connection_id: string;
+}
+
+export class ConnectionTestListSchemasReq {
+  @ApiProperty()
+  @IsString()
+  plugin: string;
+
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+}
+
+export class ConnectionTestListSchemasRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty()
+  schema_list: string[];
+}
+
+export class ConnectionTestListTablesReq {
+  @ApiProperty()
+  @IsString()
+  plugin: string;
+
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+
+  @ApiProperty()
+  @IsString()
+  schema: string;
+}
+
+export class ConnectionTestListTablesRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty()
+  table_list: string[];
+}
+
+export class GetTableMetadataReq {
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+
+  @ApiProperty()
+  @IsString()
+  plugin: string;
+
+  @ApiProperty()
+  @IsString()
+  schema: string;
+
+  @ApiProperty()
+  @IsString({ each: true })
+  table_list: string[];
+}
+
+export class GetTableMetadataRes {
+  @ApiProperty()
+  operation_result: boolean;
+  @ApiProperty({ type: [TableMetadataDto] })
+  tables_metadata: TableMetadataDto[];
+}

src/modules/connection/client.config.ts

@@ -0,0 +1,41 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { ConnectionManager } from '@dadosfera/protospack-v2';
+
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
+
+export class ConnectionClientConfiguration {
+  public name = 'ConnectionClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [
+        ConnectionManager.ProtoPackages.WritePackage,
+        ConnectionManager.ProtoPackages.ReadPackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        ConnectionManager.ProtoPaths.WriteFilePath,
+        ConnectionManager.ProtoPaths.ReadFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/connection/client.service.ts

@@ -0,0 +1,340 @@
+import { OnModuleInit, Inject } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ConnectionManager } from '@dadosfera/protospack-v2';
+import { lastValueFrom } from 'rxjs';
+import {
+  ConnectionApiConnection,
+  ConnectionRes,
+  ConnectionsRes,
+  ConnectionToCatalogDto,
+  CreateConnectionDto,
+  UpdateConnectionDto,
+} from './dtos/connection';
+import { ConnectionClientConfiguration } from './client.config';
+import {
+  GetConnectionDetailsResponse,
+  INTERNAL_GetAllConnectionsResponse,
+} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
+import { UsersService } from '../users/users.service';
+import { ConnectorClientService } from '../connector/client.service';
+import {
+  ConnectionsApiNetworkConfig,
+  NetworkConfigDto,
+} from '../network-config/dto/network-config';
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+
+export class ConnectionClientService implements OnModuleInit {
+  private connectionServiceRead: ConnectionManager.ReadService.ConnectionManagerReadServices;
+  private connectionServiceWrite: ConnectionManager.WriteService.ConnectionManagerWriteServices;
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(ConnectionClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+    private usersService: UsersService,
+    private connectorService: ConnectorClientService,
+    private dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.connectionServiceWrite =
+      this.grpcClient.getService<ConnectionManager.WriteService.ConnectionManagerWriteServices>(
+        ConnectionManager.ProtoServices.ConnectionManagerWriteServices,
+      );
+
+    this.connectionServiceRead =
+      this.grpcClient.getService<ConnectionManager.ReadService.ConnectionManagerReadServices>(
+        ConnectionManager.ProtoServices.ConnectionManagerReadServices,
+      );
+  }
+
+  async createConnection(
+    body: CreateConnectionDto,
+    metadata,
+  ): Promise<ConnectionRes> {
+    const {
+      connector_id,
+      description,
+      image_url,
+      name,
+      network_config_id,
+      properties,
+      type,
+      connector_name,
+      plugin,
+    } = body;
+
+    return lastValueFrom(
+      this.connectionServiceWrite.CreateConnection(
+        {
+          connector_id,
+          connector_name,
+          description,
+          image_url,
+          name,
+          network_config_id,
+          properties: JSON.stringify(properties),
+          type,
+          plugin,
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async updateConnection(
+    id: string,
+    body: UpdateConnectionDto,
+    metadata,
+  ): Promise<ConnectionRes> {
+    const { description, name, network_config_id, properties } = body;
+
+    return lastValueFrom(
+      this.connectionServiceWrite.UpdateConnection(
+        {
+          id,
+          description,
+          name,
+          network_config_id,
+          properties: JSON.stringify(properties),
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async deleteConnection({ body, metadata }): Promise<ConnectionRes> {
+    const { id } = body;
+
+    return lastValueFrom(
+      this.connectionServiceWrite.DeleteConnection(
+        {
+          id,
+        },
+        metadata,
+      ),
+    );
+  }
+
+  async getAllConnections({ body, metadata }): Promise<ConnectionsRes> {
+    const { search, filters, size, page } = body;
+
+    return lastValueFrom(
+      this.connectionServiceRead.GetAllConnection(
+        { search, filters: JSON.stringify(filters), size, page },
+        metadata,
+      ),
+    );
+  }
+
+  async getConnectionDetails({
+    body,
+    metadata,
+  }): Promise<GetConnectionDetailsResponse> {
+    const id = body.connection_id || body.id;
+
+    return lastValueFrom(
+      this.connectionServiceRead.GetConnectionDetails({ id }, metadata),
+    );
+  }
+
+  async INTERNAL_getAllConnections({
+    customer_name,
+  }: {
+    customer_name: string;
+  }): Promise<INTERNAL_GetAllConnectionsResponse> {
+    return lastValueFrom(
+      this.connectionServiceRead.INTERNAL_GetAllConnections({ customer_name }),
+    );
+  }
+
+  async catalogExistingConnection(data: {
+    connection?: ConnectionToCatalogDto;
+    networkConfig?: NetworkConfigDto;
+  }) {
+    return lastValueFrom(
+      this.connectionServiceRead.INTERNAL_CatalogExistingConnection({
+        data_str: JSON.stringify(data),
+      }),
+    );
+  }
+
+  async synchronizeConnections(apply: boolean) {
+    const connectors = await this.connectorService
+      .getAllConnectors({
+        search: undefined,
+        filters: undefined,
+        language: 'pt-br',
+        page: 1,
+        size: 500,
+      })
+      .then(({ connectors: connectorsStr }) =>
+        JSON.parse(connectorsStr).map((c) => c),
+      );
+    const { customers } = await this.usersService.findAllCustomers();
+
+    const customer_connections: {
+      customer_name: string;
+      customer_id: string;
+      productConnections: Record<string, any>[];
+      platformConnections: ConnectionApiConnection[];
+      productNetworkConfigs: Record<string, any>[];
+      platformNetworkConfigs: ConnectionsApiNetworkConfig[];
+    }[] = [];
+    for (const customer of customers) {
+      const { name: customer_name, id: customer_id } = customer;
+
+      const customer_connection = await this.INTERNAL_getAllConnections({
+        customer_name,
+      }).then((res) => ({
+        customer_name,
+        customer_id,
+        platformConnections: res.platformConnections
+          ? JSON.parse(res.platformConnections)
+          : [],
+        productConnections: res.productConnections
+          ? JSON.parse(res.productConnections)
+          : [],
+        productNetworkConfigs: res.productNetworkConfigs
+          ? JSON.parse(res.productNetworkConfigs)
+          : [],
+        platformNetworkConfigs: res.platformNetworkConfigs
+          ? JSON.parse(res.platformNetworkConfigs)
+          : [],
+      }));
+      customer_connections.push(customer_connection);
+    }
+    this.logger.info(
+      'DEBUG_ConnectionsMigrations - ' + JSON.stringify(customer_connections),
+    );
+
+    const migrations: {
+      customer_name: string;
+      status: any[];
+      productConnectionsMissing: ConnectionApiConnection[];
+      productNetworkConfigsMissing: ConnectionsApiNetworkConfig[];
+    }[] = [];
+    for (const customer_connection of customer_connections) {
+      const productConnectionsMissing: ConnectionApiConnection[] = [];
+      const productNetworkConfigsMissing: ConnectionsApiNetworkConfig[] = [];
+      const status: any[] = [];
+      let updatePromises: Promise<any>[] = [];
+
+      this.logger.info(
+        'DEBUG_ConnectionsMigrations - customer_connections ' +
+          JSON.stringify(customer_connection),
+      );
+      const {
+        customer_name,
+        productConnections,
+        platformConnections,
+        productNetworkConfigs,
+        platformNetworkConfigs,
+        customer_id,
+      } = customer_connection;
+
+      platformConnections?.forEach((plc) => {
+        const productConnection = productConnections.find(
+          (prc) => prc.id === plc.config_id,
+        );
+        if (!productConnection) productConnectionsMissing.push(plc);
+      });
+      platformNetworkConfigs?.forEach((platNetConfig) => {
+        const productNetworkConfig = productNetworkConfigs.find(
+          (prodNetConfig) => prodNetConfig.id === platNetConfig.config_id,
+        );
+        if (!productNetworkConfig)
+          productNetworkConfigsMissing.push(platNetConfig);
+      });
+      if (apply) {
+        for (const network_config of productNetworkConfigsMissing) {
+          const networkConfigToCatalog = {
+            id: network_config.config_id,
+            created_at: network_config.created_at,
+            updated_at: network_config.updated_at,
+            customer_name: customer_name,
+            customer_id: customer_id,
+            user_id: '',
+            network_type_id: '',
+            network_type_name: network_config.type,
+            type: network_config.type,
+            plugin: network_config.plugin,
+            name: network_config.name,
+            description: network_config.description,
+            properties: undefined,
+            keywords: undefined,
+            tags: undefined,
+          };
+
+          updatePromises.push(
+            this.catalogExistingConnection({
+              networkConfig: networkConfigToCatalog,
+            }),
+          );
+        }
+
+        this.logger.info(
+          'DEBUG_ConnectionsMigrations - Creating missing network configs for customer' +
+            customer_name +
+            ' - ' +
+            JSON.stringify(customer_connection),
+        );
+        const updateStatuses = await Promise.all(updatePromises);
+        status.push(...updateStatuses);
+        updatePromises = [];
+        for (const connection of productConnectionsMissing) {
+          const connector = connectors.find(
+            (c) => c._source.plugin === connection.plugin,
+          );
+          const connectionToCatalog = {
+            id: connection.config_id,
+            created_at: connection.created_at,
+            updated_at: new Date().toISOString(),
+            customer_id: customer_id,
+            user_id: '',
+            plugin: connection.plugin,
+            name: connection.name,
+            description: connection.description,
+            image_url: connector._source.image,
+            network_config_id: connection.network_config_id,
+            type: connection.type,
+            connector_id: connector._id,
+            connector_name: connector._source.name,
+            in_use: 1,
+            username: '',
+            customer_name,
+          };
+
+          updatePromises.push(
+            this.catalogExistingConnection({ connection: connectionToCatalog }),
+          );
+        }
+        this.logger.info(
+          'DEBUG_ConnectionsMigrations - Creating missing connections for customer' +
+            ' - ' +
+            customer_name +
+            JSON.stringify(customer_connection),
+        );
+        const updateStatuses_2 = await Promise.all(updatePromises);
+        status.push(...updateStatuses_2);
+      }
+
+      this.logger.info(
+        'DEBUG_ConnectionsMigrations - migrations for customer ' +
+          customer_name +
+          ' - ' +
+          JSON.stringify(customer_connection),
+      );
+      migrations.push({
+        customer_name,
+        status,
+        productConnectionsMissing,
+        productNetworkConfigsMissing,
+      });
+    }
+
+    return { migrations };
+  }
+}

src/modules/connection/connection.controller.ts

@@ -0,0 +1,213 @@
+import {
+  Body,
+  Controller,
+  Post,
+  Inject,
+  Put,
+  Param,
+  Delete,
+  Get,
+  Query,
+  UseFilters,
+} from '@nestjs/common';
+import { ApiExcludeEndpoint, ApiTags } from '@nestjs/swagger';
+import { ConnectionClientService } from './client.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { ValidationPipe } from '../../pipes/object-validation.pipe';
+import {
+  ConnectionDetailsRes,
+  ConnectionRes,
+  ConnectionsRes,
+  GetAllConnectionsReq,
+  UpdateConnectionDto,
+} from './dtos/connection';
+import { CreateConnectionDto } from './dtos/connection';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+
+const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
+@UseFilters(new GrpcToHttpExceptionFilter())
+@ApiTags('connections')
+@Authenticated()
+@Controller('connections')
+export class ConnectionController {
+  logger: any;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private clientService: ConnectionClientService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('connections-migration')
+  @ApiExcludeEndpoint()
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.DADOSFERA.permissions.CONNECTIONS_MIGRATION,
+  )
+  async synchronizeConnections(@Query('apply') apply: string) {
+    return this.clientService.synchronizeConnections(apply === 'true');
+  }
+
+  @Post()
+  @RequireAllPermissions(connectionPermissions.CREATE)
+  async createConnection(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: CreateConnectionDto,
+  ): Promise<ConnectionRes> {
+    this.logger.info('/connections - Create Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+    });
+
+    const response: any = await this.clientService.createConnection(
+      body,
+      metadata,
+    );
+
+    return response;
+  }
+
+  @Put('/:id')
+  @RequireAllPermissions(connectionPermissions.CREATE)
+  async updateConnection(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @Body(new ValidationPipe()) body: UpdateConnectionDto,
+  ): Promise<ConnectionRes> {
+    this.logger.info('/connections - Update Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      language,
+      username,
+    });
+
+    const response: any = await this.clientService.updateConnection(
+      id,
+      body,
+      metadata,
+    );
+
+    return response;
+  }
+
+  @Delete('/:id')
+  @RequireAllPermissions(connectionPermissions.DELETE)
+  async deleteConnection(
+    @User() user: RequestUser,
+    @Param('id') id: string,
+  ): Promise<ConnectionRes> {
+    this.logger.info('/connections - Delete Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+    });
+
+    const response: any = await this.clientService.deleteConnection({
+      body: { id },
+      metadata,
+    });
+
+    return response;
+  }
+
+  @Get()
+  async getAllConnections(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Query() queries: GetAllConnectionsReq,
+  ): Promise<ConnectionsRes> {
+    this.logger.info('/connections - Get All Connection');
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const { search, size, page, ...filters } = queries;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      language,
+      username,
+    });
+
+    const response = await this.clientService.getAllConnections({
+      body: { search, size, page, filters },
+      metadata,
+    });
+
+    return response;
+  }
+
+  @Get('/:id')
+  async getConnectionDetails(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Param('id') id: string,
+    @Query() queries,
+  ): Promise<ConnectionDetailsRes> {
+    this.logger.info('/connections - Get Connection Details');
+
+    console.log(queries);
+
+    const { details, sensitive } = queries;
+
+    const { user_id, customer_id, customer_name, username } = user;
+    const metadata = PackTheMetadata({
+      user_id,
+      customer_id,
+      customer_name,
+      username,
+      details,
+      language,
+      sensitive,
+    });
+
+    const response: any = await this.clientService.getConnectionDetails({
+      body: { id },
+      metadata,
+    });
+
+    if (details === 'true') {
+      const properties = JSON.parse(response.connection.properties);
+
+      Object.assign(response.connection, {
+        pipelines: JSON.parse(response.connection.pipelines),
+        connection_controls: properties.connection_controls,
+        config_controls: properties.config_controls,
+        connection_steps: properties.connection_steps,
+      });
+
+      delete response.connection.properties;
+    } else {
+      Object.assign(response.connection, {
+        properties: JSON.parse(response.connection.properties),
+        pipelines: JSON.parse(response.connection.pipelines),
+      });
+    }
+
+    return response;
+  }
+}

src/modules/connection/connection.module.ts

@@ -0,0 +1,22 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { ConnectionClientService } from './client.service';
+import { ConnectionClientConfiguration } from './client.config';
+import { ConnectionController } from './connection.controller';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { UsersModule } from '../users/users.module';
+import { ConnectorModule } from '../connector/connector.module';
+
+const client = new ConnectionClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    UsersModule,
+    ConnectorModule,
+  ],
+  controllers: [ConnectionController],
+  providers: [ConnectionClientService, DadosferaLogger],
+  exports: [ConnectionClientService],
+})
+export class ConnectionModule {}

src/modules/connection/dtos/connection.ts

@@ -0,0 +1,201 @@
+import {
+  ApiProperty,
+  ApiPropertyOptional,
+  OmitType,
+  PickType,
+} from '@nestjs/swagger';
+import {
+  Connection,
+  ConnectionToCatalog,
+} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
+import {
+  GetAllConnectionRequest,
+  GetAllConnectionResponse,
+  GetConnectionDetailsResponse,
+  GetConnectionResponse,
+} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
+export type ConnectionTypes = 'database' | 'file' | 'application';
+export type ConnectionCredentialsType =
+  | 'basic_auth'
+  | 'iam_user'
+  | 'role_name'
+  | 'oauth'
+  | 'api_key'
+  | 'service_account'
+  | 'headers_auth';
+export interface ConnectionApiConnection {
+  config_id: string;
+  plugin: string;
+  secret_id: string;
+  created_at: string;
+  network_config_id: null;
+  type: ConnectionTypes;
+  credentials_type: ConnectionCredentialsType;
+  credentials: Record<string, any>;
+  sensitive_fields: string[];
+  name: string;
+  customer_name: string;
+  description: string;
+  updated_at: string;
+}
+export class ConnectionDto implements Connection {
+  // ---Automatically generated information will not be sent by the frontend--- //
+  @ApiProperty()
+  id: string;
+
+  @ApiProperty()
+  customer_id: string;
+
+  @ApiProperty()
+  customer_name: string;
+
+  @ApiProperty()
+  user_id: string;
+
+  @ApiProperty()
+  updated_at: string;
+
+  @ApiProperty()
+  created_at: string;
+
+  @ApiProperty()
+  pipelines: string;
+
+  // ---Information sent by the frontend--- //
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  type: string;
+
+  @ApiPropertyOptional()
+  network_config_id: string | null;
+
+  @ApiProperty()
+  properties: DatabaseConnectionPropertiesDto | any;
+
+  // ---Information sent by the frontend (Heirs of Connector)--- //
+
+  @ApiProperty()
+  connector_id: string;
+
+  @ApiProperty()
+  connector_name: string;
+
+  @ApiProperty()
+  plugin: string;
+
+  @ApiProperty()
+  image_url: string;
+}
+
+export class ConnectionToCatalogDto implements ConnectionToCatalog {
+  // ---Automatically generated information - will not be sent by the frontend--- //
+  customer_id: string;
+  updated_at: string;
+  created_at: string;
+  // ---Information extracted from token--- //
+  user_id: string;
+  username: string;
+  customer_name: string;
+
+  // ---Information sent by the frontend--- //
+  @ApiPropertyOptional()
+  id: string;
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  type: string;
+
+  @ApiPropertyOptional()
+  network_config_id: string | null;
+
+  // ---Information sent by the frontend (Heirs of Connector)--- //
+  @ApiProperty()
+  connector_id: string;
+
+  @ApiProperty()
+  connector_name: string;
+
+  @ApiProperty()
+  plugin: string;
+
+  @ApiProperty()
+  image_url: string;
+}
+
+export class CreateConnectionDto extends OmitType(ConnectionDto, [
+  'id',
+  'customer_id',
+  'customer_name',
+  'user_id',
+  'created_at',
+  'updated_at',
+  'pipelines',
+]) {}
+
+export class UpdateConnectionDto extends PickType(ConnectionDto, [
+  'name',
+  'description',
+  'network_config_id',
+  'properties',
+]) {}
+
+export class GetAllConnectionsReq implements GetAllConnectionRequest {
+  @ApiPropertyOptional()
+  search?: string;
+  @ApiPropertyOptional()
+  page?: string;
+  @ApiPropertyOptional()
+  size?: string;
+}
+
+export class ConnectionsRes implements GetAllConnectionResponse {
+  @ApiProperty({ type: [ConnectionToCatalogDto] })
+  connections: ConnectionToCatalogDto[];
+
+  @ApiProperty()
+  message: string;
+}
+
+export class ConnectionRes implements GetConnectionResponse {
+  @ApiProperty()
+  connection?: ConnectionToCatalogDto;
+
+  @ApiProperty()
+  message: string;
+}
+
+export class ConnectionDetailsRes implements GetConnectionDetailsResponse {
+  @ApiProperty()
+  connection?: ConnectionDto;
+
+  @ApiProperty()
+  message: string;
+}
+
+export class DatabaseConnectionPropertiesDto {
+  @ApiProperty()
+  credentials_type: 'basic_auth';
+  @ApiProperty()
+  plugin: string;
+  @ApiProperty()
+  host: string;
+  @ApiProperty()
+  port: string;
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  password: string;
+  @ApiProperty()
+  database: string;
+}

src/modules/connector/client.config.ts

@@ -1,10 +1,18 @@
 import { credentials } from '@grpc/grpc-js';
-import { ClientOptions, Transport } from '@nestjs/microservices';
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
 import { ConnectorManager } from '@dadosfera/protospack-v2';
 
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
+
 export class ConnectorClientConfiguration {
-  config(): ClientOptions {
+  public name = 'ConnectorClientConfiguration';
-    return {
+  private config: GrpcOptions = {
     transport: Transport.GRPC,
     options: {
       url: process.env.INFACTORY_URL,
@@ -12,20 +20,21 @@ export class ConnectorClientConfiguration {
         ConnectorManager.ProtoPackages.WritePackage,
         ConnectorManager.ProtoPackages.ReadPackage,
       ],
-        // credentials: undefined,
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
-        credentials: process.env.LOCAL_ENV
-          ? undefined
-          : credentials.createSsl(),
       protoPath: [
         ConnectorManager.ProtoPaths.WriteFilePath,
         ConnectorManager.ProtoPaths.ReadFilePath,
       ],
       loader: {
+        keepCase: true,
         enums: String,
         objects: true,
         arrays: true,
       },
     },
   };
-  }
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
 }

src/modules/connector/client.service.ts

@@ -8,12 +8,14 @@ import {
 import { ClientGrpc } from '@nestjs/microservices';
 import { ConnectorManager } from '@dadosfera/protospack-v2';
 import { lastValueFrom } from 'rxjs';
+import { ConnectorClientConfiguration } from './client.config';
 
 export class ConnectorClientService implements OnModuleInit {
   private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices;
   private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices;
   constructor(
-    @Inject('CONNECTOR_PACKAGE') private readonly grpcClient: ClientGrpc,
+    @Inject(ConnectorClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
   ) {}
 
   onModuleInit() {
@@ -43,7 +45,6 @@ export class ConnectorClientService implements OnModuleInit {
     return lastValueFrom(
       this.connectorServiceWrite.RegisterConnector(serviceBody),
     ).catch((err) => {
-      console.log(err.details);
       throw new HttpException(
         err.details,
         err.code === 6 ? HttpStatus.CONFLICT : 400,
@@ -62,7 +63,6 @@ export class ConnectorClientService implements OnModuleInit {
 
     return lastValueFrom(this.connectorServiceWrite.UploadFile(body)).catch(
       (err) => {
-        console.log(err.details);
         throw new HttpException(
           err.details,
           err.code === 6 ? HttpStatus.CONFLICT : 400,
@@ -71,27 +71,43 @@ export class ConnectorClientService implements OnModuleInit {
     );
   }
 
-  async uploadConnectorsFile(uploadFile) {
+  async uploadConnectorsFile(uploadFile: Express.Multer.File) {
-    const connectors = JSON.parse(uploadFile.file.buffer.toString('utf8'));
+    const res = await lastValueFrom(
-
+      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage({
-    const body = connectors.map((connector) => {
+        connectors: uploadFile.buffer,
-      return JSON.stringify(connector);
+      }),
-    });
-
-    const request = {
-      connectors: body,
-    };
-
-    return lastValueFrom(
-      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage(
-        request,
-      ),
     ).catch((err) => {
       throw new HttpException(
         err.details,
         err.code === 6 ? HttpStatus.CONFLICT : 400,
       );
     });
+    const responseParsed = JSON.parse(res.message);
+    const connectorsResponse = responseParsed.items.filter(
+      (i) => i.update._index === process.env.CONNECTORS_INDEX,
+    );
+    const connectorErrors = connectorsResponse
+      .filter((i) => i.update.error)
+      .map((i) => ({
+        id: i.update?._id,
+        reason: i.update?.error,
+        status: i.error?.status,
+      }));
+    const connectorCreated = connectorsResponse
+      .filter((i) => i.update.result === 'created')
+      .map((i) => i.update._id);
+    const connectorUpdated = connectorsResponse
+      .filter((i) => i.update.result === 'updated')
+      .map((i) => i.update._id);
+    const connectorNoop = connectorsResponse
+      .filter((i) => i.update.result === 'noop')
+      .map((i) => i.update._id);
+    return {
+      errors: connectorErrors,
+      created: connectorCreated,
+      updated: connectorUpdated,
+      noop: connectorNoop,
+    };
   }
 
   async getAllConnectors(body) {
@@ -116,13 +132,13 @@ export class ConnectorClientService implements OnModuleInit {
     });
   }
 
-  async getConnectorDetails(plugin: string, language: string) {
+  async getConnectorDetails(connectorId: string, language: string) {
     const meta = new Metadata();
     meta.add('Language', language);
     return lastValueFrom(
       this.connectorServiceRead.GetConnectorDetails(
         {
-          plugin,
+          plugin: connectorId,
         },
         meta,
       ),
@@ -169,12 +185,7 @@ export class ConnectorClientService implements OnModuleInit {
         version,
         plugin,
       }),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
     );
-    });
   }
 
   async updateConnector({ plugin, changes }) {

src/modules/connector/connector.controller.ts

@@ -11,16 +11,13 @@ import {
   UploadedFile,
   UseInterceptors,
   Inject,
-  Headers,
   UploadedFiles,
 } from '@nestjs/common';
 import { FileInterceptor, FilesInterceptor } from '@nestjs/platform-express';
-import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
+import { ApiConsumes, ApiTags } from '@nestjs/swagger';
 import { ConnectorClientService } from './client.service';
 import { AddTagDto } from './dtos/add-tag';
 import { CreateConnectorDto } from './dtos/create-connector';
-import { UploadConnectorsFileDto } from './dtos/create-multiple-connector';
-import { DeleteConnectorDto } from './dtos/delete-connector';
 import { GetAllDto } from './dtos/get-all';
 import { RemoveTagDto } from './dtos/remove-tag';
 import { UpdateDto } from './dtos/update';
@@ -28,11 +25,15 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
+  RequireSomePermission,
-import { PERMISSIONS } from 'src/authentication/permissions.enum';
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('connectors')
-@ApiBearerAuth()
 @Authenticated()
 @Controller('connectors')
 export class ConnectorController {
@@ -46,7 +47,7 @@ export class ConnectorController {
   }
 
   @Post()
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.CREATE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
   @ApiConsumes('multipart/form-data')
   @UseInterceptors(FileInterceptor('file'))
   async uploadConnector(
@@ -64,7 +65,7 @@ export class ConnectorController {
   }
 
   @Post('/uploads')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.CREATE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
   @ApiConsumes('multipart/form-data')
   @UseInterceptors(FilesInterceptor('files'))
   async uploadFile(@UploadedFiles() files) {
@@ -81,26 +82,28 @@ export class ConnectorController {
   }
 
   @Post('/multiple-connectors-upload')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.CREATE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
   @ApiConsumes('multipart/form-data')
   @UseInterceptors(FileInterceptor('file'))
-  async uploadConnectorsFile(@UploadedFile() file: UploadConnectorsFileDto) {
+  async uploadConnectorsFile(@UploadedFile() file: Express.Multer.File) {
-    console.log(`/upload`, 'Upload Connector Route');
+    this.logger.info(`/upload`, 'Upload Connector Route');
 
-    const response = await this.connectorClientService.uploadConnectorsFile({
+    return await this.connectorClientService.uploadConnectorsFile(file);
-      file,
-    });
-
-    return response;
   }
 
   @Get()
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
   async getAllConnectors(
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
     @Query() queries: GetAllDto,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/GET - getAllConnectors Route');
 
     const { search, size, page, ...filters } = queries;
 
@@ -112,7 +115,7 @@ export class ConnectorController {
       page: page || 1,
     });
 
-    const connectors = JSON.parse(response.connectors).connectors;
+    const connectors = JSON.parse(response.connectors);
 
     return {
       message: response.message,
@@ -121,19 +124,31 @@ export class ConnectorController {
   }
 
   @Get('/tags')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
   async getConnectorsTags() {
     return await this.connectorClientService.getConnectorsTags();
   }
 
   @Get('/:plugin')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
   async getConnector(
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
     @Query('version') version: string,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/GET/:plugin - getConnector Route');
 
     const pluginId = `${plugin}-${version}`;
 
@@ -149,14 +164,19 @@ export class ConnectorController {
   }
 
   @Get('/:plugin/details')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CONNECTORS.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
+    PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
+  )
   async getConnectorDetails(
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
     @Query('version') version: string,
   ) {
     this.logger.info('/upload - Upload Connector Route');
-
     const pluginId = `${plugin}-${version}`;
 
     const response: any = await this.connectorClientService.getConnectorDetails(
@@ -171,7 +191,7 @@ export class ConnectorController {
   }
 
   @Put('/:plugin')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.UPDATE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
   @ApiConsumes('multipart/form-data')
   async updateConnector(
     @Param('plugin') plugin: string,
@@ -193,7 +213,7 @@ export class ConnectorController {
   }
 
   @Put('/:plugin/add-tag')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.UPDATE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
   async addTagOnConnector(
     @Param('plugin') plugin: string,
     @Body() body: AddTagDto,
@@ -220,7 +240,7 @@ export class ConnectorController {
   }
 
   @Put('/:plugin/remove-tag')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.UPDATE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
   async removeTagOnConnector(
     @Param('plugin') plugin: string,
     @Body() body: RemoveTagDto,
@@ -248,21 +268,18 @@ export class ConnectorController {
   }
 
   @Delete('/:plugin')
-  @RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.DELETE)
+  @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
   async deleteConnector(
     @Param('plugin') plugin: string,
-    @Body() { version }: DeleteConnectorDto,
+    @Query('version') version: string,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('DELETE /:plugin - Delete Connector Route');
 
-    const response: any = await this.connectorClientService.deleteConnector(
+    const response = await this.connectorClientService.deleteConnector(
       plugin,
       version,
     );
 
-    return {
+    return { message: response.message, connector: response.connector };
-      message: response.message || 'ok',
-      connector: { ...JSON.parse(response.connector) },
-    };
   }
 }

src/modules/connector/connector.module.ts

@@ -5,18 +5,12 @@ import { ConnectorClientConfiguration } from './client.config';
 import { ConnectorController } from './connector.controller';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
-const connectorClient = new ConnectorClientConfiguration();
+const client = new ConnectorClientConfiguration();
 
 @Module({
   controllers: [ConnectorController],
   providers: [ConnectorClientService, DadosferaLogger],
-  imports: [
+  imports: [ClientsModule.register([client.providerOptions])],
-    ClientsModule.register([
+  exports: [ConnectorClientService],
-      {
-        name: 'CONNECTOR_PACKAGE',
-        ...connectorClient.config(),
-      },
-    ]),
-  ],
 })
 export class ConnectorModule {}

src/modules/customers/customers.controller.ts

@@ -0,0 +1,94 @@
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import {
+  Body,
+  Controller,
+  Get,
+  HttpCode,
+  HttpStatus,
+  Inject,
+  Param,
+  Put,
+  Query,
+  UseFilters,
+} from '@nestjs/common';
+import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { CustomersService } from './customers.service';
+import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import type { StringValue } from 'ms';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+
+@ApiTags('Customers')
+@Controller('customers')
+@UseFilters(GrpcToHttpExceptionFilter)
+export class CustomersController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private customersService: CustomersService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get(':id/links')
+  @Authenticated()
+  @ApiOkResponse({ type: CustomerLinksResponse })
+  async getCustomerLinks(@Param('id') id: string) {
+    this.logger.info('getCustomerLinks', { id });
+    const links = await this.customersService.getLinks(id);
+    return { links };
+  }
+
+  @Put(':id/links')
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @ApiOkResponse()
+  @HttpCode(HttpStatus.OK)
+  async setCustomerLinks(
+    @Body() body: CustomerLinkRequest,
+    @Param('id') id: string,
+  ) {
+    const { links } = body;
+    this.logger.info('setCustomerLinks', { id, links });
+    await this.customersService.setLinks(id, links);
+  }
+
+  @Get('token')
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
+  @ApiProduces('text/plain')
+  async getCustomerToken(
+    @Query('exp') exp: StringValue,
+    @User() user: RequestUser,
+  ): Promise<string> {
+    this.logger.info('getCustomerToken', { exp, user });
+    const data = {
+      customerId: user.customer_id,
+      userId: user.user_id,
+      customerName: user.customer_name,
+    };
+    return this.customersService.generateToken(exp, data);
+  }
+
+  @Get('monitoring-dashboard')
+  @Authenticated()
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
+  )
+  async getCustomerMonitoringDashboard(
+    @User() user: RequestUser,
+  ): Promise<{ url: string }> {
+    this.logger.info('getCustomerMonitoringDashboard');
+
+    const metadata = PackTheMetadata(user);
+    return this.customersService.getMonitoringDashboardUrl(metadata);
+  }
+}

src/modules/customers/customers.module.ts

@@ -0,0 +1,23 @@
+import { Module } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import { CustomersController } from './customers.controller';
+import { CustomersService } from './customers.service';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
+
+const ducClient = new DucClient();
+const piFactoryClient = new PipelinesClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([
+      ducClient.providerOptions,
+      piFactoryClient.providerOptions,
+    ]),
+  ],
+  controllers: [CustomersController],
+  providers: [CustomersService, DadosferaLogger],
+  exports: [CustomersService],
+})
+export class CustomersModule {}

src/modules/customers/customers.service.ts

@@ -0,0 +1,159 @@
+import {
+  OnModuleInit,
+  Inject,
+  Injectable,
+  HttpException,
+  HttpStatus,
+  InternalServerErrorException,
+} from '@nestjs/common';
+
+import { firstValueFrom, lastValueFrom } from 'rxjs';
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import ErrorCodes from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import {
+  GetSecretValueCommand,
+  SecretsManagerClient,
+} from '@aws-sdk/client-secrets-manager';
+import getEnv from 'src/utils/getEnv';
+import { logger } from 'elastic-apm-node';
+import {
+  ReadService,
+  ProtoServices as PipelineProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
+import { Metadata } from '@grpc/grpc-js';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+// This function will accept any string, which may result in a bug.
+@Injectable()
+export class CustomersService implements OnModuleInit {
+  private customerService: CustomersProtoService;
+  private logger: DadosferaLogger;
+  private pipelineReadService: ReadService.PipelineV2ReadService;
+
+  constructor(
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly pipelinesGrpcClient: ClientGrpc,
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.customerService = this.grpcClient.getService<CustomersProtoService>(
+      ProtoServices.CustomersProtoService,
+    );
+    this.pipelineReadService =
+      this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
+        PipelineProtoServices.PipelineV2ReadService,
+      );
+  }
+
+  async getLinks(customerId: string) {
+    try {
+      const result = await lastValueFrom(
+        this.customerService.CustomerFindOneById({ id: customerId }),
+      );
+      return result.customer?.links || [];
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      throw err;
+    }
+  }
+
+  async setLinks(customerId: string, links: Link[]) {
+    if (!customerId || !links) {
+      throw new HttpException(null, HttpStatus.BAD_REQUEST);
+    }
+
+    try {
+      return await firstValueFrom(
+        this.customerService.CustomerUpdate({
+          id: customerId,
+          links,
+        } as CustomerUpdateRequest),
+      );
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      else throw err;
+    }
+  }
+
+  async generateToken(
+    expiresIn = '30m',
+    data: { customerId: string; userId: string; customerName: string },
+  ) {
+    const { customerId, userId: generatedById, customerName } = data;
+
+    const secretsManagerClient = new SecretsManagerClient({});
+
+    const getSecretComand = new GetSecretValueCommand({
+      SecretId: `${getEnv()}/${customerName}/jwt-signing-key`,
+    });
+    let private_key_pem;
+    await secretsManagerClient
+      .send(getSecretComand)
+      .then((res) => {
+        const secret = JSON.parse(res.SecretString);
+
+        private_key_pem = secret.private_key_pem;
+      })
+      .catch((err) => {
+        logger.error(err.stack);
+        throw new InternalServerErrorException(
+          'Error finding keys for customer',
+        );
+      });
+    let jwt_token;
+    try {
+      jwt_token = jwt.sign(
+        {
+          customerId,
+          generatedById,
+        },
+        private_key_pem,
+        {
+          algorithm: 'RS256',
+          expiresIn,
+          issuer: 'maestro',
+        },
+      );
+    } catch (err) {
+      logger.error(err.stack);
+      throw new InternalServerErrorException('Error generating token');
+    }
+    // const verify = jwt.verify(jwt_token, public_key_pem, {
+    //   algorithms: ['RS256'],
+    //   issuer: 'maestro',
+    // });
+    // const decoded = jwt.decode(jwt_token, { complete: true });
+    return jwt_token;
+  }
+  async getMonitoringDashboardUrl(metadata: Metadata) {
+    logger.info('CustomersService - getMonitoringDashboardUrl');
+
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '45',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    logger.info('Done');
+
+    return res;
+  }
+}

src/modules/customers/dtos/customers.ts

@@ -0,0 +1,23 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+
+export class CustomerLink implements Link {
+  @ApiProperty()
+  href: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+  @ApiPropertyOptional()
+  iconSrc: string;
+}
+export class CustomerLinkRequest {
+  @ApiProperty({ type: [CustomerLink] })
+  links: CustomerLink[];
+}
+
+export class CustomerLinksResponse {
+  @ApiProperty({ type: [CustomerLink] })
+  links: CustomerLink[];
+}
+

src/modules/duc/client.config.ts

@@ -0,0 +1,41 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import {
+  ProtoPackages,
+  ProtoPaths,
+} from '@dadosfera/protospack-v2/dist/lib/Duc';
+
+const isLocalConnection =
+  process.env.DUC_URL.startsWith('duc:') ||
+  process.env.DUC_URL.includes('0.0.0.0');
+
+export class DucClient {
+  public name = 'DucClient';
+
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.DUC_URL,
+      package: [ProtoPackages.WritePackage, ProtoPackages.ReadPackage],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [ProtoPaths.WriteFilePath, ProtoPaths.ReadFilePath],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+      maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem
+      maxReceiveMessageLength: 15 * 1024 * 1024,
+    },
+  };
+
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/health/health.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { HealthController } from './health.controller';
+import { HealthService } from './health.service';
+
+@Module({
+  controllers: [HealthController],
+  providers: [HealthService, DadosferaLogger],
+})
+export class HealthModule {}

src/modules/inputs/dtos/input.model.ts

@@ -1,135 +1,45 @@
-import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
-import { Info } from 'src/clients/inputs/interfaces';
-import { Method } from 'axios';
 
-export class InputModel {
+export class Column {
-  category: string;
-  plugin: string;
-  name: string;
-  cron?: string;
-  credentials?: {
-    connection_type: string;
-    oauth: string;
-    oauth_code: string;
-    start_date: string;
-  };
-  options?: {
-    oauth?: {
-      get_tokens_url: string;
-      get_tokens_method?: Method;
-      get_tokens_url_params: string;
-      get_tokens_set_response: Record<string, string>;
-      content_type: string;
-    };
-  };
-}
-
-class GoogleAnalyticsClientSecrets {
-  @ApiProperty()
-  type: string;
-  @ApiProperty()
-  project_id: string;
-  @ApiProperty()
-  private_key_id: string;
-  @ApiProperty()
-  private_key: string;
-  @ApiProperty()
-  client_email: string;
-  @ApiProperty()
-  client_id: string;
-  @ApiProperty()
-  auth_uri: string;
-  @ApiProperty()
-  token_uri: string;
-  @ApiProperty()
-  auth_provider_x509_cert_url: string;
-  @ApiProperty()
-  client_x509_cert_url: string;
-}
-class OauthObject {
-  @ApiPropertyOptional()
-  get_tokens_url: string;
-  @ApiPropertyOptional()
-  get_tokens_url_params: string;
-  @ApiPropertyOptional()
-  get_tokens_set_response: Record<string, any>;
-  @ApiPropertyOptional()
-  content_type: string;
-}
-class InputOptions {
-  @ApiPropertyOptional()
-  oauth: OauthObject;
-  @ApiPropertyOptional()
-  skip_select_columns: true;
-  @ApiPropertyOptional()
-  skip_select_entities: false;
-  @ApiPropertyOptional()
-  skip_transformation: true;
-}
-
-class CredentialsJdbc {
-  @ApiPropertyOptional()
-  jdbc_user: string;
-  @ApiPropertyOptional()
-  jdbc_password: string;
-  @ApiPropertyOptional()
-  database: string;
-  @ApiPropertyOptional()
-  endpoint: string;
-  @ApiPropertyOptional()
-  port: string;
-  @ApiPropertyOptional()
-  engine: string;
-  @ApiPropertyOptional()
-  schema: string;
-}
-class FileFormatParams {
-  @ApiPropertyOptional()
-  file_format: string;
-  @ApiPropertyOptional()
-  encoding: string;
-}
-
-class Credentials extends CredentialsJdbc {
-  @ApiProperty()
-  connection_type: string;
-
-  @ApiPropertyOptional()
-  client_aws_access_key_id: string;
-  @ApiPropertyOptional()
-  client_aws_secret_access_key: string;
-  @ApiPropertyOptional()
-  client_bucket: string;
-  @ApiPropertyOptional()
-  file_to_extract: string;
-  @ApiPropertyOptional()
-  file_format_params: FileFormatParams;
-
-  @ApiPropertyOptional()
-  view_id: string;
-  @ApiPropertyOptional()
-  client_secrets: GoogleAnalyticsClientSecrets;
-  @ApiPropertyOptional()
-  start_date: string;
-  @ApiPropertyOptional()
-  end_date: string;
-
-  @ApiPropertyOptional()
-  oauth_code: string;
-}
-class Column {
   @ApiProperty()
   name: string;
   @ApiProperty()
   type: string;
 }
-class TableColumns {
+export class TableColumns {
   @ApiProperty()
   name: string;
   @ApiProperty()
-  columns: Column[];
+  columns: string[];
   @ApiProperty()
   references: Column[];
+  @ApiProperty()
+  type: string;
+}
+export class AvailableEntity {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  replication_methods: string[];
+}
+
+export class Input {
+  @ApiProperty()
+  id: string;
+  @ApiProperty({ type: [TableColumns] })
+  tables: TableColumns[];
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  plugin: string;
+  @ApiPropertyOptional()
+  type?: string;
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  updated_at: string;
 }
 export class GetAvailableEntitiesReq {
   @ApiProperty()
@@ -137,79 +47,12 @@ export class GetAvailableEntitiesReq {
 }
 
 export class GetAvailableEntitiesRes {
-  @ApiProperty()
+  @ApiProperty({ type: [AvailableEntity] })
-  entities: string[];
+  entities: AvailableEntity[];
 }
 
-export class TestConnectionGetColumnsReq {
+export class CreateInputReq extends OmitType(Input, [
-  @ApiProperty()
+  'id',
-  plugin: string;
+  'created_at',
-  @ApiProperty()
+  'updated_at',
-  tables: string[];
+]) {}
-  @ApiPropertyOptional()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  id: string;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionGetColumnsRes {
-  @ApiProperty()
-  tables: TableColumns[];
-}
-
-export class TestConnectionReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionRes {
-  @ApiProperty()
-  connection_state: boolean;
-  @ApiProperty()
-  total_entities: number;
-  @ApiProperty()
-  database_tables: string[];
-}
-export class CreateInputReq {
-  @ApiPropertyOptional()
-  id: string;
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  category: string;
-  @ApiProperty()
-  credentials: Credentials;
-  @ApiProperty()
-  name: string;
-  @ApiPropertyOptional()
-  options: InputOptions;
-  @ApiPropertyOptional()
-  info: Info;
-}
-
-export class Input {
-  @ApiProperty()
-  id: string;
-  @ApiProperty()
-  category: string;
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  name: string;
-  @ApiProperty()
-  cron: string;
-  @ApiProperty()
-  credentials: Credentials;
-  @ApiProperty()
-  client_id: string;
-  @ApiProperty()
-  created_at: string;
-  @ApiProperty()
-  updated_at: string;
-  @ApiPropertyOptional()
-  info: Info;
-}

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,3 +1,5 @@
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+
 interface Values {
   jdbc_user: string;
   jdbc_password: string;
@@ -9,32 +11,6 @@ interface Values {
   schema: string;
 }
 
-interface Cron {
-  hours: string[];
-  hour_interval: boolean;
-  hour_resourse: boolean;
-  hour_resourse_value: number;
-  month_day: string[];
-  month_day_interval: boolean;
-  month_day_resourse: boolean;
-  month_day_resourse_value: number;
-  month: string[];
-  month_interval: boolean;
-  month_resourse: boolean;
-  month_resourse_value: number;
-  week_days: string[];
-  week_days_interval: boolean;
-  week_days_resourse: boolean;
-  week_days_resourse_value: number;
-}
-
-export interface Info {
-  customer_id: string;
-  user_id: string;
-  customer: string;
-  customer_tier: string;
-}
-
 export interface ICreateInputRequest {
   cron: string;
   name: string;
@@ -56,6 +32,7 @@ export interface UpdateInputRequest {
   values: Values;
   operation: string;
   info: Info;
+  tables: any[];
 }
 
 export interface ITestConnectionRequest {

src/modules/inputs/inputs-client.config.ts

@@ -0,0 +1,38 @@
+import { Input } from '@dadosfera/protospack-v2';
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  Transport,
+  type GrpcOptions,
+} from '@nestjs/microservices';
+
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
+export class InputsGrpcClient {
+  public readonly name = 'InputsGrpcClient';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [
+        Input.ProtoPackages.WritePackage,
+        Input.ProtoPackages.ReadPackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        Input.ProtoPaths.WriteFilePath,
+        Input.ProtoPaths.ReadFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        defaults: false,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/inputs/inputs.controller.ts

@@ -7,147 +7,123 @@ import {
   Param,
   Patch,
   Post,
+  UseFilters,
 } from '@nestjs/common';
 import { InputsService } from './inputs.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { InputsClientService } from 'src/clients/inputs/client.service';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
-import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
+import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
-import { PERMISSIONS } from '../../authentication/permissions.enum';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
 import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
 import {
   CreateInputReq,
   GetAvailableEntitiesReq,
   GetAvailableEntitiesRes,
   Input,
-  TestConnectionGetColumnsReq,
-  TestConnectionGetColumnsRes,
-  TestConnectionReq,
-  TestConnectionRes,
 } from './dtos/input.model';
+import { UpdateInputRequest } from './dtos/old_interfaces';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
-@ApiTags('inputs')
+@ApiTags('Inputs')
 @Controller('inputs')
 @AuthenticateCondition((req, user) => {
-  let action: any;
-
   switch (req.method) {
     case 'POST':
-      action = 'CREATE';
-      break;
-
     case 'PUT':
-      action = 'UPDATE';
-      break;
-
     case 'PATCH':
-      action = 'UPDATE';
+      return (
-      break;
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE.seqid,
+        ) ||
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
+        )
+      );
 
     default:
-      action = req.method;
+      return (
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE.seqid,
+        ) ||
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
+        ) ||
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.GET.seqid,
+        )
+      );
   }
-
-  if (
-    `${req.method} ${req.route.path}` ===
-    'GET /inputs/available-entities/:plugin'
-  ) {
-    action = 'GET_ENTITIES';
-  }
-
-  return user.permissions.includes(PERMISSIONS.INPUT.permissions[action].seqid);
 })
+@UseFilters(new GrpcToHttpExceptionFilter())
 export class InputsController {
-  inputService: InputsService;
   logger: DadosferaLogger;
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
-    private inputsClientService: InputsClientService,
+    private inputService: InputsService,
   ) {
     this.logger = dadosferaLogger.logger;
-    this.inputService = new InputsService(this.inputsClientService);
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('available-entities/:plugin')
   @ApiOkResponse({ type: GetAvailableEntitiesRes })
   async getAvailableEntities(
-    @Body() body,
     @Param() params: GetAvailableEntitiesReq,
+    @User() user: RequestUser,
   ) {
-    const { info } = body;
     const { plugin } = params;
-    return await this.inputService.getAvailableEntities({ info, plugin });
+    const availableEntities = await this.inputService.getAvailableEntities({
-  }
+      info: {
-
+        customer: user.customer_name,
-  @Post('/test-connection')
+        customer_id: user.customer_id,
-  @ApiOkResponse({ type: TestConnectionRes })
+        user_id: user.user_id,
-  async testConnection(@Body() data: TestConnectionReq) {
-    this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
-      user: data.info.user_id,
-      customer: data.info.customer,
-    });
-
-    const response = await this.inputService.testConnection(data);
-
-    return response;
-  }
-
-  @Post('/test-connection/get-columns')
-  @ApiOkResponse({ type: TestConnectionGetColumnsRes })
-  async getColumns(@Body() data: TestConnectionGetColumnsReq) {
-    this.logger.info(
-      `/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
-      {
-        user: data.info.user_id,
-        customer: data.info.customer,
       },
-    );
+      plugin,
-
+    });
-    const response = await this.inputService.getColumns(data);
+    return availableEntities;
-
-    return response;
   }
 
   @Post()
   @ApiOkResponse({ type: Input })
-  async create(@Body() createInputDto: CreateInputReq) {
+  async create(@Body() body: CreateInputReq, @User() user: RequestUser) {
+    const info: Info = {
+      user_id: user.user_id,
+      customer: user.customer_name,
+      customer_id: user.customer_id,
+    };
     this.logger.info(`/input - ON CREATE ROUTE`, {
-      user: createInputDto.info.user_id,
+      user: info.user_id,
-      customer: createInputDto.info.customer,
+      customer: info.customer,
     });
 
-    const response = await this.inputService.create(createInputDto);
+    const response = await this.inputService.create({ body, info });
-
-    return response;
-  }
-
-  @Post(':id')
-  @ApiOkResponse({ type: Input })
-  async reCreate(@Param('id') id: string, @Body() input: CreateInputReq) {
-    if (!input.id) input.id = id;
-    this.logger.info(`POST /${id}`, 'ON RECREATE ROUTE');
-
-    const response = await this.inputService.reCreate(input);
 
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get()
-  async findAll(@Body() body) {
+  async findAll(@User() user: RequestUser) {
+    const { user_id, customer_id, customer_name: customer } = user;
     this.logger.info(`/input - ON FIND ALL ROUTE`, {
-      user: body.info.user_id,
+      user: user.user_id,
-      customer: body.info.customer,
+      customer,
     });
-
+    const info: Info = {
-    const response = await this.inputService.findAll(body);
+      customer,
+      customer_id,
+      user_id,
+    };
+    const response = await this.inputService.findAll(info);
 
     return response;
   }
 
   @Get('/:id')
-  async findOne(@Body() body, @Param() params) {
+  async findOne(@Body() body, @Param('id') id: string) {
-    const { id } = params;
     this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
       user: body.info.user_id,
       customer: body.info.customer,
@@ -158,6 +134,7 @@ export class InputsController {
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Patch(':id')
   async update(@Body() updateInputDto: UpdateInputRequest, @Param() params) {
     const { id } = params;
@@ -173,6 +150,7 @@ export class InputsController {
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Delete(':id')
   async delete(@Body() data, @Param() params) {
     const { id } = params;

src/modules/inputs/inputs.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { InputsGrpcClient } from './inputs-client.config';
+import { InputsController } from './inputs.controller';
+import { InputsService } from './inputs.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+const client = new InputsGrpcClient();
+
+@Module({
+  controllers: [InputsController],
+  providers: [InputsService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions])],
+  exports: [InputsService],
+})
+export class InputsModule {}

src/modules/inputs/inputs.service.ts

@@ -1,20 +1,131 @@
-import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import {
-import { Timeout } from '@nestjs/schedule';
+  ForbiddenException,
-import { DecodeGrpcStruct } from 'protospack';
+  HttpException,
+  HttpStatus,
+  Inject,
+  Injectable,
+} from '@nestjs/common';
 import CronParser, { CronExpression } from 'cron-parser';
-import { InputsClientService } from 'src/clients/inputs/client.service';
+import { ClientGrpc } from '@nestjs/microservices';
-import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
+import { lastValueFrom } from 'rxjs';
+import { objectCamelToSnake } from 'src/utils/CaseConverter';
+import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
+import { Input } from '@dadosfera/protospack-v2';
+import {
+  GetAvailableEntitiesRequest,
+  InputCreateGenericRequest,
+  InputCreateS3Request,
+  InputNewCreateRequest,
+  TestConnectionRequest,
+} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { CreateInputReq } from './dtos/input.model';
 
 @Injectable()
+
+//TODO refactor this class to remove OLD_inputClient
 export class InputsService {
-  constructor(private inputClient: InputsClientService) {}
+  logger: DadosferaLogger;
+  inputWriteService: Input.WriteService.InputWriteService;
+  inputReadService: Input.ReadService.InputReadService;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    @Inject('InputsGrpcClient') private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  async onModuleInit() {
+    this.inputWriteService =
+      this.grpcClient.getService<Input.WriteService.InputWriteService>(
+        Input.ProtoServices.InputWriteService,
+      );
+    this.inputReadService =
+      this.grpcClient.getService<Input.ReadService.InputReadService>(
+        Input.ProtoServices.InputReadService,
+      );
+  }
+
+  OLD_inputClient = {
+    newCreate: async (createInputDto: InputNewCreateRequest) => {
+      this.logger.info('InputClientService - Create');
+
+      const input = await lastValueFrom(
+        this.inputWriteService.InputNewCreate(createInputDto),
+      );
+      objectCamelToSnake(input);
+      return input;
+    },
+    createS3Inputs: async (createInputDto: InputCreateS3Request) => {
+      this.logger.info('InputClientService - Create');
+      createInputDto;
+      const createInputResponse = await lastValueFrom(
+        this.inputWriteService.InputCreateS3(createInputDto),
+      ).catch((e) => {
+        throw new HttpException(e.details, 500);
+      });
+      this.logger.info('Done');
+      objectCamelToSnake(createInputResponse);
+      return createInputResponse;
+    },
+    update: async (updateInputDTO: UpdateInputRequest) => {
+      this.logger.info('InputClientService - Update');
+      const updateInputResponse = await lastValueFrom(
+        this.inputWriteService.InputUpdate(updateInputDTO),
+      );
+
+      return updateInputResponse;
+    },
+    testConnection: async (data: TestConnectionRequest) => {
+      this.logger.info('InputClientService - TestConnection');
+      const testConnectionResponse = await lastValueFrom(
+        this.inputReadService.NewTestConnection(data),
+      );
+
+      return testConnectionResponse;
+    },
+    getAvailableEntities: async (data) => {
+      this.logger.info('InputClientService - GetAvailableEntities');
+      data;
+      const response = await lastValueFrom(
+        this.inputReadService.GetAvailableEntities(data),
+      ).catch((err) => {
+        this.logger.info(err.details);
+        if (err.details && err.details.includes('400'))
+          throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
+        throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
+      });
+      objectCamelToSnake(response);
+
+      return response;
+    },
+  };
+
   secondsInADay = 60 * 60 * 24;
   secondsInAnHour = 60 * 60;
 
   adjustInputPayload(payload) {
-    if (payload?.input_generic) return DecodeGrpcStruct(payload.input_generic);
+    if (payload) {
-    if (!payload.input_s3 && !payload.input_jdbc) return payload;
+      if (payload.input_generic) {
-    return payload?.input_s3 || payload?.input_jdbc;
+        const { credentials } = payload.input_generic;
+        return {
+          ...payload.input_generic,
+          credentials: credentials && JSON.parse(credentials),
+        };
+      }
+      if (payload.input_s3) return payload.input_s3;
+      if (payload.input_jdbc) return payload.input_jdbc;
+
+      const { credentials } = payload;
+      return {
+        ...payload,
+        credentials: credentials && JSON.parse(credentials),
+      };
+    }
+    return payload;
   }
   getDifferenceInSeconds(date1: Date, date2: Date) {
     const diffInMs = Math.abs(date2.getTime() - date1.getTime());
@@ -37,87 +148,61 @@ export class InputsService {
     const afterNextDate = interval.next().toDate();
     const secondsApart = this.getDifferenceInSeconds(nextDate, afterNextDate);
     if (customer_tier === 'BASIC' && secondsApart < this.secondsInADay) {
-      throw new HttpException(
+      throw new ForbiddenException(
         'Intervalo de tempo não pode ser inferior a um dia.',
-        HttpStatus.FORBIDDEN,
       );
     } else if (secondsApart < this.secondsInAnHour) {
-      throw new HttpException(
+      throw new ForbiddenException(
         'Intervalo de tempo não pode ser inferior a uma hora.',
-        HttpStatus.FORBIDDEN,
       );
     }
   }
-  async create(@Body() data) {
+  async create(data: { body: CreateInputReq; info: Info }) {
-    this.validateCron(data);
+    // this.validateCron(data);
-    let response;
+    const { body, info } = data;
-    switch (data.plugin.toLowerCase()) {
+
-      case 'csv':
+    const inputCreateGenericRequest: InputCreateGenericRequest = {
-      case 'json':
+      input: {
-      case 'parquet': {
+        ...body,
-        const { info, ...input } = data;
+      },
-        const inputPayload = this.generateInputS3Payload(input);
-        response = await this.inputClient.createS3Inputs({
-          input: inputPayload,
       info,
-        });
+    };
-        break;
+
-      }
+    const { input } = await lastValueFrom(
-      case 'oracle':
+      this.inputWriteService.InputCreate(inputCreateGenericRequest),
-      case 'mysql':
-      case 'postgresql':
-      case 'sqlserver':
-        response = await this.inputClient.newCreate(data);
-        break;
-      default:
-        response = await this.inputClient.createGeneric(data);
-        break;
-    }
-    const adjustedInput = this.adjustInputPayload(response.input);
-    return { ...response, input: adjustedInput };
-  }
-  async reCreate(data) {
-    const { info, cron } = data;
-    if (cron) this.validateCron({ info, cron });
-    const response = await this.inputClient.createGeneric(data);
-    const adjustedInput = this.adjustInputPayload(response.input);
-    return { ...response, input: adjustedInput };
-  }
-  async getAvailableEntities(data): Promise<{ entities: string[] }> {
-    return await this.inputClient.getAvailableEntities(data);
-  }
-  async findAll(body) {
-    try {
-      const findAllInputResponse: any = await this.inputClient.findAll(body);
-      if (findAllInputResponse?.inputs?.length) {
-        findAllInputResponse.inputs = findAllInputResponse.inputs.map((input) =>
-          this.adjustInputPayload(input),
     );
+    const adjustedInput = this.adjustInputPayload(input);
+    return { input: adjustedInput };
   }
-      return findAllInputResponse;
+
-    } catch (err) {
+  async getAvailableEntities(data: GetAvailableEntitiesRequest) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
   }
+
+  async findAll(info: Info) {
+    const { inputs } = await lastValueFrom(
+      this.inputReadService.InputFindAll({ info }),
+    );
+    if (!inputs) return { inputs: [] };
+
+    const inputsReturn = inputs.map((i) => this.adjustInputPayload(i));
+    return { inputs: inputsReturn };
   }
 
   async findOne(idRequest: IIdRequest) {
-    try {
+    const findOneInputResponse: any = await lastValueFrom(
-      const findOneInputResponse: any = await this.inputClient.findOne(
+      this.inputReadService.InputFindOne(idRequest),
-        idRequest,
     );
     findOneInputResponse.input = this.adjustInputPayload(
       findOneInputResponse.input,
     );
     return findOneInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
   }
 
   async update(id: string, data, info: Info) {
     this.validateCron({ ...data, info });
     try {
-      const updateInputResponse: any = await this.inputClient.update({
+      const updateInputResponse: any = await this.OLD_inputClient.update({
         id,
         info,
         ...data,
@@ -133,21 +218,13 @@ export class InputsService {
   }
 
   async remove(idRequest: IIdRequest) {
-    try {
+    return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
-      const removeInputResponse = await this.inputClient.remove(idRequest);
-
-      return removeInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
   }
 
-  @Timeout(60000 * 10) // Timeout set for 10 minutes
   async testConnection(data) {
     try {
-      const testConnectionInputResponse = await this.inputClient.testConnection(
+      const testConnectionInputResponse =
-        data,
+        await this.OLD_inputClient.testConnection(data);
-      );
 
       return testConnectionInputResponse;
     } catch (err) {
@@ -155,17 +232,6 @@ export class InputsService {
     }
   }
 
-  async getColumns(data) {
-    try {
-      const testConnectionGetColumnsResponse =
-        await this.inputClient.getColumns(data);
-
-      return testConnectionGetColumnsResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
   generateInputS3Payload(payload) {
     const { credentials, plugin, cron } = payload;
     if (!credentials) return payload;

src/modules/mixpanel/mixpanel.controller.ts

@@ -0,0 +1,93 @@
+import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
+import { init } from 'mixpanel';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+
+@ApiInternalOnlyController()
+@Authenticated()
+@Controller('trackEvent')
+export class MixpanelController {
+  constructor(
+    @Inject('MIXPANEL_TOKEN')
+    private readonly mixpanelToken: string,
+  ) {}
+  @Post(':id')
+  async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
+    delete body.info;
+    const mixpanel = init(this.mixpanelToken);
+
+    const separator = user.username.includes('-') ? '-' : '.';
+    const removeValues = [
+      '.dadosferatech.dadosfera',
+      '.demo.dadosfera',
+      '.dadosferademo',
+      '.dadosferarh.dadosfera',
+      '.dadosferatech.dadosfera2',
+      '.dadosferatech.dadosfera',
+      '.dadosfera.fin',
+      '.dadosferafin.dadosfera',
+      '.praxio.dadosfera',
+      '.dadosfera.tech',
+      '.treinamentos@dadosfera.ai',
+      '.dadosfera2',
+      '.treinamentosfera',
+      '.dadosfera',
+    ];
+
+    let username = user.username;
+
+    removeValues.forEach((value) => {
+      username = username.replace(value, '');
+    });
+
+    username = username.split('@')?.[0];
+    username = username.split('+')?.[0];
+
+    let firstName = username
+      .substring(0, username.indexOf(separator))
+      .replace('dadosfera', '');
+    let lastName = username
+      .substring(username.lastIndexOf(separator) + 1)
+      .replace('dadosfera', '');
+
+    if (!firstName) {
+      firstName = lastName;
+      lastName = '';
+    }
+
+    firstName = this.capitalize(firstName);
+    lastName = this.capitalize(lastName);
+
+    await mixpanel.people.set(user.username, {
+      $first_name: firstName,
+      $last_name: lastName,
+      $name: this.getFullName(firstName, lastName),
+      $email: user.username.includes('@')
+        ? user.username
+        : user.username + '@dadosfera.ai',
+      customer_name: user.customer_name,
+    });
+
+    await mixpanel.track(id, {
+      distinct_id: user.username,
+      customer: user.customer_name,
+      env: process.env.ENV,
+      ...body,
+    });
+
+    return { id, body, user: user.username };
+  }
+
+  capitalize(sentence: string): string {
+    if (!sentence) {
+      return '';
+    }
+
+    return sentence[0].toUpperCase() + sentence.substring(1);
+  }
+
+  getFullName(firstName: string, lastName: string) {
+    return `${firstName}${lastName ? ' ' + lastName : ''}`;
+  }
+}

src/modules/mixpanel/mixpanel.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
+import { MixpanelController } from './mixpanel.controller';
+
+@Module({
+  controllers: [MixpanelController],
+  providers: [
+    {
+      provide: 'MIXPANEL_TOKEN',
+      useValue: getSecretFromSecretsManager(
+        `${process.env.ENV}/root/mixpanel_token`,
+      ),
+    },
+  ],
+})
+export class MixpanelModule {}

src/modules/network-config/dto/network-config.ts

@@ -0,0 +1,172 @@
+import {
+  NetworkConfigEntity,
+  NetworkConfigToCatalog,
+} from '@dadosfera/protospack-v2/dist/lib/NetworkConfig/interfaces/entities';
+import {
+  ApiProperty,
+  ApiPropertyOptional,
+  OmitType,
+  PickType,
+} from '@nestjs/swagger';
+
+export enum NetworkConfigTypesEnum {
+  vpn = 'vpn',
+}
+export enum NetworkConfigPluginsEnum {
+  openvpn = 'openvpn',
+}
+export interface ConnectionsApiNetworkConfig {
+  config_id: string;
+  created_at: string;
+  credentials: {
+    CERTIFICATE_FILE: string;
+    OVPN_CREDENTIALS: string;
+    TLS_AUTH_FILE: string;
+    OVPN_FILE: string;
+  };
+  customer_name: string;
+  description: string;
+  name: string;
+  plugin: string;
+  secret_id: string;
+  sensitive_fields: string[];
+  type: string;
+  updated_at: string;
+}
+export class NetworkConfigDto implements NetworkConfigEntity {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  customer_name: string;
+  @ApiProperty()
+  customer_id: string;
+  @ApiProperty()
+  properties: any;
+  @ApiProperty()
+  user_id: string;
+  @ApiProperty()
+  keywords: string;
+  @ApiProperty()
+  tags: string;
+
+  @ApiProperty()
+  network_type_id: string;
+  @ApiProperty()
+  network_type_name: string;
+  @ApiProperty({ enum: NetworkConfigTypesEnum })
+  type: string;
+  @ApiProperty({ enum: NetworkConfigPluginsEnum })
+  plugin: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+}
+
+export class CreateNetworkConfigReq extends PickType(NetworkConfigDto, [
+  'network_type_id',
+  'network_type_name',
+  'name',
+  'description',
+]) {
+  @ApiProperty({ format: 'binary' })
+  ovpn_file: string;
+}
+
+export class NetworkConfigOpenVpnProperties {
+  @ApiProperty({ enum: NetworkConfigPluginsEnum })
+  plugin: NetworkConfigPluginsEnum;
+  @ApiProperty()
+  OVPN_FILE: string;
+  @ApiPropertyOptional()
+  CERTIFICATE_FILE?: string;
+  @ApiPropertyOptional()
+  OVPN_CREDENTIALS?: string;
+  @ApiPropertyOptional()
+  TLS_AUTH_FILE?: string;
+}
+
+export class NetworkConfigToCatalogDto implements NetworkConfigToCatalog {
+  @ApiProperty()
+  id: string;
+
+  @ApiProperty()
+  customer_id: string;
+
+  @ApiProperty()
+  plugin: string;
+
+  @ApiProperty()
+  type: string;
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  created_at: string;
+
+  @ApiProperty()
+  updated_at: string;
+
+  @ApiProperty()
+  network_type_id: string;
+
+  @ApiProperty()
+  network_type_name: string;
+}
+export class FindAllNetworkConfigsRes {
+  @ApiProperty({ type: [NetworkConfigToCatalogDto] })
+  network_configs: NetworkConfigToCatalogDto[];
+
+  @ApiProperty()
+  message: string;
+}
+
+export class FindNetworkConfigByidRes {
+  @ApiProperty()
+  network_config: NetworkConfigToCatalogDto;
+}
+
+export class FindNetworkConfigByidDetails extends PickType(NetworkConfigDto, [
+  'id',
+  'name',
+  'description',
+  'type',
+  'network_type_id',
+  'customer_name',
+]) {
+  @ApiProperty()
+  properties: NetworkConfigOpenVpnProperties | string;
+}
+export class FindNetworkConfigByidDetailsRes {
+  @ApiProperty()
+  network_config: FindNetworkConfigByidDetails;
+}
+
+export class CreateNetworkConfigRes {
+  @ApiProperty()
+  network_config: NetworkConfigToCatalogDto;
+}
+
+export class UpdateNetworkConfigReq {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  description: string;
+}
+export class UpdateNetworkConfigRes {
+  @ApiProperty()
+  network_config: NetworkConfigToCatalogDto;
+}
+
+export class DeleteNetworkConfigRes {
+  @ApiProperty()
+  message: string;
+}

src/modules/network-config/network-config-client.config.ts

@@ -0,0 +1,41 @@
+import { credentials } from '@grpc/grpc-js';
+import {
+  ClientProviderOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { NetworkConfig } from '@dadosfera/protospack-v2';
+
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
+
+export class NetworkConfigGrpcClient {
+  public name = 'NetworkConfigGrpcClient';
+
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.INFACTORY_URL,
+      package: [
+        NetworkConfig.ProtoPackages.ReadPackage,
+        NetworkConfig.ProtoPackages.WritePackage,
+      ],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [
+        NetworkConfig.ProtoPaths.ReadFilePath,
+        NetworkConfig.ProtoPaths.WriteFilePath,
+      ],
+      loader: {
+        keepCase: true,
+        enums: String,
+        objects: true,
+        arrays: true,
+      },
+    },
+  };
+  providerOptions: ClientProviderOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/network-config/network-config.controller.ts

@@ -0,0 +1,151 @@
+import {
+  Body,
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Post,
+  Query,
+  UploadedFile,
+  UseInterceptors,
+} from '@nestjs/common';
+import { NetworkConfigService } from './network-config.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ApiConsumes, ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import {
+  CreateNetworkConfigReq,
+  CreateNetworkConfigRes,
+  FindAllNetworkConfigsRes,
+} from './dto/network-config';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+
+const networkConfigPermissions = PERMISSIONS_GROUPS.NETWORK_CONFIG.permissions;
+@ApiTags('Network Config')
+@Authenticated()
+@Controller('network-config')
+export class NetworkConfigController {
+  logger: any;
+  constructor(
+    private networkConfigService: NetworkConfigService,
+    @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get()
+  @ApiOkResponse({ type: FindAllNetworkConfigsRes })
+  async findAllNetworkConfigs(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Query() queries,
+  ) {
+    this.logger.info('GET /network-config', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+
+    const { user_id, customer_id, customer_name } = user;
+    const { search, size, page, ...filters } = queries;
+
+    return this.networkConfigService.findAllNetworkConfigs({
+      body: { search, size, page, filters },
+      info: { user_id, customer_id, customer_name, language },
+    });
+  }
+
+  @Get(':id')
+  @ApiOkResponse({ type: FindAllNetworkConfigsRes })
+  async findNetworkConfigById(
+    @User() user: RequestUser,
+    @Param('id') id: string,
+  ) {
+    this.logger.info('GET /network-config/:id', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    return this.networkConfigService.findNetworkConfigById(id);
+  }
+
+  @Get(':id/details')
+  @ApiOkResponse({ type: FindAllNetworkConfigsRes })
+  async findNetworkConfigByIdDetails(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+  ) {
+    this.logger.info('GET /network-config/:id', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+
+    return this.networkConfigService.findNetworkConfigByIdDetails(id, {
+      customer_name: user.customer_name,
+      customer_id: user.customer_id,
+      language,
+    });
+  }
+
+  @Post()
+  @ApiOkResponse({ type: CreateNetworkConfigRes })
+  @RequireAllPermissions(networkConfigPermissions.CREATE)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('ovpn_file'))
+  async createNetworkConfigs(
+    @User() user: RequestUser,
+    @UploadedFile() file: Express.Multer.File,
+    @Body() body: CreateNetworkConfigReq,
+  ) {
+    this.logger.info('POST /network-config', {
+      user: user.user_id,
+      customer: user.customer_name,
+    });
+    const { customer_id, customer_name, user_id } = user;
+    return this.networkConfigService.createNetworkConfig(body, file, {
+      customer_name,
+      customer_id,
+      user_id,
+    });
+  }
+
+  // @Patch(':id')
+  // @ApiOkResponse({ type: UpdateNetworkConfigRes })
+  // @RequireAllPermissions(networkConfigPermissions.CREATE)
+  // async updateNetworkConfig(
+  //   @User() user: RequestUser,
+  //   @Param('id') network_config_id: string,
+  //   @Body() body: UpdateNetworkConfigReq,
+  // ) {
+  //   this.logger.info('PATCH /network-config/:id', {
+  //     user: user.user_id,
+  //     customer: user.customer_name,
+  //   });
+  //   return this.networkConfigService.updateNetworkConfig(
+  //     body,
+  //     network_config_id,
+  //   );
+  // }
+
+  // @Delete(':id')
+  // @ApiOkResponse({ type: DeleteNetworkConfigRes })
+  // async deleteNetworkConfig(
+  //   @User() user: RequestUser,
+  //   @Param('id') network_config_id: string,
+  // ) {
+  //   this.logger.info('DELETE /network-config', {
+  //     user: user.user_id,
+  //     customer: user.customer_name,
+  //   });
+  //   return this.networkConfigService.deleteNetworkConfig(network_config_id, {
+  //     customer_name: user.customer_name,
+  //     customer_id: user.customer_id,
+  //   });
+  // }
+}

src/modules/network-config/network-config.module.ts

@@ -0,0 +1,14 @@
+import { Module } from '@nestjs/common';
+import { NetworkConfigController } from './network-config.controller';
+import { NetworkConfigService } from './network-config.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { NetworkConfigGrpcClient } from './network-config-client.config';
+const client = new NetworkConfigGrpcClient();
+
+@Module({
+  controllers: [NetworkConfigController],
+  providers: [NetworkConfigService, DadosferaLogger],
+  imports: [ClientsModule.register([client.providerOptions])],
+})
+export class NetworkConfigModule {}

src/modules/network-config/network-config.service.ts

@@ -0,0 +1,158 @@
+import { Metadata } from '@grpc/grpc-js';
+import { Inject, Injectable } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { NetworkConfig } from '@dadosfera/protospack-v2';
+import { lastValueFrom } from 'rxjs';
+import {
+  CreateNetworkConfigReq,
+  CreateNetworkConfigRes,
+  DeleteNetworkConfigRes,
+  FindAllNetworkConfigsRes,
+  FindNetworkConfigByidDetailsRes,
+  FindNetworkConfigByidRes,
+  UpdateNetworkConfigReq,
+  UpdateNetworkConfigRes,
+} from './dto/network-config';
+import { NetworkConfigGrpcClient } from './network-config-client.config';
+
+@Injectable()
+export class NetworkConfigService {
+  private networkConfigReadClient: NetworkConfig.ReadService.NetworkConfigReadService;
+  private networkConfigWriteClient: NetworkConfig.WriteService.NetworkConfigWriteService;
+  constructor(
+    @Inject(NetworkConfigGrpcClient.name)
+    private readonly grpcClient: ClientGrpc,
+  ) {
+    this.networkConfigReadClient =
+      grpcClient.getService<NetworkConfig.ReadService.NetworkConfigReadService>(
+        NetworkConfig.ProtoServices.NetworkConfigReadService,
+      );
+    this.networkConfigWriteClient =
+      grpcClient.getService<NetworkConfig.WriteService.NetworkConfigWriteService>(
+        NetworkConfig.ProtoServices.NetworkConfigWriteService,
+      );
+  }
+  async findAllNetworkConfigs({
+    body,
+    info,
+  }): Promise<FindAllNetworkConfigsRes> {
+    const { search, page, size, filters } = body;
+    const { language, customer_id, customer_name, user_id } = info;
+
+    const meta = new Metadata();
+
+    meta.add('language', language);
+    meta.add('customer_id', customer_id);
+    meta.add('customer_name', customer_name);
+    meta.add('user_id', user_id);
+
+    return lastValueFrom(
+      this.networkConfigReadClient.NetworkConfigFindAll(
+        { search, filters: JSON.stringify(filters), page, size },
+        meta,
+      ),
+    );
+  }
+
+  async findNetworkConfigById(id: string): Promise<FindNetworkConfigByidRes> {
+    const { network_config } = await lastValueFrom(
+      this.networkConfigReadClient.NetworkConfigFindOneById({
+        id,
+      }),
+    );
+    return { network_config };
+  }
+
+  async findNetworkConfigByIdDetails(
+    id: string,
+    info?,
+  ): Promise<FindNetworkConfigByidDetailsRes> {
+    const { customer_name, customer_id, language } = info;
+
+    const meta = new Metadata();
+
+    meta.add('customer_name', customer_name);
+    meta.add('customer_id', customer_id);
+    meta.add('language', language);
+
+    const { network_config } = await lastValueFrom(
+      this.networkConfigReadClient.NetworkConfigFindOneDetailsById(
+        { id },
+        meta,
+      ),
+    );
+
+    Object.assign(network_config, {
+      properties: JSON.parse(network_config.properties),
+    });
+    return { network_config };
+  }
+
+  async createNetworkConfig(
+    body: CreateNetworkConfigReq,
+    file: Express.Multer.File,
+    { customer_name, customer_id, user_id },
+  ): Promise<CreateNetworkConfigRes> {
+    const { description, name, network_type_id, network_type_name } = body;
+
+    const type = 'vpn';
+    const plugin = 'openvpn';
+
+    const OVPN_FILE_STRING = Buffer.from(file.buffer).toString();
+
+    const meta = new Metadata();
+
+    meta.add('customer_id', customer_id);
+    meta.add('customer_name', customer_name);
+    meta.add('user_id', user_id);
+
+    const { network_config } = await lastValueFrom(
+      this.networkConfigWriteClient.NetworkConfigCreate(
+        {
+          description,
+          name,
+          properties: JSON.stringify({
+            OVPN_FILE: OVPN_FILE_STRING,
+            plugin,
+          }),
+          type,
+          network_type_id,
+          network_type_name,
+          plugin,
+        },
+        meta,
+      ),
+    );
+    return { network_config };
+  }
+
+  async updateNetworkConfig(
+    body: UpdateNetworkConfigReq,
+    network_config_id: string,
+  ): Promise<UpdateNetworkConfigRes> {
+    const { description, name } = body;
+    const { network_config } = await lastValueFrom(
+      this.networkConfigWriteClient.NetworkConfigUpdate({
+        id: network_config_id,
+        description,
+        name,
+      }),
+    );
+    return { network_config };
+  }
+
+  async deleteNetworkConfig(
+    network_config_id: string,
+    { customer_name, customer_id },
+  ): Promise<DeleteNetworkConfigRes> {
+    const meta = new Metadata();
+    meta.add('customer_id', customer_id);
+    meta.add('customer_name', customer_name);
+    return lastValueFrom(
+      this.networkConfigWriteClient.NetworkConfigDelete(
+        { id: network_config_id },
+        meta,
+      ),
+    );
+  }
+}

src/modules/oauth/oauth.controller.ts

@@ -1,27 +1,29 @@
 import { Controller, Get, Redirect, Req, UseGuards } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import { AuthGuard } from '@nestjs/passport';
-import { InputsClientService } from 'src/clients/inputs/client.service';
+import { ConnectionClientService } from '../connection/client.service';
-import { InputsService } from '../inputs/inputs.service';
+import jwt from 'jsonwebtoken';
-import { Request } from 'express';
+import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
-
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 @ApiTags('oauth')
 @Controller('oauth')
 export class OauthController {
-  inputService: InputsService;
+  redirectUrl: string;
-  frontendRedirectUri = '';
+
-  constructor(private inputsClientService: InputsClientService) {
+  constructor(
-    this.inputService = new InputsService(this.inputsClientService);
+    private connectionService: ConnectionClientService,
+    private logger: DadosferaLogger,
+  ) {
     switch (process.env.ENV) {
-      case 'dev':
       case 'stg':
-        this.frontendRedirectUri = `https://app.${process.env.ENV}.dadosfera.ai/coletix/auth-callback`;
+        this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/collect/auth-callback`;
         break;
       case 'prd':
-        this.frontendRedirectUri = `https://app.dadosfera.ai/coletix/auth-callback`;
+        this.redirectUrl = `https://app.dadosfera.ai/collect/auth-callback`;
         break;
       default:
-        this.frontendRedirectUri = `http://localhost:4200/coletix/auth-callback`;
+        this.redirectUrl = `http://localhost:4200/collect/auth-callback`;
     }
   }
   @Get('hubspot')
@@ -30,11 +32,12 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('hubspot/callback')
   @UseGuards(AuthGuard('hubspot'))
   @Redirect()
   async oauthHubspotCallback(@Req() req) {
-    return await this.callback(req, 'hubspot');
+    return await this.callback(req);
   }
 
   @Get('googleads')
@@ -43,11 +46,12 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('googleads/callback')
   @UseGuards(AuthGuard('google'))
   @Redirect()
   async oauthGoogleAdsCallback(@Req() req) {
-    return await this.callback(req, 'googleads');
+    return await this.callback(req);
   }
 
   @Get('google-sheets')
@@ -56,11 +60,12 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('google-sheets/callback')
   @UseGuards(AuthGuard('google'))
   @Redirect()
   async oauthGoogleSheetsCallback(@Req() req) {
-    return await this.callback(req, 'google-sheets');
+    return await this.callback(req);
   }
 
   @Get('facebook')
@@ -69,11 +74,12 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('facebook/callback')
   @UseGuards(AuthGuard('facebook'))
   @Redirect()
   async oauthFacebookCallback(@Req() req) {
-    return await this.callback(req, 'facebook');
+    return await this.callback(req);
   }
 
   @Get('mailchimp')
@@ -82,11 +88,12 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('mailchimp/callback')
   @UseGuards(AuthGuard('mailchimp'))
   @Redirect()
   async oauthMailchimpCallback(@Req() req) {
-    return await this.callback(req, 'mailchimp');
+    return await this.callback(req);
   }
 
   @Get('salesforce')
@@ -95,36 +102,78 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('salesforce/callback')
   @UseGuards(AuthGuard('salesforce'))
   @Redirect()
   async oauthSalesforceCallback(@Req() req) {
-    return await this.callback(req, 'salesforce');
+    return await this.callback(req);
   }
 
-  async callback(@Req() req: Request, plugin) {
+  async callback(@Req() req) {
-    const { error, state: customer_id } = req.query;
+    const { error, state } = req.query;
+    const url = new URL(this.redirectUrl);
 
-    if (error)
+    if (error) {
-      return {
+      this.logger.error(error);
-        url: `${this.frontendRedirectUri}?error=${error}`,
+      url.searchParams.set('error', 'Erro ao autorizar');
-      };
+      url.searchParams.set(
-    const request: any = req;
+        'error_description',
-    const { authInfo } = request;
+        `Credenciais inválidas. Erro: [${error}]`,
+      );
+      return { url: url.href };
+    }
 
-    if (!authInfo)
+    const { authInfo } = req;
-      return {
+
-        url: `${this.frontendRedirectUri}?error=invalid_credentials`,
+    if (!authInfo) {
-      };
+      this.logger.error('No authInfo', req);
+      url.searchParams.set('error', 'Erro ao autorizar');
+      url.searchParams.set('error_description', `Credenciais inválidas.`);
+      return { url: url.href };
+    }
 
     const { accessToken: access_token, refreshToken: refresh_token } = authInfo;
 
-    const response = await this.inputService.create({
+    let connectionInfo: any;
-      info: { customer_id, customer: 'customer', user_id: 'empty' },
+    try {
-      plugin,
+      connectionInfo = jwt.verify(state as string, process.env.JWT_PRIVATE_KEY);
-      credentials: { access_token, refresh_token },
+    } catch (error) {
-    });
+      this.logger.error('Invalid state JWT', { state, req, error });
+      url.searchParams.set('error', 'Erro ao autorizar');
+      url.searchParams.set(
+        'error_description',
+        'Ocorreu um erro ao autorizá-lo. Por favor tente novamente',
+      );
+      return { url: url.href };
+    }
 
-    return { url: `${this.frontendRedirectUri}?input_id=${response.input.id}` };
+    connectionInfo.properties.access_token = access_token;
+    connectionInfo.properties.refresh_token = refresh_token;
+
+    const { customer_id, customer_name, user_id } = connectionInfo;
+
+    try {
+      const response = connectionInfo.id
+        ? await this.connectionService.updateConnection(
+            connectionInfo.id,
+            connectionInfo,
+            PackTheMetadata({ customer_id, customer_name, user_id }),
+          )
+        : await this.connectionService.createConnection(
+            connectionInfo,
+            PackTheMetadata({ customer_id, customer_name, user_id }),
+          );
+      url.searchParams.set('connection_id', response.connection.id);
+      return { url: url.href };
+    } catch (error) {
+      this.logger.error(error);
+      url.searchParams.set('error', 'Erro ao autorizar');
+      url.searchParams.set(
+        'error_description',
+        'Ocorreu um erro interno ao autorizá-lo. Por favor tente novamente mais tarde',
+      );
+      return { url: url.href };
+    }
   }
 }

src/modules/oauth/oauth.module.ts

@@ -0,0 +1,29 @@
+import { Module } from '@nestjs/common';
+import { OauthController } from './oauth.controller';
+
+import { getOauthSecrets } from '../../utils/OauthSecrets';
+
+import { HubspotStrategy } from './passport-strategies/hubspot-strategy';
+import { FacebookStrategy } from './passport-strategies/facebook-strategy';
+import { GoogleStrategy } from './passport-strategies/google-strategy';
+import { MailChimpStrategy } from './passport-strategies/mailchimp-strategy';
+import { SalesforceStrategy } from './passport-strategies/salesforce-strategy';
+import { ConnectionModule } from '../connection/connection.module';
+import { OauthService } from './oauth.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+@Module({
+  providers: [
+    DadosferaLogger,
+    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
+    HubspotStrategy,
+    FacebookStrategy,
+    GoogleStrategy,
+    MailChimpStrategy,
+    SalesforceStrategy,
+    OauthService,
+  ],
+  controllers: [OauthController],
+  imports: [ConnectionModule],
+})
+export class OauthModule {}