Att hubspot connection

Feat/trigger catalog task

.github/workflows/deploy-manually.yml

@@ -6,12 +6,12 @@ on:
   workflow_dispatch:
     inputs:
       environment:
-        description: 'Deployment environment'
+        description: "Deployment environment"
         required: true
         type: choice
         options:
-          - dev
           - stg
+          - stg2
           - prd
 
 jobs:
@@ -26,9 +26,9 @@ jobs:
       - name: Extract Environment
         run: |
           if [ ${EVENT} == "workflow_dispatch" ]; then
-            echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
+            echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
           elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "##[set-output name=environment;]$(echo "prd")"
+            echo "environment=prd" >> $GITHUB_OUTPUT
           fi
         id: extract_environment
 
@@ -39,14 +39,13 @@ jobs:
       new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - if: github.event_name != 'workflow_dispatch'
         name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
+        uses: cycjimmy/semantic-release-action@v3
         id: semantic
         with:
-          semantic_version: 16
           extra_plugins: |
             conventional-changelog-eslint
           branches: |
@@ -56,22 +55,28 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  deploy:
-    if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
+  deploy-info:
+    if: ${{ github.event_name == 'workflow_dispatch'}}
     needs: [extract_environment, semantic_release]
-    runs-on:
-      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+    runs-on: ubuntu-latest
 
     steps:
-      - name: Printing stats
+      - name: Create summary
         env:
           EVENT: ${{ github.event_name }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ENV: ${{ needs.extract_environment.outputs.environment }}
-        run: echo ${GITHUB_REF#refs/heads/}
+        run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY
 
+  deploy:
+    if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
+    needs: [extract_environment, semantic_release]
+    runs-on:
+      [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
+
+    steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Update Pip
         run: |
@@ -90,7 +95,7 @@ jobs:
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v1-node16
         id: aws
         with:
           aws-region: us-east-1
@@ -113,10 +118,12 @@ jobs:
           ENV: ${{ needs.extract_environment.outputs.environment }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+          NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
           sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
           sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
+          sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk
@@ -136,71 +143,64 @@ jobs:
           docker system df
   api_docs:
     needs: [extract_environment, semantic_release, deploy]
-    runs-on:
-      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - name: Extract Docs PageId
+      - name: Extract Docs BlockId and PageId
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
-          DEV_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
+          STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
+          STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
+          STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
           STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
+          PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
           PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
         shell: bash
         run: |
-          if [ $ENV == "dev" ]; then
-            echo "##[set-output name=result;]$(echo $DEV_DOCS_PAGE_ID)"
+          if [ $ENV == "stg2" ]; then
+            echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
           elif [ $ENV == "stg" ]; then
-            echo "##[set-output name=result;]$(echo $STG_DOCS_PAGE_ID)"
+            echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
           elif [ $ENV == "prd" ]; then
-            echo "##[set-output name=result;]$(echo $PRD_DOCS_PAGE_ID)"
+            echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
           fi
-        id: extract_docs_page_id
-
-      - name: Extract Docs BlockId
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-          DEV_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
-          STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
-          PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
-        shell: bash
-        run: |
-          if [ $ENV == "dev" ]; then
-            echo "##[set-output name=result;]$(echo $DEV_DOCS_BLOCK_ID)"
-          elif [ $ENV == "stg" ]; then
-            echo "##[set-output name=result;]$(echo $STG_DOCS_BLOCK_ID)"
-          elif [ $ENV == "prd" ]; then
-            echo "##[set-output name=result;]$(echo $PRD_DOCS_BLOCK_ID)"
-          fi
-        id: extract_docs_block_id
+        id: extract_docs_info
 
       - name: Generate API docs
         env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
           DOCS_URL: ${{ secrets.DOCS_URL }}
           DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
-          DOCS_PAGE_ID: ${{ steps.extract_docs_page_id.outputs.result }}
-          DOCS_BLOCK_ID: ${{ steps.extract_docs_block_id.outputs.result }}
-        continue-on-error: true
+          DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
+          DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
+          EVENT: ${{ github.event_name }}
+          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
         run: |
-          SWAGGER=`cat swagger.txt`
-          curl --request POST \
-            --url $DOCS_URL \
-            --header 'Content-Type: application/json' \
-            --data @<(cat <<EOF
-            {
-            "service": "$DOCS_API_TOKEN",
-            "pageId": "$DOCS_PAGE_ID",
-            "blockId": "$DOCS_BLOCK_ID",
-            "schema":  $SWAGGER
-          }
-          EOF
-          )
-      - name: Cleanup Docker's Leftovers
-        if: always()
-        continue-on-error: true
+          if [ ${EVENT} != "workflow_dispatch" ]; then
+            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
+          fi
+          sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
+          sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
+          sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
+          curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
+
+      - name: Generate External API docs
+        env:
+          DOCS_URL: ${{ secrets.DOCS_URL }}
+          DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
+          DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}}
+          DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}}
+          EVENT: ${{ github.event_name }}
+          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
         run: |
-          docker system prune
-          docker rmi -f $(docker images -aq)
+          if [ ${EVENT} != "workflow_dispatch" ]; then
+            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json
+          fi
+          sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json
+          sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json
+          sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json
+          curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL

.github/workflows/test.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Build
         env:

README.md

@@ -1,67 +1,98 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-</p> 
+</p>
 
 # Maestro
+
 Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
 
 ## 🚀 Starting
+
 These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
 
 ### 📋 Requirements
+
 - [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
 - Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
-    - Create your Access Key on the "Security credentials" menu
-    - Set the Access Key on your local development machine
+  - Create your Access Key on the "Security credentials" menu
+  - Set the Access Key on your local development machine
 - Request access to the dev, stg and prd VPNs
 
 ### 🔧 Installation<a id="installation"></a>
-- Clone the repository
-    - SSH
-        ```
-        git clone git@github.com:dadosfera/maestro.git
-        ```
-        or
 
-    - HTTPS
-        ```
-        git clone https://github.com/dadosfera/maestro.git
-        ```
+- Clone the repository
+
+  - SSH
+
+    ```
+    git clone git@github.com:dadosfera/maestro.git
+    ```
+
+    or
+
+  - HTTPS
+    ```
+    git clone https://github.com/dadosfera/maestro.git
+    ```
 
 - Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
-    ```sh
-    nvm use
-    ```
+
+  ```sh
+  nvm use
+  ```
 
 - Install the project dependencies:
-    ```sh
-    npm i
-    ```
+
+  ```sh
+  npm i
+  ```
 
 - Setup the following enviroment variables:
-    ```
-    ENV=
-    DUC_URL=
-    INFACTORY_URL=
-    OTFACTORY_URL=
-    PIFACTORY_URL=
-    SM_OAUTH_PATH=
-    ```
+
+  ```
+  ENV=
+  DUC_URL=
+  INFACTORY_URL=
+  OTFACTORY_URL=
+  PIFACTORY_URL=
+  SM_OAUTH_PATH=
+  ```
 
 - Start the server:
-    ```sh
-    # dev mode
-    npm run start:dev
 
-    # or in debug mode
-    npm run start:debug
-    ```
-    The service should start successfully.
+  ```sh
+  # dev mode
+  npm run start:dev
+
+  # or in debug mode
+  npm run start:debug
+  ```
+
+  The service should start successfully.
+
+## 📄 Documentation
+
+NestJs makes it easy to document each route using decorators on all requests and responses properties. It automatically generates a swagger for given information and provides a route to access it http://localhost:3333/api.
+For more info check the [official documentation](https://docs.nestjs.com/openapi/introduction).
+
+### - Multiple documentations
+
+We are currently generating **2 different** documentations: Internal and External.
+
+All routes that have the decorator `@ApiInternalOnly()` will not be visible on the **External** API swagger.
+
+- When you start the application with `npm run start` it will serve and generate the swagger JSON of the **External** API
+- When you start the application with `npm run start:internal` it will serve and generate the swagger JSON of the **Internal** API
+- When you run `npm run docs` it will generate the swagger JSON of both **Internal** and **External** API, and save them to `docsfera.json` and `docsfera.external.json` respectively;
+
+It is important to run `npm run docs` before every deploy so we can always have the most updated docs published.
 
 ## Authentication decorators
+
 Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
 
 ### `@Authenticated`
+
 If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
 
 ```ts
@@ -106,6 +137,7 @@ class FooController {
 ```
 
 ### `@RequireAllPermissions`
+
 This decorator requires that **all permissions** listed are granted to the requesting user.
 
 ```ts
@@ -125,6 +157,7 @@ class FooController {
 ```
 
 ### `@RequireSomePermission`
+
 In the following case, the user is required to have **at least one** listed permission.
 
 ```ts
@@ -144,9 +177,11 @@ class FooController {
 ```
 
 ### `@AuthenticateCondition`
+
 If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
 
 In the following example:
+
 - all routes on the `FooController` controller can only be requested from localhost
 - `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
 
@@ -166,7 +201,10 @@ class FooController {
 
   @Post('bar')
   // allow requests only from a specific customer
-  @AuthenticateCondition((request: Request, user: RequestUser) => user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef')
+  @AuthenticateCondition(
+    (request: Request, user: RequestUser) =>
+      user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
+  )
   async postBar() {
     /* ... */
   }
@@ -174,51 +212,55 @@ class FooController {
 ```
 
 ### Note on authentication decorators
+
 - The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
 
-- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (*and wouldn't like to...*) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
+- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (_and wouldn't like to..._) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
 
-    ```ts
-    import {
-      RequireAllPermissions,
-      RequireSomePermission,
-      AuthenticateCondition,
-    } from '../../authentication/authentication.decorator';
-    import { PERMISSIONS } from '../../authentication/permissions.enum';
-    import { Waa, Baz } from './authenticationFunctions'
+  ```ts
+  import {
+    RequireAllPermissions,
+    RequireSomePermission,
+    AuthenticateCondition,
+  } from '../../authentication/authentication.decorator';
+  import { PERMISSIONS } from '../../authentication/permissions.enum';
+  import { Waa, Baz } from './authenticationFunctions';
 
-    @Controller('foo')
-    @RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
-    @RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.ADMIN)
-    @AuthenticateCondition(Waa)
-    @AuthenticateCondition(Baz)
-    class FooController {
+  @Controller('foo')
+  @RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
+  @RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.ADMIN)
+  @AuthenticateCondition(Waa)
+  @AuthenticateCondition(Baz)
+  class FooController {
+    /* ... */
+
+    @Get('bar')
+    async getBar() {
       /* ... */
-
-      @Get('bar')
-      async getBar() {
-        /* ... */
-      }
-
-      @Post('bar')
-      @RequireAllPermissions(Permissions.BAR.MANAGE, Permissions.BAR.USE, Permissions.BAR.REQUEST)
-      async postBar() {
-        /* ... */
-      }
     }
-    ```
+
+    @Post('bar')
+    @RequireAllPermissions(
+      Permissions.BAR.MANAGE,
+      Permissions.BAR.USE,
+      Permissions.BAR.REQUEST,
+    )
+    async postBar() {
+      /* ... */
+    }
+  }
+  ```
 
 - If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
 
-    ```ts
-    // same behavior
-    @RequireAllPermissions(Permissions.BAR.MANAGE)
-    @RequireSomePermission(Permissions.BAR.MANAGE)
-    ```
-
-
+  ```ts
+  // same behavior
+  @RequireAllPermissions(Permissions.BAR.MANAGE)
+  @RequireSomePermission(Permissions.BAR.MANAGE)
+  ```
 
 ## `@User` parameter decorator
+
 The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
 
 ```ts
@@ -264,12 +306,15 @@ class UserController {
 ```
 
 ## 📦 Development
+
 ### ⌨️ Coding Style
+
 By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
 
 **We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
 
 ### Commits pattern
+
 Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
 
 Format: `<type>[optional scope]: <description>`
@@ -277,25 +322,29 @@ Format: `<type>[optional scope]: <description>`
 Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
 
 ### Branching naming convention
+
 - **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
 
-  **It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (_) could be used to separate the ID and description.**
+  **It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (\_) could be used to separate the ID and description.**
 
-- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`,  `bugfix/JIRA-1444_gray-on-blur-fix`.
+- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
 
 - **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
 
 - **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
 
 ### Making a Pull Request
+
 1. Commit your changes
 2. Open the Pull Request on GitHub
-3. Send Pull Request link in microsfera Google Chat Group for review and possible approval
+3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
 
 ## 🛠️ Built with
+
 Some technologies used in this project:
 
 - [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
 
 ## ⚙️ Back-end Architecture
+
 The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - .env
 
   node_exporter:
-    image: prom/node-exporter:v1.3.1
+    image: ${NODE_EXPORTER_URL}
     container_name: node_exporter
     restart: always
     volumes:

environment.d.ts

@@ -0,0 +1,18 @@
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      NODE_ENV: 'test';
+      ENV: 'local' | 'stg' | 'prd' | 'test';
+      LOCAL_ENV: 'stg' | 'prd';
+
+      DUC_URL: string;
+      INFACTORY_URL: string;
+      PIFACTORY_URL: string;
+
+      INTERNAL_SWAGGER: 'true' | 'false';
+
+      AWS_REGION: string;
+    }
+  }
+}
+export {};

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.18.18",
+        "@dadosfera/protospack-v2": "3.30.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -31,11 +31,12 @@
         "dotenv": "^14.3.2",
         "elastic-apm-node": "^3.36.0",
         "helmet": "^5.1.0",
-        "jsonwebtoken": "^8.5.1",
+        "jsonwebtoken": "^9.0.0",
         "jwk-to-pem": "^2.0.5",
+        "mixpanel": "^0.17.0",
         "passport": "^0.6.0",
         "passport-facebook": "^3.0.0",
-        "passport-forcedotcom": "^0.1.5",
+        "passport-forcedotcom": "^0.2.0",
         "passport-google-oauth20": "^2.0.0",
         "passport-hubspot-oauth2": "^1.0.3",
         "passport-mailchimp": "^1.1.0",
@@ -1726,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.18.18",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.18.tgz",
-      "integrity": "sha512-4KiuMUDMZu8ouic0RBBuM7Ky9wfczeS4GKLRyD29GV371m/NOo+1ujrcLysc8OuevPdXmaqzCgE6WWi6SIt9sQ==",
+      "version": "3.30.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.30.0.tgz",
+      "integrity": "sha512-00wqZyZmjd6rMECga2S3Ow0dk6c5fZoWd6Th+Ajofq6q/kycTU66p/5xdf0sPf//WyXMGCXasqGSZa0h9i01vg==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -3543,7 +3544,6 @@
     },
     "node_modules/agent-base": {
       "version": "6.0.2",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "debug": "4"
@@ -3699,8 +3699,9 @@
     },
     "node_modules/asap": {
       "version": "2.0.6",
-      "dev": true,
-      "license": "MIT"
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
+      "dev": true
     },
     "node_modules/asn1.js": {
       "version": "5.4.1",
@@ -4494,9 +4495,10 @@
       "license": "MIT"
     },
     "node_modules/cookiejar": {
-      "version": "2.1.3",
-      "dev": true,
-      "license": "MIT"
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
+      "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==",
+      "dev": true
     },
     "node_modules/core-util-is": {
       "version": "1.0.3",
@@ -4705,9 +4707,10 @@
       }
     },
     "node_modules/dezalgo": {
-      "version": "1.0.3",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
+      "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
       "dev": true,
-      "license": "ISC",
       "dependencies": {
         "asap": "^2.0.0",
         "wrappy": "1"
@@ -5902,23 +5905,28 @@
       }
     },
     "node_modules/formidable": {
-      "version": "2.0.1",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.1.tgz",
+      "integrity": "sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "dezalgo": "1.0.3",
-        "hexoid": "1.0.0",
-        "once": "1.4.0",
-        "qs": "6.9.3"
+        "dezalgo": "^1.0.4",
+        "hexoid": "^1.0.0",
+        "once": "^1.4.0",
+        "qs": "^6.11.0"
       },
       "funding": {
         "url": "https://ko-fi.com/tunnckoCore/commissions"
       }
     },
     "node_modules/formidable/node_modules/qs": {
-      "version": "6.9.3",
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "dev": true,
-      "license": "BSD-3-Clause",
+      "dependencies": {
+        "side-channel": "^1.0.4"
+      },
       "engines": {
         "node": ">=0.6"
       },
@@ -7752,9 +7760,10 @@
       "license": "ISC"
     },
     "node_modules/json5": {
-      "version": "2.2.1",
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+      "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
       "dev": true,
-      "license": "MIT",
       "bin": {
         "json5": "lib/cli.js"
       },
@@ -7779,30 +7788,18 @@
       }
     },
     "node_modules/jsonwebtoken": {
-      "version": "8.5.1",
-      "license": "MIT",
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
+      "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
       "dependencies": {
         "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
+        "lodash": "^4.17.21",
         "ms": "^2.1.1",
-        "semver": "^5.6.0"
+        "semver": "^7.3.8"
       },
       "engines": {
-        "node": ">=4",
-        "npm": ">=1.4.28"
-      }
-    },
-    "node_modules/jsonwebtoken/node_modules/semver": {
-      "version": "5.7.1",
-      "license": "ISC",
-      "bin": {
-        "semver": "bin/semver"
+        "node": ">=12",
+        "npm": ">=6"
       }
     },
     "node_modules/jwa": {
@@ -7901,30 +7898,6 @@
       "version": "4.3.0",
       "license": "MIT"
     },
-    "node_modules/lodash.includes": {
-      "version": "4.3.0",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isboolean": {
-      "version": "3.0.3",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isinteger": {
-      "version": "4.0.4",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isnumber": {
-      "version": "3.0.3",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isplainobject": {
-      "version": "4.0.6",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isstring": {
-      "version": "4.0.1",
-      "license": "MIT"
-    },
     "node_modules/lodash.memoize": {
       "version": "4.1.2",
       "dev": true,
@@ -7935,10 +7908,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/lodash.once": {
-      "version": "4.1.1",
-      "license": "MIT"
-    },
     "node_modules/lodash.sortby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -8001,9 +7970,9 @@
       }
     },
     "node_modules/luxon": {
-      "version": "1.28.0",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
-      "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==",
+      "version": "1.28.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz",
+      "integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw==",
       "engines": {
         "node": "*"
       }
@@ -8205,6 +8174,29 @@
       "version": "1.2.6",
       "license": "MIT"
     },
+    "node_modules/mixpanel": {
+      "version": "0.17.0",
+      "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz",
+      "integrity": "sha512-DY5WeOy/hmkPrNiiZugJpWR0iMuOwuj1a3u0bgwB2eUFRV6oIew/pIahhpawdbNjb+Bye4a8ID3gefeNPvL81g==",
+      "dependencies": {
+        "https-proxy-agent": "5.0.0"
+      },
+      "engines": {
+        "node": ">=10.0"
+      }
+    },
+    "node_modules/mixpanel/node_modules/https-proxy-agent": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
+      "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
+      "dependencies": {
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/mkdirp": {
       "version": "0.5.6",
       "license": "MIT",
@@ -8229,8 +8221,9 @@
       }
     },
     "node_modules/moment-timezone": {
-      "version": "0.5.34",
-      "license": "MIT",
+      "version": "0.5.38",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.38.tgz",
+      "integrity": "sha512-nMIrzGah4+oYZPflDvLZUgoVUO4fvAqHstvG3xAUnMolWncuAiLDWNnJZj6EwJGMGfb1ZcuTFE6GI3hNOVWI/Q==",
       "dependencies": {
         "moment": ">= 2.9.0"
       },
@@ -8689,24 +8682,11 @@
       }
     },
     "node_modules/passport-forcedotcom": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.1.5.tgz",
-      "integrity": "sha512-raDRQqBJtTEHDxLkxLG+9sYE0xpPy9PqriMPbMy9jmpIcU2+mOq9PgczvUPrU8UjfcZp3gqkDHqujcUWiGOVNQ==",
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.2.0.tgz",
+      "integrity": "sha512-gRI5+zQK9iVAZ1hJgt3TkWLeDw8SYrNWUsmVk0d0RyvUviQ4mT52x3JbYRiyZefoeID8afmLsJH8VOHjWdPf3Q==",
       "dependencies": {
-        "passport-oauth2": "1.3.x"
-      },
-      "engines": {
-        "node": ">= 0.4.0"
-      }
-    },
-    "node_modules/passport-forcedotcom/node_modules/passport-oauth2": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/passport-oauth2/-/passport-oauth2-1.3.0.tgz",
-      "integrity": "sha512-AisRXcC7Vx8WSx5C9O8UNib/Y91/IOphs5tImlOMs7T+QTEJdmS+OxgXSskbnDdGQT/8mU5T3yMT6136rBjYXg==",
-      "dependencies": {
-        "oauth": "0.9.x",
-        "passport-strategy": "1.x.x",
-        "uid2": "0.0.x"
+        "passport-oauth2": "^1.6.1"
       },
       "engines": {
         "node": ">= 0.4.0"
@@ -9466,9 +9446,9 @@
       "license": "MIT"
     },
     "node_modules/semver": {
-      "version": "7.3.7",
-      "dev": true,
-      "license": "ISC",
+      "version": "7.3.8",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
+      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -10337,9 +10317,9 @@
       }
     },
     "node_modules/ts-proto": {
-      "version": "1.126.1",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.126.1.tgz",
-      "integrity": "sha512-PKrcgd0mCQJDY7tp+th349uYhnjs2Tz7wSqm9+DBcodl3YCKPt1pFtFmApOZdQ8WlghhgKAuJDIRQJBd6jD5YA==",
+      "version": "1.131.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.131.0.tgz",
+      "integrity": "sha512-oj/kWrDz3JkkPQwumQDKfwHroH9iNz2alvrbCYzNYJlb/2MAUzFNfh9T+SZUPDfoXXK6oJoW1hGCHznj67yHNg==",
       "dependencies": {
         "@types/object-hash": "^1.3.0",
         "dataloader": "^1.4.0",
@@ -10406,9 +10386,10 @@
       }
     },
     "node_modules/tsconfig-paths/node_modules/json5": {
-      "version": "1.0.1",
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
+      "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "minimist": "^1.2.0"
       },
@@ -12306,9 +12287,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.18.18",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.18.tgz",
-      "integrity": "sha512-4KiuMUDMZu8ouic0RBBuM7Ky9wfczeS4GKLRyD29GV371m/NOo+1ujrcLysc8OuevPdXmaqzCgE6WWi6SIt9sQ==",
+      "version": "3.30.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.30.0.tgz",
+      "integrity": "sha512-00wqZyZmjd6rMECga2S3Ow0dk6c5fZoWd6Th+Ajofq6q/kycTU66p/5xdf0sPf//WyXMGCXasqGSZa0h9i01vg==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -13555,7 +13536,6 @@
     },
     "agent-base": {
       "version": "6.0.2",
-      "dev": true,
       "requires": {
         "debug": "4"
       }
@@ -13652,6 +13632,8 @@
     },
     "asap": {
       "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
       "dev": true
     },
     "asn1.js": {
@@ -14211,7 +14193,9 @@
       "version": "1.0.6"
     },
     "cookiejar": {
-      "version": "2.1.3",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
+      "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==",
       "dev": true
     },
     "core-util-is": {
@@ -14349,7 +14333,9 @@
       "dev": true
     },
     "dezalgo": {
-      "version": "1.0.3",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
+      "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
       "dev": true,
       "requires": {
         "asap": "^2.0.0",
@@ -15191,18 +15177,25 @@
       }
     },
     "formidable": {
-      "version": "2.0.1",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.1.tgz",
+      "integrity": "sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==",
       "dev": true,
       "requires": {
-        "dezalgo": "1.0.3",
-        "hexoid": "1.0.0",
-        "once": "1.4.0",
-        "qs": "6.9.3"
+        "dezalgo": "^1.0.4",
+        "hexoid": "^1.0.0",
+        "once": "^1.4.0",
+        "qs": "^6.11.0"
       },
       "dependencies": {
         "qs": {
-          "version": "6.9.3",
-          "dev": true
+          "version": "6.11.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+          "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
+          "dev": true,
+          "requires": {
+            "side-channel": "^1.0.4"
+          }
         }
       }
     },
@@ -16411,7 +16404,9 @@
       "dev": true
     },
     "json5": {
-      "version": "2.2.1",
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+      "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
       "dev": true
     },
     "jsonc-parser": {
@@ -16427,23 +16422,14 @@
       }
     },
     "jsonwebtoken": {
-      "version": "8.5.1",
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
+      "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
       "requires": {
         "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
+        "lodash": "^4.17.21",
         "ms": "^2.1.1",
-        "semver": "^5.6.0"
-      },
-      "dependencies": {
-        "semver": {
-          "version": "5.7.1"
-        }
+        "semver": "^7.3.8"
       }
     },
     "jwa": {
@@ -16516,24 +16502,6 @@
     "lodash.camelcase": {
       "version": "4.3.0"
     },
-    "lodash.includes": {
-      "version": "4.3.0"
-    },
-    "lodash.isboolean": {
-      "version": "3.0.3"
-    },
-    "lodash.isinteger": {
-      "version": "4.0.4"
-    },
-    "lodash.isnumber": {
-      "version": "3.0.3"
-    },
-    "lodash.isplainobject": {
-      "version": "4.0.6"
-    },
-    "lodash.isstring": {
-      "version": "4.0.1"
-    },
     "lodash.memoize": {
       "version": "4.1.2",
       "dev": true
@@ -16542,9 +16510,6 @@
       "version": "4.6.2",
       "dev": true
     },
-    "lodash.once": {
-      "version": "4.1.1"
-    },
     "lodash.sortby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -16590,9 +16555,9 @@
       }
     },
     "luxon": {
-      "version": "1.28.0",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
-      "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
+      "version": "1.28.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz",
+      "integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw=="
     },
     "macos-release": {
       "version": "2.5.0",
@@ -16719,6 +16684,25 @@
     "minimist": {
       "version": "1.2.6"
     },
+    "mixpanel": {
+      "version": "0.17.0",
+      "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz",
+      "integrity": "sha512-DY5WeOy/hmkPrNiiZugJpWR0iMuOwuj1a3u0bgwB2eUFRV6oIew/pIahhpawdbNjb+Bye4a8ID3gefeNPvL81g==",
+      "requires": {
+        "https-proxy-agent": "5.0.0"
+      },
+      "dependencies": {
+        "https-proxy-agent": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
+          "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
+          "requires": {
+            "agent-base": "6",
+            "debug": "4"
+          }
+        }
+      }
+    },
     "mkdirp": {
       "version": "0.5.6",
       "requires": {
@@ -16736,7 +16720,9 @@
       "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
     },
     "moment-timezone": {
-      "version": "0.5.34",
+      "version": "0.5.38",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.38.tgz",
+      "integrity": "sha512-nMIrzGah4+oYZPflDvLZUgoVUO4fvAqHstvG3xAUnMolWncuAiLDWNnJZj6EwJGMGfb1ZcuTFE6GI3hNOVWI/Q==",
       "requires": {
         "moment": ">= 2.9.0"
       }
@@ -17049,23 +17035,11 @@
       }
     },
     "passport-forcedotcom": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.1.5.tgz",
-      "integrity": "sha512-raDRQqBJtTEHDxLkxLG+9sYE0xpPy9PqriMPbMy9jmpIcU2+mOq9PgczvUPrU8UjfcZp3gqkDHqujcUWiGOVNQ==",
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.2.0.tgz",
+      "integrity": "sha512-gRI5+zQK9iVAZ1hJgt3TkWLeDw8SYrNWUsmVk0d0RyvUviQ4mT52x3JbYRiyZefoeID8afmLsJH8VOHjWdPf3Q==",
       "requires": {
-        "passport-oauth2": "1.3.x"
-      },
-      "dependencies": {
-        "passport-oauth2": {
-          "version": "1.3.0",
-          "resolved": "https://registry.npmjs.org/passport-oauth2/-/passport-oauth2-1.3.0.tgz",
-          "integrity": "sha512-AisRXcC7Vx8WSx5C9O8UNib/Y91/IOphs5tImlOMs7T+QTEJdmS+OxgXSskbnDdGQT/8mU5T3yMT6136rBjYXg==",
-          "requires": {
-            "oauth": "0.9.x",
-            "passport-strategy": "1.x.x",
-            "uid2": "0.0.x"
-          }
-        }
+        "passport-oauth2": "^1.6.1"
       }
     },
     "passport-google-oauth20": {
@@ -17547,8 +17521,9 @@
       }
     },
     "semver": {
-      "version": "7.3.7",
-      "dev": true,
+      "version": "7.3.8",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
+      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
       "requires": {
         "lru-cache": "^6.0.0"
       }
@@ -18131,9 +18106,9 @@
       }
     },
     "ts-proto": {
-      "version": "1.126.1",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.126.1.tgz",
-      "integrity": "sha512-PKrcgd0mCQJDY7tp+th349uYhnjs2Tz7wSqm9+DBcodl3YCKPt1pFtFmApOZdQ8WlghhgKAuJDIRQJBd6jD5YA==",
+      "version": "1.131.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.131.0.tgz",
+      "integrity": "sha512-oj/kWrDz3JkkPQwumQDKfwHroH9iNz2alvrbCYzNYJlb/2MAUzFNfh9T+SZUPDfoXXK6oJoW1hGCHznj67yHNg==",
       "requires": {
         "@types/object-hash": "^1.3.0",
         "dataloader": "^1.4.0",
@@ -18170,7 +18145,9 @@
       },
       "dependencies": {
         "json5": {
-          "version": "1.0.1",
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
+          "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
           "dev": true,
           "requires": {
             "minimist": "^1.2.0"

package.json

@@ -15,9 +15,10 @@
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "nest start",
-    "start:dev": "nest start --watch",
-    "start:debug": "nest start --debug --watch",
+    "start:dev": "export INTERNAL_SWAGGER=true && nest start --watch",
+    "start:debug": "npm run start:dev",
     "start:prod": "node dist/main",
+    "docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "jest",
     "test:watch": "jest --watch",
@@ -28,7 +29,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.18.18",
+    "@dadosfera/protospack-v2": "3.30.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",
@@ -47,11 +48,12 @@
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.36.0",
     "helmet": "^5.1.0",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "^9.0.0",
     "jwk-to-pem": "^2.0.5",
+    "mixpanel": "^0.17.0",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
-    "passport-forcedotcom": "^0.1.5",
+    "passport-forcedotcom": "^0.2.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-hubspot-oauth2": "^1.0.3",
     "passport-mailchimp": "^1.1.0",

src/app.module.ts

@@ -24,6 +24,7 @@ import { CatalogModule } from './modules/catalog/catalog.module';
 import { ConfigModule } from '@nestjs/config';
 import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
 import { ProductboardModule } from './modules/productboard/productboard.module';
+import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
 
 @Module({
   controllers: [],
@@ -38,24 +39,25 @@ import { ProductboardModule } from './modules/productboard/productboard.module';
     ConfigModule.forRoot({
       isGlobal: true,
     }),
-    ConnectorModule,
     AuthModule,
-    PermissionsModule,
-    TermsOfUseModule,
     ConnectionModule,
     NetworkConfigModule,
+    InputsModule,
+    PipelinesV2Module,
+    CatalogModule,
+    ConnectorModule,
+    PermissionsModule,
+    TermsOfUseModule,
     ConnectionTestModule,
     PipelinesModule,
-    PipelinesV2Module,
     TransformationsModule,
-    HealthModule,
-    CatalogModule,
     UsersModule,
     RolesModule,
-    InputsModule,
     OauthModule,
-    CatalogModule,
     ProductboardModule,
+    MixpanelModule,
+    //Always leave HealthModule last, so it is on the bottom of swagger
+    HealthModule,
   ],
 })
 export class AppModule {}

src/authentication/authentication.decorator.ts

@@ -1,56 +0,0 @@
-import { Request } from 'express';
-import { CustomDecorator } from '@nestjs/common';
-import { PermissionsObjectPermission } from './permissions.enum';
-import { RequestUser } from './user.decorator';
-
-export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
-
-export type AuthenticationFunction = (req: Request, user: any) => boolean;
-
-// implementation copied from SetMetadata, but tweaked to get existing values
-// of the metadataKey and accumulate it with the new metadataValue
-function SetMultipleMetadata(metadataKey, metadataValue): CustomDecorator {
-  const decoratorFactory = (target, key, descriptor) => {
-    // .start: tweak
-    // descriptor?.value = function decorator; target = class decorator
-    const accumulatedVal =
-      Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
-    accumulatedVal.push(metadataValue);
-    // .end: tweak
-
-    if (descriptor) {
-      Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
-      return descriptor;
-    }
-
-    Reflect.defineMetadata(metadataKey, accumulatedVal, target);
-    return target;
-  };
-
-  decoratorFactory.KEY = metadataKey;
-  return decoratorFactory as any;
-}
-
-export function RequireAllPermissions(
-  ...permissions: PermissionsObjectPermission[]
-) {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
-    permissions.every(({ seqid }) => user.permissions.includes(seqid)),
-  );
-}
-
-export function RequireSomePermission(
-  ...permissions: PermissionsObjectPermission[]
-) {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
-    permissions.some(({ seqid }) => user.permissions.includes(seqid)),
-  );
-}
-
-export function AuthenticateCondition(func: AuthenticationFunction) {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, func);
-}
-
-export function Authenticated() {
-  return SetMultipleMetadata(AUTH_FUNCTION_KEY, () => true);
-}

src/authentication/authentication.guard.spec.ts

@@ -11,7 +11,7 @@ import {
   Authenticated,
   RequireAllPermissions,
   RequireSomePermission,
-} from './authentication.decorator';
+} from '../decorators/authentication.decorator';
 import { AuthenticationGuard } from './authentication.guard';
 import { PERMISSIONS_GROUPS } from './permissions.enum';
 import { AuthClientService } from '../modules/auth/auth.service';
@@ -44,7 +44,7 @@ class NoClassAuthController {
   @Get('has-all-permissions')
   @RequireAllPermissions(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
   )
   async hasAllPermissions(@Body() body) {
     return { body };
@@ -53,7 +53,7 @@ class NoClassAuthController {
   @Get('has-some-permission')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
   )
   async hasSomePermission(@Body() body) {
     return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
 
 @Controller('class-auth-condition')
 @AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
-@RequireSomePermission(PERMISSIONS_GROUPS.METABASE.permissions.OPEN)
+@RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
 class ClassAuthConditionController {
   @Get('body')
   async getBody(@Body() body) {
@@ -444,13 +444,13 @@ describe('authentication.guard', () => {
   NoClassAuthTest(tokenZ, ['zendesk']);
   ClassAuthConditionTest(tokenZ, ['zendesk']);
 
-  const tokenM = CreateToken([PERMISSIONS_GROUPS.METABASE.permissions.OPEN]);
+  const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
   NoClassAuthTest(tokenM, ['metabase']);
   ClassAuthConditionTest(tokenM, ['metabase']);
 
   const tokenZM = CreateToken([
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
   ]);
   NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
   ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);

src/authentication/authentication.guard.ts

@@ -13,8 +13,8 @@ import { AuthClientService } from '../modules/auth/auth.service';
 import {
   AuthenticationFunction,
   AUTH_FUNCTION_KEY,
-} from './authentication.decorator';
-import { RequestUser } from './user.decorator';
+} from '../decorators/authentication.decorator';
+import { RequestUser } from '../decorators/user.decorator';
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';
 

src/authentication/permissions.enum.ts

@@ -290,49 +290,70 @@ export const PERMISSIONS_GROUPS = {
         claim: 'GET /catalog',
         usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Buscar data assets',
-          'en-us': 'Search for data assets',
-          'es-es': 'Buscar data assets',
+          'pt-br': 'Acessar lista de Ativos e visualizar atributos',
+          'en-us': 'Access Assets list and its attributes',
+          'es-es': 'Acceda a la lista de Activos y vea los atributos',
         },
       },
       CREATE: {
         seqid: 19,
-        claim: 'POST /catalog',
+        claim: 'catalog:create',
         usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Criar atributos no catálogo',
-          'en-us': 'Create attributes',
-          'es-es': 'Crear atributos',
+          'pt-br': 'Criar Ativos',
+          'en-us': 'Create Assets',
+          'es-es': 'Crear Activos',
         },
       },
       UPDATE: {
         seqid: 25,
-        claim: 'PUT /catalog',
-        usage: PermissionUsages.INTERNAL,
+        claim: 'catalog:edit',
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Editar atributos no catálogo',
-          'en-us': 'Edit attributes',
-          'es-es': 'Editar atributos',
+          'pt-br': 'Criar e editar atributos no catálogo',
+          'en-us': 'Create and edit attributes on the catalog',
+          'es-es': 'Crear y editar atributos en el catálogo',
         },
       },
       DELETE: {
         seqid: 1,
-        claim: 'DELETE /catalog',
-        usage: PermissionUsages.INTERNAL,
+        claim: 'catalog:delete',
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Excluir atributos no catálogo',
-          'en-us': 'Remove attributes',
-          'es-es': 'Eliminar atributos',
+          'pt-br': 'Excluir Ativos do catálogo',
+          'en-us': 'Remove Assets from the catalog',
+          'es-es': 'Eliminar Activos del catálogo',
         },
       },
       DATA_MANAGER: {
         seqid: 42,
         claim: 'catalog:data_manager',
-        usage: PermissionUsages.PUBLIC,
+        usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Vizualizar todos os recursos do catálogo.',
-          'en-us': 'View all data assets.',
-          'es-es': 'Ver todos los activos de datos',
+          'pt-br':
+            'Gerente do catálogo. Consegue vizualizar e editar todos os ativos.',
+          'en-us': 'Catalog manager. Able to view and edit all assets.',
+          'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
+        },
+      },
+      EMBED_ANALYTICS: {
+        seqid: 44,
+        claim: 'catalog:embed',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Acessar Módulo de Incorporação de Ativos',
+          'en-us': 'Access Embedding analytics Module',
+          'es-es': 'Acceder al Módulo de Incorporación de Activos',
+        },
+      },
+      TRIGGER_CATALOG_TASK: {
+        seqid: 45,
+        claim: 'catalog:trigger-task',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Executar a catalogação de um ativo',
+          'en-us': 'Trigger an asset cataloging',
+          'es-es': 'Realizar el listado de un activo',
         },
       },
     },
@@ -417,25 +438,25 @@ export const PERMISSIONS_GROUPS = {
     permissions: {
       OPEN: {
         seqid: 16,
-        claim: 'GET /zendesk',
+        claim: 'support-chat:access',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Acessar Zendesk',
-          'en-us': 'Access Zendesk',
-          'es-es': 'Acceso Zendesk',
+          'pt-br': 'Acessar chat de suporte',
+          'en-us': 'Access support chat',
+          'es-es': 'Acceder al chat de soporte',
         },
       },
     },
   },
 
-  METABASE: {
+  ANALYZE: {
     title: {
       'pt-br': 'Analisar | Visualização',
       'en-us': 'Analyze | Visualization',
       'es-es': 'Analizar | Visualización',
     },
     permissions: {
-      OPEN: {
+      METABASE: {
         seqid: 11,
         claim: 'GET /metabase',
         usage: PermissionUsages.PUBLIC,
@@ -445,26 +466,26 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Acceso Metabase',
         },
       },
-    },
-  },
-
-  ORCHEST: {
-    title: {
-      'pt-br': 'Analisar | Inteligência',
-      'en-us': 'Analyze | Intelligence',
-      'es-es': 'Analizar | Inteligencia',
-    },
-    permissions: {
       INTELLIGENCE: {
         seqid: 31,
         claim: 'intelligence:open',
-        usage: PermissionUsages.INTERNAL,
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Acessar Orchest',
-          'en-us': 'Access Orchest',
-          'es-s': 'Acceso Orchest',
+          'pt-br': 'Acessar Módulo de Inteligência',
+          'en-us': 'Access Intelligence Module',
+          'es-s': 'Acceder Módulo de Inteligencia',
         },
       },
+    },
+  },
+
+  MODULES: {
+    title: {
+      'pt-br': 'Módulos da Dadosfera',
+      'en-us': 'Dadosfera Modules',
+      'es-es': 'Módulos de la Dadosfera',
+    },
+    permissions: {
       APP_BUILDER: {
         seqid: 32,
         claim: 'app-builder:open',
@@ -472,7 +493,7 @@ export const PERMISSIONS_GROUPS = {
         name: {
           'pt-br': 'Acessar App Builder',
           'en-us': 'Access App Builder',
-          'es-es': 'Acceso App Builder',
+          'es-es': 'Acceder App Builder',
         },
       },
       MACHINE_LEARNING: {
@@ -482,7 +503,7 @@ export const PERMISSIONS_GROUPS = {
         name: {
           'pt-br': 'Acessar Machine Learning',
           'en-us': 'Access Machine Learning',
-          'es-es': 'Acceso Machine Learning',
+          'es-es': 'Acceder Machine Learning',
         },
       },
     },
@@ -506,6 +527,25 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
+  PROCESS: {
+    title: {
+      'pt-br': 'Processar',
+      'en-us': 'Process',
+      'es-es': 'Procesar',
+    },
+    permissions: {
+      TRANSFORMATION: {
+        seqid: 43,
+        claim: 'process:open',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Módulo de Transformação',
+          'en-us': 'Access Transformation Module',
+          'es-es': 'Acceder Módulo de transformación',
+        },
+      },
+    },
+  },
   DADOSFERA: {
     title: {
       'pt-br': 'Usuários',
@@ -536,7 +576,34 @@ export const PERMISSIONS_GROUPS = {
     },
   },
 };
-
+export interface DadosferaModule {
+  name: string;
+  description: string;
+  key: string;
+  permissionSeqId: number;
+}
+export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+  {
+    name: 'Intelligence Module',
+    description: 'Orchest Module',
+    key: 'intelligence',
+    permissionSeqId: PERMISSIONS_GROUPS.ANALYZE.permissions.INTELLIGENCE.seqid,
+  },
+  {
+    name: 'Proccessing Module',
+    description: 'Proccessing Module',
+    key: 'process',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid,
+  },
+  {
+    name: 'Embedded Analytics',
+    description: 'Embedded Analytics Module',
+    key: 'embedded-analytics',
+    permissionSeqId:
+      PERMISSIONS_GROUPS.PROCESS.permissions.TRANSFORMATION.seqid,
+  },
+];
 // traverses the object searching for duplicate seqids or claims (executes at runtime)
 let nextAvailableSeqid = 0;
 const seqids = Object.values(PERMISSIONS_GROUPS).flatMap((namespace) =>
@@ -556,4 +623,9 @@ Object.values(PERMISSIONS_GROUPS).map((namespace) =>
   }),
 );
 
+DADOSFERA_MODULES.map((m) => m.key).forEach((m, i, arr) => {
+  if (arr.indexOf(m) !== i)
+    throw new Error(`DADOSFERA_MODULES[${i}] does not have a unique key`);
+});
+
 logger.log(`next available seqid ${nextAvailableSeqid + 1}`);

src/decorators/authentication.decorator.ts

@@ -0,0 +1,41 @@
+import { applyDecorators } from '@nestjs/common';
+import { ApiSecurity } from '@nestjs/swagger';
+import { type Request } from 'express';
+import { type PermissionsObjectPermission } from '../authentication/permissions.enum';
+import { type RequestUser } from './user.decorator';
+import { SetMultipleMetadata } from './shared';
+
+export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
+
+export type AuthenticationFunction = (req: Request, user: any) => boolean;
+
+export function RequireAllPermissions(
+  ...permissions: PermissionsObjectPermission[]
+) {
+  return createAuthenticatedDecorator((req, user: RequestUser) =>
+    permissions.every(({ seqid }) => user.permissions.includes(seqid)),
+  );
+}
+
+export function RequireSomePermission(
+  ...permissions: PermissionsObjectPermission[]
+) {
+  return createAuthenticatedDecorator((req, user: RequestUser) =>
+    permissions.some(({ seqid }) => user.permissions.includes(seqid)),
+  );
+}
+
+export function AuthenticateCondition(func: AuthenticationFunction) {
+  return createAuthenticatedDecorator(func);
+}
+
+export function Authenticated() {
+  return createAuthenticatedDecorator(() => true);
+}
+
+function createAuthenticatedDecorator(metadataValue: AuthenticationFunction) {
+  return applyDecorators(
+    ApiSecurity('access-token'),
+    SetMultipleMetadata(AUTH_FUNCTION_KEY, metadataValue),
+  );
+}

src/decorators/language.decorator.ts

@@ -0,0 +1,25 @@
+import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
+import { LanguageEnum } from 'src/utils/languages.enum';
+
+export const Language: () => ParameterDecorator = createParamDecorator(
+  (options: any, ctx: ExecutionContext): LanguageEnum => {
+    const headers = ctx.switchToHttp()?.getRequest()?.headers;
+
+    if (!headers) return LanguageEnum.ptbr;
+
+    let language: LanguageEnum =
+      headers['dadosfera-lang'] ?? headers['accept-language'];
+    language = language?.toLowerCase().trim() as LanguageEnum;
+
+    if (!language) return LanguageEnum.ptbr;
+
+    Object.values(LanguageEnum).some((l) => {
+      if (language === l || l.includes(language)) {
+        language = l;
+        return true;
+      }
+      language = LanguageEnum.enus;
+    });
+    return language;
+  },
+);

src/decorators/shared.ts

@@ -0,0 +1,28 @@
+import { type CustomDecorator } from '@nestjs/common';
+
+// implementation copied from SetMetadata, but tweaked to get existing values
+// of the metadataKey and accumulate it with the new metadataValue
+export function SetMultipleMetadata(
+  metadataKey,
+  metadataValue,
+): CustomDecorator {
+  const decoratorFactory: CustomDecorator = (target, key?, descriptor?) => {
+    // .start: tweak
+    // descriptor?.value = function decorator; target = class decorator
+    const accumulatedVal =
+      Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
+    accumulatedVal.push(metadataValue);
+    // .end: tweak
+
+    if (descriptor) {
+      Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
+      return descriptor;
+    }
+
+    Reflect.defineMetadata(metadataKey, accumulatedVal, target);
+    return target;
+  };
+
+  decoratorFactory.KEY = metadataKey;
+  return decoratorFactory;
+}

src/decorators/swagger.decorator.ts

@@ -0,0 +1,13 @@
+import { applyDecorators } from '@nestjs/common';
+import { ApiExcludeController, ApiExcludeEndpoint } from '@nestjs/swagger';
+
+export function ApiInternalOnlyEndpoint() {
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    return applyDecorators(ApiExcludeEndpoint());
+  return () => null;
+}
+export function ApiInternalOnlyController() {
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    return applyDecorators(ApiExcludeController());
+  return () => null;
+}

src/decorators/user.decorator.spec.ts

@@ -1,17 +1,14 @@
-import request from 'supertest';
-import { Test } from '@nestjs/testing';
-import { HttpStatus, INestApplication } from '@nestjs/common';
-import { APP_GUARD } from '@nestjs/core';
-import jwt from 'jsonwebtoken';
-
-import { Controller, Get } from '@nestjs/common';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { AuthenticationGuard } from './authentication.guard';
+import { Controller, Get, HttpStatus, INestApplication } from '@nestjs/common';
+import { APP_GUARD } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+import jwt from 'jsonwebtoken';
+import request from 'supertest';
+import { AuthenticationGuard } from '../authentication/authentication.guard';
+import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
 import { AuthClientService } from '../modules/auth/auth.service';
-
 import ErrorCodes from '../utils/errorCodes';
 import { User } from './user.decorator';
-import { PERMISSIONS_GROUPS } from './permissions.enum';
 
 const logger = {
   info: (...args) => args,
@@ -49,7 +46,7 @@ describe('user.decorator', () => {
   const fakeUserPayload = {
     user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
     username: 'super.admin',
-    permissions: [PERMISSIONS_GROUPS.METABASE.permissions.OPEN].map(
+    permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
       ({ seqid }) => seqid,
     ),
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',

src/decorators/user.decorator.ts

@@ -1,4 +1,4 @@
-import { createParamDecorator, ExecutionContext } from '@nestjs/common';
+import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
 
 import ErrorBuilder from '../utils/ErrorBuilder';
 import ErrorCodes from '../utils/errorCodes';

src/main.ts

@@ -6,6 +6,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import { AppModule } from './app.module';
 import { writeFileSync } from 'fs';
+import { execSync } from 'child_process';
+import { INestApplication } from '@nestjs/common';
 async function bootstrap() {
   DadosferaLogger.setupLogger({
     serviceName: 'maestro',
@@ -22,28 +24,60 @@ async function bootstrap() {
       optionsSuccessStatus: 204,
     },
   });
-
   app.use(helmet());
+  configureSwagger(app);
+  await app.listen(3333);
+  if (process.env.KILL_AFTER_START) await app.close();
+}
 
-  const date = new Date();
+function configureSwagger(app: INestApplication) {
+  let branchName = '';
+  try {
+    branchName = execSync('git rev-parse --abbrev-ref HEAD')
+      .toString()
+      .replace('\n', '')
+      .trim();
+  } catch (error) {
+    console.warn("Can't get branch name", error);
+  }
+  const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
 
-  const config = new DocumentBuilder()
-    .setTitle('Maestro')
+  const swaggerConfig = new DocumentBuilder()
+    .setTitle(swaggerTitle)
     .setDescription('Documentation for Maestro gateway')
-    .setVersion(
-      `${date.getDate()}.${
-        date.getMonth() + 1
-      }.${date.getFullYear()} - ${date.getHours()}:${date.getMinutes()}`,
-    )
-    .addBearerAuth()
-    .build();
+    .addSecurity('access-token', {
+      type: 'apiKey',
+      name: 'Authorization',
+      in: 'header',
+    })
+    .setDescription('This is the Maestro API');
+
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
+
+  const config = swaggerConfig.build();
 
   const document = SwaggerModule.createDocument(app, config);
 
+  const swaggerJsonFile =
+    process.env.INTERNAL_SWAGGER === 'true'
+      ? 'docsfera.json'
+      : 'docsfera.external.json';
+
   if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
 
-  await writeFileSync('swagger.txt', JSON.stringify(JSON.stringify(document)));
-
-  await app.listen(3333);
+  writeFileSync(
+    swaggerJsonFile,
+    JSON.stringify(
+      {
+        service: '${DOCS_API_TOKEN}',
+        pageId: '${DOCS_PAGE_ID}',
+        blockId: '${DOCS_BLOCK_ID}',
+        schema: document,
+      },
+      null,
+      2,
+    ),
+  );
 }
 bootstrap();

src/modules/auth/README.md

@@ -0,0 +1,38 @@
+# Auth
+
+## SSO/oAuth
+
+### Strategy
+We are using [PassportJs](https://www.passportjs.org/) to handle oAuth authentications.
+
+When the client (front end) makes a `GET /auth/oauth/{strategy}` Passport automatically redirects the user to the `strategy` login page. To do that we must configure and use a **Passport Strategy**. We must also have a callback route, conventionally `GET /auth/oauth/{strategy}/callback`, so the oAuth app can report the status of the user's login.
+
+- If the oAuth is successfull we call DUC's `AuthOauthSignIn` request that gets the tokens from Cognito and saves them on cache temporarily under a key we call `session`. Duc returns that `session` to maestro which then redirects the user to our app login page with that `session` as a query param.
+
+- If the oAuth login is not successfull for some reason or the user **does not** exist on DUC's database we redirect the user to our login page with an `error` and `error_description` as query params.
+
+### Routes
+
+So in order to have an SSO login, besides configuring the Strategy, we must have two routes for each Strategy, like in the example below:
+
+```ts
+  @Get('oauth/google')
+  @UseGuards(AuthGuard('google-login'))
+  googleOauth() {
+    this.logger.info('/oauth/google');
+    return true;
+  }
+
+  @Get('oauth/google/callback')
+  @UseGuards(AuthGuard('google-login'))
+  @Redirect()
+  async googleOauthCallback(@Req() req) {
+    const { url, email, token, language = 'pt-br' } = await this.callback(req);
+    if (url.searchParams.get('error')) {
+      this.logger.error('/oauth/google - ERROR');
+      return { url: url.href };
+    }
+    // ... Rest of the logic
+    return { url: url.href };
+  }
+```

src/modules/auth/auth.controller.ts

@@ -7,11 +7,19 @@ import {
   HttpStatus,
   Inject,
   UseFilters,
+  Get,
+  UseGuards,
+  Redirect,
+  Req,
+  Param,
 } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthSignInRequest,
-  AuthRefreshAccessTokenRequest,
+  ApiHeaders,
+  ApiOkResponse,
+  ApiSecurity,
+  ApiTags,
+} from '@nestjs/swagger';
+import {
   AuthChangePasswordRequest,
   AuthResetPasswordRequest,
   AuthVerifyResetPasswordCodeRequest,
@@ -22,18 +30,36 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
 import { AuthClientService } from './auth.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
-import { RequestUser, User } from 'src/authentication/user.decorator';
-import { Metadata } from '@grpc/grpc-js';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import {
+  AuthRefreshAccessTokenReq,
+  AuthRefreshAccessTokenRes,
+  AuthSignInReq,
+  AuthSignInRes,
+} from './dtos/login';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { AuthGuard } from '@nestjs/passport';
+import { Request } from 'express';
+import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
 @ApiTags('Auth')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('auth')
 export class AuthController {
   logger: DadosferaLogger;
+  redirectUrl: string;
 
   constructor(
     @Inject(DadosferaLogger)
@@ -41,8 +67,49 @@ export class AuthController {
     private authClient: AuthClientService,
   ) {
     this.logger = dadosferaLogger.logger;
+
+    switch (process.env.ENV) {
+      case 'stg':
+        this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/auth/login`;
+        break;
+      case 'prd':
+        this.redirectUrl = `https://app.dadosfera.ai/auth/login`;
+        break;
+      default:
+        this.redirectUrl = `http://localhost:4200/auth/login`;
+    }
   }
 
+  @Post('sign-in')
+  @HttpCode(HttpStatus.OK)
+  async signIn(
+    @Body() { username, password, totp }: AuthSignInReq,
+    @Language() language: LanguageEnum,
+  ): Promise<AuthSignInRes> {
+    this.logger.info('/auth - SignIn');
+    const metadata = PackTheMetadata({ language });
+    return this.authClient.signIn({ username, password, totp }, metadata);
+  }
+
+  @Post('refresh-access-token')
+  @HttpCode(HttpStatus.OK)
+  @ApiOkResponse({ type: AuthRefreshAccessTokenRes })
+  async refreshAccessToken(
+    @Body() body: AuthRefreshAccessTokenReq,
+    @Language() language: LanguageEnum,
+  ) {
+    this.logger.info('/auth - RefreshAccessToken');
+    const { refreshToken, customerName: customer_name } = body;
+
+    const metadata = PackTheMetadata({
+      customer_name,
+      language,
+    });
+
+    return this.authClient.refreshAccessToken({ refreshToken }, metadata);
+  }
+
+  @ApiInternalOnlyEndpoint()
   @Post('/sso/snowflake')
   @RequireAllPermissions(PERMISSIONS_GROUPS.SNOWFLAKE.permissions.OPEN)
   @HttpCode(HttpStatus.OK)
@@ -58,31 +125,7 @@ export class AuthController {
     });
   }
 
-  @Post('sign-in')
-  @HttpCode(HttpStatus.OK)
-  async signIn(
-    @Body() { username, password, totp }: AuthSignInRequest,
-    @Headers('Dadosfera-Lang') language: string,
-  ) {
-    this.logger.info('/auth - SignIn');
-    const metadata = new Metadata();
-    metadata.add('language', language || 'pt-br');
-    return this.authClient.signIn({ username, password, totp }, metadata);
-  }
-
-  @Post('refresh-access-token')
-  @HttpCode(HttpStatus.OK)
-  async refreshAccessToken(
-    @Body() { refreshToken }: AuthRefreshAccessTokenRequest,
-    @Headers('Dadosfera-Lang') language: string,
-  ) {
-    this.logger.info('/auth - RefreshAccessToken');
-    const metadata = new Metadata();
-    metadata.add('language', language || 'pt-br');
-
-    return this.authClient.refreshAccessToken({ refreshToken }, metadata);
-  }
-
+  @ApiInternalOnlyEndpoint()
   @Post('change-password')
   @HttpCode(HttpStatus.OK)
   async changePassword(
@@ -101,16 +144,24 @@ export class AuthController {
     });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('reset-password')
   @HttpCode(HttpStatus.OK)
-  async resetPassword(@Body() body: AuthResetPasswordRequest) {
+  async resetPassword(
+    @Body() body: AuthResetPasswordRequest,
+    @Language() language: LanguageEnum,
+  ) {
     this.logger.info('/auth - reset-password');
+    const metadata = PackTheMetadata({
+      language: language,
+    });
 
     const { username } = body;
 
-    return this.authClient.resetPassword({ username });
+    return this.authClient.resetPassword({ username }, metadata);
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('verify-reset-password-code')
   @HttpCode(HttpStatus.OK)
   async verifyResetPasswordCode(
@@ -123,6 +174,7 @@ export class AuthController {
     return this.authClient.verifyResetPasswordCode({ username, code });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('confirm-reset-password')
   @HttpCode(HttpStatus.OK)
   async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
@@ -137,6 +189,7 @@ export class AuthController {
     });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('enable-totp')
   @HttpCode(HttpStatus.OK)
   async enableTotpMFA(
@@ -151,6 +204,7 @@ export class AuthController {
     return this.authClient.enableTotpMFA({ accessToken, password });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('disable-totp')
   @HttpCode(HttpStatus.OK)
   async disableTotpMFA(
@@ -165,6 +219,7 @@ export class AuthController {
     return this.authClient.disableTotpMFA({ accessToken, password });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('dismiss-totp')
   @HttpCode(HttpStatus.OK)
   async dismissTotpMFA(@Headers() headers) {
@@ -175,6 +230,7 @@ export class AuthController {
     return this.authClient.dismissTotpMFA({ accessToken });
   }
 
+  @ApiInternalOnlyEndpoint()
   @Post('verify-totp')
   @HttpCode(HttpStatus.OK)
   async verifyTotp(@Body() body: AuthVerifyTotpMfaRequest, @Headers() headers) {
@@ -185,4 +241,107 @@ export class AuthController {
 
     return this.authClient.verifyTotp({ accessToken, totp });
   }
+
+  @ApiInternalOnlyEndpoint()
+  @Authenticated()
+  @ApiSecurity('access-token')
+  @Get('verify-access-token')
+  verifyAccessToken() {
+    return { access_token_status: 'valid' };
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('session/:session')
+  getSession(@Param('session') session: string) {
+    return this.authClient.getSession(session);
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('oauth/google')
+  @UseGuards(AuthGuard('google-login'))
+  googleOauth() {
+    this.logger.info('/oauth/google');
+    return true;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Get('oauth/google/callback')
+  @UseGuards(AuthGuard('google-login'))
+  @Redirect()
+  async googleOauthCallback(@Req() req) {
+    const { url, email, token, language = 'pt-br' } = await this.callback(req);
+    if (url.searchParams.get('error')) {
+      this.logger.error('/oauth/google - ERROR');
+      return { url: url.href };
+    }
+
+    await this.authClient
+      .oauthSignIn({
+        username: email,
+        token,
+      })
+      .then(({ session }) => {
+        this.logger.info('/oauth/google - SUCESS');
+        url.searchParams.set('session', session);
+      })
+      .catch((err) => {
+        this.logger.error('/oauth/google - ERROR');
+        let error = OauthErrors.INVALID_CREDENTIALS[language].error;
+        let error_description =
+          OauthErrors.INVALID_CREDENTIALS[language].error_description;
+        switch (err.details) {
+          case ErrorCodes.USER.NOT_FOUND:
+            error = OauthErrors.USER_NOT_FOUND[language].error;
+            error_description =
+              OauthErrors.USER_NOT_FOUND[language].error_description(email);
+            break;
+          case ErrorCodes.AUTH.UNAUTHORIZED:
+            error = OauthErrors.INVALID_SESSION[language].error;
+            error_description =
+              OauthErrors.INVALID_SESSION[language].error_description;
+            break;
+        }
+        url.searchParams.set('error', error);
+        url.searchParams.set('error_description', error_description);
+        return null;
+      });
+    return { url: url.href };
+  }
+
+  async callback(req: Request) {
+    const { error, state } = req.query;
+    const { authInfo } = req;
+    const url = new URL(this.redirectUrl);
+    let email, token, error_title, error_description;
+    let language: 'pt-br' | 'en-us' = 'pt-br';
+
+    const stateObject = jwt.verify(
+      state as string,
+      process.env.JWT_PRIVATE_KEY,
+    );
+    if (typeof stateObject != 'string') language = stateObject.language;
+
+    if (error || !authInfo) {
+      this.logger.error(error);
+      if (!authInfo) this.logger.error('No authInfo', { request: req });
+
+      error_title = OauthErrors.INVALID_CREDENTIALS[language].error;
+      error_description =
+        OauthErrors.INVALID_CREDENTIALS[language].error_description;
+      if (error) error_description += ` - [${error}]`;
+    } else {
+      const { accessToken } = authInfo as any;
+      const { _json: userInfo } = req.user as any;
+
+      email = userInfo.email;
+      token = accessToken;
+    }
+
+    if (error_title) {
+      url.searchParams.set('error', error_title);
+      url.searchParams.set('error_description', error_description);
+    }
+
+    return { token, email, url, language };
+  }
 }

src/modules/auth/auth.module.ts

@@ -6,12 +6,19 @@ import { AuthController } from './auth.controller';
 import { AuthClientService } from './auth.service';
 
 import { DucClient } from '../duc/client.config';
+import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
+import { getOauthSecrets } from 'src/utils/OauthSecrets';
 const client = new DucClient();
 
 @Module({
   imports: [ClientsModule.register([client.providerOptions])],
   controllers: [AuthController],
-  providers: [AuthClientService, DadosferaLogger],
+  providers: [
+    AuthClientService,
+    DadosferaLogger,
+    GoogleLoginStrategy,
+    { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
+  ],
   exports: [AuthClientService],
 })
 export class AuthModule {}

src/modules/auth/auth.service.ts

@@ -90,10 +90,15 @@ export class AuthClientService implements OnModuleInit {
     );
   }
 
-  async resetPassword({ username }: AuthResetPasswordRequest) {
+  async resetPassword(
+    { username }: AuthResetPasswordRequest,
+    metadata: Metadata,
+  ) {
     this.logger.info('resetPassword');
 
-    return lastValueFrom(this.authService.AuthResetPassword({ username }));
+    return lastValueFrom(
+      this.authService.AuthResetPassword({ username }, metadata),
+    );
   }
 
   async verifyResetPasswordCode({
@@ -152,4 +157,13 @@ export class AuthClientService implements OnModuleInit {
       this.authService.AuthVerifyTotpMfa({ accessToken, totp }),
     );
   }
+
+  async getSession(session: string) {
+    return lastValueFrom(this.authService.AuthGetSession({ session }));
+  }
+
+  async oauthSignIn(data: { username: string; token: string }) {
+    const { username, token } = data;
+    return lastValueFrom(this.authService.AuthOauthSignIn({ token, username }));
+  }
 }

src/modules/auth/dtos/login.ts

@@ -1,4 +1,8 @@
-import { ApiProperty } from '@nestjs/swagger';
+import {
+  AuthSignInRequest,
+  AuthSignInResponse,
+} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
 export class LoginDto {
   @ApiProperty()
@@ -10,3 +14,104 @@ export class LoginDto {
   @ApiProperty({ required: false })
   totp: string;
 }
+export class AuthTokens {
+  @ApiProperty()
+  accessToken: string;
+  @ApiProperty()
+  refreshToken: string;
+  @ApiPropertyOptional()
+  termsOfUseToken: string;
+  @ApiProperty({ deprecated: true })
+  idToken: string;
+}
+export enum MfaStatuses {
+  PENDING = 'pending',
+  NONE = 'none',
+  TOTP = 'totp',
+}
+
+export enum TouStatuses {
+  PENDING = 'pending',
+  REQUIRED = 'required',
+  OK = 'ok',
+}
+export class TermsOfUse {
+  @ApiProperty()
+  version: number;
+  @ApiProperty()
+  publicUrl: string;
+  @ApiProperty()
+  enforceDate: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class TermsOfUseStatus {
+  @ApiProperty({ enum: TouStatuses })
+  status: string;
+  @ApiPropertyOptional()
+  lastSigned?: TermsOfUse;
+  @ApiPropertyOptional()
+  next?: TermsOfUse;
+}
+export class AuthUser {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class AuthCustomer {
+  @ApiProperty()
+  modules: string[];
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  tier: string;
+  @ApiProperty()
+  scheduleLimit: string;
+}
+
+export class AuthSignInReq implements AuthSignInRequest {
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  password: string;
+  @ApiPropertyOptional({
+    description:
+      'TOTP code used to login when Multi-Factor Authentication is enabled',
+  })
+  totp: string;
+}
+
+export class AuthSignInRes implements AuthSignInResponse {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty({ enum: MfaStatuses })
+  mfaStatus: string;
+  @ApiPropertyOptional()
+  customer?: AuthCustomer;
+  @ApiPropertyOptional()
+  user?: AuthUser;
+  @ApiPropertyOptional()
+  tokens?: AuthTokens;
+  @ApiPropertyOptional()
+  termsOfUse?: TermsOfUseStatus;
+}
+
+export class AuthRefreshAccessTokenReq {
+  @ApiProperty()
+  refreshToken: string;
+  @ApiProperty()
+  customerName: string;
+}
+export class AuthRefreshAccessTokenRes {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty()
+  accessToken: string;
+}

src/modules/auth/passport-strategies/google-strategy.ts

@@ -0,0 +1,46 @@
+import {
+  AuthenticateOptionsGoogle,
+  Profile,
+  Strategy,
+  StrategyOptions,
+} from 'passport-google-oauth20';
+import { PassportStrategy } from '@nestjs/passport';
+import { Inject, Injectable } from '@nestjs/common';
+import { OauthSecrets } from 'src/utils/OauthSecrets';
+import { Request } from 'express';
+import jwt from 'jsonwebtoken';
+
+@Injectable()
+export class GoogleLoginStrategy extends PassportStrategy(
+  Strategy,
+  'google-login',
+) {
+  redirect_uri: string;
+  constructor(
+    @Inject('OAUTH_SECRETS')
+    private readonly oauthSecrets: OauthSecrets,
+  ) {
+    const options: StrategyOptions = {
+      clientID: oauthSecrets['google-login'].client_id,
+      clientSecret: oauthSecrets['google-login'].client_secret,
+      callbackURL: oauthSecrets['google-login'].redirect_uri,
+      scope: ['email', 'profile', 'openid'],
+    };
+    const verify = (
+      accessToken: string,
+      refreshToken: string,
+      profile: Profile,
+      done,
+    ) => {
+      return done(null, profile, { accessToken, refreshToken });
+    };
+
+    super(options, verify);
+  }
+
+  authenticate(req: Request, options: AuthenticateOptionsGoogle) {
+    const language = req.headers['dadosfera-lang'] || req.query.language;
+    options.state = jwt.sign({ language }, process.env.JWT_PRIVATE_KEY);
+    super.authenticate(req, options);
+  }
+}

src/modules/catalog/catalog-client.ts

@@ -6,6 +6,8 @@ import {
 import { credentials } from '@grpc/grpc-js';
 import { Catalog } from '@dadosfera/protospack-v2';
 
+const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+
 export class CatalogClientConfiguration {
   public name = 'CatalogClientConfiguration';
   private config: GrpcOptions = {
@@ -16,10 +18,7 @@ export class CatalogClientConfiguration {
         Catalog.ProtoPackages.ReadPackage,
         Catalog.ProtoPackages.WritePackage,
       ],
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         Catalog.ProtoPaths.ReadFilePath,
         Catalog.ProtoPaths.WriteFilePath,

src/modules/catalog/catalog.controller.ts

@@ -1,62 +1,60 @@
 import {
+  BadRequestException,
   Body,
   Controller,
   Delete,
+  ForbiddenException,
   Get,
   Headers,
-  HttpException,
   Inject,
+  NotFoundException,
   Param,
   Post,
   Put,
   Query,
-  UnsupportedMediaTypeException,
+  UseFilters,
 } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthenticateCondition,
+  ApiCreatedResponse,
+  ApiHeaders,
+  ApiOkResponse,
+  ApiTags,
+} from '@nestjs/swagger';
+import {
   Authenticated,
-} from '../../authentication/authentication.decorator';
+  RequireAllPermissions,
+  RequireSomePermission,
+} from '../../decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import {
+  GetDatasetCatalogTaskRes,
+  ICatalogAllRequest,
   ICatalogAllResponse,
   IColumnsMetadataResponse,
+  ICreateDataAsset,
   IDeleteComment,
   IDocsResponse,
   IMakeAComment,
   IOneDataAsset,
   IPreviewResponse,
   IUpdateDataRequest,
+  TriggerCatalogReq,
+  TriggerCatalogRes,
 } from './dtos';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
 @ApiTags('Catalog')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @Controller('catalog')
+@UseFilters(new GrpcToHttpExceptionFilter())
 @Authenticated()
-@AuthenticateCondition((req, user) => {
-  if (req.path.includes('/manage_permissions/')) return true;
-  let action;
-
-  switch (req.method) {
-    case 'POST':
-      action = 'CREATE';
-      break;
-
-    case 'PUT':
-      action = 'UPDATE';
-      break;
-
-    default:
-      action = req.method;
-  }
-
-  return user.permissions.includes(
-    PERMISSIONS_GROUPS.CATALOG.permissions[action].seqid,
-  );
-})
 export class CatalogController {
   logger: DadosferaLogger;
   constructor(
@@ -68,12 +66,16 @@ export class CatalogController {
   }
 
   @Get()
-  async catalogAll(
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async searchCatalog(
     @User() user: RequestUser,
-    @Query() query,
+    @Query() query: ICatalogAllRequest,
   ): Promise<ICatalogAllResponse> {
     const { user_id, customer_name, customer_id, username, permissions } = user;
-    this.logger.info(`/catalog - Catalog all data assets`, {
+    this.logger.info(`/catalog - searchCatalog`, {
       user_id,
       customer_name,
     });
@@ -93,16 +95,75 @@ export class CatalogController {
       is_data_manager,
     });
 
-    const res = await this.catalogService.getAllOpenSearchDataAssets(
+    const res = await this.catalogService.searchDataAssets(
       query,
       metadata,
       customer_id,
     );
 
-    return { ...res };
+    return res;
   }
 
+  @ApiInternalOnlyEndpoint()
+  @Get('data-asset')
+  async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
+    const { username, user_id, customer_id, customer_name, permissions } = user;
+    const { pipeline, object } = query;
+    this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
+      username,
+      customer_name,
+    });
+
+    if (!pipeline || !object) {
+      throw new BadRequestException('Query params not provided');
+    }
+
+    const is_data_manager = permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+    let has_permission = false;
+    const metadata = PackTheMetadata({
+      username,
+      user_id: undefined,
+      customer_id,
+      customer_name,
+    });
+
+    const user_roles = await this.catalogService.getUserRolesIds(user_id);
+
+    const { data_asset } =
+      await this.catalogService.getOneDataAssetByPipelineAndObject({
+        pipeline: query.pipeline,
+        object: query.object,
+        customer_id,
+        metadata,
+      });
+
+    if (data_asset?.owner === username) has_permission = true;
+
+    for (const role of user_roles) {
+      if (data_asset.p_roles.includes(role)) has_permission = true;
+    }
+
+    if (data_asset.p_users.includes(user_id)) has_permission = true;
+
+    if (is_data_manager || has_permission) {
+      delete data_asset.p_roles;
+      delete data_asset.p_users;
+      return { data_asset };
+    }
+
+    throw new ForbiddenException(
+      'You do not have permission to access this data asset.',
+    );
+  }
+
+  @ApiInternalOnlyEndpoint()
   @Get('tags')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async findAllTags(@Body() body) {
     this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
       user: body.info.user_id,
@@ -122,15 +183,18 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAsset(
     @User() user: RequestUser,
-    @Headers() headers,
-    @Param('id') id,
+    @Param('id') id: string,
+    @Query('shared') shared?: 'true',
   ) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
-    const [data_asset_type, data_asset_id] = id.split('-');
 
-    this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
+    this.logger.info(`GET /data-asset/${id}`, {
       username,
       customer_name,
     });
@@ -141,7 +205,7 @@ export class CatalogController {
     let has_permission = false;
     const metadata = PackTheMetadata({
       username,
-      user_id: undefined,
+      user_id,
       customer_id,
       customer_name,
     });
@@ -150,39 +214,41 @@ export class CatalogController {
 
     const { data_asset } = await this.catalogService.getOneDataAsset({
       customer_id,
-      data_asset_id,
-      data_asset_type,
+      id,
       metadata,
     });
+    has_permission =
+      is_data_manager ||
+      data_asset?.owner === username ||
+      (user_roles as Array<any>).some((r) => data_asset.p_roles.includes(r)) ||
+      data_asset.p_users.includes(user_id);
+    if (
+      shared === 'true' &&
+      (data_asset.share_type === undefined || data_asset.share_type === 'none')
+    )
+      throw new NotFoundException();
+    if (!has_permission)
+      throw new ForbiddenException(
+        'You do not have permission to access this data asset.',
+      );
+    delete data_asset.p_roles;
+    delete data_asset.p_users;
 
-    if (data_asset?.owner === username) has_permission = true;
-
-    for (const role of user_roles) {
-      if (data_asset.p_roles.includes(role)) has_permission = true;
-    }
-
-    if (data_asset.p_users.includes(user_id)) has_permission = true;
-
-    if (is_data_manager || has_permission) {
-      delete data_asset.p_roles;
-      delete data_asset.p_users;
-      return { data_asset };
-    }
-
-    throw new HttpException(
-      'You do not have permission to access this data asset.',
-      403,
-    );
+    return { data_asset };
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('data-asset/rls/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetRls(
     @User() user: RequestUser,
     @Headers() headers,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
-    const [data_asset_type, data_asset_id] = id.split('-');
 
     this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
       username,
@@ -204,8 +270,7 @@ export class CatalogController {
 
     const { data_asset } = await this.catalogService.getOneDataAsset({
       customer_id,
-      data_asset_id,
-      data_asset_type,
+      id,
       metadata,
     });
 
@@ -223,30 +288,24 @@ export class CatalogController {
       return { data_asset };
     }
 
-    throw new HttpException(
+    throw new ForbiddenException(
       'You do not have permission to access this data asset.',
-      403,
     );
   }
 
   @Get('data-asset/:id/columns-metadata')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
-    @Param('id') id,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
   ): Promise<IColumnsMetadataResponse> {
     const { customer_name, customer_id, user_id, username } = user;
-    const data_asset_type = id.split('-')[0];
 
-    const supported_data_assets = ['dataset'];
-
-    if (!supported_data_assets.includes(data_asset_type)) {
-      throw new UnsupportedMediaTypeException(
-        `${data_asset_type} does not have a preview function for the time being.`,
-      );
-    }
-
-    this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
+    this.logger.info(`/catalog - columns-metadata`, {
       user_id,
       customer_name,
     });
@@ -266,21 +325,16 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/preview')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetPreview(
     @User() user: RequestUser,
-    @Headers('Dadosfera-lang') language,
-    @Param('id') id,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
   ): Promise<IPreviewResponse> {
     const { customer_name, customer_id, user_id, username } = user;
-    const data_asset_type = id.split('-')[0];
-
-    const supported_data_assets = ['dataset'];
-
-    if (!supported_data_assets.includes(data_asset_type)) {
-      throw new UnsupportedMediaTypeException(
-        `${data_asset_type} does not have a preview function for the time being.`,
-      );
-    }
 
     this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
       user_id,
@@ -301,21 +355,16 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetDocs(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
-    @Param('id') id,
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
   ): Promise<IDocsResponse> {
     const { customer_name, customer_id, user_id, username } = user;
-    const data_asset_type = id.split('-')[0];
-
-    const supported_data_assets = ['dataset'];
-
-    if (!supported_data_assets.includes(data_asset_type)) {
-      throw new UnsupportedMediaTypeException(
-        `${data_asset_type} does not have a documentation function for the time being.`,
-      );
-    }
 
     this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
       user_id,
@@ -336,10 +385,14 @@ export class CatalogController {
   }
 
   @Put('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async updateDataAsset(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
-    @Param('id') id,
+    @Language() language: LanguageEnum,
+    @Param('id') data_asset_id: string,
     @Body() body: IUpdateDataRequest,
   ): Promise<IOneDataAsset> {
     const { customer_id, customer_name, user_id, username } = user;
@@ -353,7 +406,7 @@ export class CatalogController {
 
     const result = await this.catalogService.updateOneDataAsset({
       body,
-      data_asset_id: id,
+      data_asset_id,
       customer_id,
       metadata,
     });
@@ -365,14 +418,17 @@ export class CatalogController {
   }
 
   @Post('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async manageDataAssetDocs(
     @User() user: RequestUser,
     @Headers() headers,
-    @Param('id') data_asset_id,
+    @Param('id') table_id: string,
     @Body('docs') docs: string,
   ) {
     const { user_id, customer_name } = user;
-    const [data_asset_type, id] = data_asset_id.split('-');
 
     this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
       user_id,
@@ -380,7 +436,7 @@ export class CatalogController {
     });
 
     const res = await this.catalogService.createDataDocs({
-      table_id: id,
+      table_id,
       docs,
       info: {
         customer: customer_name,
@@ -390,6 +446,7 @@ export class CatalogController {
     return res;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Put('data-asset/:id/manage-permissions')
   async manageDataAssetPermissions(
     @Param('id') id: string,
@@ -411,6 +468,7 @@ export class CatalogController {
     return { data_asset: JSON.parse(response.data_asset) };
   }
 
+  @ApiInternalOnlyEndpoint()
   @Put('data-asset/:id/revoke-permissions')
   async revokeDataAssetPermissions(
     @Param('id') id: string,
@@ -432,7 +490,35 @@ export class CatalogController {
     return { message: response.message };
   }
 
+  @Post()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async createDataAsset(
+    @User() user: RequestUser,
+    @Body() body: ICreateDataAsset,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const { data_asset } = await this.catalogService.createDataAsset(
+      body,
+      metadata,
+    );
+
+    return { data_asset: JSON.parse(data_asset) };
+  }
+
   @Post('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async commentOnDataAsset(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -455,10 +541,11 @@ export class CatalogController {
   }
 
   @Delete('data-asset/:id')
-  async deleteDataAsset(
-    @Param('id') data_asset_id: string,
-    @User() user: RequestUser,
-  ) {
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
     const { customer_id, customer_name, user_id, username } = user;
     const metadata = PackTheMetadata({
       customer_id,
@@ -466,10 +553,8 @@ export class CatalogController {
       user_id,
       username,
     });
-
-    const [type, id] = data_asset_id.split('-');
     const response = await this.catalogService.deleteDataAsset(
-      { id, type },
+      { id, type: undefined },
       metadata,
     );
 
@@ -477,6 +562,10 @@ export class CatalogController {
   }
 
   @Delete('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async deleteComment(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -490,7 +579,55 @@ export class CatalogController {
       username,
     });
     const response = await this.catalogService.deleteComment(
-      { data_asset_id: id, comment: body.comment },
+      {
+        data_asset_id: id,
+        comment: { ...body.comment, created_at: undefined, message: undefined },
+      },
+      metadata,
+    );
+
+    return response;
+  }
+
+  @Post('dataset-catalog-task')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiCreatedResponse({ type: TriggerCatalogRes })
+  async triggerCatalog(
+    @User() user: RequestUser,
+    @Body() body: TriggerCatalogReq,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const session = await this.catalogService.triggerCatalog(body, metadata);
+
+    return { session };
+  }
+
+  @Get('dataset-catalog-task/:session')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiOkResponse({ type: GetDatasetCatalogTaskRes })
+  async getCatalogTask(
+    @User() user: RequestUser,
+    @Param('session') session: string,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.getDatasetCatalogTask(
+      session,
       metadata,
     );
 

src/modules/catalog/catalog.service.ts

@@ -18,6 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
 import { UsersService } from '../users/users.service';
 import { RolesService } from '../roles/roles.service';
 import { Metadata } from '@grpc/grpc-js';
+import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
 
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
@@ -58,6 +59,15 @@ class CatalogService implements OnModuleInit {
     )}.dadosfera.ai`;
   }
 
+  async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
+    this.logger.info('CatalogService - Manage Data assets permissions');
+    if (!data.embed) data.embed = undefined;
+
+    return lastValueFrom(
+      this.catalogWriteService.CreateDataAsset(data, metadata),
+    );
+  }
+
   async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
@@ -131,34 +141,12 @@ class CatalogService implements OnModuleInit {
     return roles_ids;
   }
 
-  async getUsernamesAndRolesByCustomerId(customerId: string) {
-    const resultUsers = await this.userService.findAllUsersByCustomerId(
-      customerId,
-    );
-    const resultRoles = await this.roleService.roleSearch(
-      {},
-      { customer_id: customerId },
-    );
-
-    const customer_users = resultUsers.users.map((user) => {
-      return {
-        id: user.id,
-        username: user.username,
-      };
-    });
-
-    const customer_roles = resultRoles.roles.map((role) => {
-      return {
-        id: role.id,
-        name: role.name,
-      };
-    });
-
-    return [customer_users, customer_roles];
-  }
-
-  async getAllOpenSearchDataAssets(query, metadata, customer_id) {
-    this.logger.info('CatalogService - getAllOpenSearchDashboard');
+  async searchDataAssets(
+    query: Record<string, any>,
+    metadata: Metadata,
+    customer_id: string,
+  ) {
+    this.logger.info('CatalogService - searchDataAssets');
 
     const { search, page, size, sort_by, order, ...filters } = query;
 
@@ -174,171 +162,103 @@ class CatalogService implements OnModuleInit {
         },
         metadata,
       ),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 404,
-      );
-    });
+    );
 
     const result = JSON.parse(data_assets);
 
-    const [customer_users, customer_roles] =
-      await this.getUsernamesAndRolesByCustomerId(customer_id);
+    const response = await this.getAssetsUsersAndRoles(
+      result.data_assets,
+      customer_id,
+    );
 
-    for (const asset of result.data_assets) {
-      const condition =
-        asset.owner && !asset.owner.includes('@') ? true : false;
-      const result = condition
-        ? await this.userService.findOneById(asset.owner).catch(() => null)
-        : null;
-
-      const owner = result !== null ? result.user.username : null;
-
-      const roles = [];
-      const users = [];
-
-      for (const role of customer_roles) {
-        for (const role_id of asset.roles) {
-          if (role.id === role_id) roles.push(role);
-        }
-      }
-
-      for (const user of customer_users) {
-        for (const user_id of asset.users) {
-          if (user.id === user_id) users.push(user);
-        }
-      }
-
-      Object.assign(asset, {
-        ...asset,
-        owner,
-        roles,
-        users,
-      });
-    }
-
-    return { data_assets: result.data_assets, total };
+    return { data_assets: response, total };
   }
 
-  async getOneDataAsset({
-    data_asset_id,
-    data_asset_type,
-    customer_id,
-    metadata,
+  async getOneDataAsset(data: {
+    id: string;
+    customer_id: string;
+    metadata: Metadata;
   }) {
+    const { customer_id, id, metadata } = data;
     const { data_asset } = await lastValueFrom(
       this.catalogReadService.GetOneDataAsset(
-        { id: data_asset_id, type: data_asset_type },
+        { id, type: undefined },
         metadata,
       ),
     );
-    const opensearchTable = JSON.parse(data_asset);
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
-    const result = opensearchTable.owner
-      ? await this.userService
-          .findOneById(opensearchTable.owner)
-          .catch(() => null)
-      : null;
-
-    const owner = result !== null ? result.user.username : null;
-
-    const [customer_users, customer_roles] =
-      await this.getUsernamesAndRolesByCustomerId(customer_id);
-
-    const users = [];
-    const roles = [];
-
-    for (const role of customer_roles) {
-      for (const role_id of opensearchTable.roles) {
-        if (role.id === role_id) roles.push(role);
-      }
-    }
-
-    for (const user of customer_users) {
-      for (const user_id of opensearchTable.users) {
-        if (user.id === user_id) users.push(user);
-      }
-    }
-
-    Object.assign(opensearchTable, {
-      p_roles: opensearchTable.roles,
-      p_users: opensearchTable.users,
-      roles,
-      users,
-      owner,
-    });
-
-    return { data_asset: opensearchTable };
+    return { data_asset: asset[0] };
   }
 
-  async updateOneDataAsset({ data_asset_id, customer_id, body, metadata }) {
+  async getOneDataAssetByPipelineAndObject(data: {
+    customer_id: string;
+    pipeline: string;
+    object: string;
+    metadata: Metadata;
+  }) {
+    const { customer_id, metadata, pipeline, object } = data;
+    const { data_asset } = await lastValueFrom(
+      this.catalogReadService.GetOneDataAssetByPipelineAndObject(
+        { pipeline, object },
+        metadata,
+      ),
+    );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
+
+    return { data_asset: asset[0] };
+  }
+
+  async updateOneDataAsset(data: {
+    data_asset_id: string;
+    customer_id: string;
+    body: IUpdateDataRequest;
+    metadata: Metadata;
+  }) {
+    const { body, customer_id, data_asset_id, metadata } = data;
+
     const { data_asset } = await lastValueFrom(
       this.catalogWriteService.UpdateDataAsset(
-        { id: data_asset_id, ...body },
+        { id: data_asset_id, changes: JSON.stringify(body) },
         metadata,
       ),
     );
+    let asset = JSON.parse(data_asset);
+    asset = {
+      ...asset,
+      p_roles: asset.roles,
+      p_users: asset.users,
+    };
+    asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
-    const opensearchTable = JSON.parse(data_asset);
-
-    const result = opensearchTable.owner
-      ? await this.userService
-          .findOneById(opensearchTable.owner)
-          .catch(() => null)
-      : null;
-
-    const owner = result !== null ? result.user.username : null;
-
-    const [customer_users, customer_roles] =
-      await this.getUsernamesAndRolesByCustomerId(customer_id);
-
-    const users = [];
-    const roles = [];
-
-    for (const role of customer_roles) {
-      for (const role_id of opensearchTable.roles) {
-        if (role.id === role_id) roles.push(role);
-      }
-    }
-
-    for (const user of customer_users) {
-      for (const user_id of opensearchTable.users) {
-        if (user.id === user_id) users.push(user);
-      }
-    }
-
-    Object.assign(opensearchTable, {
-      p_roles: opensearchTable.roles,
-      p_users: opensearchTable.users,
-      roles,
-      users,
-      owner,
-    });
-
-    return { data_asset: opensearchTable };
+    return { data_asset: asset[0] };
   }
 
-  async getDataDocs(data_asset_id: string, metadata: Metadata) {
-    const [data_asset_type, id] = data_asset_id.split('-');
-
+  async getDataDocs(id: string, metadata: Metadata) {
     const { documentation } = await lastValueFrom(
-      this.catalogReadService.GetDatasetDoc(
-        { id, type: data_asset_type },
-        metadata,
-      ),
+      this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
     );
     console.log(documentation);
     const docs = JSON.parse(documentation);
     return docs;
   }
 
-  async getDatasetPreview(data_asset_id: string, metadata: Metadata) {
-    const [data_asset_type, id] = data_asset_id.split('-');
-
+  async getDatasetPreview(id: string, metadata: Metadata) {
     const { preview } = await lastValueFrom(
       this.catalogReadService.GetDatasetPreview(
-        { id, type: data_asset_type },
+        { id, type: undefined },
         metadata,
       ),
     );
@@ -346,12 +266,10 @@ class CatalogService implements OnModuleInit {
     return result;
   }
 
-  async getDatasetColumnsMetadata(data_asset_id: string, metadata: Metadata) {
-    const [data_asset_type, id] = data_asset_id.split('-');
-
+  async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
     const { columns_metadata } = await lastValueFrom(
       this.catalogReadService.GetDatasetColumnsMetadata(
-        { id, type: data_asset_type },
+        { id, type: undefined },
         metadata,
       ),
     );
@@ -385,6 +303,48 @@ class CatalogService implements OnModuleInit {
 
     return response;
   }
+  async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
+    const { users: customer_users } =
+      await this.userService.findAllUsersByCustomerId(customer_id);
+    const { roles: customer_roles } = await this.roleService.roleSearch(
+      {},
+      { customer_id },
+    );
+    return data_assets.map((data_asset) => {
+      const owner = customer_users.find(
+        (u) => u.id === data_asset.owner,
+      )?.username;
+
+      const roles = [];
+      const users = [];
+      for (const role_id of data_asset.roles) {
+        const role = customer_roles.find((r) => r.id === role_id);
+        if (role) roles.push({ id: role.id, name: role.name });
+      }
+      for (const user_id of data_asset.users) {
+        const user = customer_users.find((r) => r.id === user_id);
+        if (user) users.push({ id: user.id, username: user.username });
+      }
+      return {
+        ...data_asset,
+        roles,
+        users,
+        owner,
+      } as typeof data_asset;
+    });
+  }
+  async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
+    const { session } = await lastValueFrom(
+      this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
+    );
+    return session;
+  }
+  async getDatasetCatalogTask(session: string, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
+    );
+    return res;
+  }
 }
 
 export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -1,5 +1,19 @@
-import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
+import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
+export enum DataAssetShareType {
+  none = 'none',
+  public = 'public',
+  private = 'private',
+}
+export enum OrderEnum {
+  asc = 'asc',
+  desc = 'desc',
+}
+export class EmbedObject {
+  @ApiProperty()
+  url: string;
+}
 export class IRole {
   @ApiProperty()
   name: string;
@@ -48,8 +62,8 @@ export class IDataAsset {
   data_encrypted?: string;
   @ApiPropertyOptional()
   num_rows?: number;
-  @ApiPropertyOptional()
-  display_name?: string;
+  @ApiProperty()
+  display_name: string;
   @ApiProperty()
   pipeline_id: string;
   @ApiProperty()
@@ -74,12 +88,16 @@ export class IDataAsset {
   parameters?: string[];
   @ApiPropertyOptional()
   embedding_params?: object;
-  @ApiPropertyOptional()
-  created_at?: string;
+  @ApiProperty()
+  created_at: string;
   @ApiPropertyOptional()
   updated_at?: string;
   @ApiProperty({ type: [IComment] })
   comments: IComment[];
+  @ApiPropertyOptional()
+  embed?: EmbedObject;
+  @ApiPropertyOptional({ enum: DataAssetShareType })
+  share_type?: DataAssetShareType;
 }
 
 export class IOneDataAsset {
@@ -92,9 +110,43 @@ export class IMakeAComment {
   message: string;
 }
 
+export class CommentToDelete extends PickType(IComment, ['id', 'username']) {}
 export class IDeleteComment {
   @ApiProperty()
-  comment: IComment;
+  comment: CommentToDelete;
+}
+
+export class ICatalogAllRequest {
+  @ApiPropertyOptional({
+    description: 'Texto usado para filtrar os ativos no catálogo.',
+  })
+  search;
+
+  @ApiPropertyOptional({
+    description: 'Número de registros a ser retornado',
+    maximum: 10000,
+    minimum: 1,
+  })
+  size: number;
+
+  @ApiPropertyOptional({
+    description: 'Usado para paginação, em conjunto com `size`',
+    minimum: 1,
+  })
+  page: number;
+
+  @ApiPropertyOptional({
+    description:
+      'Campo do data asset para usar na ordenação. Padrão: `created_at` ',
+  })
+  sort_by: string;
+
+  @ApiPropertyOptional({
+    enum: OrderEnum,
+    default: OrderEnum.asc,
+    description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
+  })
+  order?: OrderEnum;
 }
 
 export class ICatalogAllResponse {
@@ -126,6 +178,24 @@ export class IUpdateDataRequest {
   description: string;
   @ApiProperty()
   tags: string[];
+  @ApiPropertyOptional()
+  embed: EmbedObject;
+  @ApiPropertyOptional({ enum: DataAssetShareType })
+  share_type?: DataAssetShareType;
+}
+export class ICreateDataAsset implements CreateDataAssetRequest {
+  @ApiProperty()
+  display_name: string;
+  @ApiProperty()
+  description: string;
+  @ApiProperty()
+  data_asset_type: string;
+  @ApiProperty()
+  external_url: string;
+  @ApiProperty()
+  location: string;
+  @ApiPropertyOptional()
+  embed: EmbedObject;
 }
 
 export class IPreview {
@@ -201,3 +271,44 @@ export class IColumnsMetadataResponse {
   @ApiProperty({ type: [IColumnMetadata] })
   columns_metadata: IColumnMetadata[];
 }
+
+export class TriggerCatalogReq {
+  @ApiProperty({
+    description: 'Name of the table on Snowflake',
+    example: 'TB__4DXSD4_TEST',
+  })
+  table_name: string;
+
+  @ApiPropertyOptional({
+    description:
+      'Schema where the asset is located. If not set defaults to "PUBLIC"',
+  })
+  table_schema?: string;
+
+  @ApiPropertyOptional({
+    description: 'Id of the pipeline that generated the asset.',
+  })
+  pipeline_id: string;
+}
+
+export class TriggerCatalogRes {
+  @ApiProperty()
+  session: string;
+}
+
+export class GetDatasetCatalogTaskRes {
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  customer_name: string;
+  @ApiProperty()
+  session_id: string;
+  @ApiProperty()
+  status: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  updated_by: string;
+}

src/modules/connection-test/connection-test-client.config.ts

@@ -6,14 +6,15 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionTest } from '@dadosfera/protospack-v2';
 
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
 export class ConnectionTestClientConfiguration {
   private config: GrpcOptions = {
     transport: Transport.GRPC,
     options: {
       url: process.env.INFACTORY_URL,
       package: [ConnectionTest.ProtoPackages.ReadPackage],
-      credentials:
-        process.env.ENV === 'local' ? undefined : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [ConnectionTest.ProtoPaths.ReadFilePath],
       loader: {
         keepCase: true,

src/modules/connection-test/connection-test.controller.ts

@@ -9,7 +9,7 @@ import {
   ValidationPipe,
 } from '@nestjs/common';
 import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
-import { User, RequestUser } from 'src/authentication/user.decorator';
+import { User, RequestUser } from 'src/decorators/user.decorator';
 import { ConnectionTestService } from './connection-test.service';
 import {
   ConnectionTestPingRes,
@@ -24,9 +24,11 @@ import {
   GetTableMetadataReq,
 } from './dto/connection-test';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { Authenticated } from 'src/authentication/authentication.decorator';
+import { Authenticated } from 'src/decorators/authentication.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('Connection Test')
 @Controller('connection-test')
 @UseFilters(new GrpcToHttpExceptionFilter())

src/modules/connection-test/connection-test.service.ts

@@ -19,7 +19,7 @@ import {
   CreateConnectionDto,
   DatabaseConnectionPropertiesDto,
 } from '../connection/dtos/connection';
-import { RequestUser } from 'src/authentication/user.decorator';
+import { RequestUser } from 'src/decorators/user.decorator';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @Injectable()

src/modules/connection/client.config.ts

@@ -6,6 +6,8 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
 export class ConnectionClientConfiguration {
   public name = 'ConnectionClientConfiguration';
   private config: GrpcOptions = {
@@ -16,10 +18,7 @@ export class ConnectionClientConfiguration {
         ConnectionManager.ProtoPackages.WritePackage,
         ConnectionManager.ProtoPackages.ReadPackage,
       ],
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         ConnectionManager.ProtoPaths.WriteFilePath,
         ConnectionManager.ProtoPaths.ReadFilePath,

src/modules/connection/client.service.ts

@@ -1,9 +1,4 @@
-import {
-  OnModuleInit,
-  Inject,
-  HttpException,
-  HttpStatus,
-} from '@nestjs/common';
+import { OnModuleInit, Inject } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 import { lastValueFrom } from 'rxjs';
@@ -86,12 +81,7 @@ export class ConnectionClientService implements OnModuleInit {
         },
         metadata,
       ),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
-      );
-    });
+    );
   }
 
   async updateConnection(
@@ -112,12 +102,7 @@ export class ConnectionClientService implements OnModuleInit {
         },
         metadata,
       ),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
-      );
-    });
+    );
   }
 
   async deleteConnection({ body, metadata }): Promise<ConnectionRes> {
@@ -130,12 +115,7 @@ export class ConnectionClientService implements OnModuleInit {
         },
         metadata,
       ),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
-      );
-    });
+    );
   }
 
   async getAllConnections({ body, metadata }): Promise<ConnectionsRes> {
@@ -146,12 +126,7 @@ export class ConnectionClientService implements OnModuleInit {
         { search, filters: JSON.stringify(filters), size, page },
         metadata,
       ),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
-      );
-    });
+    );
   }
 
   async getConnectionDetails({
@@ -162,12 +137,7 @@ export class ConnectionClientService implements OnModuleInit {
 
     return lastValueFrom(
       this.connectionServiceRead.GetConnectionDetails({ id }, metadata),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
-      );
-    });
+    );
   }
 
   async INTERNAL_getAllConnections({

src/modules/connection/connection.controller.ts

@@ -7,18 +7,18 @@ import {
   Param,
   Delete,
   Get,
-  Headers,
   Query,
+  UseFilters,
 } from '@nestjs/common';
-import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
+import { ApiExcludeEndpoint, ApiTags } from '@nestjs/swagger';
 import { ConnectionClientService } from './client.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
+} from 'src/decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import { ValidationPipe } from '../../pipes/object-validation.pipe';
 import {
   ConnectionDetailsRes,
@@ -28,10 +28,14 @@ import {
 } from './dtos/connection';
 import { CreateConnectionDto } from './dtos/connection';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
 const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
+@UseFilters(new GrpcToHttpExceptionFilter())
 @ApiTags('connections')
-@ApiBearerAuth()
 @Authenticated()
 @Controller('connections')
 export class ConnectionController {
@@ -44,7 +48,9 @@ export class ConnectionController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('connections-migration')
+  @ApiExcludeEndpoint()
   @RequireAllPermissions(
     PERMISSIONS_GROUPS.DADOSFERA.permissions.CONNECTIONS_MIGRATION,
   )
@@ -80,12 +86,11 @@ export class ConnectionController {
   @RequireAllPermissions(connectionPermissions.CREATE)
   async updateConnection(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language: string,
+    @Language() language: LanguageEnum,
     @Param('id') id: string,
     @Body(new ValidationPipe()) body: UpdateConnectionDto,
   ): Promise<ConnectionRes> {
     this.logger.info('/connections - Update Connection');
-    if (!language) language = 'en-us';
 
     const { user_id, customer_id, customer_name, username } = user;
     const metadata = PackTheMetadata({
@@ -132,11 +137,10 @@ export class ConnectionController {
   @Get()
   async getAllConnections(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @Query() queries,
   ): Promise<ConnectionsRes> {
     this.logger.info('/connections - Get All Connection');
-    if (!language) language = 'en-us';
 
     const { user_id, customer_id, customer_name, username } = user;
     const { search, size, page, ...filters } = queries;
@@ -158,13 +162,16 @@ export class ConnectionController {
 
   @Get('/:id')
   async getConnectionDetails(
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
-    @Query('details') details,
+    @Param('id') id: string,
+    @Query() queries,
   ): Promise<ConnectionDetailsRes> {
     this.logger.info('/connections - Get Connection Details');
-    if (!language) language = 'en-us';
+
+    console.log(queries);
+
+    const { details, sensitive } = queries;
 
     const { user_id, customer_id, customer_name, username } = user;
     const metadata = PackTheMetadata({
@@ -174,6 +181,7 @@ export class ConnectionController {
       username,
       details,
       language,
+      sensitive,
     });
 
     const response: any = await this.clientService.getConnectionDetails({

src/modules/connector/client.config.ts

@@ -6,6 +6,8 @@ import {
 } from '@nestjs/microservices';
 import { ConnectorManager } from '@dadosfera/protospack-v2';
 
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
 export class ConnectorClientConfiguration {
   public name = 'ConnectorClientConfiguration';
   private config: GrpcOptions = {
@@ -16,10 +18,7 @@ export class ConnectorClientConfiguration {
         ConnectorManager.ProtoPackages.WritePackage,
         ConnectorManager.ProtoPackages.ReadPackage,
       ],
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         ConnectorManager.ProtoPaths.WriteFilePath,
         ConnectorManager.ProtoPaths.ReadFilePath,

src/modules/connector/client.service.ts

@@ -83,161 +83,30 @@ export class ConnectorClientService implements OnModuleInit {
       );
     });
     const responseParsed = JSON.parse(res.message);
-    const connectorErrors = responseParsed.items
-      .filter((i) => i.update._index === 'connector' && i.update.error)
+    const connectorsResponse = responseParsed.items.filter(
+      (i) => i.update._index === process.env.CONNECTORS_INDEX,
+    );
+    const connectorErrors = connectorsResponse
+      .filter((i) => i.update.error)
       .map((i) => ({
-        id: i.update._id,
-        reason: i.update.error,
-        status: i.error.status,
-      }));
-    const connectionControlsErrors = responseParsed.items
-      .filter(
-        (i) => i.update._index === 'connection_controls' && i.update.error,
-      )
-      .map((i) => ({
-        id: i.update._id,
-        reason: i.update.error,
-        status: i.error.status,
-      }));
-    const connectionStepsErrors = responseParsed.items
-      .filter((i) => i.update._index === 'connection_steps' && i.update.error)
-      .map((i) => ({
-        id: i.update._id,
-        reason: i.update.error,
-        status: i.error.status,
-      }));
-    const configControlsErrors = responseParsed.items
-      .filter((i) => i.update._index === 'config_controls' && i.update.error)
-      .map((i) => ({
-        id: i.update._id,
-        reason: i.update.error,
-        status: i.error.status,
-      }));
-    const connectorCreated = responseParsed.items
-      .filter(
-        (i) => i.update._index === 'connector' && i.update.result === 'created',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectionControlsCreated = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'connection_controls' &&
-          i.update.result === 'created',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectionStepsCreated = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'connection_steps' &&
-          i.update.result === 'created',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const configControlsCreated = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'config_controls' &&
-          i.update.result === 'created',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectorUpdated = responseParsed.items
-      .filter(
-        (i) => i.update._index === 'connector' && i.update.result === 'updated',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectionControlsUpdated = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'connection_controls' &&
-          i.update.result === 'updated',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectionStepsUpdated = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'connection_steps' &&
-          i.update.result === 'updated',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const configControlsUpdated = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'config_controls' &&
-          i.update.result === 'updated',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectorNoop = responseParsed.items
-      .filter(
-        (i) => i.update._index === 'connector' && i.update.result === 'noop',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectionControlsNoop = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'connection_controls' &&
-          i.update.result === 'noop',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const connectionStepsNoop = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'connection_steps' && i.update.result === 'noop',
-      )
-      .map((i) => ({
-        id: i.update._id,
-      }));
-    const configControlsNoop = responseParsed.items
-      .filter(
-        (i) =>
-          i.update._index === 'config_controls' && i.update.result === 'noop',
-      )
-      .map((i) => ({
-        id: i.update._id,
+        id: i.update?._id,
+        reason: i.update?.error,
+        status: i.error?.status,
       }));
+    const connectorCreated = connectorsResponse
+      .filter((i) => i.update.result === 'created')
+      .map((i) => i.update._id);
+    const connectorUpdated = connectorsResponse
+      .filter((i) => i.update.result === 'updated')
+      .map((i) => i.update._id);
+    const connectorNoop = connectorsResponse
+      .filter((i) => i.update.result === 'noop')
+      .map((i) => i.update._id);
     return {
-      errors: {
-        connectors: connectorErrors,
-        connection_controls: connectionControlsErrors,
-        connection_steps: connectionStepsErrors,
-        config_controls: configControlsErrors,
-      },
-      created: {
-        connectors: connectorCreated,
-        connection_controls: connectionControlsCreated,
-        connection_steps: connectionStepsCreated,
-        config_controls: configControlsCreated,
-      },
-      updated: {
-        connectors: connectorUpdated,
-        connection_controls: connectionControlsUpdated,
-        connection_steps: connectionStepsUpdated,
-        config_controls: configControlsUpdated,
-      },
-      noop: {
-        connectors: connectorNoop,
-        connection_controls: connectionControlsNoop,
-        connection_steps: connectionStepsNoop,
-        config_controls: configControlsNoop,
-      },
+      errors: connectorErrors,
+      created: connectorCreated,
+      updated: connectorUpdated,
+      noop: connectorNoop,
     };
   }
 
@@ -316,12 +185,7 @@ export class ConnectorClientService implements OnModuleInit {
         version,
         plugin,
       }),
-    ).catch((err) => {
-      throw new HttpException(
-        err.details,
-        err.code === 6 ? HttpStatus.CONFLICT : 400,
-      );
-    });
+    );
   }
 
   async updateConnector({ plugin, changes }) {

src/modules/connector/connector.controller.ts

@@ -11,15 +11,13 @@ import {
   UploadedFile,
   UseInterceptors,
   Inject,
-  Headers,
   UploadedFiles,
 } from '@nestjs/common';
 import { FileInterceptor, FilesInterceptor } from '@nestjs/platform-express';
-import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
+import { ApiConsumes, ApiTags } from '@nestjs/swagger';
 import { ConnectorClientService } from './client.service';
 import { AddTagDto } from './dtos/add-tag';
 import { CreateConnectorDto } from './dtos/create-connector';
-import { DeleteConnectorDto } from './dtos/delete-connector';
 import { GetAllDto } from './dtos/get-all';
 import { RemoveTagDto } from './dtos/remove-tag';
 import { UpdateDto } from './dtos/update';
@@ -28,11 +26,14 @@ import {
   Authenticated,
   RequireAllPermissions,
   RequireSomePermission,
-} from 'src/authentication/authentication.decorator';
+} from 'src/decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('connectors')
-@ApiBearerAuth()
 @Authenticated()
 @Controller('connectors')
 export class ConnectorController {
@@ -99,11 +100,10 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
   async getAllConnectors(
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
     @Query() queries: GetAllDto,
   ) {
     this.logger.info('/GET - getAllConnectors Route');
-    if (!language) language = 'en-us';
 
     const { search, size, page, ...filters } = queries;
 
@@ -144,12 +144,11 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
   async getConnector(
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
     @Query('version') version: string,
   ) {
     this.logger.info('/GET/:plugin - getConnector Route');
-    if (!language) language = 'en-us';
 
     const pluginId = `${plugin}-${version}`;
 
@@ -173,13 +172,11 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
   async getConnectorDetails(
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
     @Query('version') version: string,
   ) {
     this.logger.info('/upload - Upload Connector Route');
-    if (!language) language = 'en-us';
-
     const pluginId = `${plugin}-${version}`;
 
     const response: any = await this.connectorClientService.getConnectorDetails(
@@ -274,18 +271,15 @@ export class ConnectorController {
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
   async deleteConnector(
     @Param('plugin') plugin: string,
-    @Body() { version }: DeleteConnectorDto,
+    @Query('version') version: string,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('DELETE /:plugin - Delete Connector Route');
 
-    const response: any = await this.connectorClientService.deleteConnector(
+    const response = await this.connectorClientService.deleteConnector(
       plugin,
       version,
     );
 
-    return {
-      message: response.message || 'ok',
-      connector: { ...JSON.parse(response.connector) },
-    };
+    return { message: response.message, connector: response.connector };
   }
 }

src/modules/duc/client.config.ts

@@ -9,6 +9,8 @@ import {
   ProtoPaths,
 } from '@dadosfera/protospack-v2/dist/lib/Duc';
 
+const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0');
+
 export class DucClient {
   public name = 'DucClient';
 
@@ -16,12 +18,9 @@ export class DucClient {
     transport: Transport.GRPC,
     options: {
       url: process.env.DUC_URL,
-      package: ProtoPackages.WritePackage,
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
-      protoPath: ProtoPaths.WriteFilePath,
+      package: [ProtoPackages.WritePackage, ProtoPackages.ReadPackage],
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: [ProtoPaths.WriteFilePath, ProtoPaths.ReadFilePath],
       loader: {
         keepCase: true,
         enums: String,

src/modules/inputs/dtos/input.model.ts

@@ -1,106 +1,5 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
-import { Info } from 'protospack/dist/lib/interfaces';
 
-export class GoogleAnalyticsClientSecrets {
-  @ApiProperty()
-  type: string;
-  @ApiProperty()
-  project_id: string;
-  @ApiProperty()
-  private_key_id: string;
-  @ApiProperty()
-  private_key: string;
-  @ApiProperty()
-  client_email: string;
-  @ApiProperty()
-  client_id: string;
-  @ApiProperty()
-  auth_uri: string;
-  @ApiProperty()
-  token_uri: string;
-  @ApiProperty()
-  auth_provider_x509_cert_url: string;
-  @ApiProperty()
-  client_x509_cert_url: string;
-}
-export class OauthObject {
-  @ApiPropertyOptional()
-  get_tokens_url: string;
-  @ApiPropertyOptional()
-  get_tokens_url_params: string;
-  @ApiPropertyOptional()
-  get_tokens_set_response: Record<string, any>;
-  @ApiPropertyOptional()
-  content_type: string;
-}
-export class InputOptions {
-  @ApiPropertyOptional()
-  oauth: OauthObject;
-  @ApiPropertyOptional()
-  skip_select_columns: true;
-  @ApiPropertyOptional()
-  skip_select_entities: false;
-  @ApiPropertyOptional()
-  skip_transformation: true;
-}
-export class FileFormatParams {
-  @ApiProperty()
-  sep: string;
-  @ApiProperty()
-  header: boolean;
-  @ApiProperty()
-  encoding: string;
-}
-
-export class AuthParameters {
-  @ApiProperty()
-  aws_access_key_id: string;
-  @ApiProperty()
-  aws_secret_access_key: string;
-}
-export class CredentialsJdbc {
-  @ApiPropertyOptional()
-  jdbc_user: string;
-  @ApiPropertyOptional()
-  jdbc_password: string;
-  @ApiPropertyOptional()
-  database: string;
-  @ApiPropertyOptional()
-  endpoint: string;
-  @ApiPropertyOptional()
-  port: string;
-  @ApiPropertyOptional()
-  engine: string;
-  @ApiPropertyOptional()
-  schema: string;
-}
-export class Credentials extends CredentialsJdbc {
-  @ApiProperty()
-  connection_type: string;
-
-  @ApiPropertyOptional()
-  client_aws_access_key_id: string;
-  @ApiPropertyOptional()
-  client_aws_secret_access_key: string;
-  @ApiPropertyOptional()
-  client_bucket: string;
-  @ApiPropertyOptional()
-  file_to_extract: string;
-  @ApiPropertyOptional()
-  file_format_params: FileFormatParams;
-
-  @ApiPropertyOptional()
-  view_id: string;
-  @ApiPropertyOptional()
-  client_secrets: GoogleAnalyticsClientSecrets;
-  @ApiPropertyOptional()
-  start_date: string;
-  @ApiPropertyOptional()
-  end_date: string;
-
-  @ApiPropertyOptional()
-  oauth_code: string;
-}
 export class Column {
   @ApiProperty()
   name: string;
@@ -114,6 +13,14 @@ export class TableColumns {
   columns: string[];
   @ApiProperty()
   references: Column[];
+  @ApiProperty()
+  type: string;
+}
+export class AvailableEntity {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  replication_methods: string[];
 }
 
 export class Input {
@@ -133,26 +40,6 @@ export class Input {
   created_at: string;
   @ApiProperty()
   updated_at: string;
-
-  //deprecated attributes (for retro compatibility)
-  @ApiPropertyOptional({ deprecated: true })
-  category: string;
-  @ApiPropertyOptional({ deprecated: true })
-  credentials: Credentials;
-  @ApiPropertyOptional({ deprecated: true })
-  options: InputOptions;
-  @ApiPropertyOptional({ deprecated: true })
-  info: Info;
-  @ApiPropertyOptional({ deprecated: true })
-  cron: string;
-  @ApiPropertyOptional({ deprecated: true })
-  source_bucket: string;
-  @ApiPropertyOptional({ deprecated: true })
-  source_prefix: string;
-  @ApiPropertyOptional({ deprecated: true })
-  auth_parameters: AuthParameters;
-  @ApiPropertyOptional({ deprecated: true })
-  file_format_params: FileFormatParams;
 }
 export class GetAvailableEntitiesReq {
   @ApiProperty()
@@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq {
 }
 
 export class GetAvailableEntitiesRes {
-  @ApiProperty()
-  entities: string[];
+  @ApiProperty({ type: [AvailableEntity] })
+  entities: AvailableEntity[];
 }
 
-export class TestConnectionGetColumnsReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  tables: string[];
-  @ApiPropertyOptional()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  id: string;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionGetColumnsRes {
-  @ApiProperty({ type: [TableColumns] })
-  tables: TableColumns[];
-}
-
-export class TestConnectionReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionRes {
-  @ApiProperty()
-  connection_state: boolean;
-  @ApiProperty()
-  total_entities: number;
-  @ApiProperty()
-  database_tables: string[];
-}
 export class CreateInputReq extends OmitType(Input, [
   'id',
   'created_at',

src/modules/inputs/inputs-client.config.ts

@@ -2,10 +2,11 @@ import { Input } from '@dadosfera/protospack-v2';
 import { credentials } from '@grpc/grpc-js';
 import {
   ClientProviderOptions,
-  GrpcOptions,
   Transport,
+  type GrpcOptions,
 } from '@nestjs/microservices';
 
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
 export class InputsGrpcClient {
   public readonly name = 'InputsGrpcClient';
   private config: GrpcOptions = {
@@ -16,10 +17,7 @@ export class InputsGrpcClient {
         Input.ProtoPackages.WritePackage,
         Input.ProtoPackages.ReadPackage,
       ],
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         Input.ProtoPaths.WriteFilePath,
         Input.ProtoPaths.ReadFilePath,

src/modules/inputs/inputs.controller.ts

@@ -12,24 +12,19 @@ import {
 import { InputsService } from './inputs.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
+import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
 import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
 import {
   CreateInputReq,
   GetAvailableEntitiesReq,
   GetAvailableEntitiesRes,
   Input,
-  TestConnectionGetColumnsReq,
-  TestConnectionGetColumnsRes,
-  TestConnectionReq,
-  TestConnectionRes,
 } from './dtos/input.model';
 import { UpdateInputRequest } from './dtos/old_interfaces';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
-import ErrorBuilder from 'src/utils/ErrorBuilder';
-import ErrorCodes from 'src/utils/errorCodes';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
 @ApiTags('Inputs')
 @Controller('inputs')
@@ -72,45 +67,23 @@ export class InputsController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('available-entities/:plugin')
   @ApiOkResponse({ type: GetAvailableEntitiesRes })
   async getAvailableEntities(
-    @Body() body,
     @Param() params: GetAvailableEntitiesReq,
+    @User() user: RequestUser,
   ) {
-    const { info } = body;
     const { plugin } = params;
-    return await this.inputService.getAvailableEntities({ info, plugin });
-  }
-
-  @Post('/test-connection')
-  @ApiOkResponse({ type: TestConnectionRes })
-  async testConnection(@Body() data: TestConnectionReq) {
-    this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
-      user: data.info.user_id,
-      customer: data.info.customer,
-    });
-
-    const response = await this.inputService.testConnection(data);
-
-    return response;
-  }
-
-  @Post('/test-connection/get-columns')
-  @ApiOkResponse({ type: TestConnectionGetColumnsRes })
-  async getColumns(@Body() data: TestConnectionGetColumnsReq) {
-    throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED);
-    this.logger.info(
-      `/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
-      {
-        user: data.info.user_id,
-        customer: data.info.customer,
+    const availableEntities = await this.inputService.getAvailableEntities({
+      info: {
+        customer: user.customer_name,
+        customer_id: user.customer_id,
+        user_id: user.user_id,
       },
-    );
-
-    const response = await this.inputService.getColumns(data);
-
-    return response;
+      plugin,
+    });
+    return availableEntities;
   }
 
   @Post()
@@ -131,6 +104,7 @@ export class InputsController {
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get()
   async findAll(@User() user: RequestUser) {
     const { user_id, customer_id, customer_name: customer } = user;
@@ -149,8 +123,7 @@ export class InputsController {
   }
 
   @Get('/:id')
-  async findOne(@Body() body, @Param() params) {
-    const { id } = params;
+  async findOne(@Body() body, @Param('id') id: string) {
     this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
       user: body.info.user_id,
       customer: body.info.customer,
@@ -161,6 +134,7 @@ export class InputsController {
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Patch(':id')
   async update(@Body() updateInputDto: UpdateInputRequest, @Param() params) {
     const { id } = params;
@@ -176,6 +150,7 @@ export class InputsController {
     return response;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Delete(':id')
   async delete(@Body() data, @Param() params) {
     const { id } = params;

src/modules/inputs/inputs.service.ts

@@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
 import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
 import { Input } from '@dadosfera/protospack-v2';
 import {
+  GetAvailableEntitiesRequest,
   InputCreateGenericRequest,
   InputCreateS3Request,
   InputNewCreateRequest,
-  TestConnectionGetColumnsRequest,
   TestConnectionRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -88,32 +88,6 @@ export class InputsService {
 
       return testConnectionResponse;
     },
-    getColumns: async (data: TestConnectionGetColumnsRequest) => {
-      this.logger.info('InputClientService - TestConnection/Get-Columns');
-      const getColumnsResponse = await new Promise((resolve, reject) => {
-        this.inputReadService.GetColumns(data).subscribe({
-          next(x) {
-            resolve(objectCamelToSnake(x));
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // console.log('done');
-          },
-        });
-      })
-        .then((res) => {
-          this.logger.info('Done');
-          return res;
-        })
-        .catch((err) => {
-          this.logger.error(err.message);
-          throw new Error(err);
-        });
-
-      return getColumnsResponse;
-    },
     getAvailableEntities: async (data) => {
       this.logger.info('InputClientService - GetAvailableEntities');
       data;
@@ -188,12 +162,9 @@ export class InputsService {
     // this.validateCron(data);
     const { body, info } = data;
 
-    if (body.info) delete body.info;
-
-    const inputCreateGenericRequest: any | InputCreateGenericRequest = {
+    const inputCreateGenericRequest: InputCreateGenericRequest = {
       input: {
         ...body,
-        credentials: body.credentials && JSON.stringify(body.credentials),
       },
       info,
     };
@@ -205,8 +176,8 @@ export class InputsService {
     return { input: adjustedInput };
   }
 
-  async getAvailableEntities(data): Promise<{ entities: string[] }> {
-    return await this.OLD_inputClient.getAvailableEntities(data);
+  async getAvailableEntities(data: GetAvailableEntitiesRequest) {
+    return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
   }
 
   async findAll(info: Info) {
@@ -263,17 +234,6 @@ export class InputsService {
     }
   }
 
-  async getColumns(data) {
-    try {
-      const testConnectionGetColumnsResponse =
-        await this.OLD_inputClient.getColumns(data);
-
-      return testConnectionGetColumnsResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
   generateInputS3Payload(payload) {
     const { credentials, plugin, cron } = payload;
     if (!credentials) return payload;

src/modules/mixpanel/mixpanel.controller.ts

@@ -0,0 +1,93 @@
+import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
+import { init } from 'mixpanel';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+
+@ApiInternalOnlyController()
+@Authenticated()
+@Controller('trackEvent')
+export class MixpanelController {
+  constructor(
+    @Inject('MIXPANEL_TOKEN')
+    private readonly mixpanelToken: string,
+  ) {}
+  @Post(':id')
+  async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
+    delete body.info;
+    const mixpanel = init(this.mixpanelToken);
+
+    const separator = user.username.includes('-') ? '-' : '.';
+    const removeValues = [
+      '.dadosferatech.dadosfera',
+      '.demo.dadosfera',
+      '.dadosferademo',
+      '.dadosferarh.dadosfera',
+      '.dadosferatech.dadosfera2',
+      '.dadosferatech.dadosfera',
+      '.dadosfera.fin',
+      '.dadosferafin.dadosfera',
+      '.praxio.dadosfera',
+      '.dadosfera.tech',
+      '.treinamentos@dadosfera.ai',
+      '.dadosfera2',
+      '.treinamentosfera',
+      '.dadosfera',
+    ];
+
+    let username = user.username;
+
+    removeValues.forEach((value) => {
+      username = username.replace(value, '');
+    });
+
+    username = username.split('@')?.[0];
+    username = username.split('+')?.[0];
+
+    let firstName = username
+      .substring(0, username.indexOf(separator))
+      .replace('dadosfera', '');
+    let lastName = username
+      .substring(username.lastIndexOf(separator) + 1)
+      .replace('dadosfera', '');
+
+    if (!firstName) {
+      firstName = lastName;
+      lastName = '';
+    }
+
+    firstName = this.capitalize(firstName);
+    lastName = this.capitalize(lastName);
+
+    await mixpanel.people.set(user.username, {
+      $first_name: firstName,
+      $last_name: lastName,
+      $name: this.getFullName(firstName, lastName),
+      $email: user.username.includes('@')
+        ? user.username
+        : user.username + '@dadosfera.ai',
+      customer_name: user.customer_name,
+    });
+
+    await mixpanel.track(id, {
+      distinct_id: user.username,
+      customer: user.customer_name,
+      env: process.env.ENV,
+      ...body,
+    });
+
+    return { id, body, user: user.username };
+  }
+
+  capitalize(sentence: string): string {
+    if (!sentence) {
+      return '';
+    }
+
+    return sentence[0].toUpperCase() + sentence.substring(1);
+  }
+
+  getFullName(firstName: string, lastName: string) {
+    return `${firstName}${lastName ? ' ' + lastName : ''}`;
+  }
+}

src/modules/mixpanel/mixpanel.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
+import { MixpanelController } from './mixpanel.controller';
+
+@Module({
+  controllers: [MixpanelController],
+  providers: [
+    {
+      provide: 'MIXPANEL_TOKEN',
+      useValue: getSecretFromSecretsManager(
+        `${process.env.ENV}/root/mixpanel_token`,
+      ),
+    },
+  ],
+})
+export class MixpanelModule {}

src/modules/network-config/network-config-client.config.ts

@@ -6,6 +6,8 @@ import {
 } from '@nestjs/microservices';
 import { NetworkConfig } from '@dadosfera/protospack-v2';
 
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
 export class NetworkConfigGrpcClient {
   public name = 'NetworkConfigGrpcClient';
 
@@ -17,8 +19,7 @@ export class NetworkConfigGrpcClient {
         NetworkConfig.ProtoPackages.ReadPackage,
         NetworkConfig.ProtoPackages.WritePackage,
       ],
-      credentials:
-        process.env.ENV === 'local' ? undefined : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         NetworkConfig.ProtoPaths.ReadFilePath,
         NetworkConfig.ProtoPaths.WriteFilePath,

src/modules/network-config/network-config.controller.ts

@@ -1,12 +1,9 @@
 import {
   Body,
   Controller,
-  Delete,
   Get,
-  Headers,
   Inject,
   Param,
-  Patch,
   Post,
   Query,
   UploadedFile,
@@ -15,7 +12,7 @@ import {
 import { NetworkConfigService } from './network-config.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { ApiConsumes, ApiOkResponse, ApiTags } from '@nestjs/swagger';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import {
   CreateNetworkConfigReq,
   CreateNetworkConfigRes,
@@ -26,7 +23,9 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import {
   Authenticated,
   RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
+} from 'src/decorators/authentication.decorator';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
 
 const networkConfigPermissions = PERMISSIONS_GROUPS.NETWORK_CONFIG.permissions;
 @ApiTags('Network Config')
@@ -40,18 +39,18 @@ export class NetworkConfigController {
   ) {
     this.logger = dadosferaLogger.logger;
   }
+
   @Get()
   @ApiOkResponse({ type: FindAllNetworkConfigsRes })
   async findAllNetworkConfigs(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @Query() queries,
   ) {
     this.logger.info('GET /network-config', {
       user: user.user_id,
       customer: user.customer_name,
     });
-    if (!language) language = 'en-us';
 
     const { user_id, customer_id, customer_name } = user;
     const { search, size, page, ...filters } = queries;
@@ -79,14 +78,14 @@ export class NetworkConfigController {
   @ApiOkResponse({ type: FindAllNetworkConfigsRes })
   async findNetworkConfigByIdDetails(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @Param('id') id: string,
   ) {
     this.logger.info('GET /network-config/:id', {
       user: user.user_id,
       customer: user.customer_name,
     });
-    if (!language) language = 'en-us';
+
     return this.networkConfigService.findNetworkConfigByIdDetails(id, {
       customer_name: user.customer_name,
       customer_id: user.customer_id,

src/modules/oauth/oauth.controller.ts

@@ -5,6 +5,7 @@ import { ConnectionClientService } from '../connection/client.service';
 import jwt from 'jsonwebtoken';
 import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 @ApiTags('oauth')
 @Controller('oauth')
 export class OauthController {
@@ -15,7 +16,6 @@ export class OauthController {
     private logger: DadosferaLogger,
   ) {
     switch (process.env.ENV) {
-      case 'dev':
       case 'stg':
         this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/collect/auth-callback`;
         break;
@@ -32,6 +32,7 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('hubspot/callback')
   @UseGuards(AuthGuard('hubspot'))
   @Redirect()
@@ -45,6 +46,7 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('googleads/callback')
   @UseGuards(AuthGuard('google'))
   @Redirect()
@@ -58,6 +60,7 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('google-sheets/callback')
   @UseGuards(AuthGuard('google'))
   @Redirect()
@@ -71,6 +74,7 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('facebook/callback')
   @UseGuards(AuthGuard('facebook'))
   @Redirect()
@@ -84,6 +88,7 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('mailchimp/callback')
   @UseGuards(AuthGuard('mailchimp'))
   @Redirect()
@@ -97,6 +102,7 @@ export class OauthController {
     return true;
   }
 
+  @ApiInternalOnlyEndpoint()
   @Get('salesforce/callback')
   @UseGuards(AuthGuard('salesforce'))
   @Redirect()
@@ -148,10 +154,16 @@ export class OauthController {
     const { customer_id, customer_name, user_id } = connectionInfo;
 
     try {
-      const response = await this.connectionService.createConnection(
-        connectionInfo,
-        PackTheMetadata({ customer_id, customer_name, user_id }),
-      );
+      const response = connectionInfo.id
+        ? await this.connectionService.updateConnection(
+            connectionInfo.id,
+            connectionInfo,
+            PackTheMetadata({ customer_id, customer_name, user_id }),
+          )
+        : await this.connectionService.createConnection(
+            connectionInfo,
+            PackTheMetadata({ customer_id, customer_name, user_id }),
+          );
       url.searchParams.set('connection_id', response.connection.id);
       return { url: url.href };
     } catch (error) {

src/modules/oauth/oauth.service.ts

@@ -23,6 +23,7 @@ export class OauthService {
       user_id,
       customer_id,
       customer_name,
+      id,
     } = query;
 
     const properties = { plugin, credentials_type: 'oauth' };
@@ -45,6 +46,7 @@ export class OauthService {
       type,
       properties,
       plugin,
+      id,
     };
     return jwt.sign({ ...newConnection }, process.env.JWT_PRIVATE_KEY);
   }

src/modules/oauth/passport-strategies/facebook-strategy.ts

@@ -44,7 +44,8 @@ export class FacebookStrategy extends PassportStrategy(Strategy) {
       req,
       'application',
     );
-    options.scope = 'ads_read ads_management';
+    options.scope =
+      'pages_show_list ads_read pages_read_engagement ads_management';
     super.authenticate(req, options);
   }
 }

src/modules/oauth/passport-strategies/google-strategy.ts

@@ -29,7 +29,6 @@ export class GoogleStrategy extends PassportStrategy(Strategy, 'google') {
 
     super(options, verify);
     switch (process.env.ENV) {
-      case 'dev':
       case 'stg':
         this.redirect_uri = `https://maestro.${process.env.ENV}.dadosfera.ai/oauth`;
         break;

src/modules/oauth/passport-strategies/hubspot-strategy.ts

@@ -17,7 +17,7 @@ export class HubspotStrategy extends PassportStrategy(Strategy) {
         clientSecret: oauthSecrets.hubspot.client_secret,
         callbackURL: oauthSecrets.hubspot.redirect_uri,
         scope:
-          'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
+          'tickets automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
         passReqToCallback: true,
       },
       (accessToken, refreshToken, tokenInfo, profile, done) => {

src/modules/permissions/permissions.controller.ts

@@ -1,19 +1,16 @@
-import { Controller, Get, Headers, Inject } from '@nestjs/common';
+import { Controller, Get, Inject } from '@nestjs/common';
 import { ApiHeader, ApiOkResponse, ApiTags } from '@nestjs/swagger';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import {
-  Authenticated,
-  RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
-import {
-  PERMISSIONS_GROUPS,
-  PermissionUsages,
-} from 'src/authentication/permissions.enum';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { PermissionUsages } from 'src/authentication/permissions.enum';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { GetPublicPermissionsRes } from './dto/entities';
 import { PermissionsService } from './permissions.service';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('Permissions')
 @Authenticated()
 @Controller('permissions')
@@ -38,10 +35,10 @@ export class PermissionsController {
   @ApiOkResponse({ type: GetPublicPermissionsRes })
   getPublicPermissions(
     @User() user: RequestUser,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('getPublicPermissions', { user });
-    if (!language) language = 'en-us';
+
     const permissionGroups = this.permissionsService.getPermissionsGrouped(
       PermissionUsages.PUBLIC,
       language,

src/modules/permissions/permissions.service.ts

@@ -44,8 +44,11 @@ export class PermissionsService
       );
   }
 
-  // internally used to send permissions to duc on microservice startup
   async onApplicationBootstrap() {
+    // Do not sent permissions if running local but pointing to a remote DUC url
+    if (process.env.ENV == 'local' && !process.env.DUC_URL.includes('0.0.0.0'))
+      return;
+
     this.logger.info('sending permissions to DUC...');
 
     const permissions = Object.values(PERMISSIONS_GROUPS).flatMap((namespace) =>
@@ -57,10 +60,10 @@ export class PermissionsService
     }).catch((err: ErrorBuilder) => {
       if (
         err.code === 'No connection established' &&
-        (process.env.LOCAL_ENV === 'true' || process.env.ENV === 'local')
+        process.env.ENV === 'local'
       ) {
         return this.logger.info(
-          "couldn't connect to DUC. suppresing in local env",
+          "couldn't connect to DUC. suppressing in local env",
         );
       }
 

src/modules/pipelines/pipelines-client.ts

@@ -6,6 +6,8 @@ import {
 import { PipelinePackages, PipelineProtoFilePath } from 'protospack';
 import { credentials } from '@grpc/grpc-js';
 
+const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+
 export class PipelinesClientConfiguration {
   public name = 'PipelinesClientConfiguration';
   private config: GrpcOptions = {
@@ -13,10 +15,7 @@ export class PipelinesClientConfiguration {
     options: {
       url: process.env.PIFACTORY_URL,
       package: PipelinePackages,
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: PipelineProtoFilePath,
       loader: {
         keepCase: true,

src/modules/pipelines/pipelines.controller.ts

@@ -1,13 +1,15 @@
 import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
+import { ApiOperation, ApiTags } from '@nestjs/swagger';
 import {
   AuthenticateCondition,
   Authenticated,
-} from 'src/authentication/authentication.decorator';
+} from 'src/decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('Pipelines')
 @Controller('pipelines')
 @Authenticated()
@@ -42,8 +44,12 @@ export class PipelinesController {
   }
 
   @Post('start/:id')
-  async activate(@Param() params, @Body() body) {
-    const { id } = params;
+  @ApiOperation({
+    deprecated: true,
+    description:
+      'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
+  })
+  async activate(@Param('id') id: string, @Body() body) {
     const { info } = body;
 
     this.logger.info(
@@ -60,8 +66,12 @@ export class PipelinesController {
   }
 
   @Get(':id/status')
-  async getPipelineStatus(@Body() body, @Param() params) {
-    const { id } = params;
+  @ApiOperation({
+    deprecated: true,
+    description:
+      'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
+  })
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
     body.id = id;
 
     this.logger.info(

src/modules/pipelinesV2/interfaces.ts

@@ -10,6 +10,8 @@ export class IPipelineV2 {
   @ApiProperty()
   description: string;
   @ApiProperty()
+  type: string;
+  @ApiProperty()
   connection_id: string;
   @ApiProperty()
   cron: string;
@@ -66,3 +68,58 @@ export interface IGetPipelineLogsRequest {
   id: string;
   details: string;
 }
+
+export class IInitUploadCSVFile {
+  @ApiProperty()
+  file_name: string;
+
+  @ApiProperty()
+  parts: number;
+}
+
+export class ICompleteUploadCSVFile {
+  @ApiProperty()
+  upload_id: string;
+
+  @ApiProperty()
+  file_name: string;
+
+  @ApiProperty()
+  parts: { ETag: string; PartNumber: number }[];
+}
+
+export class ICreatePipelineCSVFile {
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  file_name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  encoding: string;
+
+  @ApiProperty()
+  sep: string;
+
+  @ApiProperty()
+  header: boolean;
+
+  @ApiProperty()
+  engine: string;
+}
+
+export class PipelineFindAllReq {
+  @ApiPropertyOptional()
+  search?: string | undefined;
+  @ApiPropertyOptional()
+  filters?: string | undefined;
+  @ApiPropertyOptional()
+  size?: string | undefined;
+  @ApiPropertyOptional()
+  page?: string | undefined;
+  @ApiPropertyOptional()
+  type?: string | undefined;
+}

src/modules/pipelinesV2/pipelines-client.ts

@@ -9,6 +9,8 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { credentials } from '@grpc/grpc-js';
 
+const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+
 export class PipelinesClientConfiguration {
   public name = 'PipelinesClientConfiguration';
   private config: GrpcOptions = {
@@ -16,10 +18,7 @@ export class PipelinesClientConfiguration {
     options: {
       url: process.env.PIFACTORY_URL,
       package: [ProtoPackages.ReadPackage, ProtoPackages.WritePackage],
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [ProtoPaths.ReadFilePath, ProtoPaths.WriteFilePath],
       loader: {
         keepCase: true,

src/modules/pipelinesV2/pipelines.controller.ts

@@ -7,31 +7,49 @@ import {
   Param,
   Post,
   Put,
-  Headers,
   Query,
   UseFilters,
   HttpCode,
   HttpStatus,
+  Patch,
+  HttpException,
+  BadRequestException,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
+  ApiHeaders,
   ApiNoContentResponse,
+  ApiOperation,
   ApiTags,
 } from '@nestjs/swagger';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
+import {
+  AuthenticateCondition,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
-import { ICreatePipelineV2Req, IPipelineV2 } from './interfaces';
+import {
+  ICompleteUploadCSVFile,
+  ICreatePipelineCSVFile,
+  ICreatePipelineV2Req,
+  IPipelineV2,
+  IInitUploadCSVFile,
+  PipelineFindAllReq,
+} from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
 
-@UseFilters(new GrpcToHttpExceptionFilter())
 @ApiTags('PipelinesV2')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
+@UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('pipelinesV2')
 @AuthenticateCondition((req, user) => {
   let action;
@@ -45,6 +63,10 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
       action = 'UPDATE';
       break;
 
+    case 'PATCH':
+      action = 'UPDATE';
+      break;
+
     default:
       action = req.method;
   }
@@ -67,12 +89,12 @@ export class PipelinesController {
   @Post()
   @ApiCreatedResponse({ type: IPipelineV2 })
   async create(
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @User() user: RequestUser,
     @Body() createPipelineDto: ICreatePipelineV2Req,
   ) {
     this.logger.info('PipelinesController - create', { user });
-    if (!language) language = 'en-us';
+
     const { customer_id, customer_name, username, user_id } = user;
     const metadata = PackTheMetadata({
       customer_id,
@@ -93,13 +115,11 @@ export class PipelinesController {
   @Get()
   async findAll(
     @User() user: RequestUser,
-    @Headers('Dadosfera-Lang') language,
-    @Query() data,
+    @Language() language: LanguageEnum,
+    @Query() data: PipelineFindAllReq,
   ) {
     this.logger.info('PipelinesController - findAll', { user });
 
-    if (!language) language = 'en-us';
-
     const { customer_id, customer_name, user_id, username } = user;
     const metadata = PackTheMetadata({
       customer_id,
@@ -116,30 +136,83 @@ export class PipelinesController {
     };
   }
 
+  @Get('/download-logs')
+  async downloadLogs(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Query('pipeline_run_id') pipeline_run_id: string,
+  ) {
+    this.logger.info('PipelinesController - downloadLogs', { user });
+
+    if (!pipeline_run_id)
+      throw new BadRequestException('Missing pipeline_run_id');
+
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const url = await this.pipelinesClientService.downloadLogs(
+      pipeline_run_id,
+      metadata,
+    );
+
+    return {
+      url,
+    };
+  }
+
   @Get(':id/config')
   async getPipelineproperties(
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     this.logger.info('PipelinesController - getPipelineproperties', { user });
-    const metadata = PackTheMetadata({
-      ...user,
-      language: language || 'pt-br',
-    });
+    const metadata = PackTheMetadata({ ...user, language: language });
     return this.pipelinesClientService.findOneProperties(id, metadata);
   }
 
+  @Get(':id/objects')
+  async getPipelineObjects(
+    @Language() language: LanguageEnum,
+    @User() user: RequestUser,
+    @Param('id') id: string,
+  ) {
+    this.logger.info('PipelinesController - getPipelineObjects', { user });
+    const metadata = PackTheMetadata({ ...user, language: language });
+    return this.pipelinesClientService.findOneObjects(id, metadata);
+  }
+
+  @Get(':id/status')
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
+    body.id = id;
+
+    this.logger.info(
+      process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
+      {
+        user: body.info.user_id,
+        customer: body.info.customer,
+      },
+    );
+
+    const response = await this.oldPipelinesService.getPipelineStatus(body);
+
+    return response;
+  }
+
   @Get('/:id')
   async findOne(
-    @Headers('Dadosfera-Lang') language,
+    @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<Messages.PipelineV2FindOneResponse> {
     this.logger.info('PipelinesController - findOne', { user });
 
-    if (!language) language = 'en-us';
-
     const { customer_id, customer_name, user_id, username } = user;
 
     const metadata = PackTheMetadata({
@@ -155,7 +228,7 @@ export class PipelinesController {
       .then((res) => {
         //{pipeline:{tables: {tables: [], input_id: ''}}}
         let tables = JSON.parse(res.pipeline.config.tables);
-        if (tables?.tables) tables = tables?.tables;
+        if (tables?.tables) tables = tables.tables;
         Object.assign(res.pipeline, {
           transformations: res.pipeline.transformations
             ? JSON.parse(res.pipeline.transformations)
@@ -173,31 +246,63 @@ export class PipelinesController {
     return result;
   }
 
-  @Put('/:id')
+  @Patch('/:id')
   async update(
+    @Language() language: LanguageEnum,
     @Body() updatePipelineDto,
-    @Param('id') id,
+    @Param('id') id: string,
     @User() user: RequestUser,
   ) {
-    this.logger.info('PipelinesController - findOne', { user });
+    this.logger.info('PipelinesController - update', { user });
     const { info } = updatePipelineDto;
     delete updatePipelineDto.info;
 
-    const response = await this.pipelinesClientService.update({
-      ...updatePipelineDto,
-      info,
-      id,
+    const { customer_id, customer_name, user_id, username } = user;
+
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
     });
 
+    const response = await this.pipelinesClientService.update(
+      {
+        ...updatePipelineDto,
+        info,
+        id,
+      },
+      metadata,
+    );
+
+    this.logger.info('PipelinesController - update: OK', { user });
+    return response;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Put('/:id')
+  @ApiOperation({
+    deprecated: true,
+    description: 'This method is deprecated. Please use PATCH instead',
+  })
+  async updateDeprecated(
+    @Language() language: LanguageEnum,
+    @Body() updatePipelineDto,
+    @Param('id') id: string,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('PipelinesController - put', { user });
+    const response = await this.update(language, updatePipelineDto, id, user);
+    this.logger.info('PipelinesController - put: OK', { user });
     return response;
   }
 
   @Delete(':id')
   @ApiNoContentResponse()
   @HttpCode(HttpStatus.NO_CONTENT)
-  async delete(@Param() params, @User() user: RequestUser) {
+  async delete(@Param('id') id: string, @User() user: RequestUser) {
     this.logger.info('PipelinesController - delete', { user });
-    const { id } = params;
     const metadata = PackTheMetadata({
       customer_id: user.customer_id,
       customer_name: user.customer_name,
@@ -206,4 +311,122 @@ export class PipelinesController {
     await this.pipelinesClientService.remove({ id, metadata, user });
     this.logger.info('PipelinesController - delete: OK');
   }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('/init-upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async initUploadFile(
+    @User() user: RequestUser,
+    @Body() body: IInitUploadCSVFile,
+  ) {
+    this.logger.info('/upload - Init Upload File Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { file_name, parts } = body;
+
+    const { urls, upload_id } =
+      await this.pipelinesClientService.initUploadFile(
+        {
+          name: file_name,
+          parts: String(parts),
+        },
+        metadata,
+      );
+
+    if (!urls) {
+      this.logger.info('pipeline/upload - Failed File pipeline');
+      throw new HttpException(
+        'Upload failed, try again in a few minutes, if the problem persists, contact support.',
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
+
+    return { urls: JSON.parse(urls), upload_id };
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('/complete-upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async completeUploadFile(
+    @User() user: RequestUser,
+    @Body() body: ICompleteUploadCSVFile,
+  ) {
+    this.logger.info('/upload - Complete Upload File Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { upload_id, parts, file_name } = body;
+
+    return await this.pipelinesClientService.completeUploadFile(
+      { upload_id, parts, file_name },
+      metadata,
+    );
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('/file')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async uploadedFile(
+    @User() user: RequestUser,
+    @Body() body: ICreatePipelineCSVFile,
+  ) {
+    this.logger.info('/file - Upload Connector Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { name, sep, header, encoding, description, file_name, engine } =
+      body;
+
+    const file_format_params = {
+      ...(sep && { sep }),
+      ...(encoding && { encoding }),
+      ...(header !== undefined ? { header } : {}),
+    };
+
+    const source_prefix = `${user.customer_name}/${file_name}`;
+
+    const upload_pipeline: ICreatePipelineV2Req = {
+      connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
+      connector_name: 'Amazon S3',
+      connector_plugin: 'aws_s3',
+      connector_version: '1.0.0',
+      image_url: `https://assets.dadosfera.ai/images/connectors/${engine}.svg`,
+      name,
+      description,
+      transformations_ids: [],
+      tags: [],
+      cron: '@once',
+      type: 'upload',
+      properties: {
+        engine,
+        source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
+        source_prefix,
+        file_format_params,
+      },
+      input_id: undefined,
+    };
+
+    const pipeline = await this.pipelinesClientService.create(
+      upload_pipeline,
+      metadata,
+    );
+
+    return pipeline;
+  }
+
+  @ApiInternalOnlyEndpoint()
+  @Post('start/:id')
+  async activate(@Param('id') id: string, @Body() body) {
+    const { info } = body;
+
+    this.logger.info(
+      process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
+      {
+        user: body.info.user_id,
+        customer: body.info.customer,
+      },
+    );
+
+    const response = await this.oldPipelinesService.runPipeline({ id, info });
+
+    return response;
+  }
 }

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,7 +1,8 @@
 import {
   BadRequestException,
+  HttpException,
+  HttpStatus,
   Inject,
-  InternalServerErrorException,
   OnModuleInit,
 } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
@@ -20,8 +21,11 @@ import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/Pipel
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
 import { InputsService } from '../inputs/inputs.service';
-import { RequestUser } from 'src/authentication/user.decorator';
+import { RequestUser } from 'src/decorators/user.decorator';
 import { TransformationsService } from '../transformations/transformations.service';
+import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
+import ErrorCodes from 'src/utils/errorCodes';
+import ErrorBuilder from 'src/utils/ErrorBuilder';
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -98,7 +102,7 @@ export class PipelinesService implements OnModuleInit {
         const [errorType, message] = error.details.split('|');
         throw new BadRequestException(message);
       }
-      throw new InternalServerErrorException(error.details);
+      throw new ErrorBuilder(error.details);
     });
     this.logger.info('Done');
 
@@ -139,11 +143,17 @@ export class PipelinesService implements OnModuleInit {
     return findOnePipelineResponse;
   }
 
-  async update(UpdatePipelineRequest: Messages.PipelineV2UpdateRequest) {
+  async update(
+    UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
+    metadata,
+  ) {
     this.logger.info('PipelinesClientService - Update');
 
     const updatePipelineResponse = await lastValueFrom(
-      this.pipelineWriteService.PipelineV2Update(UpdatePipelineRequest),
+      this.pipelineWriteService.PipelineV2Update(
+        UpdatePipelineRequest,
+        metadata,
+      ),
     );
     this.logger.info('Done');
 
@@ -157,52 +167,60 @@ export class PipelinesService implements OnModuleInit {
       customer_id: user.customer_id,
       customer: user.customer_name,
     };
+    let existsInProduct = true;
     const { pipeline } = await lastValueFrom(
       this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
-    );
+    ).catch((e) => {
+      if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false;
+      else throw new ErrorBuilder(e.details);
+
+      return { pipeline: undefined };
+    });
     await lastValueFrom(
       this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
     );
 
-    //{pipeline:{tables: {tables: [], input_id: ''}}}
-    const input = pipeline.config.tables
-      ? JSON.parse(pipeline.config.tables)
-      : null;
-    if (input)
-      await this.inputsService
-        .remove({
-          id: input.input_id,
-          info,
-        })
-        .catch((error) =>
-          this.logger.error('Could not delete input', {
-            data: { input, error },
-          }),
-        );
-
-    const transformations: { id: string }[] = pipeline.transformations
-      ? JSON.parse(pipeline.transformations)
-      : null;
-    if (transformations && transformations.length)
-      for (const transformation of transformations) {
-        await this.transformationsService
+    if (existsInProduct) {
+      //{pipeline:{tables: {tables: [], input_id: ''}}}
+      const input = pipeline.config.tables
+        ? JSON.parse(pipeline.config.tables)
+        : null;
+      if (input)
+        await this.inputsService
           .remove({
-            id: transformation.id,
+            id: input.input_id,
             info,
           })
           .catch((error) =>
-            this.logger.error('Could not delete transformation', {
-              data: { transformation, error },
+            this.logger.error('Could not delete input', {
+              data: { input, error },
             }),
           );
-      }
+
+      const transformations: { id: string }[] = pipeline.transformations
+        ? JSON.parse(pipeline.transformations)
+        : null;
+      if (transformations && transformations.length)
+        for (const transformation of transformations) {
+          await this.transformationsService
+            .remove({
+              id: transformation.id,
+              info,
+            })
+            .catch((error) =>
+              this.logger.error('Could not delete transformation', {
+                data: { transformation, error },
+              }),
+            );
+        }
+    }
   }
 
   async findOneProperties(id: string, metadata: Metadata) {
     this.logger.info('PipelinesClientService - findOneProperties');
     const { pipeline } = await this.findOne({ id }, metadata);
 
-    const pipelineProperties = JSON.parse(pipeline.properties);
+    const properties = JSON.parse(pipeline.properties);
 
     const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
 
@@ -210,27 +228,97 @@ export class PipelinesService implements OnModuleInit {
       connectorId,
       metadata.get('language')[0].toString(),
     );
-    const connector = JSON.parse(res.connector);
-    const config_controls = connector.config_controls || [];
+    const connector: { [key: string]: any; config_controls: any[] } =
+      JSON.parse(res.connector);
+    const config_controls = (connector.config_controls || []).filter(
+      (config_control) => {
+        const path: string = config_control.name;
+        if (!objHasPath({ object: { properties }, path })) return false;
 
-    for (const key in pipelineProperties) {
-      const i = config_controls
-        .map((cc) => cc.name)
-        .indexOf(`properties.${key}`);
-      if (i > -1) {
-        config_controls[i] = {
-          ...config_controls[i],
-          default_value: pipelineProperties[key],
-        };
-      }
-    }
+        let default_value = getObjValueFromPath({
+          object: { properties },
+          path,
+        });
+
+        if (default_value != undefined) {
+          if (
+            path === 'properties.spreadsheet_id' &&
+            typeof default_value === 'string'
+          )
+            default_value = 'https://docs.google.com/spreadsheets/d/'.concat(
+              default_value,
+            );
+          config_control.default_value = default_value;
+        }
+
+        return true;
+      },
+    );
 
     return {
       pipeline: {
         ...pipeline,
         config_controls,
-        properties: pipelineProperties,
+        properties,
       },
     };
   }
+
+  async findOneObjects(id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - FindOneObjects');
+
+    const { objects } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2FindObjects({ id }, metadata),
+    );
+    this.logger.info('Done');
+
+    return { objects: JSON.parse(objects) };
+  }
+
+  async initUploadFile(uploadFile, metadata) {
+    const body: Messages.PipelineV2UploadFileRequest = {
+      name: uploadFile.name,
+      parts: uploadFile.parts,
+    };
+
+    return lastValueFrom(
+      this.pipelineWriteService.PipelineV2UploadFile(body, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
+
+  async downloadLogs(pipeline_run_id: string, metadata: Metadata) {
+    this.logger.info('PipelinesClientService - DownloadLogs');
+
+    const { url } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2DownloadLogs(
+        { pipeline_run_id },
+        metadata,
+      ),
+    );
+    this.logger.info('Done');
+
+    return url;
+  }
+
+  async completeUploadFile(uploadFile, metadata) {
+    const body: Messages.PipelineV2CompleteUploadFileRequest = {
+      upload_id: uploadFile.upload_id,
+      parts: JSON.stringify(uploadFile.parts),
+      name: uploadFile.file_name,
+    };
+
+    return lastValueFrom(
+      this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
 }

src/modules/productboard/productboard.controller.ts

@@ -1,12 +1,16 @@
 import { Body, Controller, HttpException, Post } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import axios from 'axios';
-import { User, RequestUser } from 'src/authentication/user.decorator';
-import { getSecreteFromSecreteManager } from 'src/utils/SecretManager';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { User, RequestUser } from 'src/decorators/user.decorator';
+import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
 import { INote } from './dtos';
 
+@ApiInternalOnlyController()
 @ApiTags('Productboard')
 @Controller('productboard')
+@Authenticated()
 export class ProductboardController {
   @Post('notes')
   async sendNote(@Body() note: INote, @User() user: RequestUser) {
@@ -18,7 +22,7 @@ export class ProductboardController {
       : username + `@${customer_name}.default`;
 
     const path = process.env.PB_TOKEN_PATH;
-    const token = await getSecreteFromSecreteManager(path);
+    const token = await getSecretFromSecretsManager(path);
 
     const response = await axios
       .post(

src/modules/roles/dto/entities.ts

@@ -25,6 +25,8 @@ export class RoleDto {
   customer?: Customer;
   @ApiProperty()
   usage: string;
+  @ApiPropertyOptional({ description: 'Whether this role can be untoggled' })
+  canUntoggle?: boolean;
 }
 
 export class CustomerRole extends OmitType(RoleDto, ['customer']) {}
@@ -34,11 +36,17 @@ export enum TrueOrFalseEnum {
   true = 'true',
   false = 'false',
 }
+
+export enum CanUntoggleLogicEnum {
+  data_assets_sharing = 'data_assets_sharing',
+}
 export class GetRolesByCustomerReq {
   @ApiPropertyOptional()
   permissionId?: string[];
   @ApiPropertyOptional({ enum: TrueOrFalseEnum })
   getUsers?: TrueOrFalseEnum;
+  @ApiPropertyOptional({ enum: CanUntoggleLogicEnum })
+  canUntoggleLogic?: CanUntoggleLogicEnum;
 }
 
 export class GetRolesByCustomerRes {

src/modules/roles/roles.controller.ts

@@ -3,7 +3,6 @@ import {
   Controller,
   Delete,
   Get,
-  Headers,
   Inject,
   Param,
   Patch,
@@ -14,11 +13,11 @@ import {
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
-  ApiHeader,
+  ApiHeaders,
   ApiOkResponse,
   ApiTags,
 } from '@nestjs/swagger';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { RolesService } from './roles.service';
 import {
@@ -43,16 +42,14 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import {
   Authenticated,
   RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
+} from 'src/decorators/authentication.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('Roles')
-@ApiHeader({
-  name: 'dadosfera-lang',
-  enum: LanguageEnum,
-  required: false,
-  description: 'Defaults to pt-br',
-})
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @Authenticated()
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('roles')
@@ -71,12 +68,12 @@ export class RolesController {
   @ApiOkResponse({ type: GetRolesByCustomerRes })
   async searchRoles(
     @User() user: RequestUser,
-    @Query() filters: GetRolesByCustomerReq,
-    @Headers('dadosfera-lang') language,
+    @Query() params: GetRolesByCustomerReq,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('searchRoles', { user });
     this.rolesService.setLanguage(language);
-    return await this.rolesService.roleSearch(filters, user);
+    return await this.rolesService.roleSearch(params, user);
   }
 
   @Post()
@@ -85,7 +82,7 @@ export class RolesController {
   async createRole(
     @User() user: RequestUser,
     @Body() body: CreateRoleReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('createRole', { user });
     this.rolesService.setLanguage(language);
@@ -98,7 +95,7 @@ export class RolesController {
   async grantPermissionsToRole(
     @User() user: RequestUser,
     @Body() body: GrantPermissionsToRoleReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('grantPermissionsToRole', { user });
     this.rolesService.setLanguage(language);
@@ -111,7 +108,7 @@ export class RolesController {
   async revokePermissionsFromRole(
     @User() user: RequestUser,
     @Body() body: RevokePermissionsFromRoleReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('revokePermissionsToRole', { user });
     this.rolesService.setLanguage(language);
@@ -124,7 +121,7 @@ export class RolesController {
   async setRolePermissions(
     @User() user: RequestUser,
     @Body() body: SetRolePermissionsReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('setRolePermissions', { user });
     this.rolesService.setLanguage(language);
@@ -137,7 +134,7 @@ export class RolesController {
   async setRoleUsers(
     @User() user: RequestUser,
     @Body() body: SetRoleUsersReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('setRoleUsers', { user });
     this.rolesService.setLanguage(language);
@@ -150,7 +147,7 @@ export class RolesController {
   async getRoleById(
     @User() user: RequestUser,
     @Param('id') id: string,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('getRoleById', { user });
     this.rolesService.setLanguage(language);
@@ -162,7 +159,7 @@ export class RolesController {
   async deleteRole(
     @User() user: RequestUser,
     @Param('id') id: string,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('deleteRole', { user });
     this.rolesService.setLanguage(language);
@@ -176,7 +173,7 @@ export class RolesController {
     @User() user: RequestUser,
     @Param('id') id: string,
     @Body() body: UpdateRoleReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('updateRole', { user });
     this.rolesService.setLanguage(language);

src/modules/roles/roles.service.ts

@@ -18,7 +18,7 @@ import { Permission } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/ent
 import { lastValueFrom } from 'rxjs';
 import { PermissionsService } from '../permissions/permissions.service';
 import { LanguageEnum } from 'src/utils/languages.enum';
-import { RequestUser } from 'src/authentication/user.decorator';
+import { RequestUser } from 'src/decorators/user.decorator';
 import { DucClient } from '../duc/client.config';
 
 interface GetRolesPermissionsName {
@@ -70,12 +70,7 @@ export class RolesService {
     return { role: roleTreated };
   }
 
-  async roleSearch(
-    filters: GetRolesByCustomerReq,
-    { customer_id, access_token }: any,
-  ) {
-    const meta = new Metadata();
-    meta.add('access_token', access_token);
+  async roleSearch(filters: GetRolesByCustomerReq, { customer_id }) {
     let { permissionId: permissionIds } = filters;
     if (permissionIds && !Array.isArray(permissionIds))
       permissionIds = [permissionIds];
@@ -89,15 +84,13 @@ export class RolesService {
         return seqId;
       }) || [];
     const roles = await lastValueFrom(
-      this.rolesClientService.RoleSearch(
-        {
-          customerId: customer_id,
-          getUsers: isGetUsers,
-          permissionIds: [],
-          permissionSeqIds,
-        },
-        meta,
-      ),
+      this.rolesClientService.RoleSearch({
+        customerId: customer_id,
+        getUsers: isGetUsers,
+        permissionIds: [],
+        permissionSeqIds,
+        canUntoggleLogic: filters.canUntoggleLogic,
+      }),
     );
     const rolesTreated = this.getRolesPermissionsName(roles.roles);
     return { roles: rolesTreated };

src/modules/termsOfUse/termsOfUse.controller.ts

@@ -9,17 +9,22 @@ import {
   Headers,
   UseFilters,
 } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
+import { ApiHeaders, ApiTags } from '@nestjs/swagger';
 import { TermsOfUseAcceptRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
 import { TermsOfUseClientService } from './termsOfUse.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
-import { RequestUser, User } from '../../authentication/user.decorator';
+import { RequestUser, User } from '../../decorators/user.decorator';
 import { Metadata } from '@grpc/grpc-js';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
 
+@ApiInternalOnlyController()
 @ApiTags('TermsOfUse')
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('terms-of-use')
 export class TermsOfUseController {
@@ -36,7 +41,7 @@ export class TermsOfUseController {
   @Get('token')
   async getToken(
     @User({ required: true }) user: RequestUser,
-    @Headers('Dadosfera-Lang') language: string,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('/terms-of-use - get token');
     const metadata = new Metadata();

src/modules/transformations/transformations-client.ts

@@ -6,6 +6,8 @@ import {
 import { credentials } from '@grpc/grpc-js';
 import { Transformation } from '@dadosfera/protospack-v2';
 
+const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+
 export class TransformationsClientConfiguration {
   public name = 'TransformationsClientConfiguration';
   private config: GrpcOptions = {
@@ -13,10 +15,7 @@ export class TransformationsClientConfiguration {
     options: {
       url: process.env.INFACTORY_URL,
       package: [Transformation.ProtoPackages.WritePackage],
-      credentials:
-        process.env.LOCAL_ENV || process.env.ENV === 'local'
-          ? undefined
-          : credentials.createSsl(),
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [Transformation.ProtoPaths.WriteFilePath],
       loader: {
         enums: String,

src/modules/transformations/transformations.controller.ts

@@ -11,34 +11,21 @@ import {
 import { ApiTags } from '@nestjs/swagger';
 import { TransformationsClientService } from './client.service';
 import { IIdRequest } from './interfaces';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireSomePermission,
+} from 'src/decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { TransformationsService } from './transformations.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+const pipelinePermissions = PERMISSIONS_GROUPS.PIPELINE.permissions;
 
+@ApiInternalOnlyController()
 @ApiTags('Transformations')
 @Controller('transformations')
-@AuthenticateCondition((req, user) => {
-  let action;
-
-  switch (req.method) {
-    case 'POST':
-      action = 'CREATE';
-      break;
-
-    case 'PUT':
-      action = 'UPDATE';
-      break;
-
-    default:
-      action = req.method;
-  }
-
-  return user.permissions.includes(
-    PERMISSIONS_GROUPS.TRANSFORMATIONS.permissions[action].seqid,
-  );
-})
+@Authenticated()
 export class TransformationsController {
   logger: DadosferaLogger;
   constructor(
@@ -50,6 +37,7 @@ export class TransformationsController {
   }
 
   @Post()
+  @RequireSomePermission(pipelinePermissions.CREATE, pipelinePermissions.UPDATE)
   async create(@Body() createTransformationDto) {
     this.logger.info(`/transformation - ON CREATE ROUTE`, {
       user: createTransformationDto.info.user_id,
@@ -67,6 +55,7 @@ export class TransformationsController {
   }
 
   @Get()
+  @RequireSomePermission(pipelinePermissions.GET)
   async findAll(@Body() data: IIdRequest) {
     this.logger.info(`/transformation - ON Find All ROUTE`, {
       user: data.info.user_id,
@@ -82,8 +71,8 @@ export class TransformationsController {
   }
 
   @Get('/:id')
-  async findOne(@Body() data, @Param() params) {
-    const { id } = params;
+  @RequireSomePermission(pipelinePermissions.GET)
+  async findOne(@Body() data, @Param('id') id: string) {
     this.logger.info(`/transformation/${id} - ON Find One ROUTE`, {
       user: data.info.user_id,
       customer: data.info.customer,
@@ -98,6 +87,7 @@ export class TransformationsController {
   }
 
   @Put(':id')
+  @RequireSomePermission(pipelinePermissions.CREATE, pipelinePermissions.UPDATE)
   async update(
     @User() user: RequestUser,
     @Body() updateTransformationDto,
@@ -120,6 +110,7 @@ export class TransformationsController {
   }
 
   @Delete(':id')
+  @RequireSomePermission(pipelinePermissions.CREATE, pipelinePermissions.UPDATE)
   async delete(@Body() data, @Param() params) {
     const { id } = params;
     this.logger.info(`/transformation/${id} - ON DELETE ROUTE`, {

src/modules/users/users.controller.ts

@@ -1,9 +1,9 @@
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Body,
   Controller,
   Delete,
   Get,
-  Headers,
   HttpCode,
   HttpStatus,
   Inject,
@@ -16,18 +16,22 @@ import {
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
-  ApiHeader,
+  ApiHeaders,
   ApiOkResponse,
   ApiTags,
 } from '@nestjs/swagger';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import {
   Authenticated,
   RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequestUser, User } from 'src/authentication/user.decorator';
+} from 'src/decorators/authentication.decorator';
+import { Language } from 'src/decorators/language.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { RequestUser, User } from 'src/decorators/user.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import ErrorBuilder from 'src/utils/ErrorBuilder';
+import ErrorCodes from 'src/utils/errorCodes';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import {
   AssignRoleToUserReq,
@@ -40,27 +44,19 @@ import {
   GetAllHierarchiesRes,
   GetAllJobTitlesRes,
   GetAllUsersByCustomerIdRes,
+  IUserByCustomer,
   SetUserRolesReq,
   SetUserRolesRes,
   UnassignRoleToUserReq,
   UpdateUserReq,
   UpdateUserRes,
-  IUserByCustomer,
 } from './dtos/entities';
 import { UsersService } from './users.service';
-import { Metadata } from '@grpc/grpc-js';
-import ErrorBuilder from 'src/utils/ErrorBuilder';
-import ErrorCodes from 'src/utils/errorCodes';
 
-// TODO GET de hierarquias e do PATCH em usuário
+@ApiInternalOnlyController()
 @ApiTags('Users')
 @Controller('users')
-@ApiHeader({
-  name: 'dadosfera-lang',
-  enum: LanguageEnum,
-  required: false,
-  description: 'Defaults to pt-br',
-})
+@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @Authenticated()
 @UseFilters(GrpcToHttpExceptionFilter)
 export class UsersController {
@@ -75,13 +71,9 @@ export class UsersController {
   }
 
   @Get()
-  @ApiOkResponse({
-    type: GetAllUsersByCustomerIdRes,
-    description: 'List of users of a customer',
-  })
   async getAllUsersByCustomerId(
     @User() user: RequestUser,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ): Promise<GetAllUsersByCustomerIdRes> {
     this.logger.info('getAllUsersByCustomerId', { user });
     this.userService.setLanguage(language);
@@ -93,7 +85,7 @@ export class UsersController {
   @ApiOkResponse({ type: GetAllHierarchiesRes })
   async getAllHierarchies(
     @User() user: RequestUser,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ): Promise<GetAllHierarchiesRes> {
     this.logger.info('getAllHierarchies', { user });
     this.userService.setLanguage(language);
@@ -105,7 +97,7 @@ export class UsersController {
   @ApiOkResponse({ type: GetAllDepartmentsRes })
   async getAllDepartments(
     @User() user: RequestUser,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ): Promise<GetAllDepartmentsRes> {
     this.logger.info('getAllHierarchies', { user });
     this.userService.setLanguage(language);
@@ -117,7 +109,7 @@ export class UsersController {
   @ApiOkResponse({ type: GetAllDepartmentsRes })
   async getAllJobTitles(
     @User() user: RequestUser,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ): Promise<GetAllJobTitlesRes> {
     this.logger.info('getAllHierarchies', { user });
     this.userService.setLanguage(language);
@@ -131,7 +123,7 @@ export class UsersController {
   async findOneById(
     @User() user: RequestUser,
     @Param('id') id: string,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('findOneById', { user });
     this.userService.setLanguage(language);
@@ -152,14 +144,16 @@ export class UsersController {
   async createUser(
     @User() user: RequestUser,
     @Body() body: CreateUserReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
-    const meta = new Metadata();
-    meta.add('access_token', user.access_token);
+    const metadata = PackTheMetadata({
+      access_token: user.access_token,
+      language,
+    });
     this.logger.info('createUser', { user });
     this.userService.setLanguage(language);
 
-    return await this.userService.createUser(body, meta);
+    return await this.userService.createUser(body, metadata);
   }
 
   @Post('batch')
@@ -168,13 +162,17 @@ export class UsersController {
   async batchCreateUser(
     @User() user: RequestUser,
     @Body() body: BatchCreateUserReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
-    const meta = new Metadata();
-    meta.add('access_token', user.access_token);
     this.logger.info('batchCreateUser', { user });
     this.userService.setLanguage(language);
-    return await this.userService.batchCreateUser(body, meta);
+
+    const metadata = PackTheMetadata({
+      access_token: user.access_token,
+      language,
+    });
+
+    return await this.userService.batchCreateUser(body, metadata);
   }
 
   @Post(':id/resend-invite')
@@ -183,11 +181,15 @@ export class UsersController {
   async resendInvite(
     @User() user: RequestUser,
     @Param('id') id: string,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('resendInvite', { user });
     this.userService.setLanguage(language);
-    return this.userService.resendInvite({ id });
+
+    const metadata = PackTheMetadata({
+      language,
+    });
+    return this.userService.resendInvite({ id }, metadata);
   }
 
   @Delete('role')
@@ -195,7 +197,7 @@ export class UsersController {
   async unassignRoleToUser(
     @User() user: RequestUser,
     @Query() request: UnassignRoleToUserReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('unassignRoleToUser', { user });
     this.userService.setLanguage(language);
@@ -208,7 +210,7 @@ export class UsersController {
   async deleteUser(
     @User() user: RequestUser,
     @Param('id') id: string,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('deleteUser', { user });
     this.userService.setLanguage(language);
@@ -221,7 +223,7 @@ export class UsersController {
   async setUserRoles(
     @User() user: RequestUser,
     @Body() body: SetUserRolesReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('createUser', { user });
     this.userService.setLanguage(language);
@@ -233,7 +235,7 @@ export class UsersController {
   async assignRoleToUser(
     @User() user: RequestUser,
     @Body() body: AssignRoleToUserReq,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('assignRoleToUser', { user });
     this.userService.setLanguage(language);
@@ -247,7 +249,7 @@ export class UsersController {
     @User() user: RequestUser,
     @Body() body: UpdateUserReq,
     @Param('id') id: string,
-    @Headers('dadosfera-lang') language,
+    @Language() language: LanguageEnum,
   ) {
     this.logger.info('updateUser', { user });
     this.userService.setLanguage(language);

src/modules/users/users.service.ts

@@ -122,6 +122,7 @@ export class UsersService implements OnModuleInit {
     if (roleNames) {
       await this.setRoles({ roleNames, userId: id }, customerId);
     }
+
     const { user } = await lastValueFrom(
       this.usersClientService.UserUpdate({
         name,
@@ -131,6 +132,7 @@ export class UsersService implements OnModuleInit {
         hierarchy,
         jobTitle,
         email,
+        metabaseUserId: undefined,
       }),
     );
     return { user: this.adjustUsersPayload([user])[0] };
@@ -183,8 +185,10 @@ export class UsersService implements OnModuleInit {
     };
   }
 
-  async resendInvite(body: IdRequest) {
-    return lastValueFrom(this.usersClientService.UserResendInvite(body));
+  async resendInvite(body: IdRequest, metadata: Metadata) {
+    return lastValueFrom(
+      this.usersClientService.UserResendInvite(body, metadata),
+    );
   }
 
   async assignRoleToUser(body: AssignRoleToUserRequest) {

src/utils/ PackTheMetadata.ts

@@ -7,8 +7,10 @@ interface IMetadata {
   user_id?: string;
   language?: string;
   details?: string;
+  sensitive?: string;
   roles?: string[];
   is_data_manager?: boolean;
+  access_token?: string;
 }
 
 export function PackTheMetadata(info: IMetadata): Metadata {

src/utils/ErrorBuilder.ts

@@ -119,6 +119,14 @@ export function EnrichErrorCode(code: string) {
         code,
       };
 
+    case ErrorCodes.AUTH.INVALID_REFRESH_TOKEN:
+      return {
+        statusCode: HttpStatus.UNAUTHORIZED,
+        error: 'Acesso expirado!',
+        message: 'Seu acesso expirou, favor fazer login novamente.',
+        code,
+      };
+
     case ErrorCodes.AUTH.FORBIDDEN:
       return {
         statusCode: HttpStatus.FORBIDDEN,
@@ -214,6 +222,16 @@ export function EnrichErrorCode(code: string) {
         message: 'Id encontra-se num formato inválido',
         code,
       };
+    case ErrorCodes.CONNECTION.NOT_FOUND:
+    case ErrorCodes.CONNECTION.PLATFORM_NOT_FOUND:
+      return {
+        statusCode: HttpStatus.NOT_FOUND,
+        error: 'Conexão não encontrada',
+        message:
+          'Não encontramos a conexão solicitada. Caso o erro persista, entre em contato com o suporte',
+        code,
+      };
+
     case ErrorCodes.CONNECTION_TEST.ERROR:
       return {
         statusCode: HttpStatus.BAD_REQUEST,
@@ -282,6 +300,28 @@ export function EnrichErrorCode(code: string) {
         message: 'Caso o erro persista, entre em contato com o suporte',
         code,
       };
+    case ErrorCodes.CATALOG.DATA_ASSET_NOT_FOUND:
+      return {
+        statusCode: HttpStatus.NOT_FOUND,
+        error: 'Não encontramos o ativo solicitado',
+        message:
+          'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
+        code,
+      };
+    case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar prévia',
+        message: 'Ocorreu um erro devido ao tamanho da prévia do ativo.',
+        code,
+      };
+    case ErrorCodes.CATALOG.METADATA_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar informações',
+        message: 'Ocorreu um erro devido ao tamanho das informações do ativo.',
+        code,
+      };
     case ErrorCodes.INTERNAL:
     case ErrorCodes.UNKNOWN:
     default:

src/utils/OauthSecrets.ts

@@ -2,6 +2,7 @@ import {
   SecretsManagerClient,
   GetSecretValueCommand,
 } from '@aws-sdk/client-secrets-manager';
+let cachedSecrets: OauthSecrets;
 
 class OauthSecretsObject {
   client_id = '';
@@ -15,8 +16,10 @@ export class OauthSecrets {
   mailchimp = new OauthSecretsObject();
   facebook = new OauthSecretsObject();
   salesforce = new OauthSecretsObject();
+  'google-login' = new OauthSecretsObject();
 }
 export async function getOauthSecrets() {
+  if (cachedSecrets) return cachedSecrets;
   const secrets = new OauthSecrets();
   const path = process.env.SM_OAUTH_PATH;
   const secretsManagerClient = new SecretsManagerClient({});
@@ -37,5 +40,6 @@ export async function getOauthSecrets() {
       };
     }
   }
+  cachedSecrets = secrets;
   return secrets;
 }

src/utils/ObjValueFromPath.ts

@@ -0,0 +1,28 @@
+export function getObjValueFromPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): any {
+  const { object, path } = data;
+  const keys = path.split('.');
+
+  let value = JSON.parse(JSON.stringify(object));
+  for (const key of keys) {
+    if (value != undefined) value = value[key];
+  }
+  return value;
+}
+
+export function objHasPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): boolean {
+  const { object, path } = data;
+  const keys = path.split('.');
+
+  let value = JSON.parse(JSON.stringify(object));
+  for (const key of keys) {
+    if (!(key in value)) return false;
+    value = value[key];
+  }
+  return true;
+}

src/utils/SecretManager.ts

@@ -3,7 +3,7 @@ import {
   GetSecretValueCommand,
 } from '@aws-sdk/client-secrets-manager';
 
-export async function getSecreteFromSecreteManager(path: string) {
+export async function getSecretFromSecretsManager(path: string) {
   const secretsManagerClient = new SecretsManagerClient({});
 
   const getSecretComand = new GetSecretValueCommand({

src/utils/errorCodes.ts

@@ -1,5 +1,6 @@
 export const AUTH = {
   UNAUTHORIZED: 'AUTH.UNAUTHORIZED',
+  INVALID_REFRESH_TOKEN: 'AUTH.INVALID_REFRESH_TOKEN',
   FORBIDDEN: 'AUTH.FORBIDDEN',
   WRONG_CREDENTIALS: 'AUTH.WRONG_CREDENTIALS',
   WRONG_PASSWORD_CONFIRMATION: 'AUTH.WRONG_PASSWORD_CONFIRMATION',
@@ -60,9 +61,18 @@ const PIPELINE = {
 const INPUT = {
   NOT_FOUND: 'INPUT.NOT_FOUND',
 };
+const CONNECTION = {
+  NOT_FOUND: 'CONNECTION.NOT_FOUND',
+  PLATFORM_NOT_FOUND: 'CONNECTION.PLATFORM_NOT_FOUND',
+};
 const TRANSFORMATION = {
   NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
 };
+const CATALOG = {
+  DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
+  PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
+  METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
+};
 const ErrorCodes = {
   UNKNOWN: 'UNKNOWN',
   RATE_LIMIT: 'RATE_LIMIT',
@@ -77,6 +87,42 @@ const ErrorCodes = {
   PIPELINE,
   INPUT,
   TRANSFORMATION,
+  CONNECTION,
+  CATALOG,
 };
 
 export default ErrorCodes;
+
+export const OauthErrors = {
+  INVALID_CREDENTIALS: {
+    'pt-br': {
+      error: 'Erro ao autorizar',
+      error_description: 'Credenciais inválidas. Tente novamente',
+    },
+    'en-us': {
+      error: 'Authorization Error',
+      error_description: 'Invalid credentials. Please try again',
+    },
+  },
+  USER_NOT_FOUND: {
+    'pt-br': {
+      error: 'Erro ao autorizar',
+      error_description: (email) => `Usuário não existe na Dadosfera: ${email}`,
+    },
+    'en-us': {
+      error: 'Authorization Error',
+      error_description: (email) =>
+        `User does not exist on Dadosfera: ${email}`,
+    },
+  },
+  INVALID_SESSION: {
+    'pt-br': {
+      error: 'Erro ao autorizar',
+      error_description: 'Sessão inválida!',
+    },
+    'en-us': {
+      error: 'Authorization Error',
+      error_description: 'Invalid session!',
+    },
+  },
+};

tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
 }