Merge pull request #221 from dadosfera/fix/docker-compose

FIX: use docker compose instead of docker-compose

.eslintrc.js

@@ -8,7 +8,7 @@ module.exports = {
   extends: [
     'eslint:recommended',
     'plugin:@typescript-eslint/recommended',
-    'plugin:prettier/recommended',
+    'prettier',
   ],
   root: true,
   env: {

.github/workflows/deploy-manually.yml

@@ -13,6 +13,11 @@ on:
           - stg
           - stg2
           - prd
+      push_to_dockerhub:
+        description: "Push image to Dockerhub?"
+        required: true
+        type: boolean
+        default: false
 
 jobs:
   extract_environment:
@@ -39,15 +44,15 @@ jobs:
       new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - if: github.event_name != 'workflow_dispatch'
         name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v4
         id: semantic
         with:
           extra_plugins: |
-            conventional-changelog-eslint
+            conventional-changelog-eslint@4.0.0
           branches: |
             [
               'main'
@@ -76,16 +81,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Update Pip
         run: |
           python3 -m pip install --upgrade pip
 
-      - name: Install Docker Compose
-        run: |
-          python3 -m pip install docker-compose --upgrade
-
       - name: Install AWS CLI
         run: |
           python3 -m pip install awscli --upgrade
@@ -95,14 +96,14 @@ jobs:
           python3 -m pip install awsebcli --upgrade
 
       - name: Configure AWS Region
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v4
         id: aws
         with:
           aws-region: us-east-1
 
       - name: Login to AWS ECR
         id: login_ecr
-        uses: aws-actions/amazon-ecr-login@v1
+        uses: aws-actions/amazon-ecr-login@v2
 
       - name: Build, Tag, and Push Image to AWS ECR
         env:
@@ -110,8 +111,25 @@ jobs:
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
         run: |
-          docker-compose -f build.docker-compose.yml build
-          docker-compose -f build.docker-compose.yml push
+          docker compose -f build.docker-compose.yml build
+          docker compose -f build.docker-compose.yml push
+
+      - name: Login to Docker Hub
+        if: ${{inputs.push_to_dockerhub}}
+        uses: docker/login-action@v2
+        with:
+          username: dadosfera
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build, Tag, and Push Image to Dockerhub
+        if: ${{inputs.push_to_dockerhub}}
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+          IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
+          ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+        run: |
+          docker compose -f build.docker-compose.dockerhub.yml build
+          docker compose -f build.docker-compose.dockerhub.yml push
 
       - name: Create ZIP file to Deploy AWS Beanstalk
         env:
@@ -146,7 +164,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Extract Docs BlockId and PageId
         env:

.github/workflows/test.yml

@@ -9,7 +9,13 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Configure AWS
+        uses: aws-actions/configure-aws-credentials@v4
+        id: aws
+        with:
+          aws-region: us-east-1
 
       - name: Build
         env:

.nvmrc

@@ -1 +1 @@
-18.3.0
+18.10.0

.vscode/launch.json

@@ -12,6 +12,7 @@
         "start:debug"
       ],
       "runtimeExecutable": "npm",
+      "runtimeVersion": "18.10.0",
       "skipFiles": [
         "<node_internals>/**"
       ],

Dockerfile

@@ -1,5 +1,5 @@
 # install all dependencies
-FROM node:18.3.0-alpine3.15 as packages
+FROM node:18.10-alpine as packages
 WORKDIR /app
 COPY package.json package-lock.json ./
 RUN apk add --no-cache aws-cli \
@@ -7,12 +7,12 @@ RUN apk add --no-cache aws-cli \
  && npm install
 
 # unit test specific build
-FROM node:18.3.0-alpine3.15 as test
+FROM node:18.10-alpine as test
 WORKDIR /app
 COPY --from=packages /app/node_modules ./node_modules
 COPY . ./
 
-FROM node:18.3.0-alpine3.15
+FROM node:18.10-alpine
 WORKDIR /app
 COPY . /app/
 COPY --from=packages /app/node_modules ./node_modules

LICENSE

@@ -0,0 +1,10 @@
+LICENSE
+Copyright (C) 2023 Dadosfera
+
+All rights reserved.
+
+This software and related documentation are provided under a license agreement containing restrictions on use and disclosure and are protected by intellectual property laws. Except as expressly permitted in your license agreement or allowed by law, you may not use, copy, reproduce, translate, broadcast, modify, license, transmit, distribute, exhibit, perform, publish, or display any part, in any form, or by any means.
+
+Reverse engineering, disassembly, or decompilation of this software, unless required by law for interoperability, is prohibited.
+
+If you have any questions about this, please contact Dadosfera.ai at legal@dadosfera.ai

README.md

@@ -12,7 +12,7 @@ These instructions will allow you to get a working copy of the project on your l
 
 ### 📋 Requirements
 
-- [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
+- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
 - Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
   - Create your Access Key on the "Security credentials" menu
   - Set the Access Key on your local development machine

build.docker-compose.dockerhub.yml

@@ -0,0 +1,5 @@
+version: "3.8"
+services:
+  maestro:
+    build: .
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

docker-compose.dockerhub.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+services:
+  maestro:
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+    container_name: maestro
+    restart: always
+    ports:
+      - 3333:3333
+    env_file:
+      - .env
+
+  node_exporter:
+    image: ${NODE_EXPORTER_URL}
+    container_name: node_exporter
+    restart: always
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host

docsfera.external.json

@@ -241,7 +241,16 @@
         },
         "get": {
           "operationId": "ConnectionController_getConnectionDetails",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -449,7 +458,16 @@
       "/inputs/{id}": {
         "get": {
           "operationId": "InputsController_findOne",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -472,6 +490,45 @@
           ]
         }
       },
+      "/pipelinesV2/monitoring-dashboard": {
+        "get": {
+          "operationId": "PipelinesController_getMonitoringDashboard",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/pipelinesV2": {
         "post": {
           "operationId": "PipelinesController_create",
@@ -534,6 +591,46 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "filters",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -606,6 +703,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -638,6 +743,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -670,6 +783,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -709,6 +830,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -746,6 +875,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -783,6 +920,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -945,6 +1090,14 @@
                 "type": "string"
               }
             },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            },
             {
               "name": "shared",
               "required": true,
@@ -985,6 +1138,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "requestBody": {
@@ -1085,6 +1246,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1127,6 +1296,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1169,6 +1346,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1362,6 +1547,108 @@
           ]
         }
       },
+      "/catalog/dataset-catalog-task": {
+        "post": {
+          "operationId": "CatalogController_triggerCatalog",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/TriggerCatalogReq"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/TriggerCatalogRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/catalog/dataset-catalog-task/{session}": {
+        "get": {
+          "operationId": "CatalogController_getCatalogTask",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            },
+            {
+              "name": "session",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/GetDatasetCatalogTaskRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/oauth/hubspot": {
         "get": {
           "operationId": "OauthController_oauthHubspot",
@@ -1504,24 +1791,16 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/docs-refactor",
+      "title": "Maestro - feat/new-customer-monitoring",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
     },
     "tags": [],
     "servers": [
-      {
-        "url": "https://maestro.stg.dadosfera.ai",
-        "description": "Ambiente STG"
-      },
-      {
-        "url": "https://maestro-2.stg.dadosfera.ai",
-        "description": "Ambiente STG2"
-      },
       {
         "url": "https://maestro.dadosfera.ai",
-        "description": "Ambiente PRD"
+        "description": "Dadosfera API"
       }
     ],
     "components": {
@@ -1572,6 +1851,12 @@
             },
             "scheduleLimit": {
               "type": "string"
+            },
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
             }
           },
           "required": [
@@ -1579,7 +1864,8 @@
             "id",
             "name",
             "tier",
-            "scheduleLimit"
+            "scheduleLimit",
+            "links"
           ]
         },
         "AuthUser": {
@@ -2173,6 +2459,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -2213,6 +2502,7 @@
           "required": [
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -2233,6 +2523,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -2280,6 +2573,7 @@
             "id",
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -2802,6 +3096,73 @@
           "required": [
             "comment"
           ]
+        },
+        "TriggerCatalogReq": {
+          "type": "object",
+          "properties": {
+            "table_name": {
+              "type": "string",
+              "description": "Name of the table on Snowflake",
+              "example": "TB__4DXSD4_TEST"
+            },
+            "table_schema": {
+              "type": "string",
+              "description": "Schema where the asset is located. If not set defaults to \"PUBLIC\""
+            },
+            "pipeline_id": {
+              "type": "string",
+              "description": "Id of the pipeline that generated the asset."
+            }
+          },
+          "required": [
+            "table_name"
+          ]
+        },
+        "TriggerCatalogRes": {
+          "type": "object",
+          "properties": {
+            "session": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "session"
+          ]
+        },
+        "GetDatasetCatalogTaskRes": {
+          "type": "object",
+          "properties": {
+            "created_at": {
+              "type": "string"
+            },
+            "customer_name": {
+              "type": "string"
+            },
+            "session_id": {
+              "type": "string"
+            },
+            "status": {
+              "type": "string"
+            },
+            "table_name": {
+              "type": "string"
+            },
+            "updated_at": {
+              "type": "string"
+            },
+            "updated_by": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "created_at",
+            "customer_name",
+            "session_id",
+            "status",
+            "table_name",
+            "updated_at",
+            "updated_by"
+          ]
         }
       }
     }

docsfera.json

@@ -690,7 +690,16 @@
         },
         "get": {
           "operationId": "ConnectionController_getConnectionDetails",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -2574,7 +2583,16 @@
       "/inputs/{id}": {
         "get": {
           "operationId": "InputsController_findOne",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -2645,6 +2663,45 @@
           ]
         }
       },
+      "/pipelinesV2/monitoring-dashboard": {
+        "get": {
+          "operationId": "PipelinesController_getMonitoringDashboard",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/pipelinesV2": {
         "post": {
           "operationId": "PipelinesController_create",
@@ -2707,6 +2764,46 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "filters",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2779,6 +2876,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2811,6 +2916,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2843,6 +2956,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2882,6 +3003,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2919,6 +3048,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2959,6 +3096,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2996,6 +3141,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3177,6 +3330,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3207,7 +3368,16 @@
           "summary": "",
           "deprecated": true,
           "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "201": {
               "description": "",
@@ -3239,7 +3409,16 @@
           "summary": "",
           "deprecated": true,
           "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -3310,7 +3489,16 @@
       "/transformations/{id}": {
         "get": {
           "operationId": "TransformationsController_findOne",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -3618,6 +3806,14 @@
                 "type": "string"
               }
             },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            },
             {
               "name": "shared",
               "required": true,
@@ -3658,6 +3854,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "requestBody": {
@@ -3758,6 +3962,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3793,6 +4005,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3835,6 +4055,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3877,6 +4105,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -4150,6 +4386,108 @@
           ]
         }
       },
+      "/catalog/dataset-catalog-task": {
+        "post": {
+          "operationId": "CatalogController_triggerCatalog",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/TriggerCatalogReq"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/TriggerCatalogRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/catalog/dataset-catalog-task/{session}": {
+        "get": {
+          "operationId": "CatalogController_getCatalogTask",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            },
+            {
+              "name": "session",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/GetDatasetCatalogTaskRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/terms-of-use/token": {
         "get": {
           "operationId": "TermsOfUseController_getToken",
@@ -4660,6 +4998,129 @@
           ]
         }
       },
+      "/customers/{id}/links": {
+        "get": {
+          "operationId": "CustomersController_getCustomerLinks",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/CustomerLinksResponse"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        },
+        "put": {
+          "operationId": "CustomersController_setCustomerLinks",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/CustomerDto"
+                }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/customers/token": {
+        "get": {
+          "operationId": "CustomersController_getCustomerToken",
+          "parameters": [
+            {
+              "name": "exp",
+              "required": true,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/customers/monitoring-dashboard": {
+        "get": {
+          "operationId": "CustomersController_getCustomerMonitoringDashboard",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/health": {
         "get": {
           "operationId": "HealthController_check",
@@ -4676,26 +5137,13 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/docs-refactor",
+      "title": "Maestro - feat/new-customer-monitoring",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
     },
     "tags": [],
-    "servers": [
-      {
-        "url": "https://maestro.stg.dadosfera.ai",
-        "description": "Ambiente STG"
-      },
-      {
-        "url": "https://maestro-2.stg.dadosfera.ai",
-        "description": "Ambiente STG2"
-      },
-      {
-        "url": "https://maestro.dadosfera.ai",
-        "description": "Ambiente PRD"
-      }
-    ],
+    "servers": [],
     "components": {
       "securitySchemes": {
         "access-token": {
@@ -4744,6 +5192,12 @@
             },
             "scheduleLimit": {
               "type": "string"
+            },
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
             }
           },
           "required": [
@@ -4751,7 +5205,8 @@
             "id",
             "name",
             "tier",
-            "scheduleLimit"
+            "scheduleLimit",
+            "links"
           ]
         },
         "AuthUser": {
@@ -6346,6 +6801,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -6386,6 +6844,7 @@
           "required": [
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -6406,6 +6865,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -6453,6 +6915,7 @@
             "id",
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -6520,6 +6983,9 @@
             },
             "header": {
               "type": "boolean"
+            },
+            "engine": {
+              "type": "string"
             }
           },
           "required": [
@@ -6528,7 +6994,8 @@
             "description",
             "encoding",
             "sep",
-            "header"
+            "header",
+            "engine"
           ]
         },
         "IRole": {
@@ -7044,6 +7511,73 @@
             "comment"
           ]
         },
+        "TriggerCatalogReq": {
+          "type": "object",
+          "properties": {
+            "table_name": {
+              "type": "string",
+              "description": "Name of the table on Snowflake",
+              "example": "TB__4DXSD4_TEST"
+            },
+            "table_schema": {
+              "type": "string",
+              "description": "Schema where the asset is located. If not set defaults to \"PUBLIC\""
+            },
+            "pipeline_id": {
+              "type": "string",
+              "description": "Id of the pipeline that generated the asset."
+            }
+          },
+          "required": [
+            "table_name"
+          ]
+        },
+        "TriggerCatalogRes": {
+          "type": "object",
+          "properties": {
+            "session": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "session"
+          ]
+        },
+        "GetDatasetCatalogTaskRes": {
+          "type": "object",
+          "properties": {
+            "created_at": {
+              "type": "string"
+            },
+            "customer_name": {
+              "type": "string"
+            },
+            "session_id": {
+              "type": "string"
+            },
+            "status": {
+              "type": "string"
+            },
+            "table_name": {
+              "type": "string"
+            },
+            "updated_at": {
+              "type": "string"
+            },
+            "updated_by": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "created_at",
+            "customer_name",
+            "session_id",
+            "status",
+            "table_name",
+            "updated_at",
+            "updated_by"
+          ]
+        },
         "ConnectionTestPingReq": {
           "type": "object",
           "properties": {
@@ -7337,6 +7871,34 @@
             "content",
             "tags"
           ]
+        },
+        "CustomerLinksResponse": {
+          "type": "object",
+          "properties": {
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "links"
+          ]
+        },
+        "CustomerDto": {
+          "type": "object",
+          "properties": {
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "links"
+          ]
         }
       }
     }

package.json

@@ -6,7 +6,7 @@
   "private": true,
   "license": "UNLICENSED",
   "engines": {
-    "node": "18.3.0"
+    "node": "18.10.0"
   },
   "scripts": {
     "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
@@ -29,21 +29,23 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.29.0",
+    "@dadosfera/protospack-v2": "3.33.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
-    "@nestjs/common": "^8.4.7",
-    "@nestjs/config": "^1.2.1",
-    "@nestjs/core": "^8.4.7",
-    "@nestjs/mapped-types": "*",
-    "@nestjs/microservices": "^8.4.7",
-    "@nestjs/passport": "^8.2.2",
-    "@nestjs/platform-express": "^8.4.7",
-    "@nestjs/schedule": "^1.1.0",
-    "@nestjs/swagger": "^5.2.1",
+    "@nestjs/cli": "^9.4.2",
+    "@nestjs/common": "^9.4.0",
+    "@nestjs/config": "^2.3.1",
+    "@nestjs/core": "^9.4.0",
+    "@nestjs/mapped-types": "^1.2.2",
+    "@nestjs/microservices": "^9.4.0",
+    "@nestjs/passport": "^9.0.3",
+    "@nestjs/platform-express": "^9.4.0",
+    "@nestjs/schematics": "^9.1.0",
+    "@nestjs/swagger": "^6.3.0",
+    "@nestjs/testing": "^9.4.0",
     "axios": "^0.27.2",
     "class-transformer": "^0.5.1",
-    "class-validator": "^0.13.2",
+    "class-validator": "^0.14.0",
     "cron-parser": "^4.4.0",
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.36.0",
@@ -51,6 +53,7 @@
     "jsonwebtoken": "^9.0.0",
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
+    "ms": "^3.0.0-canary.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
     "passport-forcedotcom": "^0.2.0",
@@ -67,9 +70,6 @@
     "multer": "1.4.5-lts.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^8.2.8",
-    "@nestjs/schematics": "^8.0.11",
-    "@nestjs/testing": "^8.4.7",
     "@types/express": "^4.17.13",
     "@types/jest": "27.0.2",
     "@types/jsonwebtoken": "^8.5.8",
@@ -80,11 +80,10 @@
     "@types/passport-google-oauth20": "^2.0.11",
     "@types/passport-oauth2": "^1.4.11",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.29.0",
-    "@typescript-eslint/parser": "^5.29.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.18.0",
     "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-prettier": "^4.0.0",
     "jest": "^27.5.1",
     "nock": "^13.2.7",
     "prettier": "^2.7.1",
@@ -96,4 +95,4 @@
     "tsconfig-paths": "^3.14.1",
     "typescript": "^4.6.3"
   }
-}
+}

src/app.module.ts

@@ -25,6 +25,7 @@ import { ConfigModule } from '@nestjs/config';
 import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
 import { ProductboardModule } from './modules/productboard/productboard.module';
 import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
+import { CustomersModule } from './modules/customers/customers.module';
 
 @Module({
   controllers: [],
@@ -56,6 +57,7 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
     OauthModule,
     ProductboardModule,
     MixpanelModule,
+    CustomersModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
   ],

src/authentication/authentication.guard.spec.ts

@@ -44,7 +44,7 @@ class NoClassAuthController {
   @Get('has-all-permissions')
   @RequireAllPermissions(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   )
   async hasAllPermissions(@Body() body) {
     return { body };
@@ -53,7 +53,7 @@ class NoClassAuthController {
   @Get('has-some-permission')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   )
   async hasSomePermission(@Body() body) {
     return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
 
 @Controller('class-auth-condition')
 @AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
-@RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
+@RequireSomePermission(PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE)
 class ClassAuthConditionController {
   @Get('body')
   async getBody(@Body() body) {
@@ -444,13 +444,13 @@ describe('authentication.guard', () => {
   NoClassAuthTest(tokenZ, ['zendesk']);
   ClassAuthConditionTest(tokenZ, ['zendesk']);
 
-  const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
+  const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
   NoClassAuthTest(tokenM, ['metabase']);
   ClassAuthConditionTest(tokenM, ['metabase']);
 
   const tokenZM = CreateToken([
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   ]);
   NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
   ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);

src/authentication/permissions.enum.ts

@@ -55,9 +55,18 @@ export const PERMISSIONS_GROUPS = {
           'es-es': '',
         },
       },
+      GENERATE_TOKEN: {
+        seqid: 46,
+        claim: 'customer:generate-token',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Gerar Token de Acesso',
+          'en-us': 'Generate Acess Token',
+          'es-es': 'Gerar Token de Acceso',
+        },
+      },
     },
   },
-
   PIPELINE: {
     title: {
       'pt-br': 'Coletar | Pipelines',
@@ -107,7 +116,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   CONNECTION: {
     title: {
       'pt-br': 'Coletar | Fontes de dados',
@@ -147,7 +155,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   NETWORK_CONFIG: {
     title: {
       'pt-br': 'Coletar | Redes',
@@ -177,7 +184,6 @@ export const PERMISSIONS_GROUPS = {
       // },
     },
   },
-
   OUTPUT: {
     title: {
       'pt-br': 'Output',
@@ -227,7 +233,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   TRANSFORMATIONS: {
     title: {
       'pt-br': 'Micro-transformações',
@@ -277,7 +282,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   CATALOG: {
     title: {
       'pt-br': 'Explorar | Catálogo',
@@ -346,9 +350,18 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Acceder al Módulo de Incorporación de Activos',
         },
       },
+      TRIGGER_CATALOG_TASK: {
+        seqid: 45,
+        claim: 'catalog:trigger-task',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Executar a catalogação de um ativo',
+          'en-us': 'Trigger an asset cataloging',
+          'es-es': 'Realizar el listado de un activo',
+        },
+      },
     },
   },
-
   CONNECTORS: {
     title: {
       'pt-br': 'Conectores',
@@ -398,7 +411,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   SNOWFLAKE: {
     title: {
       'pt-br': 'Explorar | Consolidar',
@@ -418,7 +430,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   ZENDESK: {
     title: {
       'pt-br': 'Zendesk',
@@ -428,18 +439,17 @@ export const PERMISSIONS_GROUPS = {
     permissions: {
       OPEN: {
         seqid: 16,
-        claim: 'GET /zendesk',
+        claim: 'support-chat:access',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Acessar Zendesk',
-          'en-us': 'Access Zendesk',
-          'es-es': 'Acceso Zendesk',
+          'pt-br': 'Acessar chat de suporte',
+          'en-us': 'Access support chat',
+          'es-es': 'Acceder al chat de soporte',
         },
       },
     },
   },
-
-  ANALYZE: {
+  DATAVIZ: {
     title: {
       'pt-br': 'Analisar | Visualização',
       'en-us': 'Analyze | Visualization',
@@ -456,6 +466,15 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Acceso Metabase',
         },
       },
+    },
+  },
+  INTELLIGENCE: {
+    title: {
+      'pt-br': 'Analisar | Inteligência',
+      'en-us': 'Analyze | Intelligence',
+      'es-es': 'Analizar | Inteligencia',
+    },
+    permissions: {
       INTELLIGENCE: {
         seqid: 31,
         claim: 'intelligence:open',
@@ -468,7 +487,6 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
-
   MODULES: {
     title: {
       'pt-br': 'Módulos da Dadosfera',
@@ -565,6 +583,25 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
+  CUSTOMER: {
+    title: {
+      'pt-br': 'Organização',
+      'en-us': 'Organization',
+      'es-es': 'Organización',
+    },
+    permissions: {
+      MONITORING_DASHBOARD: {
+        seqid: 47,
+        claim: 'customer:monitoring-dashboard',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Ver o dashboard de monitoramento',
+          'en-us': 'View the monitoring dashboard',
+          'es-es': 'Ver el dashboard de monitoreo',
+        },
+      },
+    },
+  },
 };
 export interface DadosferaModule {
   name: string;
@@ -577,7 +614,8 @@ export const DADOSFERA_MODULES: Array<DadosferaModule> = [
     name: 'Intelligence Module',
     description: 'Orchest Module',
     key: 'intelligence',
-    permissionSeqId: PERMISSIONS_GROUPS.ANALYZE.permissions.INTELLIGENCE.seqid,
+    permissionSeqId:
+      PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid,
   },
   {
     name: 'Proccessing Module',

src/decorators/user.decorator.spec.ts

@@ -46,7 +46,7 @@ describe('user.decorator', () => {
   const fakeUserPayload = {
     user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
     username: 'super.admin',
-    permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
+    permissions: [PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE].map(
       ({ seqid }) => seqid,
     ),
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',

src/main.ts

@@ -42,7 +42,7 @@ function configureSwagger(app: INestApplication) {
   }
   const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
 
-  const config = new DocumentBuilder()
+  const swaggerConfig = new DocumentBuilder()
     .setTitle(swaggerTitle)
     .setDescription('Documentation for Maestro gateway')
     .addSecurity('access-token', {
@@ -50,11 +50,12 @@ function configureSwagger(app: INestApplication) {
       name: 'Authorization',
       in: 'header',
     })
-    .setDescription('This is the Maestro API')
-    .addServer('https://maestro.stg.dadosfera.ai', 'Ambiente STG')
-    .addServer('https://maestro-2.stg.dadosfera.ai', 'Ambiente STG2')
-    .addServer('https://maestro.dadosfera.ai', 'Ambiente PRD')
-    .build();
+    .setDescription('This is the Maestro API');
+
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
+
+  const config = swaggerConfig.build();
 
   const document = SwaggerModule.createDocument(app, config);
 

src/modules/auth/auth.service.ts

@@ -164,6 +164,8 @@ export class AuthClientService implements OnModuleInit {
 
   async oauthSignIn(data: { username: string; token: string }) {
     const { username, token } = data;
-    return lastValueFrom(this.authService.AuthOauthSignIn({ token, username }));
+    return lastValueFrom(
+      this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
+    );
   }
 }

src/modules/auth/dtos/login.ts

@@ -1,3 +1,4 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import {
   AuthSignInRequest,
   AuthSignInResponse,
@@ -74,6 +75,8 @@ export class AuthCustomer {
   tier: string;
   @ApiProperty()
   scheduleLimit: string;
+  @ApiProperty()
+  links: Link[];
 }
 
 export class AuthSignInReq implements AuthSignInRequest {

src/modules/catalog/catalog.controller.ts

@@ -14,9 +14,15 @@ import {
   Query,
   UseFilters,
 } from '@nestjs/common';
-import { ApiHeaders, ApiTags } from '@nestjs/swagger';
+import {
+  ApiCreatedResponse,
+  ApiHeaders,
+  ApiOkResponse,
+  ApiTags,
+} from '@nestjs/swagger';
 import {
   Authenticated,
+  RequireAllPermissions,
   RequireSomePermission,
 } from '../../decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
@@ -25,6 +31,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import {
+  GetDatasetCatalogTaskRes,
   ICatalogAllRequest,
   ICatalogAllResponse,
   IColumnsMetadataResponse,
@@ -35,6 +42,8 @@ import {
   IOneDataAsset,
   IPreviewResponse,
   IUpdateDataRequest,
+  TriggerCatalogReq,
+  TriggerCatalogRes,
 } from './dtos';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { Language } from 'src/decorators/language.decorator';
@@ -180,7 +189,7 @@ export class CatalogController {
   )
   async getDataAsset(
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
     @Query('shared') shared?: 'true',
   ) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -196,7 +205,7 @@ export class CatalogController {
     let has_permission = false;
     const metadata = PackTheMetadata({
       username,
-      user_id: undefined,
+      user_id,
       customer_id,
       customer_name,
     });
@@ -237,7 +246,7 @@ export class CatalogController {
   async getDataAssetRls(
     @User() user: RequestUser,
     @Headers() headers,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
 
@@ -292,7 +301,7 @@ export class CatalogController {
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<IColumnsMetadataResponse> {
     const { customer_name, customer_id, user_id, username } = user;
 
@@ -323,7 +332,7 @@ export class CatalogController {
   async getDataAssetPreview(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<IPreviewResponse> {
     const { customer_name, customer_id, user_id, username } = user;
 
@@ -353,7 +362,7 @@ export class CatalogController {
   async getDataAssetDocs(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<IDocsResponse> {
     const { customer_name, customer_id, user_id, username } = user;
 
@@ -383,7 +392,7 @@ export class CatalogController {
   async updateDataAsset(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') data_asset_id,
+    @Param('id') data_asset_id: string,
     @Body() body: IUpdateDataRequest,
   ): Promise<IOneDataAsset> {
     const { customer_id, customer_name, user_id, username } = user;
@@ -579,4 +588,49 @@ export class CatalogController {
 
     return response;
   }
+
+  @Post('dataset-catalog-task')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiCreatedResponse({ type: TriggerCatalogRes })
+  async triggerCatalog(
+    @User() user: RequestUser,
+    @Body() body: TriggerCatalogReq,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const session = await this.catalogService.triggerCatalog(body, metadata);
+
+    return { session };
+  }
+
+  @Get('dataset-catalog-task/:session')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiOkResponse({ type: GetDatasetCatalogTaskRes })
+  async getCatalogTask(
+    @User() user: RequestUser,
+    @Param('session') session: string,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.getDatasetCatalogTask(
+      session,
+      metadata,
+    );
+
+    return response;
+  }
 }

src/modules/catalog/catalog.service.ts

@@ -18,7 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
 import { UsersService } from '../users/users.service';
 import { RolesService } from '../roles/roles.service';
 import { Metadata } from '@grpc/grpc-js';
-import { IUpdateDataRequest } from './dtos';
+import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
 
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
@@ -333,6 +333,18 @@ class CatalogService implements OnModuleInit {
       } as typeof data_asset;
     });
   }
+  async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
+    const { session } = await lastValueFrom(
+      this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
+    );
+    return session;
+  }
+  async getDatasetCatalogTask(session: string, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
+    );
+    return res;
+  }
 }
 
 export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -271,3 +271,44 @@ export class IColumnsMetadataResponse {
   @ApiProperty({ type: [IColumnMetadata] })
   columns_metadata: IColumnMetadata[];
 }
+
+export class TriggerCatalogReq {
+  @ApiProperty({
+    description: 'Name of the table on Snowflake',
+    example: 'TB__4DXSD4_TEST',
+  })
+  table_name: string;
+
+  @ApiPropertyOptional({
+    description:
+      'Schema where the asset is located. If not set defaults to "PUBLIC"',
+  })
+  table_schema?: string;
+
+  @ApiPropertyOptional({
+    description: 'Id of the pipeline that generated the asset.',
+  })
+  pipeline_id: string;
+}
+
+export class TriggerCatalogRes {
+  @ApiProperty()
+  session: string;
+}
+
+export class GetDatasetCatalogTaskRes {
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  customer_name: string;
+  @ApiProperty()
+  session_id: string;
+  @ApiProperty()
+  status: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  updated_by: string;
+}

src/modules/connection/connection.controller.ts

@@ -164,7 +164,7 @@ export class ConnectionController {
   async getConnectionDetails(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
     @Query() queries,
   ): Promise<ConnectionDetailsRes> {
     this.logger.info('/connections - Get Connection Details');

src/modules/connector/connector.controller.ts

@@ -273,16 +273,13 @@ export class ConnectorController {
     @Param('plugin') plugin: string,
     @Query('version') version: string,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('DELETE /:plugin - Delete Connector Route');
 
-    const response: any = await this.connectorClientService.deleteConnector(
+    const response = await this.connectorClientService.deleteConnector(
       plugin,
       version,
     );
 
-    return {
-      message: response.message || 'ok',
-      connector: { ...JSON.parse(response.connector) },
-    };
+    return { message: response.message, connector: response.connector };
   }
 }

src/modules/customers/customers.controller.ts

@@ -0,0 +1,88 @@
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import {
+  Body,
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Put,
+  Query,
+  UseFilters,
+} from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { CustomersService } from './customers.service';
+import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import type { StringValue } from 'ms';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+
+@ApiInternalOnlyController()
+@ApiTags('Customers')
+@Controller('customers')
+@Authenticated()
+@UseFilters(GrpcToHttpExceptionFilter)
+export class CustomersController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private customersService: CustomersService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get(':id/links')
+  async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
+    this.logger.info('getCustomerLinks', { id });
+    const links = await this.customersService.getLinks(id);
+    return { links };
+  }
+
+  @Put(':id/links')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  setCustomerLinks(
+    @Body() body: CustomerDto,
+    @Param('id') id,
+  ): Promise<IdResponse> {
+    const { links } = body;
+    this.logger.info('setCustomerLinks', { id, links });
+    return this.customersService.setLinks(id, links);
+  }
+
+  @Get('token')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
+  async getCustomerToken(
+    @Query('exp') exp: StringValue,
+    @User() user: RequestUser,
+  ): Promise<string> {
+    this.logger.info('getCustomerToken', { exp, user });
+    const data = {
+      customerId: user.customer_id,
+      userId: user.user_id,
+      customerName: user.customer_name,
+    };
+    return this.customersService.generateToken(exp, data);
+  }
+
+  @Get('monitoring-dashboard')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
+  )
+  async getCustomerMonitoringDashboard(
+    @User() user: RequestUser,
+  ): Promise<{ url: string }> {
+    this.logger.info('getCustomerMonitoringDashboard');
+
+    const metadata = PackTheMetadata(user);
+    return this.customersService.getMonitoringDashboardUrl(metadata);
+  }
+}

src/modules/customers/customers.module.ts

@@ -0,0 +1,23 @@
+import { Module } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import { CustomersController } from './customers.controller';
+import { CustomersService } from './customers.service';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
+
+const ducClient = new DucClient();
+const piFactoryClient = new PipelinesClientConfiguration();
+
+@Module({
+  imports: [
+    ClientsModule.register([
+      ducClient.providerOptions,
+      piFactoryClient.providerOptions,
+    ]),
+  ],
+  controllers: [CustomersController],
+  providers: [CustomersService, DadosferaLogger],
+  exports: [CustomersService],
+})
+export class CustomersModule {}

src/modules/customers/customers.service.ts

@@ -0,0 +1,154 @@
+import {
+  OnModuleInit,
+  Inject,
+  Injectable,
+  HttpException,
+  HttpStatus,
+  InternalServerErrorException,
+} from '@nestjs/common';
+
+import { firstValueFrom, lastValueFrom } from 'rxjs';
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import ErrorCodes from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import {
+  GetSecretValueCommand,
+  SecretsManagerClient,
+} from '@aws-sdk/client-secrets-manager';
+import getEnv from 'src/utils/getEnv';
+import { logger } from 'elastic-apm-node';
+import {
+  ReadService,
+  ProtoServices as PipelineProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
+import { Metadata } from '@grpc/grpc-js';
+import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
+
+// This function will accept any string, which may result in a bug.
+@Injectable()
+export class CustomersService implements OnModuleInit {
+  private customerService: CustomersProtoService;
+
+  private pipelineReadService: ReadService.PipelineV2ReadService;
+
+  constructor(
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly pipelinesGrpcClient: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.customerService = this.grpcClient.getService<CustomersProtoService>(
+      ProtoServices.CustomersProtoService,
+    );
+    this.pipelineReadService =
+      this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
+        PipelineProtoServices.PipelineV2ReadService,
+      );
+  }
+
+  async getLinks(customerId: string) {
+    try {
+      const result = await lastValueFrom(
+        this.customerService.CustomerFindOneById({ id: customerId }),
+      );
+      return result.customer?.links || [];
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      throw err;
+    }
+  }
+
+  async setLinks(customerId: string, links: Link[]) {
+    if (!customerId || !links) {
+      throw new HttpException(null, HttpStatus.BAD_REQUEST);
+    }
+
+    try {
+      return await firstValueFrom(
+        this.customerService.CustomerUpdate({
+          id: customerId,
+          links,
+        } as CustomerUpdateRequest),
+      );
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      else throw err;
+    }
+  }
+
+  async generateToken(
+    expiresIn = '30m',
+    data: { customerId: string; userId: string; customerName: string },
+  ) {
+    const { customerId, userId: generatedById, customerName } = data;
+
+    const secretsManagerClient = new SecretsManagerClient({});
+
+    const getSecretComand = new GetSecretValueCommand({
+      SecretId: `${getEnv()}/${customerName}/jwt-signing-key`,
+    });
+    let private_key_pem;
+    await secretsManagerClient
+      .send(getSecretComand)
+      .then((res) => {
+        const secret = JSON.parse(res.SecretString);
+
+        private_key_pem = secret.private_key_pem;
+      })
+      .catch((err) => {
+        logger.error(err.stack);
+        throw new InternalServerErrorException(
+          'Error finding keys for customer',
+        );
+      });
+    let jwt_token;
+    try {
+      jwt_token = jwt.sign(
+        {
+          customerId,
+          generatedById,
+        },
+        private_key_pem,
+        {
+          algorithm: 'RS256',
+          expiresIn,
+          issuer: 'maestro',
+        },
+      );
+    } catch (err) {
+      logger.error(err.stack);
+      throw new InternalServerErrorException('Error generating token');
+    }
+    // const verify = jwt.verify(jwt_token, public_key_pem, {
+    //   algorithms: ['RS256'],
+    //   issuer: 'maestro',
+    // });
+    // const decoded = jwt.decode(jwt_token, { complete: true });
+    return jwt_token;
+  }
+  async getMonitoringDashboardUrl(metadata: Metadata) {
+    logger.info('CustomersService - getMonitoringDashboardUrl');
+
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '45',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    logger.info('Done');
+
+    return res;
+  }
+}

src/modules/customers/dtos/customers.ts

@@ -0,0 +1,12 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { ApiProperty } from '@nestjs/swagger';
+
+export class CustomerDto {
+  @ApiProperty()
+  links: Link[];
+}
+
+export class CustomerLinksResponse {
+  @ApiProperty()
+  links: Link[];
+}

src/modules/inputs/inputs.controller.ts

@@ -123,8 +123,7 @@ export class InputsController {
   }
 
   @Get('/:id')
-  async findOne(@Body() body, @Param() params) {
-    const { id } = params;
+  async findOne(@Body() body, @Param('id') id: string) {
     this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
       user: body.info.user_id,
       customer: body.info.customer,

src/modules/inputs/inputs.service.ts

@@ -5,7 +5,6 @@ import {
   Inject,
   Injectable,
 } from '@nestjs/common';
-import { Timeout } from '@nestjs/schedule';
 import CronParser, { CronExpression } from 'cron-parser';
 import { ClientGrpc } from '@nestjs/microservices';
 import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
@@ -222,7 +221,6 @@ export class InputsService {
     return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
   }
 
-  @Timeout(60000 * 10) // Timeout set for 10 minutes
   async testConnection(data) {
     try {
       const testConnectionInputResponse =

src/modules/oauth/passport-strategies/hubspot-strategy.ts

@@ -17,7 +17,7 @@ export class HubspotStrategy extends PassportStrategy(Strategy) {
         clientSecret: oauthSecrets.hubspot.client_secret,
         callbackURL: oauthSecrets.hubspot.redirect_uri,
         scope:
-          'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
+          'tickets automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
         passReqToCallback: true,
       },
       (accessToken, refreshToken, tokenInfo, profile, done) => {

src/modules/pipelines/pipelines.controller.ts

@@ -49,8 +49,7 @@ export class PipelinesController {
     description:
       'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
   })
-  async activate(@Param() params, @Body() body) {
-    const { id } = params;
+  async activate(@Param('id') id: string, @Body() body) {
     const { info } = body;
 
     this.logger.info(
@@ -72,8 +71,7 @@ export class PipelinesController {
     description:
       'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
   })
-  async getPipelineStatus(@Body() body, @Param() params) {
-    const { id } = params;
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
     body.id = id;
 
     this.logger.info(

src/modules/pipelinesV2/interfaces.ts

@@ -10,6 +10,8 @@ export class IPipelineV2 {
   @ApiProperty()
   description: string;
   @ApiProperty()
+  type: string;
+  @ApiProperty()
   connection_id: string;
   @ApiProperty()
   cron: string;
@@ -104,4 +106,20 @@ export class ICreatePipelineCSVFile {
 
   @ApiProperty()
   header: boolean;
+
+  @ApiProperty()
+  engine: string;
+}
+
+export class PipelineFindAllReq {
+  @ApiPropertyOptional()
+  search?: string | undefined;
+  @ApiPropertyOptional()
+  filters?: string | undefined;
+  @ApiPropertyOptional()
+  size?: string | undefined;
+  @ApiPropertyOptional()
+  page?: string | undefined;
+  @ApiPropertyOptional()
+  type?: string | undefined;
 }

src/modules/pipelinesV2/pipelines.controller.ts

@@ -14,6 +14,7 @@ import {
   Patch,
   HttpException,
   BadRequestException,
+  CacheTTL,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -40,6 +41,7 @@ import {
   ICreatePipelineV2Req,
   IPipelineV2,
   IInitUploadCSVFile,
+  PipelineFindAllReq,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
@@ -85,6 +87,18 @@ export class PipelinesController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @Get('monitoring-dashboard')
+  async getMonitoringDashboard(@User() user: RequestUser) {
+    this.logger.info('PipelinesController - getMonitoringDashboard', { user });
+
+    const metadata = PackTheMetadata(user);
+
+    const response =
+      await this.pipelinesClientService.getMonitoringDashboardUrl(metadata);
+
+    return response;
+  }
+
   @Post()
   @ApiCreatedResponse({ type: IPipelineV2 })
   async create(
@@ -115,7 +129,7 @@ export class PipelinesController {
   async findAll(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Query() data,
+    @Query() data: PipelineFindAllReq,
   ) {
     this.logger.info('PipelinesController - findAll', { user });
 
@@ -169,7 +183,7 @@ export class PipelinesController {
   async getPipelineproperties(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     this.logger.info('PipelinesController - getPipelineproperties', { user });
     const metadata = PackTheMetadata({ ...user, language: language });
@@ -180,7 +194,7 @@ export class PipelinesController {
   async getPipelineObjects(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     this.logger.info('PipelinesController - getPipelineObjects', { user });
     const metadata = PackTheMetadata({ ...user, language: language });
@@ -188,8 +202,7 @@ export class PipelinesController {
   }
 
   @Get(':id/status')
-  async getPipelineStatus(@Body() body, @Param() params) {
-    const { id } = params;
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
     body.id = id;
 
     this.logger.info(
@@ -209,7 +222,7 @@ export class PipelinesController {
   async findOne(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<Messages.PipelineV2FindOneResponse> {
     this.logger.info('PipelinesController - findOne', { user });
 
@@ -250,7 +263,7 @@ export class PipelinesController {
   async update(
     @Language() language: LanguageEnum,
     @Body() updatePipelineDto,
-    @Param('id') id,
+    @Param('id') id: string,
     @User() user: RequestUser,
   ) {
     this.logger.info('PipelinesController - update', { user });
@@ -289,7 +302,7 @@ export class PipelinesController {
   async updateDeprecated(
     @Language() language: LanguageEnum,
     @Body() updatePipelineDto,
-    @Param('id') id,
+    @Param('id') id: string,
     @User() user: RequestUser,
   ) {
     this.logger.info('PipelinesController - put', { user });
@@ -301,9 +314,8 @@ export class PipelinesController {
   @Delete(':id')
   @ApiNoContentResponse()
   @HttpCode(HttpStatus.NO_CONTENT)
-  async delete(@Param() params, @User() user: RequestUser) {
+  async delete(@Param('id') id: string, @User() user: RequestUser) {
     this.logger.info('PipelinesController - delete', { user });
-    const { id } = params;
     const metadata = PackTheMetadata({
       customer_id: user.customer_id,
       customer_name: user.customer_name,
@@ -370,30 +382,37 @@ export class PipelinesController {
     @User() user: RequestUser,
     @Body() body: ICreatePipelineCSVFile,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/file - Upload Connector Route');
     const metadata = PackTheMetadata({ ...user });
 
-    const { name, sep, header, encoding, description, file_name } = body;
+    const { name, sep, header, encoding, description, file_name, engine } =
+      body;
+
+    const file_format_params = {
+      ...(sep && { sep }),
+      ...(encoding && { encoding }),
+      ...(header !== undefined ? { header } : {}),
+    };
+
     const source_prefix = `${user.customer_name}/${file_name}`;
 
-    const upload_pipeline = {
+    const upload_pipeline: ICreatePipelineV2Req = {
       connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
       connector_name: 'Amazon S3',
       connector_plugin: 'aws_s3',
       connector_version: '1.0.0',
-      image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
+      image_url: `https://assets.dadosfera.ai/images/connectors/${engine}.svg`,
       name,
       description,
       transformations_ids: [],
       tags: [],
       cron: '@once',
-      config: { cron: '@once', tables: [] },
+      type: 'upload',
       properties: {
-        engine: 'csv',
+        engine,
         source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
         source_prefix,
-        file_format_params: { sep, encoding, header: Boolean(header) },
-        is_a_upload_csv: true,
+        file_format_params,
       },
       input_id: undefined,
     };
@@ -408,8 +427,7 @@ export class PipelinesController {
 
   @ApiInternalOnlyEndpoint()
   @Post('start/:id')
-  async activate(@Param() params, @Body() body) {
-    const { id } = params;
+  async activate(@Param('id') id: string, @Body() body) {
     const { info } = body;
 
     this.logger.info(

src/modules/pipelinesV2/pipelines.service.ts

@@ -23,7 +23,7 @@ import { ConnectorClientService } from '../connector/client.service';
 import { InputsService } from '../inputs/inputs.service';
 import { RequestUser } from 'src/decorators/user.decorator';
 import { TransformationsService } from '../transformations/transformations.service';
-import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
+import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
 
@@ -220,7 +220,7 @@ export class PipelinesService implements OnModuleInit {
     this.logger.info('PipelinesClientService - findOneProperties');
     const { pipeline } = await this.findOne({ id }, metadata);
 
-    const pipelineProperties = JSON.parse(pipeline.properties);
+    const properties = JSON.parse(pipeline.properties);
 
     const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
 
@@ -228,24 +228,38 @@ export class PipelinesService implements OnModuleInit {
       connectorId,
       metadata.get('language')[0].toString(),
     );
-    const connector = JSON.parse(res.connector);
-    const config_controls = connector.config_controls || [];
+    const connector: { [key: string]: any; config_controls: any[] } =
+      JSON.parse(res.connector);
+    const config_controls = (connector.config_controls || []).filter(
+      (config_control) => {
+        const path: string = config_control.name;
+        if (!objHasPath({ object: { properties }, path })) return false;
 
-    for (const config_control of config_controls) {
-      const path: string = config_control.name;
-      const default_value = getObjValueFromPath({
-        object: { properties: pipelineProperties },
-        path,
-      });
-      if (default_value != undefined)
-        config_control.default_value = default_value;
-    }
+        let default_value = getObjValueFromPath({
+          object: { properties },
+          path,
+        });
+
+        if (default_value != undefined) {
+          if (
+            path === 'properties.spreadsheet_id' &&
+            typeof default_value === 'string'
+          )
+            default_value = 'https://docs.google.com/spreadsheets/d/'.concat(
+              default_value,
+            );
+          config_control.default_value = default_value;
+        }
+
+        return true;
+      },
+    );
 
     return {
       pipeline: {
         ...pipeline,
         config_controls,
-        properties: pipelineProperties,
+        properties,
       },
     };
   }
@@ -307,4 +321,22 @@ export class PipelinesService implements OnModuleInit {
       );
     });
   }
+
+  async getMonitoringDashboardUrl(metadata: Metadata) {
+    this.logger.info('PipelinesClientService - getMonitoringDashboardUrl');
+
+    const res = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetDashboardUrl(
+        {
+          dashboard_id: '83',
+          exp: '15m',
+          metabase_customer_name: 'dadosferatech',
+        },
+        metadata,
+      ),
+    );
+    this.logger.info('Done');
+
+    return res;
+  }
 }

src/modules/transformations/transformations.controller.ts

@@ -72,8 +72,7 @@ export class TransformationsController {
 
   @Get('/:id')
   @RequireSomePermission(pipelinePermissions.GET)
-  async findOne(@Body() data, @Param() params) {
-    const { id } = params;
+  async findOne(@Body() data, @Param('id') id: string) {
     this.logger.info(`/transformation/${id} - ON Find One ROUTE`, {
       user: data.info.user_id,
       customer: data.info.customer,

src/utils/ErrorBuilder.ts

@@ -308,6 +308,20 @@ export function EnrichErrorCode(code: string) {
           'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
         code,
       };
+    case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar prévia',
+        message: 'Ocorreu um erro devido ao tamanho da prévia do ativo.',
+        code,
+      };
+    case ErrorCodes.CATALOG.METADATA_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar informações',
+        message: 'Ocorreu um erro devido ao tamanho das informações do ativo.',
+        code,
+      };
     case ErrorCodes.INTERNAL:
     case ErrorCodes.UNKNOWN:
     default:

src/utils/ObjValueFromPath.ts

@@ -1,4 +1,7 @@
-export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
+export function getObjValueFromPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): any {
   const { object, path } = data;
   const keys = path.split('.');
 
@@ -8,3 +11,18 @@ export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
   }
   return value;
 }
+
+export function objHasPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): boolean {
+  const { object, path } = data;
+  const keys = path.split('.');
+
+  let value = JSON.parse(JSON.stringify(object));
+  for (const key of keys) {
+    if (!(key in value)) return false;
+    value = value[key];
+  }
+  return true;
+}

src/utils/errorCodes.ts

@@ -39,6 +39,7 @@ export const CUSTOMER = {
   INVALID_ID: 'CUSTOMER.INVALID_ID',
   NOT_FOUND: 'CUSTOMER.NOT_FOUND',
   ALREADY_EXISTS: 'CUSTOMER.ALREADY_EXISTS',
+  BAD_REQUEST: 'CUSTOMER.BAD_REQUEST',
 };
 const TERMS_OF_USE = {
   UP_TO_DATE: 'TERMS_OF_USE.UP_TO_DATE',
@@ -70,6 +71,8 @@ const TRANSFORMATION = {
 };
 const CATALOG = {
   DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
+  PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
+  METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
 };
 const ErrorCodes = {
   UNKNOWN: 'UNKNOWN',

src/utils/getEnv.ts

@@ -0,0 +1,10 @@
+const availableLocalEnvs: Array<typeof process.env.ENV> = ['stg', 'test'];
+
+export default function getEnv(): typeof process.env.ENV {
+  if (process.env.ENV === 'local') {
+    if (availableLocalEnvs.includes(process.env.LOCAL_ENV))
+      return process.env.LOCAL_ENV;
+    return 'stg';
+  }
+  return process.env.ENV;
+}