Merge pull request #216 from dadosfera/feat/generate-token

Feat: generate token

.nvmrc

@@ -1 +1 @@
-18.3.0
+18.10.0

Dockerfile

@@ -1,5 +1,5 @@
 # install all dependencies
-FROM node:18.3.0-alpine3.15 as packages
+FROM node:18.10-alpine as packages
 WORKDIR /app
 COPY package.json package-lock.json ./
 RUN apk add --no-cache aws-cli \
@@ -7,12 +7,12 @@ RUN apk add --no-cache aws-cli \
  && npm install
 
 # unit test specific build
-FROM node:18.3.0-alpine3.15 as test
+FROM node:18.10-alpine as test
 WORKDIR /app
 COPY --from=packages /app/node_modules ./node_modules
 COPY . ./
 
-FROM node:18.3.0-alpine3.15
+FROM node:18.10-alpine
 WORKDIR /app
 COPY . /app/
 COPY --from=packages /app/node_modules ./node_modules

README.md

@@ -12,7 +12,7 @@ These instructions will allow you to get a working copy of the project on your l
 
 ### 📋 Requirements
 
-- [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
+- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
 - Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
   - Create your Access Key on the "Security credentials" menu
   - Set the Access Key on your local development machine

docsfera.external.json

@@ -241,7 +241,16 @@
         },
         "get": {
           "operationId": "ConnectionController_getConnectionDetails",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -449,7 +458,16 @@
       "/inputs/{id}": {
         "get": {
           "operationId": "InputsController_findOne",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -534,6 +552,46 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "filters",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -606,6 +664,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -638,6 +704,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -670,6 +744,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -709,6 +791,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -746,6 +836,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -783,6 +881,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -945,6 +1051,14 @@
                 "type": "string"
               }
             },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            },
             {
               "name": "shared",
               "required": true,
@@ -985,6 +1099,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "requestBody": {
@@ -1085,6 +1207,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1127,6 +1257,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1169,6 +1307,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1362,6 +1508,108 @@
           ]
         }
       },
+      "/catalog/dataset-catalog-task": {
+        "post": {
+          "operationId": "CatalogController_triggerCatalog",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/TriggerCatalogReq"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/TriggerCatalogRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/catalog/dataset-catalog-task/{session}": {
+        "get": {
+          "operationId": "CatalogController_getCatalogTask",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            },
+            {
+              "name": "session",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/GetDatasetCatalogTaskRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/oauth/hubspot": {
         "get": {
           "operationId": "OauthController_oauthHubspot",
@@ -1504,24 +1752,16 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/docs-refactor",
+      "title": "Maestro - feat/generate-token",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
     },
     "tags": [],
     "servers": [
-      {
-        "url": "https://maestro.stg.dadosfera.ai",
-        "description": "Ambiente STG"
-      },
-      {
-        "url": "https://maestro-2.stg.dadosfera.ai",
-        "description": "Ambiente STG2"
-      },
       {
         "url": "https://maestro.dadosfera.ai",
-        "description": "Ambiente PRD"
+        "description": "Dadosfera API"
       }
     ],
     "components": {
@@ -1572,6 +1812,12 @@
             },
             "scheduleLimit": {
               "type": "string"
+            },
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
             }
           },
           "required": [
@@ -1579,7 +1825,8 @@
             "id",
             "name",
             "tier",
-            "scheduleLimit"
+            "scheduleLimit",
+            "links"
           ]
         },
         "AuthUser": {
@@ -2173,6 +2420,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -2213,6 +2463,7 @@
           "required": [
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -2233,6 +2484,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -2280,6 +2534,7 @@
             "id",
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -2802,6 +3057,73 @@
           "required": [
             "comment"
           ]
+        },
+        "TriggerCatalogReq": {
+          "type": "object",
+          "properties": {
+            "table_name": {
+              "type": "string",
+              "description": "Name of the table on Snowflake",
+              "example": "TB__4DXSD4_TEST"
+            },
+            "table_schema": {
+              "type": "string",
+              "description": "Schema where the asset is located. If not set defaults to \"PUBLIC\""
+            },
+            "pipeline_id": {
+              "type": "string",
+              "description": "Id of the pipeline that generated the asset."
+            }
+          },
+          "required": [
+            "table_name"
+          ]
+        },
+        "TriggerCatalogRes": {
+          "type": "object",
+          "properties": {
+            "session": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "session"
+          ]
+        },
+        "GetDatasetCatalogTaskRes": {
+          "type": "object",
+          "properties": {
+            "created_at": {
+              "type": "string"
+            },
+            "customer_name": {
+              "type": "string"
+            },
+            "session_id": {
+              "type": "string"
+            },
+            "status": {
+              "type": "string"
+            },
+            "table_name": {
+              "type": "string"
+            },
+            "updated_at": {
+              "type": "string"
+            },
+            "updated_by": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "created_at",
+            "customer_name",
+            "session_id",
+            "status",
+            "table_name",
+            "updated_at",
+            "updated_by"
+          ]
         }
       }
     }

docsfera.json

@@ -690,7 +690,16 @@
         },
         "get": {
           "operationId": "ConnectionController_getConnectionDetails",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -2574,7 +2583,16 @@
       "/inputs/{id}": {
         "get": {
           "operationId": "InputsController_findOne",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -2707,6 +2725,46 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "filters",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2779,6 +2837,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2811,6 +2877,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2843,6 +2917,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2882,6 +2964,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2919,6 +3009,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2959,6 +3057,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2996,6 +3102,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3177,6 +3291,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3207,7 +3329,16 @@
           "summary": "",
           "deprecated": true,
           "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "201": {
               "description": "",
@@ -3239,7 +3370,16 @@
           "summary": "",
           "deprecated": true,
           "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -3310,7 +3450,16 @@
       "/transformations/{id}": {
         "get": {
           "operationId": "TransformationsController_findOne",
-          "parameters": [],
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
           "responses": {
             "200": {
               "description": "",
@@ -3618,6 +3767,14 @@
                 "type": "string"
               }
             },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            },
             {
               "name": "shared",
               "required": true,
@@ -3658,6 +3815,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "requestBody": {
@@ -3758,6 +3923,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3793,6 +3966,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3835,6 +4016,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -3877,6 +4066,14 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -4150,6 +4347,108 @@
           ]
         }
       },
+      "/catalog/dataset-catalog-task": {
+        "post": {
+          "operationId": "CatalogController_triggerCatalog",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/TriggerCatalogReq"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/TriggerCatalogRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/catalog/dataset-catalog-task/{session}": {
+        "get": {
+          "operationId": "CatalogController_getCatalogTask",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            },
+            {
+              "name": "session",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/GetDatasetCatalogTaskRes"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/terms-of-use/token": {
         "get": {
           "operationId": "TermsOfUseController_getToken",
@@ -4660,6 +4959,107 @@
           ]
         }
       },
+      "/customers/{id}/links": {
+        "get": {
+          "operationId": "CustomersController_getCustomerLinks",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/CustomerLinksResponse"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        },
+        "put": {
+          "operationId": "CustomersController_setCustomerLinks",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/CustomerDto"
+                }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/customers/token": {
+        "get": {
+          "operationId": "CustomersController_getCustomerToken",
+          "parameters": [
+            {
+              "name": "exp",
+              "required": true,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Customers"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/health": {
         "get": {
           "operationId": "HealthController_check",
@@ -4676,26 +5076,13 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/docs-refactor",
+      "title": "Maestro - feat/generate-token",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
     },
     "tags": [],
-    "servers": [
-      {
-        "url": "https://maestro.stg.dadosfera.ai",
-        "description": "Ambiente STG"
-      },
-      {
-        "url": "https://maestro-2.stg.dadosfera.ai",
-        "description": "Ambiente STG2"
-      },
-      {
-        "url": "https://maestro.dadosfera.ai",
-        "description": "Ambiente PRD"
-      }
-    ],
+    "servers": [],
     "components": {
       "securitySchemes": {
         "access-token": {
@@ -4744,6 +5131,12 @@
             },
             "scheduleLimit": {
               "type": "string"
+            },
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
             }
           },
           "required": [
@@ -4751,7 +5144,8 @@
             "id",
             "name",
             "tier",
-            "scheduleLimit"
+            "scheduleLimit",
+            "links"
           ]
         },
         "AuthUser": {
@@ -6346,6 +6740,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -6386,6 +6783,7 @@
           "required": [
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -6406,6 +6804,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -6453,6 +6854,7 @@
             "id",
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -6520,6 +6922,9 @@
             },
             "header": {
               "type": "boolean"
+            },
+            "engine": {
+              "type": "string"
             }
           },
           "required": [
@@ -6528,7 +6933,8 @@
             "description",
             "encoding",
             "sep",
-            "header"
+            "header",
+            "engine"
           ]
         },
         "IRole": {
@@ -7044,6 +7450,73 @@
             "comment"
           ]
         },
+        "TriggerCatalogReq": {
+          "type": "object",
+          "properties": {
+            "table_name": {
+              "type": "string",
+              "description": "Name of the table on Snowflake",
+              "example": "TB__4DXSD4_TEST"
+            },
+            "table_schema": {
+              "type": "string",
+              "description": "Schema where the asset is located. If not set defaults to \"PUBLIC\""
+            },
+            "pipeline_id": {
+              "type": "string",
+              "description": "Id of the pipeline that generated the asset."
+            }
+          },
+          "required": [
+            "table_name"
+          ]
+        },
+        "TriggerCatalogRes": {
+          "type": "object",
+          "properties": {
+            "session": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "session"
+          ]
+        },
+        "GetDatasetCatalogTaskRes": {
+          "type": "object",
+          "properties": {
+            "created_at": {
+              "type": "string"
+            },
+            "customer_name": {
+              "type": "string"
+            },
+            "session_id": {
+              "type": "string"
+            },
+            "status": {
+              "type": "string"
+            },
+            "table_name": {
+              "type": "string"
+            },
+            "updated_at": {
+              "type": "string"
+            },
+            "updated_by": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "created_at",
+            "customer_name",
+            "session_id",
+            "status",
+            "table_name",
+            "updated_at",
+            "updated_by"
+          ]
+        },
         "ConnectionTestPingReq": {
           "type": "object",
           "properties": {
@@ -7337,6 +7810,34 @@
             "content",
             "tags"
           ]
+        },
+        "CustomerLinksResponse": {
+          "type": "object",
+          "properties": {
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "links"
+          ]
+        },
+        "CustomerDto": {
+          "type": "object",
+          "properties": {
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "links"
+          ]
         }
       }
     }

package.json

@@ -6,7 +6,7 @@
   "private": true,
   "license": "UNLICENSED",
   "engines": {
-    "node": "18.3.0"
+    "node": "18.10.0"
   },
   "scripts": {
     "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
@@ -29,21 +29,23 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.29.0",
+    "@dadosfera/protospack-v2": "3.31.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
-    "@nestjs/common": "^8.4.7",
-    "@nestjs/config": "^1.2.1",
-    "@nestjs/core": "^8.4.7",
-    "@nestjs/mapped-types": "*",
-    "@nestjs/microservices": "^8.4.7",
-    "@nestjs/passport": "^8.2.2",
-    "@nestjs/platform-express": "^8.4.7",
-    "@nestjs/schedule": "^1.1.0",
-    "@nestjs/swagger": "^5.2.1",
+    "@nestjs/cli": "^9.4.2",
+    "@nestjs/common": "^9.4.0",
+    "@nestjs/config": "^2.3.1",
+    "@nestjs/core": "^9.4.0",
+    "@nestjs/mapped-types": "^1.2.2",
+    "@nestjs/microservices": "^9.4.0",
+    "@nestjs/passport": "^9.0.3",
+    "@nestjs/platform-express": "^9.4.0",
+    "@nestjs/schematics": "^9.1.0",
+    "@nestjs/swagger": "^6.3.0",
+    "@nestjs/testing": "^9.4.0",
     "axios": "^0.27.2",
     "class-transformer": "^0.5.1",
-    "class-validator": "^0.13.2",
+    "class-validator": "^0.14.0",
     "cron-parser": "^4.4.0",
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.36.0",
@@ -51,6 +53,7 @@
     "jsonwebtoken": "^9.0.0",
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
+    "ms": "^3.0.0-canary.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
     "passport-forcedotcom": "^0.2.0",
@@ -67,9 +70,6 @@
     "multer": "1.4.5-lts.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^8.2.8",
-    "@nestjs/schematics": "^8.0.11",
-    "@nestjs/testing": "^8.4.7",
     "@types/express": "^4.17.13",
     "@types/jest": "27.0.2",
     "@types/jsonwebtoken": "^8.5.8",
@@ -96,4 +96,4 @@
     "tsconfig-paths": "^3.14.1",
     "typescript": "^4.6.3"
   }
-}
+}

src/app.module.ts

@@ -25,6 +25,7 @@ import { ConfigModule } from '@nestjs/config';
 import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
 import { ProductboardModule } from './modules/productboard/productboard.module';
 import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
+import { CustomersModule } from './modules/customers/customers.module';
 
 @Module({
   controllers: [],
@@ -56,6 +57,7 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
     OauthModule,
     ProductboardModule,
     MixpanelModule,
+    CustomersModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
   ],

src/authentication/authentication.guard.spec.ts

@@ -44,7 +44,7 @@ class NoClassAuthController {
   @Get('has-all-permissions')
   @RequireAllPermissions(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   )
   async hasAllPermissions(@Body() body) {
     return { body };
@@ -53,7 +53,7 @@ class NoClassAuthController {
   @Get('has-some-permission')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   )
   async hasSomePermission(@Body() body) {
     return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
 
 @Controller('class-auth-condition')
 @AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
-@RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
+@RequireSomePermission(PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE)
 class ClassAuthConditionController {
   @Get('body')
   async getBody(@Body() body) {
@@ -444,13 +444,13 @@ describe('authentication.guard', () => {
   NoClassAuthTest(tokenZ, ['zendesk']);
   ClassAuthConditionTest(tokenZ, ['zendesk']);
 
-  const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
+  const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
   NoClassAuthTest(tokenM, ['metabase']);
   ClassAuthConditionTest(tokenM, ['metabase']);
 
   const tokenZM = CreateToken([
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
+    PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
   ]);
   NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
   ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);

src/authentication/permissions.enum.ts

@@ -55,6 +55,16 @@ export const PERMISSIONS_GROUPS = {
           'es-es': '',
         },
       },
+      GENERATE_TOKEN: {
+        seqid: 46,
+        claim: 'customer:generate-token',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Gerar Token de Acesso',
+          'en-us': 'Generate Acess Token',
+          'es-es': 'Gerar Token de Acceso',
+        },
+      },
     },
   },
 
@@ -346,6 +356,16 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Acceder al Módulo de Incorporación de Activos',
         },
       },
+      TRIGGER_CATALOG_TASK: {
+        seqid: 45,
+        claim: 'catalog:trigger-task',
+        usage: PermissionUsages.INTERNAL,
+        name: {
+          'pt-br': 'Executar a catalogação de um ativo',
+          'en-us': 'Trigger an asset cataloging',
+          'es-es': 'Realizar el listado de un activo',
+        },
+      },
     },
   },
 
@@ -428,18 +448,18 @@ export const PERMISSIONS_GROUPS = {
     permissions: {
       OPEN: {
         seqid: 16,
-        claim: 'GET /zendesk',
+        claim: 'support-chat:access',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'Acessar Zendesk',
-          'en-us': 'Access Zendesk',
-          'es-es': 'Acceso Zendesk',
+          'pt-br': 'Acessar chat de suporte',
+          'en-us': 'Access support chat',
+          'es-es': 'Acceder al chat de soporte',
         },
       },
     },
   },
 
-  ANALYZE: {
+  DATAVIZ: {
     title: {
       'pt-br': 'Analisar | Visualização',
       'en-us': 'Analyze | Visualization',
@@ -456,6 +476,15 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Acceso Metabase',
         },
       },
+    },
+  },
+  INTELLIGENCE: {
+    title: {
+      'pt-br': 'Analisar | Inteligência',
+      'en-us': 'Analyze | Intelligence',
+      'es-es': 'Analizar | Inteligencia',
+    },
+    permissions: {
       INTELLIGENCE: {
         seqid: 31,
         claim: 'intelligence:open',
@@ -577,7 +606,8 @@ export const DADOSFERA_MODULES: Array<DadosferaModule> = [
     name: 'Intelligence Module',
     description: 'Orchest Module',
     key: 'intelligence',
-    permissionSeqId: PERMISSIONS_GROUPS.ANALYZE.permissions.INTELLIGENCE.seqid,
+    permissionSeqId:
+      PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid,
   },
   {
     name: 'Proccessing Module',

src/decorators/user.decorator.spec.ts

@@ -46,7 +46,7 @@ describe('user.decorator', () => {
   const fakeUserPayload = {
     user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
     username: 'super.admin',
-    permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
+    permissions: [PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE].map(
       ({ seqid }) => seqid,
     ),
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',

src/main.ts

@@ -42,7 +42,7 @@ function configureSwagger(app: INestApplication) {
   }
   const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
 
-  const config = new DocumentBuilder()
+  const swaggerConfig = new DocumentBuilder()
     .setTitle(swaggerTitle)
     .setDescription('Documentation for Maestro gateway')
     .addSecurity('access-token', {
@@ -50,11 +50,12 @@ function configureSwagger(app: INestApplication) {
       name: 'Authorization',
       in: 'header',
     })
-    .setDescription('This is the Maestro API')
-    .addServer('https://maestro.stg.dadosfera.ai', 'Ambiente STG')
-    .addServer('https://maestro-2.stg.dadosfera.ai', 'Ambiente STG2')
-    .addServer('https://maestro.dadosfera.ai', 'Ambiente PRD')
-    .build();
+    .setDescription('This is the Maestro API');
+
+  if (process.env.INTERNAL_SWAGGER !== 'true')
+    swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
+
+  const config = swaggerConfig.build();
 
   const document = SwaggerModule.createDocument(app, config);
 

src/modules/auth/dtos/login.ts

@@ -1,3 +1,4 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import {
   AuthSignInRequest,
   AuthSignInResponse,
@@ -74,6 +75,8 @@ export class AuthCustomer {
   tier: string;
   @ApiProperty()
   scheduleLimit: string;
+  @ApiProperty()
+  links: Link[];
 }
 
 export class AuthSignInReq implements AuthSignInRequest {

src/modules/catalog/catalog.controller.ts

@@ -14,9 +14,15 @@ import {
   Query,
   UseFilters,
 } from '@nestjs/common';
-import { ApiHeaders, ApiTags } from '@nestjs/swagger';
+import {
+  ApiCreatedResponse,
+  ApiHeaders,
+  ApiOkResponse,
+  ApiTags,
+} from '@nestjs/swagger';
 import {
   Authenticated,
+  RequireAllPermissions,
   RequireSomePermission,
 } from '../../decorators/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
@@ -25,6 +31,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import {
+  GetDatasetCatalogTaskRes,
   ICatalogAllRequest,
   ICatalogAllResponse,
   IColumnsMetadataResponse,
@@ -35,6 +42,8 @@ import {
   IOneDataAsset,
   IPreviewResponse,
   IUpdateDataRequest,
+  TriggerCatalogReq,
+  TriggerCatalogRes,
 } from './dtos';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { Language } from 'src/decorators/language.decorator';
@@ -180,7 +189,7 @@ export class CatalogController {
   )
   async getDataAsset(
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
     @Query('shared') shared?: 'true',
   ) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -196,7 +205,7 @@ export class CatalogController {
     let has_permission = false;
     const metadata = PackTheMetadata({
       username,
-      user_id: undefined,
+      user_id,
       customer_id,
       customer_name,
     });
@@ -237,7 +246,7 @@ export class CatalogController {
   async getDataAssetRls(
     @User() user: RequestUser,
     @Headers() headers,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
 
@@ -292,7 +301,7 @@ export class CatalogController {
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<IColumnsMetadataResponse> {
     const { customer_name, customer_id, user_id, username } = user;
 
@@ -323,7 +332,7 @@ export class CatalogController {
   async getDataAssetPreview(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<IPreviewResponse> {
     const { customer_name, customer_id, user_id, username } = user;
 
@@ -353,7 +362,7 @@ export class CatalogController {
   async getDataAssetDocs(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<IDocsResponse> {
     const { customer_name, customer_id, user_id, username } = user;
 
@@ -383,7 +392,7 @@ export class CatalogController {
   async updateDataAsset(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Param('id') data_asset_id,
+    @Param('id') data_asset_id: string,
     @Body() body: IUpdateDataRequest,
   ): Promise<IOneDataAsset> {
     const { customer_id, customer_name, user_id, username } = user;
@@ -579,4 +588,49 @@ export class CatalogController {
 
     return response;
   }
+
+  @Post('dataset-catalog-task')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiCreatedResponse({ type: TriggerCatalogRes })
+  async triggerCatalog(
+    @User() user: RequestUser,
+    @Body() body: TriggerCatalogReq,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const session = await this.catalogService.triggerCatalog(body, metadata);
+
+    return { session };
+  }
+
+  @Get('dataset-catalog-task/:session')
+  @RequireAllPermissions(
+    PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
+  )
+  @ApiOkResponse({ type: GetDatasetCatalogTaskRes })
+  async getCatalogTask(
+    @User() user: RequestUser,
+    @Param('session') session: string,
+  ) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+    const response = await this.catalogService.getDatasetCatalogTask(
+      session,
+      metadata,
+    );
+
+    return response;
+  }
 }

src/modules/catalog/catalog.service.ts

@@ -18,7 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
 import { UsersService } from '../users/users.service';
 import { RolesService } from '../roles/roles.service';
 import { Metadata } from '@grpc/grpc-js';
-import { IUpdateDataRequest } from './dtos';
+import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
 
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
@@ -333,6 +333,18 @@ class CatalogService implements OnModuleInit {
       } as typeof data_asset;
     });
   }
+  async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
+    const { session } = await lastValueFrom(
+      this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
+    );
+    return session;
+  }
+  async getDatasetCatalogTask(session: string, metadata: Metadata) {
+    const res = await lastValueFrom(
+      this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
+    );
+    return res;
+  }
 }
 
 export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -271,3 +271,44 @@ export class IColumnsMetadataResponse {
   @ApiProperty({ type: [IColumnMetadata] })
   columns_metadata: IColumnMetadata[];
 }
+
+export class TriggerCatalogReq {
+  @ApiProperty({
+    description: 'Name of the table on Snowflake',
+    example: 'TB__4DXSD4_TEST',
+  })
+  table_name: string;
+
+  @ApiPropertyOptional({
+    description:
+      'Schema where the asset is located. If not set defaults to "PUBLIC"',
+  })
+  table_schema?: string;
+
+  @ApiPropertyOptional({
+    description: 'Id of the pipeline that generated the asset.',
+  })
+  pipeline_id: string;
+}
+
+export class TriggerCatalogRes {
+  @ApiProperty()
+  session: string;
+}
+
+export class GetDatasetCatalogTaskRes {
+  @ApiProperty()
+  created_at: string;
+  @ApiProperty()
+  customer_name: string;
+  @ApiProperty()
+  session_id: string;
+  @ApiProperty()
+  status: string;
+  @ApiProperty()
+  table_name: string;
+  @ApiProperty()
+  updated_at: string;
+  @ApiProperty()
+  updated_by: string;
+}

src/modules/connection/connection.controller.ts

@@ -164,7 +164,7 @@ export class ConnectionController {
   async getConnectionDetails(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
     @Query() queries,
   ): Promise<ConnectionDetailsRes> {
     this.logger.info('/connections - Get Connection Details');

src/modules/connector/connector.controller.ts

@@ -273,16 +273,13 @@ export class ConnectorController {
     @Param('plugin') plugin: string,
     @Query('version') version: string,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('DELETE /:plugin - Delete Connector Route');
 
-    const response: any = await this.connectorClientService.deleteConnector(
+    const response = await this.connectorClientService.deleteConnector(
       plugin,
       version,
     );
 
-    return {
-      message: response.message || 'ok',
-      connector: { ...JSON.parse(response.connector) },
-    };
+    return { message: response.message, connector: response.connector };
   }
 }

src/modules/customers/customers.controller.ts

@@ -0,0 +1,74 @@
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import {
+  Body,
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Put,
+  Query,
+  UseFilters,
+} from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/decorators/authentication.decorator';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { CustomersService } from './customers.service';
+import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
+import { RequestUser, User } from 'src/decorators/user.decorator';
+import type { StringValue } from 'ms';
+
+@ApiInternalOnlyController()
+@ApiTags('Customers')
+@Controller('customers')
+@Authenticated()
+@UseFilters(GrpcToHttpExceptionFilter)
+export class CustomersController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private customersService: CustomersService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get(':id/links')
+  async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
+    this.logger.info('getCustomerLinks', { id });
+    const links = await this.customersService.getLinks(id);
+    return { links };
+  }
+
+  @Put(':id/links')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  setCustomerLinks(
+    @Body() body: CustomerDto,
+    @Param('id') id,
+  ): Promise<IdResponse> {
+    const { links } = body;
+    this.logger.info('setCustomerLinks', { id, links });
+    return this.customersService.setLinks(id, links);
+  }
+
+  @Get('token')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
+  async getCustomerToken(
+    @Query('exp') exp: StringValue,
+    @User() user: RequestUser,
+  ): Promise<string> {
+    this.logger.info('getCustomerToken', { exp, user });
+    const data = {
+      customerId: user.customer_id,
+      userId: user.user_id,
+      customerName: user.customer_name,
+    };
+    return this.customersService.generateToken(exp, data);
+  }
+}

src/modules/customers/customers.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ClientsModule } from '@nestjs/microservices';
+import { DucClient } from '../duc/client.config';
+import { CustomersController } from './customers.controller';
+import { CustomersService } from './customers.service';
+
+const client = new DucClient();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [CustomersController],
+  providers: [CustomersService, DadosferaLogger],
+  exports: [CustomersService],
+})
+export class CustomersModule {}

src/modules/customers/customers.service.ts

@@ -0,0 +1,123 @@
+import {
+  OnModuleInit,
+  Inject,
+  Injectable,
+  HttpException,
+  HttpStatus,
+  InternalServerErrorException,
+} from '@nestjs/common';
+
+import { firstValueFrom, lastValueFrom } from 'rxjs';
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { DucClient } from '../duc/client.config';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
+import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import ErrorCodes from 'src/utils/errorCodes';
+import jwt from 'jsonwebtoken';
+import {
+  GetSecretValueCommand,
+  SecretsManagerClient,
+} from '@aws-sdk/client-secrets-manager';
+import getEnv from 'src/utils/getEnv';
+import { logger } from 'elastic-apm-node';
+
+// This function will accept any string, which may result in a bug.
+@Injectable()
+export class CustomersService implements OnModuleInit {
+  private customerService: CustomersProtoService;
+
+  constructor(
+    @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.customerService = this.grpcClient.getService<CustomersProtoService>(
+      ProtoServices.CustomersProtoService,
+    );
+  }
+
+  async getLinks(customerId: string) {
+    try {
+      const result = await lastValueFrom(
+        this.customerService.CustomerFindOneById({ id: customerId }),
+      );
+      return result.customer?.links || [];
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      throw err;
+    }
+  }
+
+  async setLinks(customerId: string, links: Link[]) {
+    if (!customerId || !links) {
+      throw new HttpException(null, HttpStatus.BAD_REQUEST);
+    }
+
+    try {
+      return await firstValueFrom(
+        this.customerService.CustomerUpdate({
+          id: customerId,
+          links,
+        } as CustomerUpdateRequest),
+      );
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      else throw err;
+    }
+  }
+
+  async generateToken(
+    expiresIn = '30m',
+    data: { customerId: string; userId: string; customerName: string },
+  ) {
+    const { customerId, userId: generatedById, customerName } = data;
+
+    const secretsManagerClient = new SecretsManagerClient({});
+
+    const getSecretComand = new GetSecretValueCommand({
+      SecretId: `${getEnv()}/${customerName}/jwt-signing-key`,
+    });
+    let private_key_pem;
+    await secretsManagerClient
+      .send(getSecretComand)
+      .then((res) => {
+        const secret = JSON.parse(res.SecretString);
+
+        private_key_pem = secret.private_key_pem;
+      })
+      .catch((err) => {
+        logger.error(err.stack);
+        throw new InternalServerErrorException(
+          'Error finding keys for customer',
+        );
+      });
+    let jwt_token;
+    try {
+      jwt_token = jwt.sign(
+        {
+          customerId,
+          generatedById,
+        },
+        private_key_pem,
+        {
+          algorithm: 'RS256',
+          expiresIn,
+          issuer: 'maestro',
+        },
+      );
+    } catch (err) {
+      logger.error(err.stack);
+      throw new InternalServerErrorException('Error generating token');
+    }
+    // const verify = jwt.verify(jwt_token, public_key_pem, {
+    //   algorithms: ['RS256'],
+    //   issuer: 'maestro',
+    // });
+    // const decoded = jwt.decode(jwt_token, { complete: true });
+    return jwt_token;
+  }
+}

src/modules/customers/dtos/customers.ts

@@ -0,0 +1,12 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
+import { ApiProperty } from '@nestjs/swagger';
+
+export class CustomerDto {
+  @ApiProperty()
+  links: Link[];
+}
+
+export class CustomerLinksResponse {
+  @ApiProperty()
+  links: Link[];
+}

src/modules/inputs/inputs.controller.ts

@@ -123,8 +123,7 @@ export class InputsController {
   }
 
   @Get('/:id')
-  async findOne(@Body() body, @Param() params) {
-    const { id } = params;
+  async findOne(@Body() body, @Param('id') id: string) {
     this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
       user: body.info.user_id,
       customer: body.info.customer,

src/modules/inputs/inputs.service.ts

@@ -5,7 +5,6 @@ import {
   Inject,
   Injectable,
 } from '@nestjs/common';
-import { Timeout } from '@nestjs/schedule';
 import CronParser, { CronExpression } from 'cron-parser';
 import { ClientGrpc } from '@nestjs/microservices';
 import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
@@ -222,7 +221,6 @@ export class InputsService {
     return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
   }
 
-  @Timeout(60000 * 10) // Timeout set for 10 minutes
   async testConnection(data) {
     try {
       const testConnectionInputResponse =

src/modules/oauth/passport-strategies/hubspot-strategy.ts

@@ -17,7 +17,7 @@ export class HubspotStrategy extends PassportStrategy(Strategy) {
         clientSecret: oauthSecrets.hubspot.client_secret,
         callbackURL: oauthSecrets.hubspot.redirect_uri,
         scope:
-          'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
+          'tickets automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
         passReqToCallback: true,
       },
       (accessToken, refreshToken, tokenInfo, profile, done) => {

src/modules/pipelines/pipelines.controller.ts

@@ -49,8 +49,7 @@ export class PipelinesController {
     description:
       'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
   })
-  async activate(@Param() params, @Body() body) {
-    const { id } = params;
+  async activate(@Param('id') id: string, @Body() body) {
     const { info } = body;
 
     this.logger.info(
@@ -72,8 +71,7 @@ export class PipelinesController {
     description:
       'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
   })
-  async getPipelineStatus(@Body() body, @Param() params) {
-    const { id } = params;
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
     body.id = id;
 
     this.logger.info(

src/modules/pipelinesV2/interfaces.ts

@@ -10,6 +10,8 @@ export class IPipelineV2 {
   @ApiProperty()
   description: string;
   @ApiProperty()
+  type: string;
+  @ApiProperty()
   connection_id: string;
   @ApiProperty()
   cron: string;
@@ -104,4 +106,20 @@ export class ICreatePipelineCSVFile {
 
   @ApiProperty()
   header: boolean;
+
+  @ApiProperty()
+  engine: string;
+}
+
+export class PipelineFindAllReq {
+  @ApiPropertyOptional()
+  search?: string | undefined;
+  @ApiPropertyOptional()
+  filters?: string | undefined;
+  @ApiPropertyOptional()
+  size?: string | undefined;
+  @ApiPropertyOptional()
+  page?: string | undefined;
+  @ApiPropertyOptional()
+  type?: string | undefined;
 }

src/modules/pipelinesV2/pipelines.controller.ts

@@ -40,6 +40,7 @@ import {
   ICreatePipelineV2Req,
   IPipelineV2,
   IInitUploadCSVFile,
+  PipelineFindAllReq,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
@@ -115,7 +116,7 @@ export class PipelinesController {
   async findAll(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Query() data,
+    @Query() data: PipelineFindAllReq,
   ) {
     this.logger.info('PipelinesController - findAll', { user });
 
@@ -169,7 +170,7 @@ export class PipelinesController {
   async getPipelineproperties(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     this.logger.info('PipelinesController - getPipelineproperties', { user });
     const metadata = PackTheMetadata({ ...user, language: language });
@@ -180,7 +181,7 @@ export class PipelinesController {
   async getPipelineObjects(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ) {
     this.logger.info('PipelinesController - getPipelineObjects', { user });
     const metadata = PackTheMetadata({ ...user, language: language });
@@ -188,8 +189,7 @@ export class PipelinesController {
   }
 
   @Get(':id/status')
-  async getPipelineStatus(@Body() body, @Param() params) {
-    const { id } = params;
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
     body.id = id;
 
     this.logger.info(
@@ -209,7 +209,7 @@ export class PipelinesController {
   async findOne(
     @Language() language: LanguageEnum,
     @User() user: RequestUser,
-    @Param('id') id,
+    @Param('id') id: string,
   ): Promise<Messages.PipelineV2FindOneResponse> {
     this.logger.info('PipelinesController - findOne', { user });
 
@@ -250,7 +250,7 @@ export class PipelinesController {
   async update(
     @Language() language: LanguageEnum,
     @Body() updatePipelineDto,
-    @Param('id') id,
+    @Param('id') id: string,
     @User() user: RequestUser,
   ) {
     this.logger.info('PipelinesController - update', { user });
@@ -289,7 +289,7 @@ export class PipelinesController {
   async updateDeprecated(
     @Language() language: LanguageEnum,
     @Body() updatePipelineDto,
-    @Param('id') id,
+    @Param('id') id: string,
     @User() user: RequestUser,
   ) {
     this.logger.info('PipelinesController - put', { user });
@@ -301,9 +301,8 @@ export class PipelinesController {
   @Delete(':id')
   @ApiNoContentResponse()
   @HttpCode(HttpStatus.NO_CONTENT)
-  async delete(@Param() params, @User() user: RequestUser) {
+  async delete(@Param('id') id: string, @User() user: RequestUser) {
     this.logger.info('PipelinesController - delete', { user });
-    const { id } = params;
     const metadata = PackTheMetadata({
       customer_id: user.customer_id,
       customer_name: user.customer_name,
@@ -370,30 +369,37 @@ export class PipelinesController {
     @User() user: RequestUser,
     @Body() body: ICreatePipelineCSVFile,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/file - Upload Connector Route');
     const metadata = PackTheMetadata({ ...user });
 
-    const { name, sep, header, encoding, description, file_name } = body;
+    const { name, sep, header, encoding, description, file_name, engine } =
+      body;
+
+    const file_format_params = {
+      ...(sep && { sep }),
+      ...(encoding && { encoding }),
+      ...(header !== undefined ? { header } : {}),
+    };
+
     const source_prefix = `${user.customer_name}/${file_name}`;
 
-    const upload_pipeline = {
+    const upload_pipeline: ICreatePipelineV2Req = {
       connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
       connector_name: 'Amazon S3',
       connector_plugin: 'aws_s3',
       connector_version: '1.0.0',
-      image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
+      image_url: `https://assets.dadosfera.ai/images/connectors/${engine}.svg`,
       name,
       description,
       transformations_ids: [],
       tags: [],
       cron: '@once',
-      config: { cron: '@once', tables: [] },
+      type: 'upload',
       properties: {
-        engine: 'csv',
+        engine,
         source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
         source_prefix,
-        file_format_params: { sep, encoding, header: Boolean(header) },
-        is_a_upload_csv: true,
+        file_format_params,
       },
       input_id: undefined,
     };
@@ -408,8 +414,7 @@ export class PipelinesController {
 
   @ApiInternalOnlyEndpoint()
   @Post('start/:id')
-  async activate(@Param() params, @Body() body) {
-    const { id } = params;
+  async activate(@Param('id') id: string, @Body() body) {
     const { info } = body;
 
     this.logger.info(

src/modules/pipelinesV2/pipelines.service.ts

@@ -23,7 +23,7 @@ import { ConnectorClientService } from '../connector/client.service';
 import { InputsService } from '../inputs/inputs.service';
 import { RequestUser } from 'src/decorators/user.decorator';
 import { TransformationsService } from '../transformations/transformations.service';
-import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
+import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
 
@@ -220,7 +220,7 @@ export class PipelinesService implements OnModuleInit {
     this.logger.info('PipelinesClientService - findOneProperties');
     const { pipeline } = await this.findOne({ id }, metadata);
 
-    const pipelineProperties = JSON.parse(pipeline.properties);
+    const properties = JSON.parse(pipeline.properties);
 
     const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
 
@@ -228,24 +228,38 @@ export class PipelinesService implements OnModuleInit {
       connectorId,
       metadata.get('language')[0].toString(),
     );
-    const connector = JSON.parse(res.connector);
-    const config_controls = connector.config_controls || [];
+    const connector: { [key: string]: any; config_controls: any[] } =
+      JSON.parse(res.connector);
+    const config_controls = (connector.config_controls || []).filter(
+      (config_control) => {
+        const path: string = config_control.name;
+        if (!objHasPath({ object: { properties }, path })) return false;
 
-    for (const config_control of config_controls) {
-      const path: string = config_control.name;
-      const default_value = getObjValueFromPath({
-        object: { properties: pipelineProperties },
-        path,
-      });
-      if (default_value != undefined)
-        config_control.default_value = default_value;
-    }
+        let default_value = getObjValueFromPath({
+          object: { properties },
+          path,
+        });
+
+        if (default_value != undefined) {
+          if (
+            path === 'properties.spreadsheet_id' &&
+            typeof default_value === 'string'
+          )
+            default_value = 'https://docs.google.com/spreadsheets/d/'.concat(
+              default_value,
+            );
+          config_control.default_value = default_value;
+        }
+
+        return true;
+      },
+    );
 
     return {
       pipeline: {
         ...pipeline,
         config_controls,
-        properties: pipelineProperties,
+        properties,
       },
     };
   }

src/modules/transformations/transformations.controller.ts

@@ -72,8 +72,7 @@ export class TransformationsController {
 
   @Get('/:id')
   @RequireSomePermission(pipelinePermissions.GET)
-  async findOne(@Body() data, @Param() params) {
-    const { id } = params;
+  async findOne(@Body() data, @Param('id') id: string) {
     this.logger.info(`/transformation/${id} - ON Find One ROUTE`, {
       user: data.info.user_id,
       customer: data.info.customer,

src/utils/ErrorBuilder.ts

@@ -308,6 +308,20 @@ export function EnrichErrorCode(code: string) {
           'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
         code,
       };
+    case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar prévia',
+        message: 'Ocorreu um erro devido ao tamanho da prévia do ativo.',
+        code,
+      };
+    case ErrorCodes.CATALOG.METADATA_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar informações',
+        message: 'Ocorreu um erro devido ao tamanho das informações do ativo.',
+        code,
+      };
     case ErrorCodes.INTERNAL:
     case ErrorCodes.UNKNOWN:
     default:

src/utils/ObjValueFromPath.ts

@@ -1,4 +1,7 @@
-export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
+export function getObjValueFromPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): any {
   const { object, path } = data;
   const keys = path.split('.');
 
@@ -8,3 +11,18 @@ export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
   }
   return value;
 }
+
+export function objHasPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): boolean {
+  const { object, path } = data;
+  const keys = path.split('.');
+
+  let value = JSON.parse(JSON.stringify(object));
+  for (const key of keys) {
+    if (!(key in value)) return false;
+    value = value[key];
+  }
+  return true;
+}

src/utils/errorCodes.ts

@@ -39,6 +39,7 @@ export const CUSTOMER = {
   INVALID_ID: 'CUSTOMER.INVALID_ID',
   NOT_FOUND: 'CUSTOMER.NOT_FOUND',
   ALREADY_EXISTS: 'CUSTOMER.ALREADY_EXISTS',
+  BAD_REQUEST: 'CUSTOMER.BAD_REQUEST',
 };
 const TERMS_OF_USE = {
   UP_TO_DATE: 'TERMS_OF_USE.UP_TO_DATE',
@@ -70,6 +71,8 @@ const TRANSFORMATION = {
 };
 const CATALOG = {
   DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
+  PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
+  METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
 };
 const ErrorCodes = {
   UNKNOWN: 'UNKNOWN',

src/utils/getEnv.ts

@@ -0,0 +1,10 @@
+const availableLocalEnvs: Array<typeof process.env.ENV> = ['stg', 'test'];
+
+export default function getEnv(): typeof process.env.ENV {
+  if (process.env.ENV === 'local') {
+    if (availableLocalEnvs.includes(process.env.LOCAL_ENV))
+      return process.env.LOCAL_ENV;
+    return 'stg';
+  }
+  return process.env.ENV;
+}