FEAT: update deployment to include firebase base url

Dockerfile

@@ -1,5 +1,5 @@
 FROM node:20-alpine AS base_image
-RUN npm install -g npm@10.8.2
+RUN npm install -g npm@latest
 
 FROM base_image AS build_base
 WORKDIR /app

docsfera.json

@@ -900,9 +900,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -952,9 +949,6 @@
             "connections"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -1003,9 +997,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -1042,9 +1033,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -1078,9 +1066,6 @@
             "connections"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2418,9 +2403,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2504,9 +2486,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2546,9 +2525,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2588,9 +2564,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2628,9 +2601,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2670,9 +2640,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2714,9 +2681,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -2758,9 +2722,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3110,9 +3071,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3162,9 +3120,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3235,9 +3190,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3278,9 +3230,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3321,9 +3270,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3364,9 +3310,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3400,16 +3343,20 @@
           ],
           "responses": {
             "200": {
-              "description": ""
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
             }
           },
           "tags": [
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3457,9 +3404,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3505,9 +3449,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3556,9 +3497,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3597,9 +3535,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3658,9 +3593,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3709,52 +3641,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
-      "/pipelinesV2/{id}/upgrade": {
-        "patch": {
-          "operationId": "PipelinesController_upgradeConnector",
-          "parameters": [
-            {
-              "name": "dadosfera-lang",
-              "in": "header",
-              "required": false,
-              "schema": {
-                "enum": [
-                  "pt-br",
-                  "en-us"
-                ],
-                "type": "string"
-              }
-            },
-            {
-              "name": "id",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "responses": {
-            "204": {
-              "description": ""
-            }
-          },
-          "tags": [
-            "PipelinesV2"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3797,9 +3683,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3849,9 +3732,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3901,9 +3781,6 @@
             "PipelinesV2"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -3950,6 +3827,85 @@
           "tags": [
             "PipelinesV2"
           ],
+          "security": [
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/pipelines/start/{id}": {
+        "post": {
+          "operationId": "PipelinesController_activate",
+          "summary": "",
+          "deprecated": true,
+          "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Pipelines"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/pipelines/{id}/status": {
+        "get": {
+          "operationId": "PipelinesController_getPipelineStatus",
+          "summary": "",
+          "deprecated": true,
+          "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Pipelines"
+          ],
           "security": [
             {
               "access-token": []
@@ -4124,9 +4080,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4152,9 +4105,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4191,9 +4141,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4228,9 +4175,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4265,9 +4209,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4295,9 +4236,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4325,9 +4263,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4355,9 +4290,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4394,9 +4326,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4433,9 +4362,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4463,9 +4389,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4493,9 +4416,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4533,9 +4453,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4572,9 +4489,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4619,9 +4533,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4658,9 +4569,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4705,9 +4613,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4744,9 +4649,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4781,9 +4683,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4820,9 +4719,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4859,9 +4755,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4899,9 +4792,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4929,9 +4819,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -4959,9 +4846,6 @@
             "Platform API"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5373,9 +5257,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5419,9 +5300,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5531,9 +5409,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5566,9 +5441,6 @@
             "Catalog"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5611,9 +5483,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5707,9 +5576,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5768,9 +5634,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5819,9 +5682,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5915,9 +5775,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -5968,9 +5825,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6029,9 +5883,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6085,72 +5936,6 @@
             "Catalog"
           ],
           "security": [
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            },
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
-      "/catalog/data-asset/{id}/certification-status": {
-        "put": {
-          "operationId": "CatalogController_updateDataAssetCertificationStatus",
-          "parameters": [
-            {
-              "name": "dadosfera-lang",
-              "in": "header",
-              "required": false,
-              "schema": {
-                "enum": [
-                  "pt-br",
-                  "en-us"
-                ],
-                "type": "string"
-              }
-            },
-            {
-              "name": "id",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "requestBody": {
-            "required": true,
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
-                }
-              }
-            }
-          },
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Catalog"
-          ],
-          "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             },
@@ -6194,9 +5979,6 @@
             "Catalog"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6237,9 +6019,6 @@
             "Catalog"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6300,9 +6079,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6361,9 +6137,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -6740,9 +6513,6 @@
             {
               "access-token": []
             },
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -7196,9 +6966,6 @@
             "Connection Test"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -7235,9 +7002,6 @@
             "Connection Test"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -7274,9 +7038,6 @@
             "Connection Test"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -7313,9 +7074,6 @@
             "Connection Test"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -7352,9 +7110,6 @@
             "Connection Test"
           ],
           "security": [
-            {
-              "access-token": []
-            },
             {
               "access-token": []
             }
@@ -11270,23 +11025,6 @@
             "data_asset"
           ]
         },
-        "IUpdateCertificationStatusRequest": {
-          "type": "object",
-          "properties": {
-            "certification_status": {
-              "type": "string",
-              "enum": [
-                "draft",
-                "in_review",
-                "approved",
-                "deprecated"
-              ]
-            }
-          },
-          "required": [
-            "certification_status"
-          ]
-        },
         "ICreateDataAsset": {
           "type": "object",
           "properties": {

package-lock.json

@@ -16,7 +16,8 @@
         "@aws-sdk/lib-dynamodb": "^3.414.0",
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "^3.40.0-beta.10",
+        "@dadosfera/protospack": "2.5.3",
+        "@dadosfera/protospack-v2": "3.40.0-beta.8",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1734,10 +1735,20 @@
         "winston-log2gelf": "^2.4.0"
       }
     },
+    "node_modules/@dadosfera/protospack": {
+      "version": "2.5.3",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
+      "integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
+      "license": "ISC",
+      "dependencies": {
+        "rxjs": "^7.5.5"
+      }
+    },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.40.0-beta.10",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
-      "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
+      "version": "3.40.0-beta.8",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz",
+      "integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -34,7 +34,8 @@
     "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "^3.40.0-beta.10",
+    "@dadosfera/protospack": "2.5.3",
+    "@dadosfera/protospack-v2": "3.40.0-beta.8",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/app.module.ts

@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
 import { NetworkConfigModule } from './modules/network-config/network-config.module';
 import { InputsModule } from './modules/inputs/inputs.module';
 import { OauthModule } from './modules/oauth/oauth.module';
+import { PipelinesModule } from './modules/pipelines/pipelines.module';
 import { TransformationsModule } from './modules/transformations/transformations.module';
 import { HealthModule } from './modules/health/health.module';
 import { CatalogModule } from './modules/catalog/catalog.module';
@@ -59,6 +60,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
     PermissionsModule,
     TermsOfUseModule,
     ConnectionTestModule,
+    PipelinesModule,
     TransformationsModule,
     UsersModule,
     RolesModule,

src/authentication/permissions.enum.ts

@@ -357,16 +357,6 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Crear y editar atributos en el catálogo',
         },
       },
-      CERTIFY: {
-        seqid: 53,
-        claim: 'catalog:certify',
-        usage: PermissionUsages.PUBLIC,
-        name: {
-          'pt-br': 'Alterar o status de certificação dos Ativos',
-          'en-us': "Change Assets' certification status",
-          'es-es': 'Cambiar el estado de certificación de los Activos',
-        },
-      },
       DELETE: {
         seqid: 1,
         claim: 'catalog:delete',
@@ -722,8 +712,6 @@ export const DADOSFERA_MODULES_KEYS = {
   PII: 'pii',
   EMBED: 'embedded-analytics',
   EMBED_ASSIGNED: 'embed-assigned',
-  CATALOG: 'catalog',
-  COLLECT: 'collect',
 }
 
 export const DADOSFERA_MODULES: Array<DadosferaModule> = [

src/modules/catalog/catalog.controller.ts

@@ -17,7 +17,6 @@ import {
   HttpStatus,
   Res,
 } from '@nestjs/common';
-import { ValidationPipe } from '../../pipes/object-validation.pipe';
 import {
   ApiCreatedResponse,
   ApiHeaders,
@@ -47,7 +46,6 @@ import {
   IMakeAComment,
   IOneDataAsset,
   IPreviewResponse,
-  IUpdateCertificationStatusRequest,
   IUpdateDataRequest,
   TriggerCatalogReq,
   TriggerCatalogRes,
@@ -85,9 +83,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async searchCatalog(
     @User() user: RequestUser,
     @Query() query: ICatalogAllRequest,
@@ -127,9 +122,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async dowloadAsserts(
     @User() user: RequestUser,
     @Query() query: ICatalogAllRequest,
@@ -173,9 +165,6 @@ export class CatalogController {
 
   @ApiInternalOnlyEndpoint()
   @Get('data-asset')
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
     const { pipeline, object } = query;
@@ -234,9 +223,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async findAllTags(@Body() body) {
     this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
       user: body.info.user_id,
@@ -288,9 +274,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async getDataAsset(
     @User() user: RequestUser,
     @Param('id') id: string,
@@ -402,9 +385,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -436,9 +416,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async getDataAssetPreview(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -470,9 +447,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async getDataAssetDocs(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -504,9 +478,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async updateDataAsset(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -522,8 +493,6 @@ export class CatalogController {
       language,
     });
 
-    delete (body as any).certification_status;
-
     const result = await this.catalogService.updateOneDataAsset({
       body,
       data_asset_id,
@@ -537,44 +506,11 @@ export class CatalogController {
     return result;
   }
 
-  @Put('data-asset/:id/certification-status')
-  @RequireSomePermission(
-    PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
-    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
-  )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
-  async updateDataAssetCertificationStatus(
-    @User() user: RequestUser,
-    @Language() language: LanguageEnum,
-    @Param('id') data_asset_id: string,
-    @Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
-  ): Promise<IUpdateCertificationStatusRequest> {
-    const { customer_id, customer_name, user_id, username } = user;
-    const metadata = PackTheMetadata({
-      customer_id,
-      customer_name,
-      user_id,
-      username,
-      language,
-    });
-
-    return this.catalogService.updateCertificationStatus({
-      body,
-      data_asset_id,
-      metadata,
-    });
-  }
-
   @Post('data-asset/:id/docs')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async manageDataAssetDocs(
     @User() user: RequestUser,
     @Headers() headers,
@@ -612,9 +548,6 @@ export class CatalogController {
 
   @ApiInternalOnlyEndpoint()
   @Put('data-asset/:id/manage-permissions')
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async manageDataAssetPermissions(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -637,9 +570,6 @@ export class CatalogController {
 
   @ApiInternalOnlyEndpoint()
   @Put('data-asset/:id/revoke-permissions')
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async revokeDataAssetPermissions(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -665,9 +595,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async createDataAsset(
     @User() user: RequestUser,
     @Body() body: ICreateDataAsset,
@@ -692,9 +619,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async commentOnDataAsset(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -721,9 +645,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
     const { customer_id, customer_name, user_id, username } = user;
     const metadata = PackTheMetadata({
@@ -745,9 +666,6 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async deleteComment(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -901,9 +819,6 @@ export class CatalogController {
 
   @Get('nimbus-dashboards')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.CATALOG
-  )
   async getNimbusDashboards(
     @User() user: RequestUser,
     @Body() body: GetNimbusDashboardsRequest,

src/modules/catalog/catalog.module.ts

@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { CatalogController } from './catalog.controller';
 import { CatalogClientConfiguration } from './catalog-client';
 import { ClientsModule } from '@nestjs/microservices';
+import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { UsersModule } from '../users/users.module';
 import { RolesModule } from '../roles/roles.module';
 import { CustomersModule } from '../customers/customers.module';
 import { ShareModule } from './share/share.module';
 import { CatalogService } from './catalog.service';
+import { MixpanelModule } from '../mixpanel/mixpanel.module';
 
 const client = new CatalogClientConfiguration();
 
 @Module({
   imports: [
     ClientsModule.register([client.providerOptions]),
+    OldPipelineModule,
     UsersModule,
     RolesModule,
     CustomersModule,

src/modules/catalog/catalog.service.ts

@@ -29,7 +29,6 @@ import {
   AssetReporter,
   BatchRemoveRlsRulesRequest,
   CreateDataDocsDTO,
-  IUpdateCertificationStatusRequest,
   IUpdateDataRequest,
   TriggerCatalogReq,
 } from './dtos';
@@ -385,28 +384,6 @@ class CatalogService implements OnModuleInit {
     return { data_asset: asset[0] };
   }
 
-  async updateCertificationStatus(data: {
-    data_asset_id: string;
-    body: IUpdateCertificationStatusRequest;
-    metadata: Metadata;
-  }) {
-    const { body, data_asset_id, metadata } = data;
-
-    await lastValueFrom(
-      this.catalogWriteService.UpdateDataAsset(
-        {
-          id: data_asset_id,
-          changes: JSON.stringify({
-            certification_status: body.certification_status,
-          }),
-        },
-        metadata,
-      ),
-    );
-
-    return { certification_status: body.certification_status };
-  }
-
   async updateOneDataAsset(data: {
     data_asset_id: string;
     customer_id: string;

src/modules/catalog/dtos/index.ts

@@ -1,5 +1,4 @@
 import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
-import { IsEnum } from 'class-validator';
 import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 export enum DataAssetShareType {
@@ -7,12 +6,6 @@ export enum DataAssetShareType {
   public = 'public',
   private = 'private',
 }
-export enum CertificationStatus {
-  draft = 'draft',
-  in_review = 'in_review',
-  approved = 'approved',
-  deprecated = 'deprecated',
-}
 export enum OrderEnum {
   asc = 'asc',
   desc = 'desc',
@@ -212,13 +205,6 @@ export class IUpdateDataRequest {
   @ApiPropertyOptional()
   docs?: string;
 }
-
-export class IUpdateCertificationStatusRequest {
-  @ApiProperty({ enum: CertificationStatus })
-  @IsEnum(CertificationStatus)
-  certification_status: CertificationStatus;
-}
-
 export class ICreateDataAsset implements CreateDataAssetRequest {
   @ApiProperty()
   display_name: string;

src/modules/connection-test/connection-test.controller.ts

@@ -24,19 +24,15 @@ import {
   GetTableMetadataReq,
 } from './dto/connection-test';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
+import { Authenticated } from 'src/decorators/authentication.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
-import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
 
 @ApiInternalOnlyController()
 @ApiTags('Connection Test')
 @Controller('connection-test')
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Authenticated()
-@RequireModule(
-  DADOSFERA_MODULES_KEYS.COLLECT
-)
 export class ConnectionTestController {
   logger: any;
   constructor(

src/modules/connection/connection.controller.ts

@@ -16,9 +16,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
-  RequireModule,
 } from 'src/decorators/authentication.decorator';
-import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { ValidationPipe } from '../../pipes/object-validation.pipe';
 import {
@@ -40,9 +39,6 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
 @ApiTags('connections')
 @Authenticated()
 @Controller('connections')
-@RequireModule(
-  DADOSFERA_MODULES_KEYS.COLLECT
-)
 export class ConnectionController {
   logger: any;
   constructor(

src/modules/connector/connector.controller.ts

@@ -25,10 +25,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
-  RequireModule,
   RequireSomePermission,
 } from 'src/decorators/authentication.decorator';
-import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -100,9 +99,6 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async getAllConnectors(
     @Language() language: LanguageEnum,
     @Query() queries: GetAllDto,
@@ -135,9 +131,6 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async getConnectorsTags() {
     return await this.connectorClientService.getConnectorsTags();
   }
@@ -150,9 +143,6 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async getConnector(
     @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
@@ -181,9 +171,6 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async getConnectorDetails(
     @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
@@ -206,9 +193,6 @@ export class ConnectorController {
   @Put('/:plugin')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
   @ApiConsumes('multipart/form-data')
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async updateConnector(
     @Param('plugin') plugin: string,
     @Body() body: UpdateDto,
@@ -230,9 +214,6 @@ export class ConnectorController {
 
   @Put('/:plugin/add-tag')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async addTagOnConnector(
     @Param('plugin') plugin: string,
     @Body() body: AddTagDto,
@@ -260,9 +241,6 @@ export class ConnectorController {
 
   @Put('/:plugin/remove-tag')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async removeTagOnConnector(
     @Param('plugin') plugin: string,
     @Body() body: RemoveTagDto,
@@ -291,9 +269,6 @@ export class ConnectorController {
 
   @Delete('/:plugin')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
-  @RequireModule(
-    DADOSFERA_MODULES_KEYS.COLLECT
-  )
   async deleteConnector(
     @Param('plugin') plugin: string,
     @Query('version') version: string,

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,8 +1,4 @@
-export interface Info {
-  user_id: string;
-  customer_id: string;
-  customer: string;
-}
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
 interface Values {
   jdbc_user: string;

src/modules/inputs/inputs.service.ts

@@ -23,7 +23,6 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 import { CreateInputReq } from './dtos/input.model';
-import { Metadata } from '@grpc/grpc-js';
 
 @Injectable()
 
@@ -74,10 +73,10 @@ export class InputsService {
       objectCamelToSnake(createInputResponse);
       return createInputResponse;
     },
-    update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
+    update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
       this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
       const updateInputResponse = await lastValueFrom(
-        this.inputWriteService.InputUpdate(updateInputDTO, metadata),
+        this.inputWriteService.InputUpdate(updateInputDTO),
       );
 
       return updateInputResponse;
@@ -207,7 +206,7 @@ export class InputsService {
     return findOneInputResponse;
   }
 
-  async update(id: string, data, info: Info, metadata?: Metadata) {
+  async update(id: string, data, info: Info) {
     // this.validateCron({ ...data, info });
     try {
       const {
@@ -218,7 +217,7 @@ export class InputsService {
         id,
         ...data,
         info,
-      }, metadata);
+      });
 
       const updateInputResponse = this.adjustInputPayload(
         input,

src/modules/pipelines/client.service.ts

@@ -0,0 +1,78 @@
+import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import {
+  PipelineServicesNames,
+  PipelinesServiceInterface,
+} from '@dadosfera/protospack';
+import { lastValueFrom } from 'rxjs';
+
+import { IIdRequest } from './interfaces';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { PipelinesClientConfiguration } from './pipelines-client';
+
+export class PipelinesClientService implements OnModuleInit {
+  private pipelineService: PipelinesServiceInterface;
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+      dadosferaLogger: DadosferaLogger,
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.pipelineService =
+      this.grpcClient.getService<PipelinesServiceInterface>(
+        PipelineServicesNames.PipelineService,
+      );
+  }
+
+  async getPipelineStatus(data) {
+    this.logger.info('PipelinesClientService - GetPipelineStatus');
+
+    const statusPipelineResponse = await lastValueFrom(
+      this.pipelineService.getPipelineStatus(data),
+    )
+      .then((res) => {
+        const statusArray =
+          res.status?.sort((a, b) => {
+            if (a.id < b.id) {
+              return 1;
+            } else {
+              return -1;
+            }
+          }) || [];
+        return { status: statusArray };
+      })
+      .catch((err) => {
+        this.logger.error(err.message);
+        throw new Error(err);
+      });
+    this.logger.info('Done');
+
+    return statusPipelineResponse;
+  }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    this.logger.info('PipelinesClientService - RunPipeline');
+    const statusPipelineResponse = await lastValueFrom(
+      this.pipelineService.triggerPipeline({ id, info }),
+    ).catch((err) => {
+      this.logger.error(err.message);
+      throw new Error(err);
+    });
+
+    if (statusPipelineResponse.status == false) {
+      throw new ConflictException(
+        'This pipeline is not ready yet to execute, Try again later!',
+      );
+    }
+
+    this.logger.info('Done');
+    return statusPipelineResponse;
+  }
+}

src/modules/pipelines/interfaces.d.ts

@@ -0,0 +1,36 @@
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+
+export interface ICreatePipelineDto {
+  input: IdRequest;
+  transformations: IdRequest[];
+  output: IdRequest;
+  tags: string[];
+  name: string;
+  description: string;
+  info: Info;
+}
+
+export interface IdRequest {
+  id: string;
+}
+
+export interface IIdRequest {
+  id: string;
+  info: Info;
+}
+
+export interface IUpdatePipelineRequest {
+  input: IdRequest;
+  transformations: IdRequest[];
+  output: IdRequest;
+  tags: string[];
+  name: string;
+  description: string;
+  id: string;
+  info: Info;
+}
+
+export interface IGetPipelineLogsRequest {
+  id: string;
+  details: string;
+}

src/modules/pipelines/pipelines-client.ts

@@ -0,0 +1,33 @@
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
+import { credentials } from '@grpc/grpc-js';
+
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  process.env.PIFACTORY_URL.includes('0.0.0.0');
+
+export class PipelinesClientConfiguration {
+  public name = 'PipelinesClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.PIFACTORY_URL,
+      package: PipelinePackages,
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: PipelineProtoFilePath,
+      loader: {
+        keepCase: true,
+        enums: String,
+        defaults: false,
+      },
+    },
+  };
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/pipelines/pipelines.controller.ts

@@ -0,0 +1,72 @@
+import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
+import { ApiOperation, ApiTags } from '@nestjs/swagger';
+import {
+  AuthenticateCondition,
+  Authenticated,
+  RequireSomePermission,
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { PipelinesService } from './pipelines.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+
+@ApiInternalOnlyController()
+@ApiTags('Pipelines')
+@Controller('pipelines')
+@Authenticated()
+export class PipelinesController {
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private pipelineService: PipelinesService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post('start/:id')
+  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  @ApiOperation({
+    deprecated: true,
+    description:
+      'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
+  })
+  async activate(@Param('id') id: string, @Body() body) {
+    const { info } = body;
+
+    this.logger.info(
+      process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
+      {
+        user: body.info.user_id,
+        customer: body.info.customer,
+      },
+    );
+
+    const response = await this.pipelineService.runPipeline({ id, info });
+
+    return response;
+  }
+
+  @Get(':id/status')
+  @ApiOperation({
+    deprecated: true,
+    description:
+      'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
+  })
+  @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
+    body.id = id;
+
+    this.logger.info(
+      process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
+      {
+        user: body.info.user_id,
+        customer: body.info.customer,
+      },
+    );
+
+    const response = await this.pipelineService.getPipelineStatus(body);
+
+    return response;
+  }
+}

src/modules/pipelines/pipelines.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { PipelinesController } from './pipelines.controller';
+import { PipelinesService } from './pipelines.service';
+
+import { PipelinesClientConfiguration } from './pipelines-client';
+import { PipelinesClientService } from './client.service';
+
+const client = new PipelinesClientConfiguration();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [PipelinesController],
+  providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
+  exports: [PipelinesService],
+})
+export class PipelinesModule {}

src/modules/pipelines/pipelines.service.ts

@@ -0,0 +1,33 @@
+import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import { PipelinesClientService } from './client.service';
+import { IIdRequest } from './interfaces';
+import { objectCamelToSnake } from 'src/utils/CaseConverter';
+
+@Injectable()
+export class PipelinesService {
+  constructor(private pipelineClient: PipelinesClientService) {}
+
+  async getPipelineStatus(data: IIdRequest) {
+    try {
+      const pipelineStatusResponse =
+        await this.pipelineClient.getPipelineStatus(data);
+
+      return objectCamelToSnake(pipelineStatusResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    try {
+      const triggerPipelineResponse = await this.pipelineClient.runPipeline({
+        id,
+        info,
+      });
+
+      return objectCamelToSnake(triggerPipelineResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
+}

src/modules/pipelinesV2/interfaces.ts

@@ -1,4 +1,5 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
 export class PipelineInputsDTO {
   @ApiProperty()
@@ -61,12 +62,6 @@ export interface IIdRequest {
   info: Info;
 }
 
-export interface Info {
-  user_id: string;
-  customer_id: string;
-  customer: string;
-}
-
 export interface IUpdatePipelineRequest {
   input: IdRequest;
   transformations: IdRequest[];

src/modules/pipelinesV2/pipelines.controller.ts

@@ -25,16 +25,16 @@ import {
 } from '@nestjs/swagger';
 import {
   RequireAllPermissions,
-  RequireModule,
   RequireSomePermission,
 } from 'src/decorators/authentication.decorator';
-import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 
+import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import {
   ICompleteUploadCSVFile,
   ICreatePipelineCSVFile,
@@ -58,15 +58,14 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('pipelinesV2')
-@RequireModule(
-  DADOSFERA_MODULES_KEYS.COLLECT
-)
 export class PipelinesController {
   logger: DadosferaLogger;
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
+
     private pipelinesClientService: PipelinesService,
+    private oldPipelinesService: OldPipelineService,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -202,7 +201,7 @@ export class PipelinesController {
       customer: body.info.customer,
     });
 
-    const response = await this.pipelinesClientService.getPipelineStatus(body);
+    const response = await this.oldPipelinesService.getPipelineStatus(body);
 
     return response;
   }
@@ -321,6 +320,7 @@ export class PipelinesController {
   ) {
     this.logger.info('PipelinesController - update', { user });
 
+    const { customer_id, customer_name, user_id, username } = user;
     const info: Info = {
       user_id: user.user_id,
       customer: user.customer_name,
@@ -328,7 +328,13 @@ export class PipelinesController {
       pipeline_id: pipelineId
     };
 
-    const metadata = PackTheMetadata(user);
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
 
     const response = await this.pipelinesClientService.updatePipelineInput(
       pipelineId,
@@ -362,21 +368,6 @@ export class PipelinesController {
     return response;
   }
 
-  @Patch('/:id/upgrade')
-  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
-  @HttpCode(HttpStatus.NO_CONTENT)
-  async upgradeConnector(
-    @Language() language: LanguageEnum,
-    @Param('id') id: string,
-    @User() user: RequestUser
-  ) {
-    this.logger.info('PipelinesController - upgrade connector');
-
-    const metadata = PackTheMetadata(user);
-
-    await this.pipelinesClientService.upgrade(id, metadata);
-  }
-
   @Delete(':id')
   @ApiNoContentResponse()
   @HttpCode(HttpStatus.NO_CONTENT)
@@ -506,7 +497,7 @@ export class PipelinesController {
       },
     );
 
-    const response = await this.pipelinesClientService.runPipeline({ id, info });
+    const response = await this.oldPipelinesService.runPipeline({ id, info });
 
     return response;
   }

src/modules/pipelinesV2/pipelines.module.ts

@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
 
 import { PipelinesClientConfiguration } from './pipelines-client';
 
+import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
@@ -20,6 +21,7 @@ const client = new PipelinesClientConfiguration();
 @Module({
   imports: [
     ClientsModule.register([client.providerOptions]),
+    OldPipelineModule,
     ConnectorModule,
     InputsModule,
     TransformationsModule,

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,7 +1,6 @@
 /* eslint-disable no-async-promise-executor */
 import {
   BadRequestException,
-  ConflictException,
   HttpException,
   HttpStatus,
   Inject,
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
 
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PipelinesClientConfiguration } from './pipelines-client';
-import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
+import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
@@ -177,15 +176,6 @@ export class PipelinesService implements OnModuleInit {
     return updatePipelineResponse;
   }
 
-  async upgrade(id: string, metadata: Metadata) {
-    await lastValueFrom(
-      this.pipelineWriteService.Upgrade(
-        { id },
-        metadata,
-      ),
-    );
-  }
-
   async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
     const { id, metadata, user } = data;
     const info = {
@@ -383,8 +373,7 @@ export class PipelinesService implements OnModuleInit {
     const updateInputResponse = await this.inputsService.update(
       inputId,
       updateInputDTO,
-      info,
-      metadata
+      info
     );
 
     const inputRollback = () => {
@@ -405,36 +394,34 @@ export class PipelinesService implements OnModuleInit {
 
     const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
 
-    if (user.customer_modules.includes('catalog')) {
-      nimbusUpdates.forEach(update => {
-        const nimbusRollback = () => {
-          return this.nimbusService.renameTable(
-            info.customer,
-            update.database,
-            {
-              table_name: update.table_name,
-              table_schema: update.table_schema
-            },
-            {
-              table_name: update.old_table_name,
-              table_schema: update.old_table_schema
-            }
-          );
-        }
-        rollback.push(nimbusRollback);
-      });
-  
-      try {
-        await this.updateNimbus(info.customer, nimbusUpdates);
-      } catch (error) {
-        this.logger.error(error);
-        if (error instanceof AxiosError) {
-          this.logger.error(JSON.stringify(error.response.data));
-        }
-        await this.executeRenameRollback(rollback);
-  
-        throw new Error("Error Nimbus updating tables");
+    nimbusUpdates.forEach(update => {
+      const nimbusRollback = () => {
+        return this.nimbusService.renameTable(
+          info.customer,
+          update.database,
+          {
+            table_name: update.table_name,
+            table_schema: update.table_schema
+          },
+          {
+            table_name: update.old_table_name,
+            table_schema: update.old_table_schema
+          }
+        );
       }
+      rollback.push(nimbusRollback);
+    });
+
+    try {
+      await this.updateNimbus(info.customer, nimbusUpdates);
+    } catch (error) {
+      this.logger.error(error);
+      if (error instanceof AxiosError) {
+        this.logger.error(JSON.stringify(error.response.data));
+      }
+      await this.executeRenameRollback(rollback);
+
+      throw new Error("Error Nimbus updating tables");
     }
 
     try {
@@ -636,49 +623,4 @@ export class PipelinesService implements OnModuleInit {
     return assets;
   }
 
-  async getPipelineStatus(data) {
-    this.logger.info('PipelinesClientService - GetPipelineStatus');
-
-    const statusPipelineResponse = await lastValueFrom(
-      this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
-    )
-      .then((res) => {
-        const statusArray =
-          res.status?.sort((a, b) => {
-            if (a.id < b.id) {
-              return 1;
-            } else {
-              return -1;
-            }
-          }) || [];
-        return { status: statusArray };
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-    this.logger.info('Done');
-
-    return statusPipelineResponse;
-  }
-
-  async runPipeline({ id, info }: IIdRequest) {
-    this.logger.info('PipelinesClientService - RunPipeline');
-    const statusPipelineResponse = await lastValueFrom(
-      this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
-    ).catch((err) => {
-      this.logger.error(err.message);
-      throw new Error(err);
-    });
-
-    if (statusPipelineResponse.status == false) {
-      throw new ConflictException(
-        'This pipeline is not ready yet to execute, Try again later!',
-      );
-    }
-
-    this.logger.info('Done');
-    return statusPipelineResponse;
-  }
-
 }

src/modules/platform-api/platform-api.controller.ts

@@ -20,11 +20,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
-  RequireModule,
 } from '../../decorators/authentication.decorator';
 import { User, RequestUser } from '../../decorators/user.decorator';
 import { PlatformApiService } from './platform-api.service';
-import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
@@ -50,7 +49,6 @@ type RenameTablesBody = {
 
 @ApiTags('Platform API')
 @Controller('platform')
-@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
 export class PlatformApiController {
   private logger: any;
 

src/modules/release_note/release_note.controller.spec.ts

@@ -1,7 +1,6 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteController } from './release_note.controller';
 import { ReleaseNoteService } from './release_note.service';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteController', () => {
   let controller: ReleaseNoteController;
@@ -9,7 +8,7 @@ describe('ReleaseNoteController', () => {
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
       controllers: [ReleaseNoteController],
-      providers: [ReleaseNoteService, DadosferaLogger],
+      providers: [ReleaseNoteService],
     }).compile();
 
     controller = module.get<ReleaseNoteController>(ReleaseNoteController);

src/modules/release_note/release_note.service.spec.ts

@@ -1,13 +1,12 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteService } from './release_note.service';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteService', () => {
   let service: ReleaseNoteService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [ReleaseNoteService, DadosferaLogger],
+      providers: [ReleaseNoteService],
     }).compile();
 
     service = module.get<ReleaseNoteService>(ReleaseNoteService);

src/modules/transformations/interfaces.d.ts

@@ -1,8 +1,5 @@
-export interface Info {
-  user_id: string;
-  customer_id: string;
-  customer: string;
-}
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+
 export interface ICreateTransformationsRequest {
   transformations: Transformation[];
   info: Info;

src/utils/PackTheMetadata.ts

@@ -9,7 +9,6 @@ interface IMetadata {
   details?: string;
   sensitive?: string;
   roles?: string[];
-  customer_modules?: string[];
   is_data_manager?: boolean;
   access_token?: string;
   host?: string;