fix(auth): environment-scoped session cookies and resilient /auth/me refresh

Apps behind an nginx auth_request (Orchest / Intelligence / Process) rely on
the ddf-* cookies and on GET /auth/me. Several problems made them lose the
session or show 500 pages:

- Cookies were written with a hard-coded Domain=.dadosfera.ai in every
  environment, so a staging login overwrote the production session (and
  vice versa) in the same browser. Names now take AUTH_COOKIE_SUFFIX and the
  domain comes from COOKIE_DOMAIN (staging: -stg / .stg.dadosfera.ai).
- maxAge was the token's absolute epoch (~56 years). It is now a real
  duration, SESSION_COOKIE_MAX_AGE_DAYS (default 30).
- /auth/me answered 500 when a refresh failed for anything but
  NotAuthorizedException, when the refreshed token failed validation, or
  when DUC hung (no deadline). It now answers 401 and expires the dead
  cookies for definitive failures, 503 for DUC transport errors/timeouts
  (AUTH_RPC_TIMEOUT_MS, default 10s), keeps 403 answers instead of masking
  them with a refresh, refreshes when only the refresh cookie is left, and
  stops logging request headers (they contain the cookies).
- /auth/refresh-access-token rewrites all three cookies from the session the
  frontend just proved valid, healing stale cookies from another login or
  environment. /auth/sign-out expires all three and always responds.

Helm values wire the new variables for prd and stg.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AUEXmydTBX9G2CGJ97YQKL

.dockerignore

@@ -0,0 +1,12 @@
+node_modules
+dist
+.git
+*.log
+npm-debug.log*
+.DS_Store
+.env
+.env.*
+coverage
+.nyc_output
+*.tgz
+!protospack.tgz

.github/workflows/validate-k8s.yml

@@ -71,6 +71,11 @@ jobs:
           sudo mv helmfile /usr/local/bin/
           helmfile --version
 
+      - name: Install Helm Diff plugin
+        run: |
+          helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
+          helm diff version
+
       - name: Debug Helm env
         run: |
           helm env
@@ -102,4 +107,5 @@ jobs:
       - name: Run Helmfile Diff
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
+          HELM_PLUGINS: /home/runner/.local/share/helm/plugins
         run: helmfile -f deploy/helmfiles/${ENV}.yaml diff

Dockerfile

@@ -1,4 +1,5 @@
-FROM node:18.17-alpine AS base_image
+FROM node:20-alpine AS base_image
+RUN npm install -g npm@10.8.2
 
 FROM base_image AS build_base
 WORKDIR /app
@@ -21,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
 # run aws cli without mounting secret, because CI already has AWS credentials
 FROM build_base AS ci_image
 RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
-RUN npm ci
+RUN npm ci --ignore-scripts
 COPY . .
 
 
@@ -36,7 +37,7 @@ FROM build_base AS dev
 RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
   aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
 # flag --build-from-source is required to force-build sqlite3
-RUN npm ci
+RUN npm ci --ignore-scripts
 COPY . .
 ENTRYPOINT npm run start:dev
 

Dockerfile.local

@@ -0,0 +1,47 @@
+FROM node:22-alpine AS base_image
+RUN npm install -g npm@latest
+
+FROM base_image AS build_base
+WORKDIR /app
+RUN apk update
+RUN apk add --no-cache \
+      aws-cli \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont
+COPY package*.json ./
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
+
+# Local build with secrets
+FROM build_base AS build
+RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
+  aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+RUN  npm ci --ignore-scripts
+COPY . .
+RUN npm run build
+
+
+FROM base_image
+WORKDIR /app
+COPY --from=build /app/dist ./dist
+COPY --from=build /app/node_modules ./node_modules
+COPY --from=build /app/package*.json ./
+RUN apk update
+RUN apk add --no-cache \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
+ENTRYPOINT ["npm", "run", "start:prod"]

README.md

@@ -4,6 +4,7 @@
 
 # Maestro
 
+
 Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
 
 ```mermaid

deploy/helm-chart/templates/deployment.yaml

@@ -48,6 +48,9 @@ spec:
             {{- toYaml .Values.resources | nindent 12 }}
           {{- end }}
           env:
+            # Auth Provider Configuration (cognito or keycloak)
+            - name: AUTH_PROVIDER
+              value: {{ .Values.maestro.auth_provider | default "cognito" | quote }}
             - name: AWS_IDENTITY_POOL_ID
               value: {{ .Values.maestro.aws_identity_pool_id }}
             - name: AWS_REGION
@@ -98,6 +101,17 @@ spec:
               value: {{ .Values.maestro.dedicated_proxy }}
             - name: COOKIE_SECRET
               value: {{ .Values.maestro.cookie_secret }}
+            # Browser session cookies (ddf-auth*, ddf-refresh-auth*, ddf-user-id*).
+            # Scoped per environment so a staging login never overwrites a
+            # production session in the same browser (both live under dadosfera.ai).
+            - name: COOKIE_DOMAIN
+              value: {{ .Values.maestro.cookie_domain | default ".dadosfera.ai" | quote }}
+            - name: AUTH_COOKIE_SUFFIX
+              value: {{ .Values.maestro.auth_cookie_suffix | default "" | quote }}
+            - name: SESSION_COOKIE_MAX_AGE_DAYS
+              value: {{ .Values.maestro.session_cookie_max_age_days | default "30" | quote }}
+            - name: AUTH_RPC_TIMEOUT_MS
+              value: {{ .Values.maestro.auth_rpc_timeout_ms | default "10000" | quote }}
             - name: REDIS_DATABASE
               value: "{{ .Values.maestro.redis_database }}"
             - name: REDIS_HOST
@@ -108,6 +122,12 @@ spec:
               value: "{{ .Values.maestro.redis_tls }}"
             - name: PLATFORM_API_URL
               value: {{ .Values.maestro.platform_api_url }}
+            - name: CONNECTIONS_API_URL
+              value: {{ .Values.maestro.connections_api_url | default "" | quote }}
+            - name: STORAGE_EXPLORER_API_URL
+              value: {{ .Values.maestro.storage_explorer_api_url | quote }}
+            - name: FIREBASE_BASE_URL
+              value: {{ .Values.maestro.firebase_base_url }}
             - name: JWT_PRIVATE_KEY
               valueFrom:
                 secretKeyRef:

deploy/helm-chart/values-stg.yaml

@@ -7,8 +7,17 @@ maestro:
   open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
   open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
   cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
+  # Session cookies: staging gets its own names AND domain so that
+  # app.stg.dadosfera.ai and app.dadosfera.ai sessions never collide.
+  cookie_domain: ".stg.dadosfera.ai"
+  auth_cookie_suffix: "-stg"
+  session_cookie_max_age_days: "30"
+  auth_rpc_timeout_ms: "10000"
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
+  connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com
+  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
+  firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
 
 hostname: maestro.stg.dadosfera.ai
 

deploy/helm-chart/values.yaml

@@ -27,6 +27,9 @@ resources:
     cpu: 2000m
     memory: 2Gi
 maestro:
+  # Auth provider: "cognito" (default) or "keycloak"
+  # Note: maestro doesn't connect to Keycloak directly, only duc does
+  auth_provider: "cognito"
   aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
   duc_url: duc.dadosfera.ai
   in_factory_url: in-factory.dadosfera.ai
@@ -44,6 +47,7 @@ maestro:
   open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
   open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
   platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
+  storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
   dedicated_proxy: ""
   restricted_ip: ""
   redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
@@ -51,6 +55,14 @@ maestro:
   redis_database: "0"
   redis_tls: "true"
   cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
+  # Session cookies: production keeps the historical names on the apex domain.
+  cookie_domain: ".dadosfera.ai"
+  auth_cookie_suffix: ""
+  session_cookie_max_age_days: "30"
+  auth_rpc_timeout_ms: "10000"
+  firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
+  connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com
+
 autoscaling:
   enabled: false
   minReplicas: 1

jest.config.ts

@@ -11,6 +11,7 @@ const config: Config.InitialOptions = {
     '<rootDir>/node_modules/',
     '.*\\.module\\.[jt]s$',
   ],
+  setupFiles: ['<rootDir>/jest.setup.ts'],
   // moduleDirectories: ['node_modules'], // default is already 'node_modules'
   // rootDir: '.', //No need
   // testEnvironment: 'node', //Defaults to 'node'

jest.setup.ts

@@ -0,0 +1,3 @@
+process.env.DUC_URL="duc:50051"
+process.env.INFACTORY_URL="in-factory:50052"
+process.env.PIFACTORY_URL="pi-factory:50053"

package.json

@@ -10,7 +10,7 @@
   },
   "scripts": {
     "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
-    "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
+    "proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
     "prebuild": "rimraf dist",
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
@@ -30,12 +30,11 @@
     "@aws-crypto/sha256-js": "^5.2.0",
     "@aws-sdk/client-dynamodb": "^3.414.0",
     "@aws-sdk/client-secrets-manager": "^3.414.0",
-    "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/credential-provider-node": "^3.940.0",
+    "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.38.0-beta.16",
+    "@dadosfera/protospack-v2": "3.40.0",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",
@@ -49,7 +48,7 @@
     "@nestjs/schematics": "^9.2.0",
     "@nestjs/swagger": "^6.3.0",
     "@nestjs/testing": "^9.4.3",
-    "axios": "^0.27.2",
+    "axios": "0.30.3",
     "cache-manager": "^5.1.4",
     "cache-manager-ioredis-yet": "^1.1.0",
     "class-transformer": "^0.5.1",
@@ -65,6 +64,7 @@
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
     "ms": "^3.0.0-canary.1",
+    "multer": "^2.0.2",
     "openid-client": "^5.7.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
@@ -80,7 +80,13 @@
     "swagger-ui-express": "^4.6.3"
   },
   "overrides": {
-    "multer": "1.4.5-lts.1"
+    "axios": "0.30.3",
+    "form-data": "^4.0.4",
+    "body-parser": "^1.20.3",
+    "cross-spawn": "^7.0.5",
+    "glob": "^10.5.0",
+    "path-to-regexp": "^3.3.0",
+    "semver": "^7.5.2"
   },
   "devDependencies": {
     "@types/cache-manager": "^4.0.6",
@@ -111,5 +117,8 @@
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^3.14.2",
     "typescript": "^4.9.5"
+  },
+  "resolutions": {
+    "axios": "0.30.3"
   }
 }

src/app.module.ts

@@ -17,7 +17,6 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
 import { NetworkConfigModule } from './modules/network-config/network-config.module';
 import { InputsModule } from './modules/inputs/inputs.module';
 import { OauthModule } from './modules/oauth/oauth.module';
-import { PipelinesModule } from './modules/pipelines/pipelines.module';
 import { TransformationsModule } from './modules/transformations/transformations.module';
 import { HealthModule } from './modules/health/health.module';
 import { CatalogModule } from './modules/catalog/catalog.module';
@@ -34,6 +33,9 @@ import { AssignModule } from './modules/assign/assign.module';
 import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
 import { ApiKeyModule } from './modules/api-key/api-key.module';
 import { PlatformApiModule } from './modules/platform-api/platform-api.module';
+import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
+import { ReleaseNoteModule } from './modules/release_note/release_note.module';
+
 
 @Module({
   providers: [
@@ -57,7 +59,6 @@ import { PlatformApiModule } from './modules/platform-api/platform-api.module';
     PermissionsModule,
     TermsOfUseModule,
     ConnectionTestModule,
-    PipelinesModule,
     TransformationsModule,
     UsersModule,
     RolesModule,
@@ -75,8 +76,10 @@ import { PlatformApiModule } from './modules/platform-api/platform-api.module';
     IdentityProviderModule,
     NetworkPolicyModule,
     PlatformApiModule,
+    StorageExplorerModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
+    ReleaseNoteModule,
   ],
 })
 export class AppModule {}

src/authentication/authentication.guard.ts

@@ -153,6 +153,7 @@ export class AuthenticationGuard
       user_id: accessTokenPayload.user_id,
       username: accessTokenPayload.username,
       permissions: accessTokenPayload.permissions,
+      roles: accessTokenPayload.roles,
       customer_id: accessTokenPayload.customer_id,
       customer_name: accessTokenPayload.customer_name,
       customer_tier: accessTokenPayload.customer_tier,

src/authentication/extract-user.ts

@@ -11,6 +11,7 @@ export function extractUserFrom(aRawJwt: string) {
     user_id: payload.user_id,
     username: payload.username,
     permissions: payload.permissions,
+    roles: payload.roles,
     customer_id: payload.customer_id,
     customer_name: payload.customer_name,
     customer_tier: payload.customer_tier,

src/authentication/permissions.enum.ts

@@ -357,6 +357,16 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Crear y editar atributos en el catálogo',
         },
       },
+      CERTIFY: {
+        seqid: 53,
+        claim: 'catalog:certify',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Alterar o status de certificação dos Ativos',
+          'en-us': "Change Assets' certification status",
+          'es-es': 'Cambiar el estado de certificación de los Activos',
+        },
+      },
       DELETE: {
         seqid: 1,
         claim: 'catalog:delete',
@@ -668,6 +678,35 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
+  STORAGE_EXPLORER: {
+    title: {
+      'pt-br': 'Storage Explorer',
+      'en-us': 'Storage Explorer',
+      'es-es': 'Storage Explorer',
+    },
+    permissions: {
+      READ: {
+        seqid: 51,
+        claim: 'storage-explorer:read',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Ler dados do Storage Explorer',
+          'en-us': 'Read Storage Explorer data',
+          'es-es': 'Leer datos del Storage Explorer',
+        },
+      },
+      WRITE: {
+        seqid: 52,
+        claim: 'storage-explorer:write',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Escrever dados no Storage Explorer',
+          'en-us': 'Write Storage Explorer data',
+          'es-es': 'Escribir datos en Storage Explorer',
+        },
+      },
+    },
+  },
 };
 export interface DadosferaModule {
   name: string;
@@ -683,6 +722,8 @@ export const DADOSFERA_MODULES_KEYS = {
   PII: 'pii',
   EMBED: 'embedded-analytics',
   EMBED_ASSIGNED: 'embed-assigned',
+  CATALOG: 'catalog',
+  COLLECT: 'collect',
 }
 
 export const DADOSFERA_MODULES: Array<DadosferaModule> = [

src/decorators/user.decorator.ts

@@ -12,6 +12,7 @@ export interface RequestUser {
   customer_tier: string;
   access_token: string;
   customer_modules: string[];
+  roles: string[];
 }
 
 export const User: (options?: { required?: boolean }) => ParameterDecorator =

src/guards/pipeline-execution.guard.ts

@@ -0,0 +1,65 @@
+import {
+  BadRequestException,
+  CanActivate,
+  ExecutionContext,
+  Inject,
+  Injectable,
+  OnModuleInit,
+} from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { map, Observable } from 'rxjs';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import {
+  ReadService,
+  ProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
+import { PipelinesClientConfiguration } from 'src/modules/pipelinesV2/pipelines-client';
+import { PlatformApiService } from 'src/modules/platform-api/platform-api.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+@Injectable()
+export class PipelineExecutionGuard implements CanActivate {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private readonly platformApiService: PlatformApiService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  async canActivate(context: ExecutionContext): Promise<boolean> {
+    try {
+      this.logger.info(
+        'PipelineExecutionGuard: Checking if pipeline can be executed...',
+      );
+      const request = context.switchToHttp().getRequest();
+      const pipelineId = request.params.pipelineId;
+      const user = request.user;
+      const idRegex = /[^0-9a-zA-Z_$]+/g;
+      const convertedId = pipelineId.replace(idRegex, '_');
+  
+      const status = await this.platformApiService.proxy(
+        'GET',
+        `/pipeline/${convertedId}/pipeline_run`,
+        user,
+      );
+
+      const currentStatus = status[status.length - 1]
+
+      this.logger.info('Pipeline current status response:' +  JSON.stringify(currentStatus));
+  
+      if (currentStatus.last_status.toLowerCase() === 'running') {
+        this.logger.error('Pipeline is running, cannot update input now');
+        throw new BadRequestException('Pipeline is running, cannot update input now');
+      } else {
+        return true;
+      }
+    } catch (error) {
+      this.logger.error('Error in PipelineExecutionGuard: ' + error.message);
+      throw new BadRequestException('Error checking pipeline status: ' + error.message);
+    }
+
+  }
+}

src/main.ts

@@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) {
   );
 }
 bootstrap();
+

src/modules/auth/README.md

@@ -35,4 +35,45 @@ So in order to have an SSO login, besides configuring the Strategy, we must have
     // ... Rest of the logic
     return { url: url.href };
   }
-```
+```
+
+## Browser session cookies
+
+Besides the tokens returned in the sign-in body (kept by the frontend in
+`localStorage` and sent as `Authorization`), Maestro also persists the session
+as httpOnly cookies so that apps which cannot hold a token themselves —
+Orchest / Intelligence / Process behind an nginx `auth_request` pointing at
+`GET /auth/me` — can authenticate the browser.
+
+| Cookie | Content |
+| --- | --- |
+| `ddf-auth<suffix>` | DUC access token |
+| `ddf-refresh-auth<suffix>` | IdP refresh token |
+| `ddf-user-id<suffix>` | DUC user id |
+
+All three are `HttpOnly; Secure; SameSite=None; Path=/` and live for
+`SESSION_COOKIE_MAX_AGE_DAYS` (default 30 — keep it aligned with the IdP
+refresh-token validity). See `session-cookies.ts`.
+
+They are **environment-scoped** so that a login on `app.stg.dadosfera.ai`
+never overwrites a production session in the same browser:
+
+| Env | `AUTH_COOKIE_SUFFIX` | `COOKIE_DOMAIN` |
+| --- | --- | --- |
+| prd | `""` | `.dadosfera.ai` |
+| stg | `-stg` | `.stg.dadosfera.ai` |
+
+`GET /auth/me` behaviour:
+
+- valid access token → `200` + user (permissions are the raw seqids from the JWT);
+- expired/invalid access token but refresh token present → refresh, rewrite the
+  cookies (`Set-Cookie`, access token first — nginx `auth_request` only relays
+  the first one) and `200`;
+- refresh impossible (expired/revoked/foreign token, unknown user) → `401` and the
+  cookies are expired so the browser stops presenting a dead session;
+- DUC unreachable / slower than `AUTH_RPC_TIMEOUT_MS` (default 10s) → `503`,
+  cookies untouched (transient, retry);
+- resource belongs to another customer → `403`; new terms of use pending → `403`.
+
+`POST /auth/refresh-access-token` re-writes all three cookies from the session
+the frontend is actually using, and `POST /auth/sign-out` expires them.

src/modules/auth/auth.controller.ts

@@ -14,6 +14,8 @@ import {
   Param,
   Res,
   UnauthorizedException,
+  HttpException,
+  ServiceUnavailableException,
 } from '@nestjs/common';
 import {
   ApiHeaders,
@@ -37,6 +39,7 @@ import {
   RequireAllPermissions,
 } from 'src/decorators/authentication.decorator';
 import { AuthClientService } from './auth.service';
+import { UserDTO } from './dtos/login';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
 import { RequestUser, User } from 'src/decorators/user.decorator';
@@ -50,11 +53,15 @@ import {
 import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 import { AuthGuard } from '@nestjs/passport';
 import { Request, Response } from 'express';
+import { status as GrpcStatus } from '@grpc/grpc-js';
 import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
+import ErrorBuilder from 'src/utils/ErrorBuilder';
+import { clearSessionCookies, readAuthSession } from './session-cookies';
 import jwt, { JwtPayload } from 'jsonwebtoken';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 type CookiesValues = {
   accessToken?: string;
@@ -74,6 +81,7 @@ export class AuthController {
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
     private authClient: AuthClientService,
+    private apiKeyService: ApiKeyService,
   ) {
     this.logger = dadosferaLogger.logger;
 
@@ -126,14 +134,16 @@ export class AuthController {
   ) {
     try {
       this.logger.info('/auth - SignOut');
-   
-      this.authClient.cleanUpAuthSession(res);
 
-      return res.send();
+      this.authClient.cleanUpAuthSession(res);
     } catch (error) {
-      this.logger.error('/auth - SignIn - ERROR', error);
+      this.logger.error('/auth - SignOut - ERROR', error);
     }
 
+    // Always answer: a sign-out that hangs leaves the frontend logout
+    // waiting on a request that never completes.
+    return res.send();
+
   }
 
   @Post('refresh-access-token')
@@ -146,7 +156,7 @@ export class AuthController {
     @Res() res: Response,
   ) {
     this.logger.info('/auth - RefreshAccessToken');
-    const frontHost = origin.replace(/^https?:\/\//, '');
+    const frontHost = (origin || '').replace(/^https?:\/\//, '');
     const { refreshToken, userId } = body;
 
     const metadata = PackTheMetadata({
@@ -156,10 +166,14 @@ export class AuthController {
 
     const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
 
+    // Re-sync ALL session cookies with the session the frontend is actually
+    // using. The IdP does not rotate refresh tokens (DUC returns an empty
+    // refreshToken), so fall back to the one that just proved valid — this
+    // heals cookies left behind by an older login or by another environment.
     this.authClient.writeAuthSession(res, {
       accessToken: data.accessToken,
-      refreshToken: data.refreshToken,
-      userId
+      refreshToken: data.refreshToken || refreshToken,
+      userId,
     });
 
     return res.send(data);
@@ -190,13 +204,14 @@ export class AuthController {
   ) {
     this.logger.info('/auth - change-password');
 
-    const { oldPassword, newPassword } = body;
+    const { oldPassword, newPassword, totpCode } = body;
     const { authorization: accessToken } = headers;
 
     return this.authClient.changePassword({
       accessToken,
       oldPassword,
       newPassword,
+      totpCode,
     });
   }
 
@@ -214,7 +229,8 @@ export class AuthController {
 
     const { username } = body;
 
-    return this.authClient.resetPassword({ username }, metadata);
+    await this.authClient.resetPassword({ username }, metadata);
+    return { authProvider: process.env.AUTH_PROVIDER || 'cognito' };
   }
 
   @ApiInternalOnlyEndpoint()
@@ -473,37 +489,134 @@ export class AuthController {
 
   @Get('me')
   async getMe(@Req() req: Request, @Res() res: Response) {
-    this.logger.info('GET /auth/me ')
+    const resourceHost = (req.headers['x-original-url'] as string) || '';
+    // Never dump req.headers here: it contains the session cookies.
+    this.logger.info('GET /auth/me', { resourceHost });
 
-    // Get token and headers
-    const accessToken = req.cookies['ddf-auth'];
-    const refreshToken = req.cookies['ddf-refresh-auth']; 
-    const userId = req.cookies['ddf-user-id']; 
-    const resourceHost = req.headers["host"]
+    // Check for API key header first
+    const apiKey = req.get('X-Api-key');
+    if (apiKey) {
+      this.logger.info('Authenticating via X-Api-key header');
+      let api_key: Awaited<ReturnType<ApiKeyService['get']>>['api_key'];
+      try {
+        ({ api_key } = await this.apiKeyService.get(apiKey));
+      } catch (error) {
+        if (this.isTransportError(error)) {
+          throw this.identityProviderUnavailable(error);
+        }
+        this.logger.error('/auth/me - invalid API key', {
+          error: error?.message,
+        });
+        throw new UnauthorizedException('Invalid API key');
+      }
 
-    const hasUserSession = Boolean(accessToken) && Boolean(userId);
-    this.logger.info('Has User Session: ' + hasUserSession);
+      const userDto: UserDTO = {
+        id: api_key.user_id,
+        name: api_key.username,
+        email: api_key.username,
+        customer: {
+          id: api_key.customer_id,
+          name: api_key.customer_name,
+          tier: api_key.customer_tier,
+        },
+        permissions: [],
+      };
 
-    if (!hasUserSession) {
-      throw new UnauthorizedException()
+      return res.status(200).json(userDto);
+    }
+
+    const { accessToken, refreshToken, userId } = readAuthSession(req);
+    const canRefresh = Boolean(refreshToken) && Boolean(userId);
+    this.logger.info('Has User Session: ' + Boolean(accessToken || canRefresh));
+
+    if (!accessToken && !canRefresh) {
+      throw new UnauthorizedException();
+    }
+
+    if (accessToken) {
+      try {
+        const userDto = await this.authClient.validateUserSession(
+          accessToken,
+          resourceHost,
+        );
+        return res.status(200).json(userDto);
+      } catch (error) {
+        // Definitive answers (e.g. 403: resource belongs to another
+        // customer) must not be masked by a refresh attempt.
+        if (error instanceof HttpException) throw error;
+        if (this.isTransportError(error)) {
+          throw this.identityProviderUnavailable(error);
+        }
+        if (!canRefresh) {
+          this.logger.error(
+            '/auth/me - access token rejected and no refresh token',
+            { error: error?.message },
+          );
+          clearSessionCookies(res);
+          throw new ErrorBuilder(ErrorCodes.AUTH.INVALID_REFRESH_TOKEN);
+        }
+        this.logger.info('/auth/me - access token rejected, refreshing', {
+          error: error?.message,
+        });
+      }
     }
 
     try {
-      const userDto = await this.authClient.validateUserSession(accessToken, resourceHost);
-      return res.status(200).json(userDto);
-    } catch (error) {
-
-      if (!refreshToken) {
-        this.logger.error('Invalid refresh token or customer name');
-        throw new UnauthorizedException("Invalid refresh token or customer name");
-      };
-
-      const {
-        authSession,
-        user
-      } = await this.authClient.refreshUserSession(refreshToken, userId, resourceHost);
-      this.authClient.writeAuthSession(res, authSession);
+      const { authSession, user } = await this.authClient.refreshUserSession(
+        refreshToken,
+        userId,
+        resourceHost,
+      );
+      // Refresh tokens are not rotated by the IdP (DUC returns ''), so keep
+      // the one that just worked; this also slides the cookie lifetime.
+      this.authClient.writeAuthSession(res, {
+        ...authSession,
+        refreshToken: authSession.refreshToken || refreshToken,
+      });
       return res.status(200).json(user);
+    } catch (error) {
+      if (error instanceof HttpException) throw error;
+      if (this.isTransportError(error)) {
+        throw this.identityProviderUnavailable(error);
+      }
+
+      const code =
+        typeof error?.details === 'string' ? error.details : undefined;
+      if (code === ErrorCodes.TERMS_OF_USE.REQUIRED) {
+        // The session is fine; the user has to accept the new terms first.
+        throw new ErrorBuilder(code);
+      }
+
+      // Expired, revoked, or minted by another environment / user pool.
+      // Answer 401 (never 500) and drop the dead cookies so the browser
+      // does not keep presenting them on every request.
+      this.logger.error('/auth/me - session refresh failed', {
+        code,
+        error: error?.message,
+      });
+      clearSessionCookies(res);
+      throw new ErrorBuilder(ErrorCodes.AUTH.INVALID_REFRESH_TOKEN);
     }
   }
+
+  /** DUC unreachable or too slow — a transient failure, not an auth answer. */
+  private isTransportError(error: any): boolean {
+    return (
+      error?.name === 'TimeoutError' ||
+      error?.code === GrpcStatus.UNAVAILABLE ||
+      error?.code === GrpcStatus.DEADLINE_EXCEEDED
+    );
+  }
+
+  private identityProviderUnavailable(error: any): HttpException {
+    this.logger.error('/auth/me - identity provider unavailable', {
+      code: error?.code,
+      error: error?.message,
+    });
+    // 503 (not 500): callers such as nginx auth_request / Orchest's
+    // auth-server treat it as "retry later" instead of a broken session.
+    return new ServiceUnavailableException(
+      'Identity provider temporarily unavailable',
+    );
+  }
 }

src/modules/auth/auth.module.ts

@@ -8,10 +8,11 @@ import { AuthClientService } from './auth.service';
 import { DucClient } from '../duc/client.config';
 import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
 import { getOauthSecrets } from 'src/utils/OauthSecrets';
+import { ApiKeyModule } from '../api-key/api-key.module';
 const client = new DucClient();
 
 @Module({
-  imports: [ClientsModule.register([client.providerOptions])],
+  imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule],
   controllers: [AuthController],
   providers: [
     AuthClientService,

src/modules/auth/auth.service.ts

@@ -8,7 +8,7 @@ import {
 } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { lastValueFrom } from 'rxjs';
+import { lastValueFrom, timeout } from 'rxjs';
 
 import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
 import {
@@ -35,12 +35,24 @@ import { BulkEditResponse, UserDTO } from './dtos/login';
 import jwt, { JwtPayload } from 'jsonwebtoken';
 import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 import { Request, Response } from 'express';
+import {
+  AuthSession,
+  clearSessionCookies,
+  writeSessionCookies,
+} from './session-cookies';
 
-type AuthSession = {
-  accessToken?: string;
-  refreshToken?: string;
-  userId?: string;
-};
+export type { AuthSession };
+
+/**
+ * Upper bound for the DUC round-trips on the session-validation path
+ * (`GET /auth/me`). Without it a hung DUC keeps every `auth_request`
+ * subrequest open until the ingress gives up (minutes), which surfaces to
+ * users as 500/504 pages instead of a fast, retryable 503.
+ */
+function authRpcTimeoutMs(): number {
+  const ms = Number(process.env.AUTH_RPC_TIMEOUT_MS);
+  return Number.isFinite(ms) && ms > 0 ? ms : 10_000;
+}
 
 @Injectable()
 export class AuthClientService implements OnModuleInit {
@@ -70,7 +82,9 @@ export class AuthClientService implements OnModuleInit {
   async getPublicKeys() {
     this.logger.info('GetPublicKeys');
 
-    return lastValueFrom(this.authService.AuthGetPublicKeys({}));
+    return lastValueFrom(
+      this.authService.AuthGetPublicKeys({}).pipe(timeout(authRpcTimeoutMs())),
+    );
   }
 
   async snowflakeSignIn(input: AuthSnowflakeSignInRequest) {
@@ -132,10 +146,9 @@ export class AuthClientService implements OnModuleInit {
     this.logger.info('RefreshAccessToken');
 
     return lastValueFrom(
-      this.authService.AuthRefreshAccessToken(
-        { refreshToken, userId },
-        metadata,
-      ),
+      this.authService
+        .AuthRefreshAccessToken({ refreshToken, userId }, metadata)
+        .pipe(timeout(authRpcTimeoutMs())),
     );
   }
 
@@ -143,6 +156,7 @@ export class AuthClientService implements OnModuleInit {
     accessToken,
     oldPassword,
     newPassword,
+    totpCode,
   }: AuthChangePasswordRequest) {
     this.logger.info('ChangePassword');
 
@@ -151,6 +165,7 @@ export class AuthClientService implements OnModuleInit {
         accessToken,
         oldPassword,
         newPassword,
+        totpCode,
       }),
     );
   }
@@ -344,67 +359,18 @@ export class AuthClientService implements OnModuleInit {
     };
   }
 
+  /**
+   * Persists the session in the browser as httpOnly cookies. Names, domain
+   * and lifetime are environment-scoped — see `session-cookies.ts`.
+   */
   public writeAuthSession(res: Response, data: AuthSession) {
-    let exp = 1000 * 60 * 5; // 5 minutes
-
-    if (data.accessToken) {
-      const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
-      exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
-
-      this.logger.info('Set Cookie ddf-auth');
-      res.cookie('ddf-auth', data.accessToken, {
-        domain: '.dadosfera.ai',
-        maxAge: exp,
-        httpOnly: true,
-        secure: true,
-        sameSite: 'none', // Necessário para cookies em requisições cross-site
-      });
-    }
-
-    if (data.refreshToken) {
-      this.logger.info('Set Cookie ddf-refresh-auth');
-      res.cookie('ddf-refresh-auth', data.refreshToken, {
-        domain: '.dadosfera.ai',
-        maxAge: exp,
-        httpOnly: true,
-        secure: true,
-        sameSite: 'none', // Necessário para cookies em requisições cross-site
-      });
-    }
-
-    if (data.userId) {
-      this.logger.info('Set Cookie ddf-refresh-auth');
-      res.cookie('ddf-user-id', data.userId, {
-        domain: '.dadosfera.ai',
-        maxAge: exp,
-        httpOnly: true,
-        secure: true,
-        sameSite: 'none', // Necessário para cookies em requisições cross-site
-      });
-    }
+    this.logger.info('Set session cookies');
+    writeSessionCookies(res, data);
   }
 
+  /** Expires every session cookie this environment writes. */
   public cleanUpAuthSession(res: Response) {
-    const exp = 1000 * 60 * 3;
-
-    res.cookie('ddf-auth', '', {
-      domain: 'dadosfera.ai',
-      maxAge: Date.now() - exp,
-      expires: new Date(),
-      httpOnly: true,
-      secure: true,
-      sameSite: 'none', // Necessário para cookies em requisições cross-site
-    });
-
-    res.cookie('ddf-refresh-auth', '', {
-      domain: 'dadosfera.ai',
-      maxAge: Date.now() - exp,
-      expires: new Date(),
-      httpOnly: true,
-      secure: true,
-      sameSite: 'none', // Necessário para cookies em requisições cross-site
-    });
-
+    clearSessionCookies(res);
     this.logger.info('Clean cookie sessions');
   }
 
@@ -430,12 +396,20 @@ export class AuthClientService implements OnModuleInit {
     });
 
     const { user } = await lastValueFrom(
-      this.userService.UserFindOneById({ id: payload.user_id }, metadata),
+      this.userService
+        .UserFindOneById({ id: payload.user_id }, metadata)
+        .pipe(timeout(authRpcTimeoutMs())),
     );
 
+    if (!user) {
+      // Deleted/unknown user: a definitive "no session", not a crash.
+      throw new HttpException('User not found', HttpStatus.UNAUTHORIZED);
+    }
+
     const userDto: UserDTO = {
       id: user.id,
-      name: user.username,
+      name: user.name,
+      email: user.email,
       jobTitle: user?.jobTitle || null,
       department: user?.department || null,
       hierarchy: user?.hierarchy || null,
@@ -444,6 +418,9 @@ export class AuthClientService implements OnModuleInit {
         name: payload.customer_name,
         tier: payload.customer_tier,
       },
+      // Raw permission seqids from the JWT. Consumers own the seqid->meaning
+      // mapping (e.g. Orchest's auth-server); Maestro reports them as-is.
+      permissions: payload.permissions ?? [],
     };
 
     return userDto;

src/modules/auth/dtos/login.ts

@@ -144,6 +144,7 @@ export interface BulkEditResponse {
 export type UserDTO = {
   id: string,
   name: string,
+  email: string,
   jobTitle?: string,
   department?: string,
   hierarchy?: string,
@@ -151,5 +152,6 @@ export type UserDTO = {
     id: string,
     name: string,
     tier: string,
-  }
+  },
+  permissions: number[],
 }

src/modules/auth/session-cookies.spec.ts

@@ -0,0 +1,103 @@
+import {
+  clearSessionCookies,
+  readAuthSession,
+  sessionCookieMaxAgeMs,
+  sessionCookieNames,
+  sessionCookieOptions,
+  writeSessionCookies,
+} from './session-cookies';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+function fakeResponse() {
+  const cookie = jest.fn();
+  const clearCookie = jest.fn();
+  return { cookie, clearCookie };
+}
+
+describe('session cookies', () => {
+  it('uses the legacy names and domain when nothing is configured (production)', () => {
+    const env = {};
+    expect(sessionCookieNames(env)).toEqual({
+      accessToken: 'ddf-auth',
+      refreshToken: 'ddf-refresh-auth',
+      userId: 'ddf-user-id',
+    });
+    expect(sessionCookieOptions(env)).toMatchObject({
+      domain: '.dadosfera.ai',
+      path: '/',
+      httpOnly: true,
+      secure: true,
+      sameSite: 'none',
+    });
+  });
+
+  it('scopes names and domain per environment so staging never clobbers production', () => {
+    const env = { AUTH_COOKIE_SUFFIX: '-stg', COOKIE_DOMAIN: '.stg.dadosfera.ai' };
+    expect(sessionCookieNames(env)).toEqual({
+      accessToken: 'ddf-auth-stg',
+      refreshToken: 'ddf-refresh-auth-stg',
+      userId: 'ddf-user-id-stg',
+    });
+    expect(sessionCookieOptions(env).domain).toBe('.stg.dadosfera.ai');
+  });
+
+  it('treats maxAge as a duration (30 days by default), not as the token epoch', () => {
+    expect(sessionCookieMaxAgeMs({})).toBe(30 * DAY_MS);
+    expect(sessionCookieMaxAgeMs({ SESSION_COOKIE_MAX_AGE_DAYS: '7' })).toBe(7 * DAY_MS);
+    expect(sessionCookieMaxAgeMs({ SESSION_COOKIE_MAX_AGE_DAYS: 'nope' })).toBe(30 * DAY_MS);
+    expect(sessionCookieMaxAgeMs({ SESSION_COOKIE_MAX_AGE_DAYS: '-1' })).toBe(30 * DAY_MS);
+  });
+
+  it('reads the session using the environment-specific names', () => {
+    const env = { AUTH_COOKIE_SUFFIX: '-stg' };
+    const req = {
+      cookies: {
+        'ddf-auth': 'prod-token',
+        'ddf-auth-stg': 'stg-token',
+        'ddf-refresh-auth-stg': 'stg-refresh',
+        'ddf-user-id-stg': 'u-stg',
+      },
+    };
+    expect(readAuthSession(req as any, env)).toEqual({
+      accessToken: 'stg-token',
+      refreshToken: 'stg-refresh',
+      userId: 'u-stg',
+    });
+    expect(readAuthSession({ cookies: undefined } as any, env)).toEqual({
+      accessToken: undefined,
+      refreshToken: undefined,
+      userId: undefined,
+    });
+  });
+
+  it('writes the access-token cookie first and only the parts that are present', () => {
+    const res = fakeResponse();
+    writeSessionCookies(res as any, { accessToken: 'a', userId: 'u' }, {});
+
+    expect(res.cookie.mock.calls.map((c) => c[0])).toEqual(['ddf-auth', 'ddf-user-id']);
+    expect(res.cookie.mock.calls[0][2]).toMatchObject({
+      domain: '.dadosfera.ai',
+      path: '/',
+      maxAge: 30 * DAY_MS,
+      httpOnly: true,
+      secure: true,
+      sameSite: 'none',
+    });
+  });
+
+  it('clears all three cookies with the same domain and path they were set with', () => {
+    const res = fakeResponse();
+    const env = { AUTH_COOKIE_SUFFIX: '-stg', COOKIE_DOMAIN: '.stg.dadosfera.ai' };
+    clearSessionCookies(res as any, env);
+
+    expect(res.clearCookie.mock.calls.map((c) => c[0])).toEqual([
+      'ddf-auth-stg',
+      'ddf-refresh-auth-stg',
+      'ddf-user-id-stg',
+    ]);
+    for (const call of res.clearCookie.mock.calls) {
+      expect(call[1]).toMatchObject({ domain: '.stg.dadosfera.ai', path: '/' });
+    }
+  });
+});

src/modules/auth/session-cookies.ts

@@ -0,0 +1,131 @@
+import { CookieOptions, Request, Response } from 'express';
+
+/**
+ * Browser session cookies written by Maestro and consumed by every app that
+ * authenticates through `GET /auth/me` (Orchest auth-server, nginx
+ * `auth_request` ingresses, ...).
+ *
+ * The names and the domain are environment-scoped so that a staging login
+ * never overwrites (or is confused with) a production session in the same
+ * browser: both platforms live under `dadosfera.ai`, so a cookie on the
+ * registrable domain is sent to *both* environments.
+ *
+ *   ENV      | AUTH_COOKIE_SUFFIX | COOKIE_DOMAIN
+ *   prd      | ""                 | .dadosfera.ai       -> ddf-auth
+ *   stg      | "-stg"             | .stg.dadosfera.ai   -> ddf-auth-stg
+ */
+export type AuthSession = {
+  accessToken?: string;
+  refreshToken?: string;
+  userId?: string;
+};
+
+export type SessionCookieNames = {
+  accessToken: string;
+  refreshToken: string;
+  userId: string;
+};
+
+type Env = Record<string, string | undefined>;
+
+const DEFAULT_COOKIE_DOMAIN = '.dadosfera.ai';
+const DEFAULT_SESSION_COOKIE_MAX_AGE_DAYS = 30;
+
+export function cookieDomain(env: Env = process.env): string {
+  return env.COOKIE_DOMAIN || DEFAULT_COOKIE_DOMAIN;
+}
+
+export function cookieSuffix(env: Env = process.env): string {
+  return env.AUTH_COOKIE_SUFFIX || '';
+}
+
+export function sessionCookieNames(env: Env = process.env): SessionCookieNames {
+  const suffix = cookieSuffix(env);
+  return {
+    accessToken: `ddf-auth${suffix}`,
+    refreshToken: `ddf-refresh-auth${suffix}`,
+    userId: `ddf-user-id${suffix}`,
+  };
+}
+
+/**
+ * How long the browser keeps the session cookies. This is a *duration*
+ * (express `maxAge` is in milliseconds from now), independent from the
+ * access token's own `exp`: an expired access token in the cookie is
+ * harmless — `/auth/me` transparently refreshes it — while a refresh token
+ * that outlives the cookie would make the browser forget a still-valid
+ * session. Keep it aligned with the IdP refresh-token validity.
+ */
+export function sessionCookieMaxAgeMs(env: Env = process.env): number {
+  const days = Number(env.SESSION_COOKIE_MAX_AGE_DAYS);
+  const safeDays =
+    Number.isFinite(days) && days > 0
+      ? days
+      : DEFAULT_SESSION_COOKIE_MAX_AGE_DAYS;
+  return safeDays * 24 * 60 * 60 * 1000;
+}
+
+export function sessionCookieOptions(env: Env = process.env): CookieOptions {
+  return {
+    domain: cookieDomain(env),
+    path: '/',
+    httpOnly: true,
+    secure: true,
+    // Required: the cookies are sent cross-site (app.* -> maestro.*) and
+    // from embedded iframes (app.* embedding app-intelligence-*.*).
+    sameSite: 'none',
+  };
+}
+
+export function readAuthSession(
+  req: Pick<Request, 'cookies'>,
+  env: Env = process.env,
+): AuthSession {
+  const names = sessionCookieNames(env);
+  const cookies = req.cookies ?? {};
+  return {
+    accessToken: cookies[names.accessToken] || undefined,
+    refreshToken: cookies[names.refreshToken] || undefined,
+    userId: cookies[names.userId] || undefined,
+  };
+}
+
+/**
+ * (Re)writes whichever parts of the session are present. The access-token
+ * cookie is written first on purpose: nginx `auth_request` relays only the
+ * first `Set-Cookie` of the auth subrequest (`$upstream_http_set_cookie`),
+ * and the access token is the one that changes on refresh.
+ */
+export function writeSessionCookies(
+  res: Pick<Response, 'cookie'>,
+  session: AuthSession,
+  env: Env = process.env,
+): void {
+  const names = sessionCookieNames(env);
+  const options: CookieOptions = {
+    ...sessionCookieOptions(env),
+    maxAge: sessionCookieMaxAgeMs(env),
+  };
+
+  if (session.accessToken) {
+    res.cookie(names.accessToken, session.accessToken, options);
+  }
+  if (session.refreshToken) {
+    res.cookie(names.refreshToken, session.refreshToken, options);
+  }
+  if (session.userId) {
+    res.cookie(names.userId, session.userId, options);
+  }
+}
+
+/** Expires all three session cookies (same name/domain/path they were set with). */
+export function clearSessionCookies(
+  res: Pick<Response, 'clearCookie'>,
+  env: Env = process.env,
+): void {
+  const names = sessionCookieNames(env);
+  const options = sessionCookieOptions(env);
+  res.clearCookie(names.accessToken, options);
+  res.clearCookie(names.refreshToken, options);
+  res.clearCookie(names.userId, options);
+}

src/modules/catalog/catalog.controller.ts

@@ -17,6 +17,7 @@ import {
   HttpStatus,
   Res,
 } from '@nestjs/common';
+import { ValidationPipe } from '../../pipes/object-validation.pipe';
 import {
   ApiCreatedResponse,
   ApiHeaders,
@@ -46,6 +47,7 @@ import {
   IMakeAComment,
   IOneDataAsset,
   IPreviewResponse,
+  IUpdateCertificationStatusRequest,
   IUpdateDataRequest,
   TriggerCatalogReq,
   TriggerCatalogRes,
@@ -83,6 +85,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async searchCatalog(
     @User() user: RequestUser,
     @Query() query: ICatalogAllRequest,
@@ -122,6 +127,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async dowloadAsserts(
     @User() user: RequestUser,
     @Query() query: ICatalogAllRequest,
@@ -165,6 +173,9 @@ export class CatalogController {
 
   @ApiInternalOnlyEndpoint()
   @Get('data-asset')
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
     const { username, user_id, customer_id, customer_name, permissions } = user;
     const { pipeline, object } = query;
@@ -223,6 +234,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async findAllTags(@Body() body) {
     this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
       user: body.info.user_id,
@@ -241,11 +255,59 @@ export class CatalogController {
     return res;
   }
 
+  @Get('schemas')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async findSchemas(@User() user: RequestUser) {
+    const { username, user_id, customer_id, customer_name } = user;
+    this.logger.info(`/catalog - ON FIND SCHEMAS ROUTE`, {
+      username,
+      customer_name,
+    });
+
+    const metadata = PackTheMetadata({
+      username,
+      user_id,
+      customer_id,
+      customer_name,
+    });
+
+    try {
+      const res = await this.catalogService.findSchemas(metadata);
+      return res;
+    } catch (error) {
+      throw new HttpException(error.message, HttpStatus.NOT_FOUND);
+    }
+
+  }
+
+  @Get('custom-properties')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getCustomPropertyDefinitions(@User() user: RequestUser) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+
+    return this.catalogService.getCustomPropertyDefinitions(metadata);
+  }
+
   @Get('data-asset/:id')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async getDataAsset(
     @User() user: RequestUser,
     @Param('id') id: string,
@@ -357,6 +419,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -388,6 +453,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async getDataAssetPreview(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -419,6 +487,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async getDataAssetDocs(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -450,6 +521,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async updateDataAsset(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
@@ -465,6 +539,8 @@ export class CatalogController {
       language,
     });
 
+    delete (body as any).certification_status;
+
     const result = await this.catalogService.updateOneDataAsset({
       body,
       data_asset_id,
@@ -478,11 +554,44 @@ export class CatalogController {
     return result;
   }
 
+  @Put('data-asset/:id/certification-status')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
+  async updateDataAssetCertificationStatus(
+    @User() user: RequestUser,
+    @Language() language: LanguageEnum,
+    @Param('id') data_asset_id: string,
+    @Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
+  ): Promise<IUpdateCertificationStatusRequest> {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    return this.catalogService.updateCertificationStatus({
+      body,
+      data_asset_id,
+      metadata,
+    });
+  }
+
   @Post('data-asset/:id/docs')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async manageDataAssetDocs(
     @User() user: RequestUser,
     @Headers() headers,
@@ -520,6 +629,9 @@ export class CatalogController {
 
   @ApiInternalOnlyEndpoint()
   @Put('data-asset/:id/manage-permissions')
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async manageDataAssetPermissions(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -542,6 +654,9 @@ export class CatalogController {
 
   @ApiInternalOnlyEndpoint()
   @Put('data-asset/:id/revoke-permissions')
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async revokeDataAssetPermissions(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -567,6 +682,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async createDataAsset(
     @User() user: RequestUser,
     @Body() body: ICreateDataAsset,
@@ -591,6 +709,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async commentOnDataAsset(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -617,6 +738,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
     const { customer_id, customer_name, user_id, username } = user;
     const metadata = PackTheMetadata({
@@ -638,6 +762,9 @@ export class CatalogController {
     PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
     PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async deleteComment(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -791,6 +918,9 @@ export class CatalogController {
 
   @Get('nimbus-dashboards')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.CATALOG
+  )
   async getNimbusDashboards(
     @User() user: RequestUser,
     @Body() body: GetNimbusDashboardsRequest,
@@ -943,4 +1073,4 @@ export class CatalogController {
       this.logger.error(error.message);
     }
   }
-}
+}

src/modules/catalog/catalog.module.ts

@@ -5,20 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { CatalogController } from './catalog.controller';
 import { CatalogClientConfiguration } from './catalog-client';
 import { ClientsModule } from '@nestjs/microservices';
-import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { UsersModule } from '../users/users.module';
 import { RolesModule } from '../roles/roles.module';
 import { CustomersModule } from '../customers/customers.module';
 import { ShareModule } from './share/share.module';
 import { CatalogService } from './catalog.service';
-import { MixpanelModule } from '../mixpanel/mixpanel.module';
 
 const client = new CatalogClientConfiguration();
 
 @Module({
   imports: [
     ClientsModule.register([client.providerOptions]),
-    OldPipelineModule,
     UsersModule,
     RolesModule,
     CustomersModule,

src/modules/catalog/catalog.service.ts

@@ -29,6 +29,7 @@ import {
   AssetReporter,
   BatchRemoveRlsRulesRequest,
   CreateDataDocsDTO,
+  IUpdateCertificationStatusRequest,
   IUpdateDataRequest,
   TriggerCatalogReq,
 } from './dtos';
@@ -41,7 +42,6 @@ import {
 import { TypeParser } from 'src/utils/FileParser/parser-types';
 import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
 
-
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
@@ -58,7 +58,6 @@ class CatalogService implements OnModuleInit {
     this.logger = dadosferaLogger.logger;
   }
 
-
   onModuleInit() {
     this.catalogReadService =
       this.grpcClient.getService<ReadService.CatalogReadServices>(
@@ -74,81 +73,69 @@ class CatalogService implements OnModuleInit {
       );
   }
 
-
   _getNimbusUrl(body) {
     this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
-
     if (process.env.ENV === 'prd') {
       return `https://nimbus-${customer}.dadosfera.ai`;
     }
 
-
     return `https://nimbus-${customer}.${process.env.ENV.replace(
       'local',
       'stg',
     )}.dadosfera.ai`;
   }
 
-
   async getPiiReporter(metadata: Metadata, type: TypeParser) {
-    this.logger.info('getPiiReporter: ' + type)
+    this.logger.info('getPiiReporter: ' + type);
     try {
-      const {
-        data
-      } = await lastValueFrom(
-        this.catalogWriteService.GetPiiReporter({}, metadata)
-      )
-      this.logger.info("Finish grpc call")
-
+      const { data } = await lastValueFrom(
+        this.catalogWriteService.GetPiiReporter({}, metadata),
+      );
+      this.logger.info('Finish grpc call');
 
       const parser = ParserBuilder.build<PiiMetadata>(type);
 
-
-      this.logger.info('parser file to: ' + type)
-      const file = await parser.parse(data)
-      this.logger.info('finish parser')
+      this.logger.info('parser file to: ' + type);
+      const file = await parser.parse(data);
+      this.logger.info('finish parser');
       const mimeTypes: Record<TypeParser, string> = {
-        'csv': 'text/csv',
-        'html': 'text/html',
-        'pdf': 'application/pdf'
-      }
-
+        csv: 'text/csv',
+        html: 'text/html',
+        pdf: 'application/pdf',
+      };
 
       const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
       const filename = `relatorio-pii-${timestamp}.${type}`;
 
-
       return {
         file,
         filename: filename,
-        type: mimeTypes[type]
-      }
+        type: mimeTypes[type],
+      };
     } catch (error) {
       this.logger.error(error.message);
       throw error;
     }
-
-
   }
 
+  async getCustomPropertyDefinitions(metadata: Metadata) {
+    return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
+  }
 
   async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
     if (!data.embed) data.embed = undefined;
 
-
     return lastValueFrom(
       this.catalogWriteService.CreateDataAsset(data, metadata),
     );
   }
 
-
   async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
-
     return lastValueFrom(
       this.catalogWriteService.ManagePermission(data, metadata),
     ).catch((err) => {
@@ -159,11 +146,9 @@ class CatalogService implements OnModuleInit {
     });
   }
 
-
   async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
-
     return lastValueFrom(
       this.catalogWriteService.RevokePermission(data, metadata),
     ).catch((err) => {
@@ -174,11 +159,9 @@ class CatalogService implements OnModuleInit {
     });
   }
 
-
   async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
-
     return lastValueFrom(
       this.catalogWriteService.MakeAComment(data, metadata),
     ).catch((err) => {
@@ -189,11 +172,9 @@ class CatalogService implements OnModuleInit {
     });
   }
 
-
   async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
-
     return lastValueFrom(
       this.catalogWriteService.UpdateAComment(data, metadata),
     ).catch((err) => {
@@ -204,11 +185,9 @@ class CatalogService implements OnModuleInit {
     });
   }
 
-
   async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
 
-
     return lastValueFrom(
       this.catalogWriteService.DeleteDataAsset(data, metadata),
     ).catch((err) => {
@@ -219,30 +198,96 @@ class CatalogService implements OnModuleInit {
     });
   }
 
-
   async getUserRolesIds(userId: string) {
     const result = await this.userService.findOneById(userId).catch(() => null);
 
-
     if (result) {
       return result.user.roles.map((role) => role.id);
     }
 
-
     return [];
   }
 
-
   async searchDataAssets(
     query: Record<string, any>,
     metadata: Metadata,
     customer_id: string,
   ) {
-    this.logger.info('CatalogService - searchDataAssets');
-
+    this.logger.info('CatalogService - searchDataAssets', { query });
 
     const { search, page, size, sort_by, order, ...filters } = query;
 
+    this.logger.debug('Extracted filters:', { filters });
+
+    if (
+      filters.manually !== undefined &&
+      filters.manually !== null &&
+      filters.manually !== ''
+    ) {
+      filters.manually = Number(filters.manually); // 1 ou 0
+    } else {
+      delete filters.manually;
+    }
+
+    if (filters.owner) {
+      const { users: customer_users } =
+        await this.userService.findAllUsersByCustomerId(customer_id);
+
+      this.logger.info('Available users in database count:', {
+        count: customer_users.length,
+      });
+      this.logger.info('First 5 users:', {
+        users: customer_users
+          .slice(0, 5)
+          .map((u) => ({ id: u.id, email: u.email, name: u.name })),
+      });
+
+      const ownerValues = Array.isArray(filters.owner)
+        ? filters.owner
+        : typeof filters.owner === 'string' && filters.owner.includes(',')
+        ? filters.owner.split(',').map((o: string) => o.trim())
+        : [filters.owner];
+
+      this.logger.info('Owner values to convert:', {
+        ownerValues,
+        ownerFiltersOriginal: filters.owner,
+      });
+
+      const ownerIds = ownerValues
+        .map((ownerValue: string) => {
+          const normalizedOwner = ownerValue.replace(/\s/g, '+');
+          const user = customer_users.find((u) => {
+            const isIdMatch = u.id === ownerValue;
+            const isEmailMatch =
+              u.email === ownerValue || u.email === normalizedOwner;
+            const isNameMatch =
+              u.name === ownerValue || u.name === normalizedOwner;
+            this.logger.info('Comparing:', {
+              userId: u.id,
+              userEmail: u.email,
+              userName: u.name,
+              filterValue: ownerValue,
+              normalizedFilter: normalizedOwner,
+              idMatch: isIdMatch,
+              emailMatch: isEmailMatch,
+              nameMatch: isNameMatch,
+            });
+            return isIdMatch || isEmailMatch || isNameMatch;
+          });
+          this.logger.info('Looking for owner result:', {
+            ownerValue,
+            found: !!user,
+            userId: user?.id,
+          });
+          return user?.id || ownerValue;
+        })
+        .filter((id: string) => id);
+
+      if (ownerIds.length > 0) {
+        filters.owner = ownerIds;
+      }
+    }
+
     const { data_assets, total } = await lastValueFrom(
       this.catalogReadService.GetAllDataAssets(
         {
@@ -257,20 +302,18 @@ class CatalogService implements OnModuleInit {
       ),
     );
 
+    console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
 
     const result = JSON.parse(data_assets);
 
-
     const response = await this.getAssetsUsersAndRoles(
       result.data_assets,
       customer_id,
     );
 
-
     return { data_assets: response, total };
   }
 
-
   async downloadAssets(
     query: Record<string, any>,
     metadata: Metadata,
@@ -278,33 +321,27 @@ class CatalogService implements OnModuleInit {
   ) {
     const data = await this.searchDataAssets(query, metadata, customer_id);
 
-
-    const formatData = data.data_assets.map(asset => ({
+    const formatData = data.data_assets.map((asset) => ({
       id: asset.id,
       display_name: asset.display_name,
       data_asset_type: asset.data_asset_type,
       created_at: asset.created_at,
-      tags: '[' + asset.tags.join(', ') + ']'
-    }))
-
+      tags: '[' + asset.tags.join(', ') + ']',
+    }));
 
     const parser = ParserBuilder.build<AssetReporter>('csv');
 
-
     const file = await parser.parse(formatData);
 
-
     const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
     const filename = `dadosfera_assets_${timestamp}.csv`;
 
-
     return {
       file,
-      filename
-    }
+      filename,
+    };
   }
 
-
   async getOneDataAsset(data: {
     id: string;
     customer_id: string;
@@ -325,11 +362,9 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
-
     return { data_asset: asset[0] };
   }
 
-
   async getOneDataAssetByPipelineAndObject(data: {
     customer_id: string;
     pipeline: string;
@@ -351,10 +386,30 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
-
     return { data_asset: asset[0] };
   }
 
+  async updateCertificationStatus(data: {
+    data_asset_id: string;
+    body: IUpdateCertificationStatusRequest;
+    metadata: Metadata;
+  }) {
+    const { body, data_asset_id, metadata } = data;
+
+    await lastValueFrom(
+      this.catalogWriteService.UpdateDataAsset(
+        {
+          id: data_asset_id,
+          changes: JSON.stringify({
+            certification_status: body.certification_status,
+          }),
+        },
+        metadata,
+      ),
+    );
+
+    return { certification_status: body.certification_status };
+  }
 
   async updateOneDataAsset(data: {
     data_asset_id: string;
@@ -364,7 +419,6 @@ class CatalogService implements OnModuleInit {
   }) {
     const { body, customer_id, data_asset_id, metadata } = data;
 
-
     const { data_asset } = await lastValueFrom(
       this.catalogWriteService.UpdateDataAsset(
         { id: data_asset_id, changes: JSON.stringify(body) },
@@ -379,7 +433,6 @@ class CatalogService implements OnModuleInit {
     };
     asset = await this.getAssetsUsersAndRoles([asset], customer_id);
 
-
     return { data_asset: asset[0] };
   }
 
@@ -387,12 +440,11 @@ class CatalogService implements OnModuleInit {
     const { documentation } = await lastValueFrom(
       this.catalogReadService.GetDatasetDoc({ id }, metadata),
     );
- 
+
     const docs = JSON.parse(documentation);
     return docs;
   }
 
-
   async getDatasetPreview(id: string, metadata: Metadata) {
     const { preview } = await lastValueFrom(
       this.catalogReadService.GetDatasetPreview(
@@ -404,7 +456,6 @@ class CatalogService implements OnModuleInit {
     return result;
   }
 
-
   async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
     const { columns_metadata } = await lastValueFrom(
       this.catalogReadService.GetDatasetColumnsMetadata(
@@ -420,7 +471,7 @@ class CatalogService implements OnModuleInit {
     if (body.asset_type === 'table' || body.asset_type === 'view') {
       return this.createDataDocsViaNimbus(body);
     }
-    
+
     return this.createDataDocsViaGrpc(body, metadata);
   }
 
@@ -434,7 +485,6 @@ class CatalogService implements OnModuleInit {
     return data;
   }
 
-
   private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
     this.logger.info('Creating data docs via gRPC for other asset types');
     try {
@@ -447,9 +497,8 @@ class CatalogService implements OnModuleInit {
           metadata,
         ),
       );
-      
+
       return response;
-      
     } catch (error) {
       this.logger.error('Error creating data asset docs:', error);
       throw new HttpException(
@@ -462,7 +511,6 @@ class CatalogService implements OnModuleInit {
   async findAllTags(data, metadata) {
     this.logger.info('CatalogService - findAllCustomerTags');
 
-
     const response = await lastValueFrom(
       this.catalogReadService.GetCustomerTags(data, metadata),
     )
@@ -475,9 +523,25 @@ class CatalogService implements OnModuleInit {
         throw new Error(err);
       });
 
-
     return response;
   }
+
+  async findSchemas(metadata: Metadata) {
+    this.logger.info('CatalogService - findSchemas');
+
+    try {
+      const response = await lastValueFrom(
+        this.catalogReadService.GetSchemas({}, metadata),
+      );
+
+      return response;
+
+    } catch (error) {
+      this.logger.error('Error fetching schemas:', error);
+      throw error;
+    }
+  }
+
   async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
     const { users: customer_users } =
       await this.userService.findAllUsersByCustomerId(customer_id);
@@ -490,17 +554,15 @@ class CatalogService implements OnModuleInit {
         (u) => u.id === data_asset.owner,
       )?.email;
 
-
       const roles = [];
       const users = [];
-      const data_asset_roles = data_asset?.roles || []
+      const data_asset_roles = data_asset?.roles || [];
       for (const role_id of data_asset_roles) {
         const role = customer_roles.find((r) => r.id === role_id);
         if (role) roles.push({ id: role.id, name: role.name });
       }
 
-
-      const data_asset_users = data_asset?.users || []
+      const data_asset_users = data_asset?.users || [];
       for (const user_id of data_asset_users) {
         const user = customer_users.find((r) => r.id === user_id);
         if (user) users.push({ id: user.id, email: user.email });
@@ -514,7 +576,6 @@ class CatalogService implements OnModuleInit {
     });
   }
 
-
   async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
     const { session } = await lastValueFrom(
       this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -528,7 +589,6 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
-
   async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogWriteService.AddRlsRule(data, metadata),
@@ -536,7 +596,6 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
-
   async removeRlsRule(id: number, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogWriteService.RemoveRlsRule({ id }, metadata),
@@ -544,14 +603,12 @@ class CatalogService implements OnModuleInit {
     return res;
   }
 
-
   async batchRemoveRlsRule(
     query: BatchRemoveRlsRulesRequest,
     metadata: Metadata,
   ) {
     const { id_rls, nimbus_dashboard_id } = query;
 
-
     if (id_rls && nimbus_dashboard_id) {
       throw new BadRequestException(
         "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
@@ -572,7 +629,6 @@ class CatalogService implements OnModuleInit {
     return 'OK';
   }
 
-
   async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogReadService.GetRlsRules(data, metadata),
@@ -580,7 +636,6 @@ class CatalogService implements OnModuleInit {
     return res.rls_rules;
   }
 
-
   async getOneRlsRule(id: number, metadata: Metadata) {
     const res = await lastValueFrom(
       this.catalogReadService.GetOneRlsRule({ id }, metadata),
@@ -588,7 +643,6 @@ class CatalogService implements OnModuleInit {
     return res.rls_rule;
   }
 
-
   async getNimbusDashboards(
     data: GetNimbusDashboardsRequest,
     metadata: Metadata,
@@ -599,88 +653,99 @@ class CatalogService implements OnModuleInit {
     return res.dashboards;
   }
 
-
   async createTableMetadata(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
-    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
-
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
 
     const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
 
-
-    this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
-    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
-    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
-
+    this.logger.info(
+      `Creating table metadata for table ${body.table_metadata.table_name}`,
+      { ...body.logMetadata },
+    );
+    this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
+    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
+      ...body.logMetadata,
+    });
 
     try {
-      const { data, status } = await axios.post(endpoint, {...body.table_metadata});
-
+      const { data, status } = await axios.post(endpoint, {
+        ...body.table_metadata,
+      });
 
       this.logger.info(
         `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
-        {...body.logMetadata},
+        { ...body.logMetadata },
       );
       return data.id;
     } catch (error) {
       this.logger.error(
         `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
           error.response?.status
-        } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
+        } because of ${JSON.stringify(error.response?.data) || error.message}`,
+        { ...body.logMetadata },
+      );
       throw new Error(error.response?.data?.message || error.message);
     }
   }
 
-
   async createColumnMetadata(body: any): Promise<number[]> {
     const nimbusUrl = this._getNimbusUrl(body);
     this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
     const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
 
-
- 
-
-
     try {
-      this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
-      this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
-      this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
-      const { data, status } = await axios.post(endpoint, body.column_metadata);
-
+      this.logger.info(
+        `Creating column metadata for table ${body.column_metadata.table_name}`,
+        { ...body.logMetadata },
+      );
+      this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
+      this.logger.debug(
+        `Payload: ${JSON.stringify(body.column_metadata)}`,
+        { ...body.logMetadata },
+      );
+      const { data, status } = await axios.post(
+        endpoint,
+        body.column_metadata,
+      );
 
       this.logger.info(
         `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
-        {...body.logMetadata},
+        { ...body.logMetadata },
       );
       return data.map((column) => column.id);
     } catch (error) {
       this.logger.error(
         `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
           error.response?.status
-        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
+        } because of ${error.response?.data || error.message}`,
+        { ...body.logMetadata },
+      );
       throw new Error(error.response?.data?.message || error.message);
     }
   }
 
-
   async createDataPreview(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
-    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
     const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
 
-
-    this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
-    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
-    this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
-
+    this.logger.info(
+      `Creating data preview for table ${body.data_preview.table_name}`,
+      { ...body.logMetadata },
+    );
+    this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
+    this.logger.debug(
+      `Payload: ${JSON.stringify(body.data_preview)}`,
+      { ...body.logMetadata },
+    );
 
     try {
       const { data, status } = await axios.post(endpoint, body.data_preview);
 
-
       this.logger.info(
         `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
-        {...body.logMetadata},
+        { ...body.logMetadata },
       );
       return data.id;
     } catch (error) {
@@ -688,17 +753,73 @@ class CatalogService implements OnModuleInit {
         `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
           error.response?.status
         } because of ${error.response?.data || error.message}`,
-        {...body.logMetadata},
+        { ...body.logMetadata },
       );
       throw new Error(error.response?.data?.message || error.message);
     }
   }
 
+  async renameTableOnNimbus(
+    nimbusUrl: string,
+    nimbusId: number,
+    changes: { table_name?: string; table_schema?: string; display_name?: string },
+  ): Promise<void> {
+    const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
+    this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
+    await axios.patch(endpoint, changes);
+  }
+
+  async renameColumnMetadataOnNimbus(
+    nimbusUrl: string,
+    databaseName: string,
+    oldTableName: string,
+    oldTableSchema: string,
+    newTableName: string,
+    newTableSchema: string,
+  ): Promise<void> {
+    const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
+    this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
+    const { data: columns } = await axios.get(listEndpoint);
+
+    const filtered = Array.isArray(columns) ? columns : [];
+
+    for (const column of filtered) {
+      const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
+      await axios.patch(patchEndpoint, {
+        table_name: newTableName,
+        table_schema: newTableSchema,
+      });
+    }
+    this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
+  }
+
+  async renameDataPreviewOnNimbus(
+    nimbusUrl: string,
+    databaseName: string,
+    oldTableName: string,
+    oldTableSchema: string,
+    newTableName: string,
+    newTableSchema: string,
+  ): Promise<void> {
+    const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
+    this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
+    const { data: previews } = await axios.get(listEndpoint);
+
+    const filtered = Array.isArray(previews) ? previews : [];
+
+    for (const preview of filtered) {
+      const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
+      await axios.patch(patchEndpoint, {
+        table_name: newTableName,
+        table_schema: newTableSchema,
+      });
+    }
+    this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
+  }
 
   async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
     const customer_name_raw = metadata.get('customer_name');
 
-
     const customer_name = customer_name_raw?.[0]?.toString();
     if (!customer_name) {
       throw new BadRequestException('Customer name not found in metadata');
@@ -721,5 +842,4 @@ class CatalogService implements OnModuleInit {
   }
 }
 
-
 export { CatalogService };

src/modules/catalog/dtos/index.ts

@@ -1,4 +1,10 @@
 import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
+import {
+  IsEnum,
+  IsNotEmpty,
+  IsOptional,
+  IsString,
+} from 'class-validator';
 import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 export enum DataAssetShareType {
@@ -6,6 +12,12 @@ export enum DataAssetShareType {
   public = 'public',
   private = 'private',
 }
+export enum CertificationStatus {
+  draft = 'draft',
+  in_review = 'in_review',
+  approved = 'approved',
+  deprecated = 'deprecated',
+}
 export enum OrderEnum {
   asc = 'asc',
   desc = 'desc',
@@ -98,6 +110,8 @@ export class IDataAsset {
   embed?: EmbedObject;
   @ApiPropertyOptional({ enum: DataAssetShareType })
   share_type?: DataAssetShareType;
+  @ApiPropertyOptional()
+  docs?: string;
 }
 
 export class IOneDataAsset {
@@ -189,6 +203,27 @@ export class IData {
   day_opening: number;
 }
 
+
+export enum CustomPropertyType {
+  TEXT = 'text',
+  NUMBER = 'number',
+  DATE = 'date',
+  BOOLEAN = 'boolean',
+}
+
+export class CustomPropertyDto {
+  @ApiProperty()
+  key: string;
+  @ApiProperty()
+  value: string;
+  @ApiProperty({ enum: CustomPropertyType })
+  type: CustomPropertyType;
+  @ApiPropertyOptional()
+  color?: string;
+  @ApiPropertyOptional()
+  emoji?: string;
+}
+
 export class IUpdateDataRequest {
   @ApiProperty()
   name: string;
@@ -200,7 +235,18 @@ export class IUpdateDataRequest {
   embed: EmbedObject;
   @ApiPropertyOptional({ enum: DataAssetShareType })
   share_type?: DataAssetShareType;
+  @ApiPropertyOptional()
+  docs?: string;
+  @ApiPropertyOptional({ type: [CustomPropertyDto] })
+  custom_properties?: CustomPropertyDto[];
 }
+
+export class IUpdateCertificationStatusRequest {
+  @ApiProperty({ enum: CertificationStatus })
+  @IsEnum(CertificationStatus)
+  certification_status: CertificationStatus;
+}
+
 export class ICreateDataAsset implements CreateDataAssetRequest {
   @ApiProperty()
   display_name: string;
@@ -214,6 +260,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
   location: string;
   @ApiPropertyOptional()
   embed: EmbedObject;
+  @ApiPropertyOptional()
+  docs: string;
 }
 
 export class IPreview {
@@ -352,4 +400,4 @@ export type CreateDataDocsDTO = {
   docs: string;
   asset_type: string;
 
-}
+}

src/modules/catalog/share/share.service.ts

@@ -262,6 +262,7 @@ export class ShareService implements OnModuleInit {
         user_id: accessTokenPayload.user_id,
         username: accessTokenPayload.username,
         permissions: accessTokenPayload.permissions,
+        roles: accessTokenPayload.roles,
         customer_id: accessTokenPayload.customer_id,
         customer_name: accessTokenPayload.customer_name,
         customer_tier: accessTokenPayload.customer_tier,

src/modules/connection-test/connection-test.controller.ts

@@ -22,17 +22,24 @@ import {
   ConnectionTestListTablesRes,
   GetTableMetadataRes,
   GetTableMetadataReq,
+  RefreshCatalogReq,
+  RefreshCatalogRes,
+  RefreshCatalogStatusReq,
 } from './dto/connection-test';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { Authenticated } from 'src/decorators/authentication.decorator';
+import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
 
 @ApiInternalOnlyController()
 @ApiTags('Connection Test')
 @Controller('connection-test')
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Authenticated()
+@RequireModule(
+  DADOSFERA_MODULES_KEYS.COLLECT
+)
 export class ConnectionTestController {
   logger: any;
   constructor(
@@ -84,7 +91,7 @@ export class ConnectionTestController {
     });
     return this.connectionTestService.connectionTestListSchemas(
       body,
-      user.customer_name,
+      user,
     );
   }
 
@@ -101,7 +108,7 @@ export class ConnectionTestController {
     });
     return this.connectionTestService.connectionTestListTables(
       body,
-      user.customer_name,
+      user,
     );
   }
 
@@ -118,7 +125,38 @@ export class ConnectionTestController {
     });
     return this.connectionTestService.getTableMetadata(
       body,
-      user.customer_name,
+      user,
     );
   }
+
+  @Post('refresh-catalog')
+  @ApiOkResponse({ type: RefreshCatalogRes })
+  @HttpCode(HttpStatus.ACCEPTED)
+  async refreshCatalog(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: RefreshCatalogReq,
+  ) {
+    this.logger.info('/connection-test/refresh-catalog', {
+      user: user.user_id,
+      customer: user.customer_name,
+      connection: body.connection_id,
+    });
+    return this.connectionTestService.refreshCatalog(body, user);
+  }
+
+  @Post('refresh-catalog/status')
+  @ApiOkResponse({ type: RefreshCatalogRes })
+  @HttpCode(HttpStatus.OK)
+  async refreshCatalogStatus(
+    @User() user: RequestUser,
+    @Body(new ValidationPipe()) body: RefreshCatalogStatusReq,
+  ) {
+    this.logger.info('/connection-test/refresh-catalog/status', {
+      user: user.user_id,
+      customer: user.customer_name,
+      connection: body.connection_id,
+      session: body.session_id,
+    });
+    return this.connectionTestService.refreshCatalogStatus(body, user);
+  }
 }

src/modules/connection-test/connection-test.module.ts

@@ -5,10 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { ClientsModule } from '@nestjs/microservices';
 import { ConnectionTestClientConfiguration } from './connection-test-client.config';
 import { ConnectionModule } from '../connection/connection.module';
+import { ConnectionsApiModule } from '../connections-api/connections-api.module';
+import { PlatformApiModule } from '../platform-api/platform-api.module';
 const client = new ConnectionTestClientConfiguration();
 @Module({
   controllers: [ConnectionTestController],
   providers: [ConnectionTestService, DadosferaLogger],
-  imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    ConnectionModule,
+    ConnectionsApiModule,
+    PlatformApiModule,
+  ],
 })
 export class ConnectionTestModule {}

src/modules/connection-test/connection-test.service.spec.ts

@@ -0,0 +1,205 @@
+import { ConnectionTestService } from './connection-test.service';
+import { RequestUser } from 'src/decorators/user.decorator';
+
+describe('ConnectionTestService catalog cache', () => {
+  const user: RequestUser = {
+    user_id: 'user-id',
+    username: 'user@example.com',
+    permissions: [],
+    customer_id: 'customer-id',
+    customer_name: 'customer-name',
+    customer_tier: 'standard',
+    access_token: 'token',
+    customer_modules: [],
+    roles: [],
+  };
+  const grpcClient = { getService: jest.fn().mockReturnValue({}) };
+  const connectionsService = {};
+  const connectionsApiService = { proxy: jest.fn() };
+  const platformApiService = { proxy: jest.fn() };
+  let service: ConnectionTestService;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    service = new ConnectionTestService(
+      grpcClient as any,
+      connectionsService as any,
+      connectionsApiService as any,
+      platformApiService as any,
+    );
+  });
+
+  it('keeps the existing schemas response contract', async () => {
+    connectionsApiService.proxy.mockResolvedValue({
+      schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }],
+    });
+
+    await expect(
+      service.connectionTestListSchemas(
+        { connection_id: 'config-id', plugin: 'postgresql' },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      schema_list: ['analytics', 'public'],
+    });
+  });
+
+  it('keeps the existing tables response contract', async () => {
+    connectionsApiService.proxy.mockResolvedValue({
+      tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
+    });
+
+    await expect(
+      service.connectionTestListTables(
+        {
+          connection_id: 'config-id',
+          plugin: 'postgresql',
+          schema: 'public',
+        },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      table_list: ['customers', 'orders'],
+    });
+  });
+
+  it('maps cached columns to the existing table metadata contract', async () => {
+    connectionsApiService.proxy.mockResolvedValue({
+      columns: [
+        {
+          column_name: 'id',
+          data_type: 'bigint',
+          is_primary_key: true,
+        },
+      ],
+    });
+
+    await expect(
+      service.getTableMetadata(
+        {
+          connection_id: 'config-id',
+          plugin: 'postgresql',
+          schema: 'public',
+          table_list: ['customers'],
+        },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      tables_metadata: [
+        {
+          table_name: 'customers',
+          columns: [
+            {
+              name: 'id',
+              type: 'bigint',
+              is_primary_key: true,
+            },
+          ],
+          references: [],
+        },
+      ],
+    });
+    expect(connectionsApiService.proxy).toHaveBeenCalledWith(
+      'GET',
+      '/connection_catalog/config-id/schemas/public/tables/customers/columns',
+      user,
+    );
+  });
+
+  it('submits a catalog refresh without holding the request open', async () => {
+    platformApiService.proxy.mockResolvedValue({
+      session_id: 'session-id',
+      date: '20260731',
+    });
+
+    await expect(
+      service.refreshCatalog(
+        { connection_id: 'config-id', plugin: 'postgresql' },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      status: 'PENDING',
+      session_id: 'session-id',
+      date: '20260731',
+    });
+
+    expect(platformApiService.proxy).toHaveBeenCalledWith(
+      'POST',
+      '/connection_test',
+      user,
+      {
+        customer_id: user.customer_name,
+        plugin: 'postgresql',
+        task: {
+          task_type: 'refresh_catalog',
+          connection: {
+            provider: 'connection_manager',
+            config_id: 'config-id',
+          },
+        },
+      },
+    );
+  });
+
+  it('keeps polling without changing the catalog pointer while pending', async () => {
+    platformApiService.proxy.mockResolvedValue({ status: 'PENDING' });
+
+    await expect(
+      service.refreshCatalogStatus(
+        {
+          connection_id: 'config-id',
+          plugin: 'postgresql',
+          session_id: 'session-id',
+          date: '20260731',
+        },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: false,
+      status: 'PENDING',
+      session_id: 'session-id',
+      date: '20260731',
+    });
+
+    expect(connectionsApiService.proxy).not.toHaveBeenCalled();
+  });
+
+  it('publishes the catalog pointer after the refresh finishes', async () => {
+    platformApiService.proxy.mockResolvedValue({ status: 'DONE' });
+    connectionsApiService.proxy.mockResolvedValue({
+      last_catalog_refresh_status: 'SUCCESS',
+    });
+
+    await expect(
+      service.refreshCatalogStatus(
+        {
+          connection_id: 'config/id',
+          plugin: 'postgresql',
+          session_id: 'session-id',
+          date: '20260731',
+        },
+        user,
+      ),
+    ).resolves.toEqual({
+      operation_result: true,
+      status: 'DONE',
+      session_id: 'session-id',
+      date: '20260731',
+    });
+
+    expect(connectionsApiService.proxy).toHaveBeenCalledWith(
+      'PUT',
+      '/connection_config/config%2Fid/catalog_metadata',
+      user,
+      {
+        last_catalog_refresh_status: 'SUCCESS',
+        last_catalog_connection_test_date: '20260731',
+        last_catalog_connection_test_session_id: 'session-id',
+      },
+    );
+  });
+});

src/modules/connection-test/connection-test.service.ts

@@ -1,4 +1,4 @@
-import { Inject, Injectable } from '@nestjs/common';
+import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import { ConnectionTest } from '@dadosfera/protospack-v2';
 import { lastValueFrom } from 'rxjs';
@@ -13,6 +13,9 @@ import {
   ConnectionTestPingRes,
   GetTableMetadataReq,
   GetTableMetadataRes,
+  RefreshCatalogReq,
+  RefreshCatalogRes,
+  RefreshCatalogStatusReq,
 } from './dto/connection-test';
 import { ConnectionClientService } from '../connection/client.service';
 import {
@@ -21,6 +24,8 @@ import {
 } from '../connection/dtos/connection';
 import { RequestUser } from 'src/decorators/user.decorator';
 import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { ConnectionsApiService } from '../connections-api/connections-api.service';
+import { PlatformApiService } from '../platform-api/platform-api.service';
 
 @Injectable()
 export class ConnectionTestService {
@@ -28,6 +33,8 @@ export class ConnectionTestService {
   constructor(
     @Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
     private connectionsService: ConnectionClientService,
+    private connectionsApiService: ConnectionsApiService,
+    private platformApiService: PlatformApiService,
   ) {
     this.connectionTestReadClient =
       grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
@@ -147,45 +154,137 @@ export class ConnectionTestService {
   }
   async connectionTestListSchemas(
     body: ConnectionTestListSchemasReq,
-    customer_name: string,
+    user: RequestUser,
   ): Promise<ConnectionTestListSchemasRes> {
-    const { connection_id, plugin } = body;
-    return lastValueFrom(
-      this.connectionTestReadClient.ListSchemas({
-        connection_id,
-        customer_name,
-        plugin,
-      }),
+    const result = await this.connectionsApiService.proxy(
+      'GET',
+      `/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`,
+      user,
     );
+    return {
+      operation_result: true,
+      schema_list: result.schemas.map((schema) => schema.schema_name),
+    };
   }
+
   async connectionTestListTables(
     body: ConnectionTestListTablesReq,
-    customer_name: string,
+    user: RequestUser,
   ): Promise<ConnectionTestListTablesRes> {
-    const { connection_id, plugin, schema } = body;
-    return lastValueFrom(
-      this.connectionTestReadClient.ListTables({
-        connection_id,
-        customer_name,
-        plugin,
-        schema,
-      }),
+    const result = await this.connectionsApiService.proxy(
+      'GET',
+      `/connection_catalog/${encodeURIComponent(body.connection_id)}` +
+        `/schemas/${encodeURIComponent(body.schema)}/tables`,
+      user,
     );
+    return {
+      operation_result: true,
+      table_list: result.tables.map((table) => table.table_name),
+    };
   }
 
   async getTableMetadata(
     body: GetTableMetadataReq,
-    customer_name: string,
+    user: RequestUser,
   ): Promise<GetTableMetadataRes> {
-    const { schema, plugin, table_list, connection_id } = body;
-    return lastValueFrom(
-      this.connectionTestReadClient.GetTableMetadata({
-        connection_id,
-        customer_name,
-        plugin,
-        schema,
-        table_list,
+    const tables_metadata = await Promise.all(
+      body.table_list.map(async (table_name) => {
+        const result = await this.connectionsApiService.proxy(
+          'GET',
+          `/connection_catalog/${encodeURIComponent(body.connection_id)}` +
+            `/schemas/${encodeURIComponent(body.schema)}` +
+            `/tables/${encodeURIComponent(table_name)}/columns`,
+          user,
+        );
+        return {
+          table_name,
+          columns: result.columns.map((column) => ({
+            name: column.column_name,
+            type: column.data_type,
+            is_primary_key: column.is_primary_key,
+          })),
+          references: [],
+        };
       }),
     );
+    return { operation_result: true, tables_metadata };
+  }
+
+  async refreshCatalog(
+    body: RefreshCatalogReq,
+    user: RequestUser,
+  ): Promise<RefreshCatalogRes> {
+    const task = await this.platformApiService.proxy(
+      'POST',
+      '/connection_test',
+      user,
+      {
+        customer_id: user.customer_name,
+        plugin: body.plugin,
+        task: {
+          task_type: 'refresh_catalog',
+          connection: {
+            provider: 'connection_manager',
+            config_id: body.connection_id,
+          },
+        },
+      },
+    );
+
+    if (!task.session_id || !task.date) {
+      throw new HttpException(
+        'Platform API did not return a catalog refresh task identifier',
+        HttpStatus.BAD_GATEWAY,
+      );
+    }
+
+    return {
+      operation_result: true,
+      status: 'PENDING',
+      session_id: task.session_id,
+      date: task.date,
+    };
+  }
+
+  async refreshCatalogStatus(
+    body: RefreshCatalogStatusReq,
+    user: RequestUser,
+  ): Promise<RefreshCatalogRes> {
+    const result = await this.platformApiService.proxy(
+      'POST',
+      '/connection_test/status',
+      user,
+      {
+        session_id: body.session_id,
+        date: body.date,
+      },
+    );
+
+    if (result.status === 'DONE') {
+      await this.connectionsApiService.proxy(
+        'PUT',
+        `/connection_config/${encodeURIComponent(
+          body.connection_id,
+        )}/catalog_metadata`,
+        user,
+        {
+          last_catalog_refresh_status: 'SUCCESS',
+          last_catalog_connection_test_date: body.date,
+          last_catalog_connection_test_session_id: body.session_id,
+        },
+      );
+    } else if (result.status === 'ERROR' || result.status === 'EXPIRED') {
+      throw new HttpException(
+        `Catalog refresh finished with status ${result.status}`,
+        HttpStatus.BAD_GATEWAY,
+      );
+    }
+
+    return {
+      operation_result: result.status === 'DONE',
+      status: result.status,
+      session_id: body.session_id,
+      date: body.date,
+    };
   }
 }

src/modules/connection-test/dto/connection-test.ts

@@ -1,5 +1,5 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
-import { IsString, IsOptional } from 'class-validator';
+import { IsIn, IsString, IsOptional } from 'class-validator';
 import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
 import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
 export class ColumnDto {
@@ -7,6 +7,8 @@ export class ColumnDto {
   name: string;
   @ApiProperty()
   type: string;
+  @ApiProperty()
+  is_primary_key: boolean;
 }
 export class TableMetadataDto {
   @ApiProperty()
@@ -131,3 +133,37 @@ export class GetTableMetadataRes {
   @ApiProperty({ type: [TableMetadataDto] })
   tables_metadata: TableMetadataDto[];
 }
+
+export class RefreshCatalogReq {
+  @ApiProperty()
+  @IsString()
+  connection_id: string;
+
+  @ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] })
+  @IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver'])
+  plugin: string;
+}
+
+export class RefreshCatalogStatusReq extends RefreshCatalogReq {
+  @ApiProperty()
+  @IsString()
+  session_id: string;
+
+  @ApiProperty()
+  @IsString()
+  date: string;
+}
+
+export class RefreshCatalogRes {
+  @ApiProperty()
+  operation_result: boolean;
+
+  @ApiProperty()
+  status: string;
+
+  @ApiProperty()
+  session_id: string;
+
+  @ApiProperty()
+  date: string;
+}

src/modules/connection/connection.controller.ts

@@ -16,8 +16,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
+  RequireModule,
 } from 'src/decorators/authentication.decorator';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { ValidationPipe } from '../../pipes/object-validation.pipe';
 import {
@@ -39,6 +40,9 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
 @ApiTags('connections')
 @Authenticated()
 @Controller('connections')
+@RequireModule(
+  DADOSFERA_MODULES_KEYS.COLLECT
+)
 export class ConnectionController {
   logger: any;
   constructor(

src/modules/connections-api/connections-api.config.ts

@@ -0,0 +1,11 @@
+export const CONNECTIONS_API_CONFIG = {
+  getUrl: (): string => {
+    const url = process.env.CONNECTIONS_API_URL;
+    if (!url) {
+      throw new Error('CONNECTIONS_API_URL environment variable is not set');
+    }
+    return url;
+  },
+  region: process.env.AWS_REGION || 'us-east-1',
+  timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10),
+};

src/modules/connections-api/connections-api.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { ConnectionsApiService } from './connections-api.service';
+
+@Module({
+  providers: [ConnectionsApiService, DadosferaLogger],
+  exports: [ConnectionsApiService],
+})
+export class ConnectionsApiModule {}

src/modules/connections-api/connections-api.service.ts

@@ -0,0 +1,99 @@
+import { Injectable, Inject, HttpException } from '@nestjs/common';
+import { SignatureV4 } from '@aws-sdk/signature-v4';
+import { Sha256 } from '@aws-crypto/sha256-js';
+import { defaultProvider } from '@aws-sdk/credential-provider-node';
+import axios, { AxiosResponse, Method } from 'axios';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { RequestUser } from '../../decorators/user.decorator';
+import { CONNECTIONS_API_CONFIG } from './connections-api.config';
+
+@Injectable()
+export class ConnectionsApiService {
+  private signer: SignatureV4;
+  private logger: any;
+
+  constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) {
+    this.logger = dadosferaLogger.logger;
+    this.signer = new SignatureV4({
+      service: 'execute-api',
+      region: CONNECTIONS_API_CONFIG.region,
+      credentials: defaultProvider(),
+      sha256: Sha256,
+    });
+  }
+
+  async proxy(
+    method: string,
+    path: string,
+    user: RequestUser,
+    body?: any,
+    query?: Record<string, string>,
+  ): Promise<any> {
+    const baseUrl = CONNECTIONS_API_CONFIG.getUrl();
+    const url = new URL(`${baseUrl}${path}`);
+
+    if (query) {
+      Object.entries(query).forEach(([key, value]) => {
+        if (value !== undefined && value !== null) {
+          url.searchParams.set(key, String(value));
+        }
+      });
+    }
+    const headers: Record<string, string> = {
+      host: url.hostname,
+      'content-type': 'application/json',
+      customer_name: user.customer_name || '',
+      customer_id: user.customer_id || '',
+      'x-user-id': user.user_id || '',
+      'x-username': user.username || '',
+      'x-customer-tier': user.customer_tier || '',
+      'x-customer-id': user.customer_id || '',
+    };
+    const requestToSign = {
+      method: method.toUpperCase(),
+      protocol: url.protocol,
+      hostname: url.hostname,
+      port: url.port ? parseInt(url.port, 10) : undefined,
+      path: url.pathname + url.search,
+      headers,
+      body: body ? JSON.stringify(body) : undefined,
+    };
+
+    try {
+      const signedRequest = await this.signer.sign(requestToSign);
+      const response: AxiosResponse = await axios({
+        method: method as Method,
+        url: url.href,
+        headers: signedRequest.headers as Record<string, string>,
+        data: body,
+        timeout: CONNECTIONS_API_CONFIG.timeout,
+        validateStatus: () => true,
+      });
+
+      if (response.status >= 400) {
+        throw new HttpException(response.data, response.status);
+      }
+      return response.data;
+    } catch (error) {
+      this.logger.error('Connections API proxy error', {
+        error: error.message,
+        path,
+        method: method.toUpperCase(),
+      });
+      if (error instanceof HttpException) {
+        throw error;
+      }
+      if (error.response) {
+        throw new HttpException(error.response.data, error.response.status);
+      }
+      if (error.code === 'ECONNREFUSED') {
+        throw new HttpException('Connections API service unavailable', 503);
+      }
+      if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
+        throw new HttpException('Connections API request timeout', 504);
+      }
+      throw new HttpException('Internal server error', 500);
+    }
+  }
+}

src/modules/connector/connector.controller.ts

@@ -25,9 +25,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import {
   Authenticated,
   RequireAllPermissions,
+  RequireModule,
   RequireSomePermission,
 } from 'src/decorators/authentication.decorator';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -99,6 +100,9 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async getAllConnectors(
     @Language() language: LanguageEnum,
     @Query() queries: GetAllDto,
@@ -131,6 +135,9 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async getConnectorsTags() {
     return await this.connectorClientService.getConnectorsTags();
   }
@@ -143,6 +150,9 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async getConnector(
     @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
@@ -171,6 +181,9 @@ export class ConnectorController {
     PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
     PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
   )
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async getConnectorDetails(
     @Language() language: LanguageEnum,
     @Param('plugin') plugin: string,
@@ -193,6 +206,9 @@ export class ConnectorController {
   @Put('/:plugin')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
   @ApiConsumes('multipart/form-data')
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async updateConnector(
     @Param('plugin') plugin: string,
     @Body() body: UpdateDto,
@@ -214,6 +230,9 @@ export class ConnectorController {
 
   @Put('/:plugin/add-tag')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async addTagOnConnector(
     @Param('plugin') plugin: string,
     @Body() body: AddTagDto,
@@ -241,6 +260,9 @@ export class ConnectorController {
 
   @Put('/:plugin/remove-tag')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async removeTagOnConnector(
     @Param('plugin') plugin: string,
     @Body() body: RemoveTagDto,
@@ -269,6 +291,9 @@ export class ConnectorController {
 
   @Delete('/:plugin')
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
+  @RequireModule(
+    DADOSFERA_MODULES_KEYS.COLLECT
+  )
   async deleteConnector(
     @Param('plugin') plugin: string,
     @Query('version') version: string,

src/modules/customers/customers.controller.ts

@@ -136,4 +136,32 @@ export class CustomersController {
     const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
     return result;
   }
+
+  @Get(':id/organization-info')
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @ApiOkResponse({ description: 'Organization information' })
+  async getOrganizationInfo(@Param('id') id: string) {
+    this.logger.info('getOrganizationInfo', { id });
+    return this.customersService.getOrganizationInfo(id);
+  }
+
+  @Put(':id/organization-info')
+  @Authenticated()
+  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
+  @HttpCode(HttpStatus.OK)
+  @ApiOkResponse({ description: 'Organization information updated' })
+  async updateOrganizationInfo(
+    @Param('id') id: string,
+    @Body() body: {
+      companyName: string;
+      companySite: string;
+      domain: string;
+      cnpj: string;
+      description: string;
+    },
+  ) {
+    return this.customersService.updateOrganizationInfo(id, body);
+  }
+
 }

src/modules/customers/customers.service.ts

@@ -9,12 +9,12 @@ import {
 } from '@nestjs/common';
 
 import { firstValueFrom, lastValueFrom } from 'rxjs';
-import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import { DucClient } from '../duc/client.config';
 import { ClientGrpc } from '@nestjs/microservices';
 import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { CustomerLinksConfig } from './dtos/customers';
 import ErrorCodes from 'src/utils/errorCodes';
 import jwt from 'jsonwebtoken';
 import {
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
     )
   }
 
-  async getLinks(customerId: string) {
+  async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
     try {
       const result = await lastValueFrom(
-        this.customerService.CustomerFindOneById({ id: customerId }),
+        this.customerService.CustomerGetLinks({ customerId }),
       );
-      return result.customer?.links || [];
+      return (result.links as CustomerLinksConfig) || null;
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
         throw new HttpException(err.details, HttpStatus.NOT_FOUND);
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
     }
   }
 
-  async setLinks(customerId: string, links: Link[]) {
+  async setLinks(customerId: string, links: CustomerLinksConfig) {
     if (!customerId || !links) {
       throw new HttpException(null, HttpStatus.BAD_REQUEST);
     }
 
     try {
       return await firstValueFrom(
-        this.customerService.CustomerUpdate({
-          id: customerId,
-          links,
-        } as CustomerUpdateRequest),
+        this.customerService.CustomerSetLinks({
+          customerId,
+          links: links as CustomerSetLinksRequest['links'],
+        }),
       );
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
@@ -223,4 +223,57 @@ export class CustomersService implements OnModuleInit {
       })
     )
   }
-}
+
+  async updateOrganizationInfo(
+    customerId: string,
+    data: {
+      companyName: string;
+      companySite: string;
+      domain: string;
+      cnpj: string;
+      description: string;
+    },
+  ) {
+    try {
+      const result = await lastValueFrom(
+        this.customerService.OrganizationUpdate({
+          customerId,
+          companyName: data.companyName || '',
+          companySite: data.companySite || '',
+          domain: data.domain || '',
+          cnpj: data.cnpj || '',
+          description: data.description || '',
+        }),
+      );
+
+      return result;
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      else throw err;
+    }
+  }
+
+  async getOrganizationInfo(customerId: string) {
+    try {
+      const customerResponse = await lastValueFrom(
+        this.customerService.CustomerFindOneById({ id: customerId })
+      );
+
+      const customer = customerResponse.customer;
+      
+      return {
+        companyName: customer.companyName || '',
+        companySite: customer.companySite || '',
+        domain: customer.domain || '',
+        cnpj: customer.cnpj || '',
+        description: customer.description || ''
+      };
+    } catch (err) {
+      if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+        throw new HttpException(err.details, HttpStatus.NOT_FOUND);
+      else throw err;
+    }
+  }
+
+}

src/modules/customers/dtos/customers.ts

@@ -1,7 +1,6 @@
-import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
-export class CustomerLink implements Link {
+export class CustomerLinkItem {
   @ApiProperty()
   href: string;
   @ApiProperty()
@@ -9,15 +8,59 @@ export class CustomerLink implements Link {
   @ApiProperty()
   description: string;
   @ApiPropertyOptional()
-  iconSrc: string;
+  iconSrc?: string;
 }
+
+export class CustomerSidebarLinkItem {
+  @ApiProperty()
+  type: 'link';
+  @ApiProperty({ type: Object })
+  title: Record<string, string>;
+  @ApiProperty()
+  link: string;
+  @ApiPropertyOptional()
+  icon?: string;
+}
+
+export class CustomerSidebarMenuItem {
+  @ApiProperty()
+  type: 'menu';
+  @ApiProperty({ type: Object })
+  title: Record<string, string>;
+  @ApiPropertyOptional()
+  icon?: string;
+  @ApiProperty({ type: [CustomerSidebarLinkItem] })
+  items: CustomerSidebarLinkItem[];
+}
+
+export class CustomerSidebarSection {
+  @ApiProperty({ type: Object })
+  title: Record<string, string>;
+  @ApiProperty({
+    type: 'array',
+    items: {
+      oneOf: [
+        { $ref: '#/components/schemas/CustomerSidebarMenuItem' },
+        { $ref: '#/components/schemas/CustomerSidebarLinkItem' },
+      ],
+    },
+  })
+  items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
+}
+
+export class CustomerLinksConfig {
+  @ApiPropertyOptional({ type: [CustomerLinkItem] })
+  home?: CustomerLinkItem[];
+  @ApiPropertyOptional({ type: [CustomerSidebarSection] })
+  sidebar?: CustomerSidebarSection[];
+}
+
 export class CustomerLinkRequest {
-  @ApiProperty({ type: [CustomerLink] })
-  links: CustomerLink[];
+  @ApiProperty({ type: CustomerLinksConfig })
+  links: CustomerLinksConfig;
 }
 
 export class CustomerLinksResponse {
-  @ApiProperty({ type: [CustomerLink] })
-  links: CustomerLink[];
-}
-
+  @ApiPropertyOptional({ type: CustomerLinksConfig })
+  links?: CustomerLinksConfig;
+}

src/modules/customers/dtos/organization.ts

@@ -0,0 +1,27 @@
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+
+export class OrganizationUpdateRequest {
+  @ApiProperty()
+  name: string;
+  @ApiPropertyOptional()
+  companySite: string;
+  @ApiProperty()
+  domain: string;
+  @ApiPropertyOptional()
+  info: string;
+  @ApiPropertyOptional()
+  cnpj: string;
+}
+
+export class OrganizationResponse {
+  @ApiProperty()
+  name: string;
+  @ApiPropertyOptional()
+  companySite: string;
+  @ApiProperty()
+  domain: string;
+  @ApiPropertyOptional()
+  info: string;
+  @ApiPropertyOptional()
+  cnpj: string;
+}

src/modules/inputs/dtos/input.model.ts

@@ -11,10 +11,19 @@ export class TableColumns {
   name: string;
   @ApiProperty()
   columns: string[];
-  @ApiProperty()
+  @ApiPropertyOptional({ type: [Column] })
   references: Column[];
   @ApiProperty()
+  destination: Record<'raw' | 'qualify', {
+    table_name: string;
+    table_schema: string;
+  }> | null;
+  @ApiProperty()
   type: string;
+  @ApiPropertyOptional({ type: [String] })
+  identifier_columns?: string[];
+  @ApiPropertyOptional({ type: Column })
+  reference_column?: Column;
 }
 export class AvailableEntity {
   @ApiProperty()

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,4 +1,8 @@
-import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+export interface Info {
+  user_id: string;
+  customer_id: string;
+  customer: string;
+}
 
 interface Values {
   jdbc_user: string;

src/modules/inputs/inputs.controller.ts

@@ -99,6 +99,7 @@ export class InputsController {
       customer: info.customer,
     });
 
+    this.logger.info(JSON.stringify(body))
     const response = await this.inputService.create({ body, info });
 
     return response;

src/modules/inputs/inputs.service.ts

@@ -17,10 +17,13 @@ import {
   InputCreateGenericRequest,
   InputCreateS3Request,
   InputNewCreateRequest,
+  InputUpdateResponse,
+  RollbackInputRequest,
   TestConnectionRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 import { CreateInputReq } from './dtos/input.model';
+import { Metadata } from '@grpc/grpc-js';
 
 @Injectable()
 
@@ -71,10 +74,10 @@ export class InputsService {
       objectCamelToSnake(createInputResponse);
       return createInputResponse;
     },
-    update: async (updateInputDTO: UpdateInputRequest) => {
-      this.logger.info('InputClientService - Update');
+    update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
+      this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
       const updateInputResponse = await lastValueFrom(
-        this.inputWriteService.InputUpdate(updateInputDTO),
+        this.inputWriteService.InputUpdate(updateInputDTO, metadata),
       );
 
       return updateInputResponse;
@@ -164,6 +167,11 @@ export class InputsService {
     const inputCreateGenericRequest: InputCreateGenericRequest = {
       input: {
         ...body,
+        tables: (body.tables || []).map((table) => ({
+          ...table,
+          identifier_columns: table.identifier_columns || [],
+          reference_column: table.reference_column || table.references?.[0],
+        })),
       },
       info,
     };
@@ -199,24 +207,47 @@ export class InputsService {
     return findOneInputResponse;
   }
 
-  async update(id: string, data, info: Info) {
-    this.validateCron({ ...data, info });
+  async update(id: string, data, info: Info, metadata?: Metadata) {
+    // this.validateCron({ ...data, info });
     try {
-      const updateInputResponse: any = await this.OLD_inputClient.update({
+      const {
+        tablesUpdate,
+        dataAssetUpdate,
+        input
+      } = await this.OLD_inputClient.update({
         id,
-        info,
         ...data,
-      });
+        info,
+      }, metadata);
 
-      updateInputResponse.input = this.adjustInputPayload(
-        updateInputResponse?.input,
+      const updateInputResponse = this.adjustInputPayload(
+        input,
       );
-      return updateInputResponse;
+      return {
+        input: updateInputResponse,
+        tablesUpdate,
+        dataAssetUpdate
+      };
     } catch (err) {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
 
+  async rollbackUpdate(
+    data: RollbackInputRequest
+  ) {
+    this.logger.info('PipelinesClientService - rollbackUpdate');
+    this.logger.info('Rolling back input update with data: ' + JSON.stringify(data));
+    const updatePipelineResponse = await lastValueFrom(
+      this.inputWriteService.RollbackInputUpdate(
+        data
+      ),
+    );
+    this.logger.info('Done');
+
+    return updatePipelineResponse;
+  }
+
   async remove(idRequest: IIdRequest) {
     return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
   }
@@ -258,4 +289,12 @@ export class InputsService {
     };
     return formatedPayload;
   }
+
+  async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
+    return lastValueFrom(this.inputWriteService.MarkTableDeleted(data));
+  }
+
+  async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
+    return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data));
+  }
 }

src/modules/permissions/permissions.controller.spec.ts

@@ -1,6 +1,6 @@
 import { ClientsModule } from '@nestjs/microservices';
 import { Test, TestingModule } from '@nestjs/testing';
-// import { DucClient } from 'src/clients/duc/client.config';
+// import { DucClient } from '../duc/client.config'
 import { PermissionsController } from './permissions.controller';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PermissionsService } from './permissions.service';
@@ -22,7 +22,7 @@ describe.skip('PermissionsController', () => {
           provide: DadosferaLogger,
           useValue: { logger },
         },
-        PermissionsService,
+        PermissionsService
       ],
     }).compile();
 

src/modules/pipelines/client.service.ts

@@ -1,78 +0,0 @@
-import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
-import { ClientGrpc } from '@nestjs/microservices';
-import {
-  PipelineServicesNames,
-  PipelinesServiceInterface,
-} from '@dadosfera/protospack';
-import { lastValueFrom } from 'rxjs';
-
-import { IIdRequest } from './interfaces';
-
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { PipelinesClientConfiguration } from './pipelines-client';
-
-export class PipelinesClientService implements OnModuleInit {
-  private pipelineService: PipelinesServiceInterface;
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-      dadosferaLogger: DadosferaLogger,
-    @Inject(PipelinesClientConfiguration.name)
-    private readonly grpcClient: ClientGrpc,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  onModuleInit() {
-    this.pipelineService =
-      this.grpcClient.getService<PipelinesServiceInterface>(
-        PipelineServicesNames.PipelineService,
-      );
-  }
-
-  async getPipelineStatus(data) {
-    this.logger.info('PipelinesClientService - GetPipelineStatus');
-
-    const statusPipelineResponse = await lastValueFrom(
-      this.pipelineService.getPipelineStatus(data),
-    )
-      .then((res) => {
-        const statusArray =
-          res.status?.sort((a, b) => {
-            if (a.id < b.id) {
-              return 1;
-            } else {
-              return -1;
-            }
-          }) || [];
-        return { status: statusArray };
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-    this.logger.info('Done');
-
-    return statusPipelineResponse;
-  }
-
-  async runPipeline({ id, info }: IIdRequest) {
-    this.logger.info('PipelinesClientService - RunPipeline');
-    const statusPipelineResponse = await lastValueFrom(
-      this.pipelineService.triggerPipeline({ id, info }),
-    ).catch((err) => {
-      this.logger.error(err.message);
-      throw new Error(err);
-    });
-
-    if (statusPipelineResponse.status == false) {
-      throw new ConflictException(
-        'This pipeline is not ready yet to execute, Try again later!',
-      );
-    }
-
-    this.logger.info('Done');
-    return statusPipelineResponse;
-  }
-}

src/modules/pipelines/interfaces.d.ts

@@ -1,36 +0,0 @@
-import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
-
-export interface ICreatePipelineDto {
-  input: IdRequest;
-  transformations: IdRequest[];
-  output: IdRequest;
-  tags: string[];
-  name: string;
-  description: string;
-  info: Info;
-}
-
-export interface IdRequest {
-  id: string;
-}
-
-export interface IIdRequest {
-  id: string;
-  info: Info;
-}
-
-export interface IUpdatePipelineRequest {
-  input: IdRequest;
-  transformations: IdRequest[];
-  output: IdRequest;
-  tags: string[];
-  name: string;
-  description: string;
-  id: string;
-  info: Info;
-}
-
-export interface IGetPipelineLogsRequest {
-  id: string;
-  details: string;
-}

src/modules/pipelines/pipelines-client.ts

@@ -1,33 +0,0 @@
-import {
-  ClientsProviderAsyncOptions,
-  GrpcOptions,
-  Transport,
-} from '@nestjs/microservices';
-import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
-import { credentials } from '@grpc/grpc-js';
-
-const isLocalConnection =
-  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
-  process.env.PIFACTORY_URL.includes('0.0.0.0');
-
-export class PipelinesClientConfiguration {
-  public name = 'PipelinesClientConfiguration';
-  private config: GrpcOptions = {
-    transport: Transport.GRPC,
-    options: {
-      url: process.env.PIFACTORY_URL,
-      package: PipelinePackages,
-      credentials: isLocalConnection ? undefined : credentials.createSsl(),
-      protoPath: PipelineProtoFilePath,
-      loader: {
-        keepCase: true,
-        enums: String,
-        defaults: false,
-      },
-    },
-  };
-  providerOptions: ClientsProviderAsyncOptions = {
-    name: this.name,
-    ...this.config,
-  };
-}

src/modules/pipelines/pipelines.controller.ts

@@ -1,72 +0,0 @@
-import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
-import { ApiOperation, ApiTags } from '@nestjs/swagger';
-import {
-  AuthenticateCondition,
-  Authenticated,
-  RequireSomePermission,
-} from 'src/decorators/authentication.decorator';
-import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
-import { PipelinesService } from './pipelines.service';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
-
-@ApiInternalOnlyController()
-@ApiTags('Pipelines')
-@Controller('pipelines')
-@Authenticated()
-export class PipelinesController {
-  logger: DadosferaLogger;
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private pipelineService: PipelinesService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  @Post('start/:id')
-  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
-  @ApiOperation({
-    deprecated: true,
-    description:
-      'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
-  })
-  async activate(@Param('id') id: string, @Body() body) {
-    const { info } = body;
-
-    this.logger.info(
-      process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
-      {
-        user: body.info.user_id,
-        customer: body.info.customer,
-      },
-    );
-
-    const response = await this.pipelineService.runPipeline({ id, info });
-
-    return response;
-  }
-
-  @Get(':id/status')
-  @ApiOperation({
-    deprecated: true,
-    description:
-      'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
-  })
-  @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getPipelineStatus(@Body() body, @Param('id') id: string) {
-    body.id = id;
-
-    this.logger.info(
-      process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
-      {
-        user: body.info.user_id,
-        customer: body.info.customer,
-      },
-    );
-
-    const response = await this.pipelineService.getPipelineStatus(body);
-
-    return response;
-  }
-}

src/modules/pipelines/pipelines.module.ts

@@ -1,19 +0,0 @@
-import { Module } from '@nestjs/common';
-import { ClientsModule } from '@nestjs/microservices';
-import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-
-import { PipelinesController } from './pipelines.controller';
-import { PipelinesService } from './pipelines.service';
-
-import { PipelinesClientConfiguration } from './pipelines-client';
-import { PipelinesClientService } from './client.service';
-
-const client = new PipelinesClientConfiguration();
-
-@Module({
-  imports: [ClientsModule.register([client.providerOptions])],
-  controllers: [PipelinesController],
-  providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
-  exports: [PipelinesService],
-})
-export class PipelinesModule {}

src/modules/pipelines/pipelines.service.ts

@@ -1,33 +0,0 @@
-import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
-import { PipelinesClientService } from './client.service';
-import { IIdRequest } from './interfaces';
-import { objectCamelToSnake } from 'src/utils/CaseConverter';
-
-@Injectable()
-export class PipelinesService {
-  constructor(private pipelineClient: PipelinesClientService) {}
-
-  async getPipelineStatus(data: IIdRequest) {
-    try {
-      const pipelineStatusResponse =
-        await this.pipelineClient.getPipelineStatus(data);
-
-      return objectCamelToSnake(pipelineStatusResponse);
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
-  async runPipeline({ id, info }: IIdRequest) {
-    try {
-      const triggerPipelineResponse = await this.pipelineClient.runPipeline({
-        id,
-        info,
-      });
-
-      return objectCamelToSnake(triggerPipelineResponse);
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-}

src/modules/pipelinesV2/interfaces.ts

@@ -1,5 +1,13 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
-import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+
+export class PipelineInputsDTO {
+  @ApiProperty()
+  tables: Array<{
+    name: string,
+    type: string,
+    
+  }>
+}
 
 export class IPipelineV2 {
   @ApiProperty()
@@ -53,6 +61,12 @@ export interface IIdRequest {
   info: Info;
 }
 
+export interface Info {
+  user_id: string;
+  customer_id: string;
+  customer: string;
+}
+
 export interface IUpdatePipelineRequest {
   input: IdRequest;
   transformations: IdRequest[];
@@ -123,3 +137,30 @@ export class PipelineFindAllReq {
   @ApiPropertyOptional()
   type?: string | undefined;
 }
+
+export interface UpdateTableDTO {
+  name: string;
+  type: string;
+  columns: string[];
+  destinations: {
+    raw: {
+      table_schema: string;
+      table_name: string;
+    };
+    qualify: {
+      table_schema: string;
+      table_name: string;
+    };
+  };
+  identifier_columns: string[];
+  reference_column: {
+    name: string;
+    type: string;
+  };
+  memory: number;
+}
+
+export interface UpdatePlatformInputRequest {
+  cron: string;
+  tables: Array<UpdateTableDTO>;
+}

src/modules/pipelinesV2/pipelines.controller.ts

@@ -14,7 +14,7 @@ import {
   Patch,
   HttpException,
   BadRequestException,
-  CacheTTL,
+  UseGuards,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -24,18 +24,17 @@ import {
   ApiTags,
 } from '@nestjs/swagger';
 import {
-  AuthenticateCondition,
   RequireAllPermissions,
+  RequireModule,
   RequireSomePermission,
 } from 'src/decorators/authentication.decorator';
-import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 
-import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import {
   ICompleteUploadCSVFile,
   ICreatePipelineCSVFile,
@@ -43,23 +42,31 @@ import {
   IPipelineV2,
   IInitUploadCSVFile,
   PipelineFindAllReq,
+  UpdatePlatformInputRequest,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
+
+type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any };
+type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
 
 @ApiTags('PipelinesV2')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
 @UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('pipelinesV2')
+@RequireModule(
+  DADOSFERA_MODULES_KEYS.COLLECT
+)
 export class PipelinesController {
   logger: DadosferaLogger;
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
     private pipelinesClientService: PipelinesService,
-    private oldPipelinesService: OldPipelineService,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -187,6 +194,7 @@ export class PipelinesController {
   @Get(':id/status')
   @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getPipelineStatus(@Body() body, @Param('id') id: string) {
+
     body.id = id;
 
     this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
@@ -194,7 +202,7 @@ export class PipelinesController {
       customer: body.info.customer,
     });
 
-    const response = await this.oldPipelinesService.getPipelineStatus(body);
+    const response = await this.pipelinesClientService.getPipelineStatus(body);
 
     return response;
   }
@@ -218,26 +226,50 @@ export class PipelinesController {
       language,
     });
 
-    const result = await this.pipelinesClientService
-      .findOne({ id }, metadata)
-      .then((res) => {
-        //{pipeline:{tables: {tables: [], input_id: ''}}}
-        let tables = JSON.parse(res.pipeline.config.tables);
-        if (tables?.tables) tables = tables.tables;
-        Object.assign(res.pipeline, {
-          transformations: res.pipeline.transformations
-            ? JSON.parse(res.pipeline.transformations)
-            : [],
-          config: {
-            cron: res.pipeline.config.cron,
-            tables,
-          },
-          properties: res.pipeline.properties
-            ? JSON.parse(res.pipeline.properties)
-            : {},
-        });
-        return res;
-      });
+    const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata);
+
+    const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables);
+    const input_id = parsed.input_id;
+    const tables: PipelineTable[] = parsed.tables ?? [];
+
+    Object.assign(pipelineRes.pipeline, {
+      transformations: pipelineRes.pipeline.transformations
+        ? JSON.parse(pipelineRes.pipeline.transformations)
+        : [],
+      config: {
+        cron: pipelineRes.pipeline.config.cron,
+        tables,
+        input_id,
+      },
+      properties: pipelineRes.pipeline.properties
+        ? JSON.parse(pipelineRes.pipeline.properties)
+        : {},
+    });
+
+    return pipelineRes;
+  }
+
+  @Get("/:id/data-assets")
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET
+  )
+  async findAllDataAssetByPipeline(
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @User() user: RequestUser,
+    @Query('object') object: string
+  ) {
+    const payload = {
+      pipeline: id,
+      object: object, 
+    };
+
+    this.logger.info(`GET pipelinesV2/:id/data-assets` + JSON.stringify(payload));
+    
+    const result =
+      await this.pipelinesClientService.findAllDataAssetByPipeline(payload, user);
 
     return result;
   }
@@ -277,6 +309,40 @@ export class PipelinesController {
     return response;
   }
 
+  @Patch('/:pipelineId/inputs/:id')
+  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  @UseGuards(PipelineExecutionGuard)
+  async updatePipelineInput(
+    @Language() language: LanguageEnum,
+    @Body() pipelineInputDTO: UpdatePlatformInputRequest,
+    @Param('id') inputId: string,
+    @Param('pipelineId') pipelineId: string,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('PipelinesController - update', { user });
+
+    const info: Info = {
+      user_id: user.user_id,
+      customer: user.customer_name,
+      customer_id: user.customer_id,
+      pipeline_id: pipelineId
+    };
+
+    const metadata = PackTheMetadata(user);
+
+    const response = await this.pipelinesClientService.updatePipelineInput(
+      pipelineId,
+      inputId,
+      pipelineInputDTO,
+      info,
+      user,
+      metadata,
+    );
+
+    this.logger.info('PipelinesController - update: OK', { user });
+    return response;
+  }
+
   @ApiInternalOnlyEndpoint()
   @Put('/:id')
   @ApiOperation({
@@ -296,6 +362,21 @@ export class PipelinesController {
     return response;
   }
 
+  @Patch('/:id/upgrade')
+  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  @HttpCode(HttpStatus.NO_CONTENT)
+  async upgradeConnector(
+    @Language() language: LanguageEnum,
+    @Param('id') id: string,
+    @User() user: RequestUser
+  ) {
+    this.logger.info('PipelinesController - upgrade connector');
+
+    const metadata = PackTheMetadata(user);
+
+    await this.pipelinesClientService.upgrade(id, metadata);
+  }
+
   @Delete(':id')
   @ApiNoContentResponse()
   @HttpCode(HttpStatus.NO_CONTENT)
@@ -425,7 +506,7 @@ export class PipelinesController {
       },
     );
 
-    const response = await this.oldPipelinesService.runPipeline({ id, info });
+    const response = await this.pipelinesClientService.runPipeline({ id, info });
 
     return response;
   }

src/modules/pipelinesV2/pipelines.module.ts

@@ -7,23 +7,28 @@ import { PipelinesService } from './pipelines.service';
 
 import { PipelinesClientConfiguration } from './pipelines-client';
 
-import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
+import { PlatformApiModule } from '../platform-api/platform-api.module';
+import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
+import { NimbusService } from 'src/services/nimbus/nimbus.service';
+import { CatalogModule } from '../catalog/catalog.module';
 
 const client = new PipelinesClientConfiguration();
 
 @Module({
   imports: [
     ClientsModule.register([client.providerOptions]),
-    OldPipelineModule,
     ConnectorModule,
     InputsModule,
     TransformationsModule,
+    PlatformApiModule,
+    NimbusServicesModule,
+    CatalogModule
   ],
   controllers: [PipelinesController],
-  providers: [PipelinesService, DadosferaLogger],
+  providers: [PipelinesService, DadosferaLogger, NimbusService],
   exports: [PipelinesService],
 })
 export class PipelinesV2Module {}

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,5 +1,7 @@
+/* eslint-disable no-async-promise-executor */
 import {
   BadRequestException,
+  ConflictException,
   HttpException,
   HttpStatus,
   Inject,
@@ -16,7 +18,7 @@ import { lastValueFrom } from 'rxjs';
 
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PipelinesClientConfiguration } from './pipelines-client';
-import { ICreatePipelineV2Req } from './interfaces';
+import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
@@ -26,6 +28,17 @@ import { TransformationsService } from '../transformations/transformations.servi
 import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
+import { PlatformApiService } from '../platform-api/platform-api.service';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
+import { AxiosError } from 'axios';
+import { NimbusService } from 'src/services/nimbus/nimbus.service';
+import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
+import { PackTheMetadata } from 'src/utils/PackTheMetadata';
+import { IDataAsset } from '../catalog/dtos';
+import { CatalogService } from '../catalog/catalog.service';
+
+type RollbackPromise = () => Promise<any>;
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -39,6 +52,9 @@ export class PipelinesService implements OnModuleInit {
     private readonly connectorService: ConnectorClientService,
     private readonly inputsService: InputsService,
     private readonly transformationsService: TransformationsService,
+    private readonly platformAPI: PlatformApiService,
+    private readonly nimbusService: NimbusService,
+    private readonly catalogService: CatalogService
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -138,6 +154,7 @@ export class PipelinesService implements OnModuleInit {
     const findOnePipelineResponse = await lastValueFrom(
       this.pipelineReadService.PipelineV2FindOne(data, metadata),
     );
+    console.log('pipeline find one response', findOnePipelineResponse);
     this.logger.info('Done');
 
     return findOnePipelineResponse;
@@ -160,6 +177,15 @@ export class PipelinesService implements OnModuleInit {
     return updatePipelineResponse;
   }
 
+  async upgrade(id: string, metadata: Metadata) {
+    await lastValueFrom(
+      this.pipelineWriteService.Upgrade(
+        { id },
+        metadata,
+      ),
+    );
+  }
+
   async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
     const { id, metadata, user } = data;
     const info = {
@@ -339,4 +365,320 @@ export class PipelinesService implements OnModuleInit {
 
     return res;
   }
+
+  async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
+    this.logger.info('InputClientService - Update');
+
+    const {
+      input: oldInput
+    } = await this.inputsService.findOne({
+      id: inputId,
+      info: info
+    });
+
+    this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
+    const pipelineIdFormat = pipelineId.split('-').join('_');
+    const rollback: RollbackPromise[] = [];
+
+    const updateInputResponse = await this.inputsService.update(
+      inputId,
+      updateInputDTO,
+      info,
+      metadata
+    );
+
+    const inputRollback = () => {
+      this.logger.info("exec rollback to input: " + JSON.stringify(oldInput));
+      return this.inputsService.rollbackUpdate(
+        {
+          id: inputId,
+          dataAssetUpdate: updateInputResponse.dataAssetUpdate,
+          tables: oldInput.tables,
+          info
+        }
+      ) as Promise<any>;
+    }
+
+    rollback.push(inputRollback);
+
+    this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse))
+
+    const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
+
+    if (user.customer_modules.includes('catalog')) {
+      nimbusUpdates.forEach(update => {
+        const nimbusRollback = () => {
+          return this.nimbusService.renameTable(
+            info.customer,
+            update.database,
+            {
+              table_name: update.table_name,
+              table_schema: update.table_schema
+            },
+            {
+              table_name: update.old_table_name,
+              table_schema: update.old_table_schema
+            }
+          );
+        }
+        rollback.push(nimbusRollback);
+      });
+  
+      try {
+        await this.updateNimbus(info.customer, nimbusUpdates);
+      } catch (error) {
+        this.logger.error(error);
+        if (error instanceof AxiosError) {
+          this.logger.error(JSON.stringify(error.response.data));
+        }
+        await this.executeRenameRollback(rollback);
+  
+        throw new Error("Error Nimbus updating tables");
+      }
+    }
+
+    try {
+      await this.updatePlatformJobs(
+        pipelineIdFormat,
+        updateInputResponse.input.type,
+        updateInputDTO,
+        user
+      );
+    } catch (error) {
+      this.logger.error(error);
+      await this.executeRenameRollback(rollback)
+      throw new Error("Error Platform API updating jobs");
+    }
+
+    return updateInputResponse;
+  }
+
+  private async executeRenameRollback(request: RollbackPromise[]) {
+    this.logger.info('rollback steps: ' + request.length)
+    const result = await Promise.allSettled(request.map(func => func()));
+    result.forEach(promise => {
+      this.logger.info("Promise finish with status: " + promise.status)
+
+      if (promise.status === "rejected") {
+        this.logger.error("reject with: " + JSON.stringify(promise.reason || {}))
+      }
+
+      if (promise.status === "fulfilled") {
+        this.logger.info("success with: " + JSON.stringify(promise.value || {}))
+      }
+    });
+
+  }
+
+  private async updateNimbus(customer: string, changes: TableUpdate[]) {
+    // throw new Error("teste error nimbus");
+    this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
+    if(!changes || changes.length === 0) return;
+
+    const requests = changes.map(change => {
+      return this.nimbusService.renameTable(customer, change.database, {
+        table_name: change.old_table_name,
+        table_schema: change.old_table_schema
+      }, {
+        table_name: change.table_name,
+        table_schema: change.table_schema
+      });
+    })
+
+    const values = await Promise.allSettled(requests);
+
+    const success = values.map(request => request.status === "fulfilled")
+
+    this.logger.info("Updates with succes: " + success.length);
+
+    values.forEach(promise => {
+      this.logger.info("Promise finish with status: " + promise.status)
+
+      if (promise.status === "rejected") {
+        this.logger.error("reject with: " + JSON.stringify(promise.reason || {}));
+        throw new Error(promise.reason );
+      }
+
+      if (promise.status === "fulfilled") {
+        this.logger.info("success with: " + JSON.stringify(promise.value || {}));
+      }
+    });
+
+  }
+
+  async updatePlatformJobs(pipelineId: string, pipelineType: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
+    const jobsUpdated = [];
+
+    for (const [index, table] of updateInputDTO.tables.entries()) {
+      const jobUpdate = {
+        job_id: `${pipelineId}_${index}`,
+      }
+
+      if (table.type !== "incremental_with_qualify") {
+        delete table.destinations?.qualify;
+      }
+
+      if (table.memory) {
+        jobUpdate["memory"] = {
+          amount: table.memory * 1000
+        }
+      }
+      
+      this.logger.info('Updating input reference for table: ' + table.name);
+      let hasUpdateSyncMode = false;
+
+      const jobSyncMode = {}
+
+      if (table.columns) {
+        hasUpdateSyncMode = true;
+        jobSyncMode['column_include_list'] = table.columns;
+      }
+
+      if (table.reference_column) {
+        hasUpdateSyncMode = true;
+        jobSyncMode['incremental_column_name'] = table.reference_column.name;
+        jobSyncMode['incremental_column_type'] = table.reference_column.type;
+      }
+
+      if (table.identifier_columns) {
+        hasUpdateSyncMode = true;
+        jobSyncMode['primary_keys'] = table.identifier_columns;
+      }
+
+      if (table.type) {
+        hasUpdateSyncMode = true;
+        
+        jobSyncMode['target_load_type'] = table.type;
+      }
+
+      if(hasUpdateSyncMode) {
+        jobUpdate["sync_mode"] = jobSyncMode;
+      }
+
+      if (Object.keys(table.destinations).length > 1) {
+        let hasChanges = false
+        const jobRenameTables = {
+            raw: {},
+            qualify: {}
+        }
+
+        if (Object.keys(table.destinations.raw).length > 1) {
+          hasChanges = true;
+          jobRenameTables.raw = table.destinations.raw;
+        }
+
+        if (Object.keys(table.destinations.qualify).length > 1) {
+          hasChanges = true;
+          jobRenameTables.qualify = table.destinations.qualify;
+        }
+
+        if (hasChanges) {
+          jobUpdate['rename_tables'] = jobRenameTables;
+        }
+      }
+
+      jobsUpdated.push(jobUpdate);
+    }
+
+    this.logger.info('Request body:' + JSON.stringify({
+      jobs_updated: jobsUpdated
+    }));
+
+    const response = await this.platformAPI.proxy(
+      'PUT',
+        `/pipeline/${pipelineId}/jobs`,
+      user,
+      {
+        job_updates: jobsUpdated
+      }
+    )
+    this.logger.info('Platform api response: ' + JSON.stringify(response));
+  }
+
+  async findAllDataAssetByPipeline(data: {
+    pipeline: string,
+    object?: string
+  }, user: RequestUser) {
+    const metadata = PackTheMetadata(user);
+
+    const isDataAdmin = user.permissions.includes(
+      PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
+    );
+
+    let has_permission = false;
+
+    const {
+      data_assets: resultString
+    } = await lastValueFrom(
+      this.pipelineReadService.FindAllDataAssetByPipeline(data, metadata)
+    );
+
+    const result = JSON.parse(resultString) as any;
+    const data_assets: IDataAsset[] = []
+    result.forEach(data_asset => {
+      if (data_asset?.owner === user.username) has_permission = true;
+
+      for (const role of user.roles) {
+        if (data_asset.roles.includes(role)) has_permission = true;
+      }
+  
+      if (data_asset.users.includes(user.user_id)) has_permission = true;
+  
+      if (isDataAdmin || has_permission) {
+        delete data_asset.p_roles;
+        delete data_asset.p_users;
+        data_assets.push(data_asset as IDataAsset);
+      }
+    });
+
+    const assets = await this.catalogService.getAssetsUsersAndRoles(data_assets, user.customer_id);
+
+    return assets;
+  }
+
+  async getPipelineStatus(data) {
+    this.logger.info('PipelinesClientService - GetPipelineStatus');
+
+    const statusPipelineResponse = await lastValueFrom(
+      this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
+    )
+      .then((res) => {
+        const statusArray =
+          res.status?.sort((a, b) => {
+            if (a.id < b.id) {
+              return 1;
+            } else {
+              return -1;
+            }
+          }) || [];
+        return { status: statusArray };
+      })
+      .catch((err) => {
+        this.logger.error(err.message);
+        throw new Error(err);
+      });
+    this.logger.info('Done');
+
+    return statusPipelineResponse;
+  }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    this.logger.info('PipelinesClientService - RunPipeline');
+    const statusPipelineResponse = await lastValueFrom(
+      this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
+    ).catch((err) => {
+      this.logger.error(err.message);
+      throw new Error(err);
+    });
+
+    if (statusPipelineResponse.status == false) {
+      throw new ConflictException(
+        'This pipeline is not ready yet to execute, Try again later!',
+      );
+    }
+
+    this.logger.info('Done');
+    return statusPipelineResponse;
+  }
+
 }

src/modules/platform-api/platform-api.controller.ts

@@ -10,24 +10,47 @@ import {
   Query,
   Inject,
   BadRequestException,
+  HttpException,
+  NotFoundException,
+  UseGuards,
 } from '@nestjs/common';
-import { ApiTags, ApiOperation } from '@nestjs/swagger';
+import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import {
   Authenticated,
   RequireAllPermissions,
+  RequireModule,
 } from '../../decorators/authentication.decorator';
 import { User, RequestUser } from '../../decorators/user.decorator';
 import { PlatformApiService } from './platform-api.service';
-import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
+import { CatalogService } from '../catalog/catalog.service';
+import { PackTheMetadata } from '../../utils/PackTheMetadata';
+import { ValidationTableDTO } from './platform-api.dto';
+import { InputsService } from '../inputs/inputs.service';
+import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
+
+
+type ValidateTablesDTO = {
+  tables: Array<{
+    table_schema: string,
+    table_name: string
+  }>
+}
+
+type RenameTablesBody = {
+  raw?: { table_name: string; table_schema: string };
+  qualify?: { table_name: string; table_schema: string };
+}
 
 @ApiTags('Platform API')
 @Controller('platform')
+@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
 export class PlatformApiController {
   private logger: any;
 
@@ -36,6 +59,8 @@ export class PlatformApiController {
     private readonly elasticsearchService: ElasticsearchService,
     private readonly dynamoDBService: DynamoDBService,
     private readonly customersService: CustomersService,
+    private readonly catalogService: CatalogService,
+    private readonly inputsService: InputsService,
     @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
   ) {
     this.logger = dadosferaLogger.logger;
@@ -49,6 +74,10 @@ export class PlatformApiController {
     return id?.replace(/-/g, '_') || '';
   }
 
+  private decodePathParam(value: string): string {
+    return value ? decodeURIComponent(value) : '';
+  }
+
   /**
    * Denormalize ID back to UUID format (replace _ with -).
    * Used when we receive a normalized ID but need the original UUID.
@@ -67,6 +96,23 @@ export class PlatformApiController {
     return jobId?.replace(/-/g, '_') || '';
   }
 
+  private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
+    const connectorTypes = ['jdbc', 'singer', 's3'];
+    for (const type of connectorTypes) {
+      try {
+        const job = await this.platformApiService.proxy(
+          'GET',
+          `/jobs/${type}/${normalizedJobId}`,
+          user,
+        );
+        return job;
+      } catch (error) {
+        // Continue to next connector type
+      }
+    }
+    throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
+  }
+
   /**
    * Extract the pipeline ID (base UUID) from a job ID.
    * Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
@@ -94,7 +140,7 @@ export class PlatformApiController {
   }
 
   private readonly VALID_CONNECTORS = ['jdbc', 'singer', 's3'];
-  private readonly MAX_MEMORY_MB = 12000; // 12GB maximum memory per pipeline/job
+  private readonly MAX_MEMORY_MB = 12000; // 12GB maximum memory per pipelines/job
 
   /**
    * Validate that connector is provided and is a valid type.
@@ -365,55 +411,9 @@ export class PlatformApiController {
     }
   }
 
-  /**
-   * Sync sync-mode changes to DynamoDB for JDBC connectors.
-   * Always passes both target_load_type and incremental_column_name to ensure proper sync.
-   */
-  private async syncJdbcSyncModeToDynamoDB(
-    jobId: string,
-    body: any,
-    user: RequestUser,
-  ): Promise<void> {
-    // JDBC sync mode uses target_load_type field
-    const changes: any = {};
-
-    if ('target_load_type' in body) {
-      changes.target_load_type = body.target_load_type;
-    }
-
-    // Handle incremental_column_name:
-    // - If provided in body, use that value
-    // - If changing to full_load, explicitly clear it
-    if ('incremental_column_name' in body) {
-      changes.incremental_column_name = body.incremental_column_name;
-      changes.incremental_column_type = body.incremental_column_type;
-    } else if (body.target_load_type === 'full_load') {
-      // Changing to full_load without specifying incremental_column - clear it
-      changes.incremental_column_name = null;
-    }
-
-    await this.syncJobInputToDynamoDB(jobId, changes, user, 'jdbc');
-  }
-
-  /**
-   * Sync sync-mode changes to DynamoDB for Singer connectors.
-   */
-  private async syncSingerSyncModeToDynamoDB(
-    jobId: string,
-    body: any,
-    user: RequestUser,
-  ): Promise<void> {
-    // Singer sync mode uses replication_method field
-    // Map to DynamoDB type: FULL_TABLE -> full_load, INCREMENTAL -> incremental
-    if ('replication_method' in body) {
-      const type = body.replication_method === 'INCREMENTAL' ? 'incremental' : 'full_load';
-      await this.syncJobInputToDynamoDB(jobId, { load_type: type }, user, 'singer');
-    }
-  }
-
   // ==================== PIPELINE ROUTES ====================
 
-  @Post('pipeline')
+  @Post('pipelines')
   @ApiOperation({ summary: 'Create a new pipeline' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
   async createPipeline(@Body() body: any, @User() user: RequestUser) {
@@ -534,7 +534,7 @@ export class PlatformApiController {
     );
   }
 
-  @Get('pipeline/:pipelineId')
+  @Get('pipelines/:pipelineId')
   @ApiOperation({ summary: 'Get pipeline by ID' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getPipeline(
@@ -545,7 +545,7 @@ export class PlatformApiController {
     return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
   }
 
-  @Patch('pipeline/:pipelineId')
+  @Patch('pipelines/:pipelineId')
   @ApiOperation({ summary: 'Update pipeline by ID' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async updatePipeline(
@@ -596,7 +596,7 @@ export class PlatformApiController {
     return result;
   }
 
-  @Delete('pipeline/:pipelineId')
+  @Delete('pipelines/:pipelineId')
   @ApiOperation({ summary: 'Delete pipeline by ID' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
   async deletePipeline(
@@ -626,7 +626,7 @@ export class PlatformApiController {
     return result;
   }
 
-  @Post('pipeline/execute')
+  @Post('pipelines/execute')
   @ApiOperation({ summary: 'Execute a pipeline' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async executePipeline(@Body() body: any, @User() user: RequestUser) {
@@ -636,10 +636,10 @@ export class PlatformApiController {
       ...body,
       customer_id: user.customer_name,
     };
-    return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
+    return this.platformApiService.proxy('POST', '/pipelines/execute', user, enrichedBody);
   }
 
-  @Post('pipeline/pause')
+  @Post('pipelines/pause')
   @ApiOperation({ summary: 'Pause a pipeline' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async pausePipeline(@Body() body: any, @User() user: RequestUser) {
@@ -652,7 +652,7 @@ export class PlatformApiController {
     return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
   }
 
-  @Post('pipeline/unpause')
+  @Post('pipelines/unpause')
   @ApiOperation({ summary: 'Unpause a pipeline' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
@@ -665,7 +665,7 @@ export class PlatformApiController {
     return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
   }
 
-  @Put('pipeline/:pipelineId/memory')
+  @Put('pipelines/:pipelineId/memory')
   @ApiOperation({ summary: 'Update pipeline memory configuration' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async updatePipelineMemory(
@@ -688,7 +688,7 @@ export class PlatformApiController {
 
   // ==================== PIPELINE METADATA ROUTES ====================
 
-  @Put('pipeline/:pipelineId/metadata')
+  @Put('pipelines/:pipelineId/metadata')
   @ApiOperation({ summary: 'Update pipeline metadata' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async updatePipelineMetadata(
@@ -720,9 +720,44 @@ export class PlatformApiController {
     );
   }
 
+  // ==================== PIPELINE VALIDATION ====================
+
+  @Get('pipelines/catalog/schemas')
+  @ApiOperation({ summary: 'Get available schemas' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getAvailableSchemas(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'GET',
+      `/catalog/schemas`,
+      user,
+      undefined,
+      query,
+    );
+  }
+
+  @Post('pipelines/catalog/tables/validate')
+  @ApiOperation({ summary: 'Validate Table and Schema' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async validateTableAndSchema(
+    @Body() payload: ValidationTableDTO,
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'POST',
+      `/catalog/tables/validate`,
+      user,
+      payload,
+      query,
+    );
+  }
+
   // ==================== PIPELINE RUN ROUTES ====================
 
-  @Get('pipeline/:pipelineId/pipeline_run')
+  @Get('pipelines/:pipelineId/pipeline_run')
   @ApiOperation({ summary: 'Get pipeline runs for a pipeline' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getPipelineRuns(
@@ -740,7 +775,7 @@ export class PlatformApiController {
     );
   }
 
-  @Get('pipeline/:pipelineId/pipeline_run/:runId')
+  @Get('pipelines/:pipelineId/pipeline_run/:runId')
   @ApiOperation({ summary: 'Get specific pipeline run' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getPipelineRun(
@@ -757,7 +792,7 @@ export class PlatformApiController {
     );
   }
 
-  @Get('pipeline/pipeline_run/:runId/logs')
+  @Get('pipelines/pipeline_run/:runId/logs')
   @ApiOperation({ summary: 'Get pipeline run logs' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getPipelineRunLogs(
@@ -775,6 +810,68 @@ export class PlatformApiController {
     );
   }
 
+  @Post('pipelines/:pipelineId/pipeline_run/:runId/cancel')
+  @ApiOperation({ summary: 'Cancel a running pipeline run' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  async cancelPipelineRun(
+    @Param('pipelineId') pipelineId: string,
+    @Param('runId') runId: string,
+    @User() user: RequestUser,
+  ) {
+    const normalizedPipelineId = this.normalizePipelineId(pipelineId);
+    const normalizedRunId = this.normalizePipelineId(runId);
+
+    const status = await this.platformApiService.proxy(
+      'GET',
+      `/pipeline/${normalizedPipelineId}/pipeline_run`,
+      user,
+    );
+
+    if (status.length === 1) {
+      throw new BadRequestException('The first pipeline cannot be canceled');
+    }
+
+    return this.platformApiService.proxy(
+      'POST',
+      `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
+      user,
+    );
+  }
+
+  @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
+  @ApiOperation({
+    summary: 'Get pipeline run jobs',
+    description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
+  })
+  @ApiOkResponse({
+    description: 'DB-backed job runs for the selected pipeline run.',
+    schema: {
+      type: 'object',
+      properties: {
+        jobs: {
+          type: 'array',
+          items: { type: 'object' },
+        },
+      },
+      required: ['jobs'],
+    },
+  })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getPipelineRunJobs(
+    @Param('pipelineId') pipelineId: string,
+    @Param('runId') runId: string,
+    @User() user: RequestUser,
+  ) {
+    const normalizedPipelineId = this.normalizePipelineId(pipelineId);
+    const decodedRunId = this.decodePathParam(runId);
+
+    return this.platformApiService.proxy(
+      'GET',
+      `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
+      user,
+    );
+  }
+
   // ==================== JOBS - COLUMN EDITING ROUTES ====================
 
   @Put('jobs/:jobId/input')
@@ -874,41 +971,54 @@ export class PlatformApiController {
     );
   }
 
-  // ==================== JOBS - JDBC SYNC MODE ROUTES ====================
-
-  @Get('jobs/jdbc/:jobId')
-  @ApiOperation({ summary: 'Get JDBC job details' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getJdbcJob(@Param('jobId') jobId: string, @User() user: RequestUser) {
-    // Normalize job ID for Platform API (replace - with _)
-    const normalizedJobId = this.normalizeJobId(jobId);
-    return this.platformApiService.proxy('GET', `/jobs/jdbc/${normalizedJobId}`, user);
-  }
-
-  @Post('jobs/jdbc/:jobId/sync-mode')
-  @ApiOperation({ summary: 'Update JDBC job sync mode' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
-  async updateJdbcSyncMode(
-    @Param('jobId') jobId: string,
-    @Body() body: any,
+  @Delete('pipelines/:pipelineId/inputs/:inputId')
+  @ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
+  @UseGuards(PipelineExecutionGuard)
+  async deleteTable(
+    @Param('pipelineId') pipelineId: string,
+    @Param('inputId') inputId: string,
+    @Body() body: { table_name: string },
     @User() user: RequestUser,
   ) {
-    // Normalize job ID for Platform API (replace - with _)
-    const normalizedJobId = this.normalizeJobId(jobId);
+    const tableName = body.table_name;
+    const info = {
+      customer_id: user.customer_id,
+      customer: user.customer_name,
+      user_id: user.user_id,
+    };
 
-    const result = await this.platformApiService.proxy(
-      'POST',
-      `/jobs/jdbc/${normalizedJobId}/sync-mode`,
-      user,
-      body,
-    );
+    this.logger.info('deleteTable: marking table as deleted', { inputId, tableName });
+    const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
+    this.logger.info('deleteTable: table marked as deleted', { inputId, tableName });
 
-    // Sync to DynamoDB (pass raw jobId for pipeline extraction)
-    await this.syncJdbcSyncModeToDynamoDB(jobId, body, user);
+    try {
+      const normalizedPipelineId = this.normalizePipelineId(pipelineId);
+      this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId });
+      const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
+      this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length });
 
-    return result;
+      const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName);
+      if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`);
+
+      this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id });
+      await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
+      this.logger.info('deleteTable: job deleted', { jobId: job.job_id });
+
+      return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at };
+    } catch (error) {
+      this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message });
+      try {
+        await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info });
+      } catch (rollbackError) {
+        this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message });
+      }
+      throw error;
+    }
   }
 
+  // ==================== JOBS - JDBC SYNC MODE ROUTES ====================
+
   @Get('jobs/jdbc/configs/allowed_datatypes')
   @ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
@@ -920,52 +1030,6 @@ export class PlatformApiController {
     );
   }
 
-  // ==================== JOBS - SINGER REPLICATION ROUTES ====================
-
-  @Get('jobs/singer/:jobId')
-  @ApiOperation({ summary: 'Get Singer job details' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getSingerJob(@Param('jobId') jobId: string, @User() user: RequestUser) {
-    // Normalize job ID for Platform API (replace - with _)
-    const normalizedJobId = this.normalizeJobId(jobId);
-    return this.platformApiService.proxy('GET', `/jobs/singer/${normalizedJobId}`, user);
-  }
-
-  @Post('jobs/singer/:jobId/sync-mode')
-  @ApiOperation({ summary: 'Update Singer job sync mode' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
-  async updateSingerSyncMode(
-    @Param('jobId') jobId: string,
-    @Body() body: any,
-    @User() user: RequestUser,
-  ) {
-    // Normalize job ID for Platform API (replace - with _)
-    const normalizedJobId = this.normalizeJobId(jobId);
-
-    const result = await this.platformApiService.proxy(
-      'POST',
-      `/jobs/singer/${normalizedJobId}/sync-mode`,
-      user,
-      body,
-    );
-
-    // Sync to DynamoDB (pass raw jobId for pipeline extraction)
-    await this.syncSingerSyncModeToDynamoDB(jobId, body, user);
-
-    return result;
-  }
-
-  // ==================== JOBS - S3 ROUTES ====================
-
-  @Get('jobs/s3/:jobId')
-  @ApiOperation({ summary: 'Get S3 job details' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getS3Job(@Param('jobId') jobId: string, @User() user: RequestUser) {
-    // Normalize job ID for Platform API (replace - with _)
-    const normalizedJobId = this.normalizeJobId(jobId);
-    return this.platformApiService.proxy('GET', `/jobs/s3/${normalizedJobId}`, user);
-  }
-
   // ==================== HEALTH ROUTE ====================
 
   @Get('health')

src/modules/platform-api/platform-api.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ValidationTableDTO {
+    @ApiProperty()
+    tables: Array<{
+        table_name: string;
+        table_schema: string;
+    }>
+}

src/modules/platform-api/platform-api.module.ts

@@ -7,9 +7,11 @@ import { PlatformApiService } from './platform-api.service';
 import { ElasticsearchModule } from '../../services/elasticsearch';
 import { DynamoDBModule } from '../../services/dynamodb';
 import { CustomersModule } from '../customers/customers.module';
+import { CatalogModule } from '../catalog/catalog.module';
+import { InputsModule } from '../inputs/inputs.module';
 
 @Module({
-  imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
+  imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule],
   controllers: [PlatformApiController],
   providers: [PlatformApiService, DadosferaLogger],
   exports: [PlatformApiService],

src/modules/platform-api/platform-api.service.ts

@@ -89,6 +89,12 @@ export class PlatformApiService {
 
       // Propagate non-2xx responses as HttpExceptions
       if (response.status >= 400) {
+        this.logger.error('Platform API upstream error' + JSON.stringify({
+          status: response.status,
+          data: response.data,
+          path,
+          method: method.toUpperCase(),
+        }));
         throw new HttpException(response.data, response.status);
       }
 
@@ -101,6 +107,8 @@ export class PlatformApiService {
         method: method.toUpperCase(),
       });
 
+      this.logger.error(error)
+
       if (error instanceof HttpException) {
         throw error;
       }

src/modules/release_note/dto/release_note.dto.ts

@@ -0,0 +1,14 @@
+export type ReleaseNoteDTO = {
+    id: string;
+    date: string;
+    tag: string;
+    title: string;
+    visible: boolean;
+    expiryDate: string;
+    content: string;
+    showEmojis: boolean;
+    image?: string;
+    link?: string;
+    linkText?: string;
+  };
+  

src/modules/release_note/release_note.controller.spec.ts

@@ -0,0 +1,31 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ReleaseNoteController } from './release_note.controller';
+import { ReleaseNoteService } from './release_note.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+describe('ReleaseNoteController', () => {
+  let controller: ReleaseNoteController;
+  const logger = {
+    info: (...args) => args,
+    error: (...args) => args,
+  };
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [ReleaseNoteController],
+      providers: [
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        ReleaseNoteService
+      ],
+    }).compile();
+
+    controller = module.get<ReleaseNoteController>(ReleaseNoteController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

src/modules/release_note/release_note.controller.ts

@@ -0,0 +1,26 @@
+import { Controller, Get, Inject } from '@nestjs/common';
+import { ReleaseNoteService } from './release_note.service';
+import { Authenticated } from 'src/decorators/authentication.decorator';
+import { Language } from 'src/decorators/language.decorator';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+@Controller('release_note')
+@Authenticated()
+export class ReleaseNoteController {
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private readonly releaseNoteService: ReleaseNoteService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Get()
+  async getLatestReleaseNote(@Language() language: LanguageEnum) {
+    this.logger.info(`Fetching latest release note for language: ${language}`);
+    return await this.releaseNoteService.getLatestReleaseNote(language);
+  }
+}

src/modules/release_note/release_note.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { ReleaseNoteService } from './release_note.service';
+import { ReleaseNoteController } from './release_note.controller';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+@Module({
+  controllers: [ReleaseNoteController],
+  providers: [ReleaseNoteService, DadosferaLogger]
+})
+export class ReleaseNoteModule {}

src/modules/release_note/release_note.service.spec.ts

@@ -0,0 +1,29 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ReleaseNoteService } from './release_note.service';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+describe('ReleaseNoteService', () => {
+  let service: ReleaseNoteService;
+  const logger = {
+    info: (...args) => args,
+    error: (...args) => args,
+  };
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        {
+          provide: DadosferaLogger,
+          useValue: { logger },
+        },
+        ReleaseNoteService
+      ],
+    }).compile();
+
+    service = module.get<ReleaseNoteService>(ReleaseNoteService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

src/modules/release_note/release_note.service.ts

@@ -0,0 +1,46 @@
+import { Inject, Injectable } from '@nestjs/common';
+import axios, { AxiosInstance } from 'axios';
+import { LanguageEnum } from 'src/utils/languages.enum';
+import { ReleaseNoteDTO } from './dto/release_note.dto';
+import DadosferaLogger from '@dadosfera/dadosfera-logs';
+
+@Injectable()
+export class ReleaseNoteService {
+  client: AxiosInstance;
+    logger: DadosferaLogger;
+
+  constructor(
+        @Inject(DadosferaLogger)
+        dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+      this.client = axios.create({
+          baseURL: process.env.FIREBASE_BASE_URL,
+      });
+  }
+
+  async getLatestReleaseNote(lang: LanguageEnum) {
+    try {
+      const lng = lang.split('-');
+      const language = lng[0] + '-' + lng[1].toUpperCase();
+
+      const endpoint = `/release_note/${language}.json`;
+      const {
+        data,
+        status,
+        config
+      } = await this.client.get<ReleaseNoteDTO>(endpoint)
+      this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`);
+      this.logger.info(`Request URL: ${config.baseURL}/${config.url}`);
+
+      return data;
+    } catch (error) {
+      this.logger.error(`Error fetching release note: ${error.message}`);
+
+      if (axios.isAxiosError(error)) {
+        this.logger.error(`Axios error details: ${error.toJSON()}`);
+      }
+    }
+
+  }
+}

src/modules/roles/roles.service.ts

@@ -229,27 +229,33 @@ export class RolesService {
     const [roleTreated] = this.getRolesPermissionsName([role.role]);
     return { role: roleTreated };
   }
+
   getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
     const newRoles: RoleDto[] = [];
     for (const role of roles) {
-      const allPermissions = this.permissionsService.getAllPermissions(
-        this.language,
-      );
-      const newPermissions = role.permissions.map((p) => {
-        const permission = allPermissions.find((per) => per.seqid === p.seqid);
-        return {
-          ...p,
-          name: permission.name,
-          id: p.seqid,
-        };
-      });
-      const newRole: RoleDto = {
-        ...role,
-        permissions: newPermissions,
-        isPublic: role.isPublic,
-      };
+      const newRole: RoleDto = this.formatRole(role);
       newRoles.push(newRole);
     }
     return newRoles;
   }
+
+  private formatRole(role: GetRolesPermissionsName): RoleDto {
+    const allPermissions = this.permissionsService.getAllPermissions(
+      this.language
+    );
+    const newPermissions = role.permissions.map((p) => {
+      const permission = allPermissions.find((per) => per.seqid === p.seqid);
+      return {
+        ...p,
+        name: permission.name,
+        id: p.seqid,
+      };
+    });
+
+    return {
+      ...role,
+      permissions: newPermissions,
+      isPublic: role.isPublic,
+    };;
+  }
 }

src/modules/storage-explorer/storage-explorer.config.ts

@@ -0,0 +1,13 @@
+export const STORAGE_EXPLORER_CONFIG = {
+  getUrl: (customerName: string): string => {
+    const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
+    if (!urlTemplate) {
+      throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
+    }
+    // Replace {customer_id} placeholder with actual customer ID
+    // For local: http://172.17.0.1:8000/api (no placeholder)
+    // For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
+    return urlTemplate.replace('{customer}', customerName);
+  },
+  timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
+};

src/modules/storage-explorer/storage-explorer.controller.ts

@@ -0,0 +1,383 @@
+import {
+  Controller,
+  Get,
+  Post,
+  Put,
+  Param,
+  Body,
+  Query,
+  Inject,
+  UseInterceptors,
+  UploadedFiles,
+  Headers,
+} from '@nestjs/common';
+import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
+import { FilesInterceptor } from '@nestjs/platform-express';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import FormData from 'form-data';
+
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from '../../decorators/authentication.decorator';
+import { User, RequestUser } from '../../decorators/user.decorator';
+import { StorageExplorerService } from './storage-explorer.service';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+
+@ApiTags('Storage Explorer')
+@Controller('storage-explorer')
+@Authenticated()
+export class StorageExplorerController {
+  private logger: any;
+
+  constructor(
+    private readonly storageExplorerService: StorageExplorerService,
+    @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  // ============================================
+  // TABLE OPERATIONS
+  // ============================================
+
+  @ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' })
+  @Post('tables/validate-name')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
+  async validateTableName(
+    @Body() body: any,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'POST',
+      '/tables/validate-name',
+      user,
+      
+      body,
+    );
+  }
+
+  @ApiOperation({ summary: 'Create a new table' })
+  @Post('tables')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
+  async createTable(
+    @Body() body: any,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'POST',
+      '/tables/',
+      user,
+      
+      body,
+    );
+  }
+
+  @ApiOperation({ summary: 'List all tables with pagination' })
+  @Get('tables')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async listTables(
+    @Query('page') page: number,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      '/tables/',
+      user,
+      
+      undefined,
+      { page },
+    );
+  }
+
+  @ApiOperation({ summary: 'Get table details by ID' })
+  @Get('tables/:tableId')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async getTable(
+    @Param('tableId') tableId: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      `/tables/${tableId}`,
+      user,
+    );
+  }
+
+  @ApiOperation({ summary: 'Link a dataset to a table' })
+  @Post('tables/:tableId/datasets/:datasetId')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
+  async linkDatasetToTable(
+    @Param('tableId') tableId: string,
+    @Param('datasetId') datasetId: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'POST',
+      `/tables/${tableId}/datasets/${datasetId}`,
+      user,
+      
+    );
+  }
+
+  @ApiOperation({ summary: 'Get all datasets linked to a table' })
+  @Get('tables/:tableId/datasets')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async getTableDatasets(
+    @Param('tableId') tableId: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      `/tables/${tableId}/datasets`,
+      user,
+      
+    );
+  }
+
+  @ApiOperation({ summary: 'Get table schema' })
+  @Get('tables/:tableId/schema')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async getTableSchema(
+    @Param('tableId') tableId: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      `/tables/${tableId}/schema`,
+      user,
+      
+    );
+  }
+
+  @ApiOperation({ summary: 'Validate schema compatibility between table and dataset' })
+  @Post('tables/:tableId/validate-compatibility/:datasetId')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async validateSchemaCompatibility(
+    @Param('tableId') tableId: string,
+    @Param('datasetId') datasetId: string,
+    @User() user: RequestUser,
+  ) {
+    return this.storageExplorerService.proxy(
+      'POST',
+      `/tables/${tableId}/validate-compatibility/${datasetId}`,
+      user,
+      
+    );
+  }
+
+  // ============================================
+  // DATASET OPERATIONS
+  // ============================================
+
+  @ApiOperation({ summary: 'Get dataset preview data' })
+  @Get('datasets/:datasetId/preview')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async getDatasetPreview(
+    @Param('datasetId') datasetId: string,
+    @Query('limit') limit: number,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      `/datasets/${datasetId}/preview`,
+      user,
+      
+      undefined,
+      { limit },
+    );
+  }
+
+  @ApiOperation({ summary: 'Get dataset schema information' })
+  @Get('datasets/:datasetId/schema')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async getDatasetSchema(
+    @Param('datasetId') datasetId: string,
+    @Query('force_refresh') forceRefresh: boolean,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      `/datasets/${datasetId}/schema`,
+      user,
+      
+      undefined,
+      { force_refresh: forceRefresh },
+    );
+  }
+
+  @ApiOperation({ summary: 'List all datasets for a specific upload' })
+  @Get('datasets/upload/:uploadId')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async listDatasetsByUpload(
+    @Param('uploadId') uploadId: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      `/datasets/upload/${uploadId}`,
+      user,
+      
+    );
+  }
+
+  @ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' })
+  @Put('datasets/:datasetId/refresh-schema')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
+  async refreshDatasetSchema(
+    @Param('datasetId') datasetId: string,
+    @Body() body: any,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'PUT',
+      `/datasets/${datasetId}/refresh-schema`,
+      user,
+      
+      body,
+    );
+  }
+
+  // ============================================
+  // STORAGE OPERATIONS
+  // ============================================
+
+  @ApiOperation({ summary: 'List file explorer uploads with pagination' })
+  @Get('storage/uploads/history')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async listFileExplorerUploads(
+    @Query('page') page: number,
+    @Query('limit') limit: number,
+    @Query('folder_path') folderPath: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      '/storage/uploads/history',
+      user,
+      
+      undefined,
+      { page, limit, folder_path: folderPath },
+    );
+  }
+
+  @ApiOperation({ summary: 'Browse folders and files in storage' })
+  @Get('storage/browse')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async browseStorage(
+    @Query('path') path: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      '/storage/browse',
+      user,
+      
+      undefined,
+      { path },
+    );
+  }
+
+  @ApiOperation({ summary: 'Upload multiple files to storage' })
+  @Post('storage/upload/batch')
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FilesInterceptor('files'))
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
+  async batchUpload(
+    @UploadedFiles() files: Array<Express.Multer.File>,
+    @Body('folder_path') folderPath: string,
+    @User() user: RequestUser,
+    
+  ) {
+    // Create FormData to forward files to storage-explorer API
+    const formData = new FormData();
+    
+    // Add files
+    if (files && files.length > 0) {
+      files.forEach((file) => {
+        formData.append('files', file.buffer, {
+          filename: file.originalname,
+          contentType: file.mimetype,
+        });
+      });
+    }
+    
+    // Add folder_path
+    if (folderPath) {
+      formData.append('folder_path', folderPath);
+    }
+
+    return this.storageExplorerService.proxyFormData(
+      'POST',
+      '/storage/upload/batch',
+      user,
+      
+      formData,
+    );
+  }
+
+  @ApiOperation({ summary: 'Create a new folder in storage' })
+  @Post('storage/folder/create')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
+  async createFolder(
+    @Body() body: any,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'POST',
+      '/storage/folder/create',
+      user,
+      
+      body,
+    );
+  }
+
+  @ApiOperation({ summary: 'Download a file from storage' })
+  @Get('storage/download')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async downloadFile(
+    @Query('file_path') filePath: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      '/storage/download',
+      user,
+      
+      undefined,
+      { file_path: filePath },
+    );
+  }
+
+  @ApiOperation({ summary: 'Get detailed file metadata' })
+  @Get('storage/metadata')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
+  async getFileMetadata(
+    @Query('file_path') filePath: string,
+    @User() user: RequestUser,
+    
+  ) {
+    return this.storageExplorerService.proxy(
+      'GET',
+      '/storage/metadata',
+      user,
+      undefined,
+      { file_path: filePath },
+    );
+  }
+}

src/modules/storage-explorer/storage-explorer.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { StorageExplorerController } from './storage-explorer.controller';
+import { StorageExplorerService } from './storage-explorer.service';
+
+@Module({
+  imports: [],
+  controllers: [StorageExplorerController],
+  providers: [StorageExplorerService, DadosferaLogger],
+  exports: [StorageExplorerService],
+})
+export class StorageExplorerModule {}

src/modules/storage-explorer/storage-explorer.service.ts

@@ -0,0 +1,178 @@
+import { Injectable, Inject, HttpException } from '@nestjs/common';
+import axios, { AxiosResponse, Method } from 'axios';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { RequestUser } from '../../decorators/user.decorator';
+import { STORAGE_EXPLORER_CONFIG } from './storage-explorer.config';
+
+@Injectable()
+export class StorageExplorerService {
+  private logger: any;
+
+  constructor(
+    @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  async proxy(
+    method: string,
+    path: string,
+    user: RequestUser,
+    body?: any,
+    query?: Record<string, any>
+  ): Promise<any> {
+    // Validate customer_id is present for multi-tenant isolation
+    if (!user.customer_id) {
+      throw new HttpException('Customer ID is required for storage operations', 400);
+    }
+
+    // Get customer-specific storage-explorer URL
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
+    const url = new URL(`${baseUrl}${path}`);
+
+    // Add query params
+    if (query) {
+      Object.entries(query).forEach(([key, value]) => {
+        if (value !== undefined && value !== null) {
+          url.searchParams.set(key, String(value));
+        }
+      });
+    }
+
+    const headers: Record<string, string> = {
+      'content-type': 'application/json',
+    };
+
+    this.logger.info('Proxying request to storage-explorer', {
+      method: method.toUpperCase(),
+      path,
+      customer_id: user.customer_id,
+      storage_url: baseUrl,
+      user_id: user.user_id,
+    });
+
+    try {
+      const response: AxiosResponse = await axios({
+        method: method as Method,
+        url: url.href,
+        headers,
+        data: body,
+        timeout: STORAGE_EXPLORER_CONFIG.timeout,
+        validateStatus: () => true, // Don't throw on non-2xx
+      });
+
+      // Propagate non-2xx responses as HttpExceptions
+      if (response.status >= 400) {
+        throw new HttpException(response.data, response.status);
+      }
+
+      return response.data;
+    } catch (error) {
+      this.logger.error('Storage Explorer API proxy error', {
+        error: error.message,
+        status: error.response?.status,
+        path,
+        storage_url: baseUrl,
+        method: method.toUpperCase(),
+      });
+
+      if (error instanceof HttpException) {
+        throw error;
+      }
+
+      if (error.response) {
+        throw new HttpException(error.response.data, error.response.status);
+      }
+
+      if (error.code === 'ECONNREFUSED') {
+        throw new HttpException('Storage Explorer API service unavailable', 503);
+      }
+
+      if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
+        throw new HttpException('Storage Explorer API request timeout', 504);
+      }
+
+      throw new HttpException('Internal server error', 500);
+    }
+  }
+
+  /**
+   * Proxy with file upload support (multipart/form-data)
+   */
+  async proxyFormData(
+    method: string,
+    path: string,
+    user: RequestUser,
+    formData: any,
+    query?: Record<string, any>,
+  ): Promise<any> {
+    // Validate customer_id is present for multi-tenant isolation
+    if (!user.customer_id) {
+      throw new HttpException('Customer ID is required for storage operations', 400);
+    }
+
+    // Get customer-specific storage-explorer URL
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
+    const url = new URL(`${baseUrl}${path}`);
+
+    // Add query params
+    if (query) {
+      Object.entries(query).forEach(([key, value]) => {
+        if (value !== undefined && value !== null) {
+          url.searchParams.set(key, String(value));
+        }
+      });
+    }
+
+    const headers: Record<string, string> = {
+      // Let axios set Content-Type for multipart/form-data with boundary
+      ...formData.getHeaders?.(),
+    };
+
+    this.logger.info('Proxying form data request to storage-explorer', {
+      method: method.toUpperCase(),
+      path,
+      customer_id: user.customer_id,
+      storage_url: baseUrl,
+      user_id: user.user_id,
+    });
+
+    try {
+      const response: AxiosResponse = await axios({
+        method: method as Method,
+        url: url.href,
+        headers,
+        data: formData,
+        timeout: STORAGE_EXPLORER_CONFIG.timeout,
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        validateStatus: () => true,
+      });
+
+      if (response.status >= 400) {
+        throw new HttpException(response.data, response.status);
+      }
+
+      return response.data;
+    } catch (error) {
+      this.logger.error('Storage Explorer API form data proxy error', {
+        error: error.message,
+        status: error.response?.status,
+        path,
+        storage_url: baseUrl,
+        method: method.toUpperCase(),
+      });
+
+      if (error instanceof HttpException) {
+        throw error;
+      }
+
+      if (error.response) {
+        throw new HttpException(error.response.data, error.response.status);
+      }
+
+      throw new HttpException('Internal server error', 500);
+    }
+  }
+}

src/modules/transformations/interfaces.d.ts

@@ -1,5 +1,8 @@
-import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
-
+export interface Info {
+  user_id: string;
+  customer_id: string;
+  customer: string;
+}
 export interface ICreateTransformationsRequest {
   transformations: Transformation[];
   info: Info;

src/modules/users/dtos/entities.ts

@@ -38,6 +38,12 @@ export class User {
   department?: string;
   @ApiProperty()
   hierarchy?: string;
+  @ApiProperty()
+  bio?: string;
+  @ApiProperty()
+  companyName?: string;
+  @ApiProperty()
+  personalSite?: string;
   @ApiPropertyOptional()
   customer?: Customer;
   @ApiProperty()
@@ -58,6 +64,8 @@ export class UserNoRolesAndCustomer extends OmitType(UserNoRoles, [
 export class IUserByCustomer extends OmitType(User, ['customer']) {
   @ApiPropertyOptional()
   permissions?: string[];
+  @ApiPropertyOptional()
+  authProvider?: string;
 }
 
 export class CreateUserReq {
@@ -109,6 +117,12 @@ export class UpdateUserReq {
   @ApiPropertyOptional()
   hierarchy?: string;
   @ApiPropertyOptional()
+  bio?: string;
+  @ApiPropertyOptional()
+  personalSite?: string;
+  @ApiPropertyOptional()
+  companyName?: string;
+  @ApiPropertyOptional()
   roleNames?: string[];
 }
 

src/modules/users/users.controller.ts

@@ -266,7 +266,6 @@ export class UsersController {
   }
 
   @Patch(':id')
-  @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
   @ApiOkResponse({ type: UpdateUserRes })
   async updateUser(
     @User() user: RequestUser,
@@ -274,6 +273,18 @@ export class UsersController {
     @Param('id') id: string,
     @Language() language: LanguageEnum,
   ) {
+
+    const isSameUser = user.user_id === id;
+    const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
+    if (!isSameUser && !isSuperAdmin) {
+      throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
+    }
+
+    if (isSameUser && !isSuperAdmin && body.roleNames) {
+      // Prevent users from updating their own roles
+      delete body.roleNames;
+    }
+
     this.logger.info('updateUser', { user });
     this.userService.setLanguage(language);
     return await this.userService.updateUser(body, id, user.customer_id);

src/modules/users/users.service.ts

@@ -125,6 +125,7 @@ export class UsersService implements OnModuleInit {
       return { permissions };
     });
     res.user.permissions = permissions;
+    res.user.authProvider = process.env.AUTH_PROVIDER || 'cognito';
     return res;
   }
 
@@ -148,20 +149,23 @@ export class UsersService implements OnModuleInit {
   }
 
   async updateUser(req: UpdateUserReq, id: string, customerId: string) {
-    const { department, hierarchy, jobTitle, name, roleNames, email } = req;
-    if (roleNames) {
+    const { roleNames, ...updateUserDTO } = req;
+    if (roleNames && roleNames.length > 0) {
       await this.setRoles({ roleNames, userId: id }, customerId);
     }
 
     const { user } = await lastValueFrom(
       this.usersClientService.UserUpdate({
-        name,
+        department: updateUserDTO.department,
+        email: updateUserDTO.email,
+        hierarchy: updateUserDTO.hierarchy,
+        jobTitle: updateUserDTO.jobTitle,
+        name: updateUserDTO.name,
+        bio: updateUserDTO.bio,
+        companyName: updateUserDTO.companyName,
+        personalSite: updateUserDTO.personalSite,
         customerId,
         id,
-        department,
-        hierarchy,
-        jobTitle,
-        email,
         metabaseUserId: undefined,
       }),
     );

src/pipes/object-validation.pipe.ts

@@ -14,7 +14,10 @@ export class ValidationPipe implements PipeTransform<any> {
       return value;
     }
     const object = plainToInstance(metatype, value);
-    const errors = await validate(object);
+    const errors = await validate(object, {
+      forbidUnknownValues: false,
+      whitelist: true,
+    });
     if (errors.length > 0) {
       const errorMessages = errors.map((err) => err.constraints);
       throw new BadRequestException(errorMessages);

src/services/dynamodb/dynamodb.service.ts

@@ -125,9 +125,13 @@ export class DynamoDBService {
       },
     });
 
-    const { Item } = await this.documentClient.send(getCommand);
-
-    return Item as InputDocument | null;
+    try {
+      const { Item } = await this.documentClient.send(getCommand);
+      return Item as InputDocument | null;
+    } catch (error) {
+      this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message });
+      throw error;
+    }
   }
 
   async deleteInput(clientId: string, inputId: string): Promise<void> {
@@ -203,7 +207,6 @@ export class DynamoDBService {
         updatedTable.reference_column = changes.reference_column;
       }
     }
-
     tables[tableIndex] = updatedTable;
 
     // Save updated document
@@ -231,4 +234,5 @@ export class DynamoDBService {
       throw error;
     }
   }
+
 }

src/services/elasticsearch/elasticsearch.service.ts

@@ -358,6 +358,81 @@ export class ElasticsearchService {
     }
   }
 
+  private getDataAssetIndex(customerName: string): string {
+    return `${customerName}_data_assets_catalog`;
+  }
+
+  async findDataAssetByTable(
+    customerName: string,
+    tableName: string,
+    tableSchema: string,
+  ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
+    const index = this.getDataAssetIndex(customerName);
+
+    this.logger.info('Elasticsearch: Searching data asset', {
+      index,
+      tableName,
+      tableSchema,
+    });
+
+    try {
+      const response = await this.client.post(`/${index}/_search`, {
+        query: {
+          bool: {
+            must: [
+              { term: { 'table_name.keyword': tableName.toUpperCase() } },
+              { term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
+            ],
+          },
+        },
+        size: 1,
+      });
+
+      const hits = response.data.hits?.hits || [];
+      if (hits.length === 0) {
+        this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
+        return null;
+      }
+
+      return { ...hits[0]._source, _es_id: hits[0]._id };
+    } catch (error) {
+      this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
+      throw error;
+    }
+  }
+
+  async updateDataAsset(
+    customerName: string,
+    assetId: string,
+    updates: Record<string, any>,
+  ): Promise<any> {
+    const index = this.getDataAssetIndex(customerName);
+
+    this.logger.info('Elasticsearch: Updating data asset', {
+      index,
+      assetId,
+      fields: Object.keys(updates),
+    });
+
+    try {
+      const response = await this.client.post(
+        `/${index}/_update/${assetId}`,
+        { doc: updates },
+        { params: { refresh: 'wait_for' } },
+      );
+
+      this.logger.info('Elasticsearch: Data asset updated', {
+        assetId,
+        result: response.data.result,
+      });
+
+      return response.data;
+    } catch (error) {
+      this.handleError('updateDataAsset', error, { assetId, index });
+      throw error;
+    }
+  }
+
   private handleError(
     operation: string,
     error: any,

src/services/nimbus/nimbus.module.ts

@@ -0,0 +1,9 @@
+import { Module } from "@nestjs/common";
+import { NimbusService } from "./nimbus.service";
+import DadosferaLogger from "@dadosfera/dadosfera-logs";
+
+@Module({
+  providers: [NimbusService, DadosferaLogger],
+  exports: [NimbusService],
+})
+export class NimbusServicesModule {}

src/services/nimbus/nimbus.service.ts

@@ -0,0 +1,56 @@
+import DadosferaLogger from "@dadosfera/dadosfera-logs";
+import { Inject, Injectable } from "@nestjs/common";
+import axios from "axios";
+
+type TableUpdate = {
+    table_schema: string;
+    table_name: string;
+}
+
+@Injectable()
+export class NimbusService {
+    private logger: DadosferaLogger;
+  
+    constructor(
+      @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
+    ) {
+      this.logger = dadosferaLogger.logger;
+    }
+
+    private buildUrl(customerName: string) {
+        if (process.env.ENV === 'prd') {
+            return `https://nimbus-${customerName}.dadosfera.ai`;
+          }
+      
+          return `https://nimbus-${customerName}.${process.env.ENV.replace(
+            'local',
+            'stg',
+          )}.dadosfera.ai`;
+    }
+
+    async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) {
+        const nimbusUrl = this.buildUrl(customerName);
+
+        const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
+
+        try {
+            this.logger.info("Request for PATCH " + nimbusUrl + path);
+            this.logger.info("Payload: " + JSON.stringify(update));
+            const { data } = await axios.patch(nimbusUrl + path, {
+                table_name: update.table_name,
+                table_schema: update.table_schema
+            })
+
+            return data;
+        } catch (error) {
+            this.logger.error(error);
+            return {
+                message: error.message,
+                database,
+                old, 
+                update
+            }
+        }
+
+    }
+}

src/utils/PackTheMetadata.ts

@@ -9,6 +9,7 @@ interface IMetadata {
   details?: string;
   sensitive?: string;
   roles?: string[];
+  customer_modules?: string[];
   is_data_manager?: boolean;
   access_token?: string;
   host?: string;

tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.*.ts"]
 }