From ec810828779f8b1c35f690457f3dc13c622a342e Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Thu, 13 Nov 2025 17:48:38 -0300 Subject: [PATCH 01/30] FIX: resolve cors from localhost --- src/main.ts | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/main.ts b/src/main.ts index 70161e2..f990f13 100644 --- a/src/main.ts +++ b/src/main.ts @@ -18,25 +18,35 @@ async function bootstrap() { }); const logger = new DadosferaLogger(); + const corsOrigins = []; + + if (process.env.ENV === 'local') { + corsOrigins.push('http://localhost:4200'); + } else { + corsOrigins.push( + 'https://app.stg.dadosfera.ai', + 'https://app.dadosfera.ai', + 'https://unimed.dadosfera.ai', + 'https://boston-scientific.dadosfera.ai', + 'https://plataforma.dadosfera.ai' + ); + } + const app = await NestFactory.create(AppModule, { logger, cors: { - origin: [ - 'https://app.stg.dadosfera.ai', - 'https://app.dadosfera.ai', - 'https://unimed.dadosfera.ai', - 'https://boston-scientific.dadosfera.ai', - 'https://plataforma.dadosfera.ai' - ], + origin: corsOrigins, methods: 'GET,HEAD,PUT,PATCH,POST,DELETE', preflightContinue: false, optionsSuccessStatus: 204, credentials: true, }, }); + app.use(helmet()); app.use(cookieParser(process.env.COOKIE_SECRET)); - if (process.env.ENV === 'prd') { + + if (process.env.ENV !== 'local') { app.use('/catalog/register-dataset', json({ limit: '10mb' })); app.use( '/catalog/register-dataset', From dd0d08ad6e1128298a4d9b032fb2c8cc3f310a69 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Thu, 13 Nov 2025 17:48:55 -0300 Subject: [PATCH 02/30] FEAT: add permissions to lineage module --- src/authentication/permissions.enum.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 2426df7..3eeb2a7 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -390,6 +390,25 @@ export const PERMISSIONS_GROUPS = { }, }, }, + LINEAGE: { + title: { + 'pt-br': 'Explorar | Linhagem', + 'en-us': 'Explore | Lineage', + 'es-es': 'Explorar | Linaje', + }, + permissions: { + VIEW: { + seqid: 50, + claim: 'lineage:view', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Acessar ao módulo de Linhagem', + 'en-us': 'Access to Lineage module', + 'es-es': 'Acceda al módulo de Linaje', + }, + } + }, + }, EMBED: { title: { 'pt-br': 'Analisar | Incorporação', From d4451153a37a2d7a0f89e62c5e640c03194e6a97 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Thu, 13 Nov 2025 18:14:47 -0300 Subject: [PATCH 03/30] FIX: change wget to curl --- .github/workflows/k8s-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-deploy.yml b/.github/workflows/k8s-deploy.yml index d3d469d..c83898c 100644 --- a/.github/workflows/k8s-deploy.yml +++ b/.github/workflows/k8s-deploy.yml @@ -105,7 +105,7 @@ jobs: - name: Install Helmfile run: | - wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz tar -xzf helmfile_0.148.0_linux_amd64.tar.gz sudo mv helmfile /usr/local/bin/ helmfile --version From f00d2bf41d4bd6942d94fb5856ef79d63600bec2 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Fri, 21 Nov 2025 09:55:39 -0300 Subject: [PATCH 04/30] FIX: reset cstumer theme --- docsfera.json | 69 ++++++++++++++++++++---- package-lock.json | 77 +++++++++++++++------------ package.json | 4 +- src/main.ts | 1 + src/modules/theme/theme.controller.ts | 20 +++++++ src/modules/theme/theme.service.ts | 12 +++++ 6 files changed, 137 insertions(+), 46 deletions(-) diff --git a/docsfera.json b/docsfera.json index a18dad5..ab3a2c1 100644 --- a/docsfera.json +++ b/docsfera.json @@ -42,6 +42,33 @@ ] } }, + "/auth/sign-out": { + "post": { + "operationId": "AuthController_signOut", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "" + } + }, + "tags": [ + "Auth" + ] + } + }, "/auth/refresh-access-token": { "post": { "operationId": "AuthController_refreshAccessToken", @@ -3415,9 +3442,6 @@ "PipelinesV2" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -3467,9 +3491,6 @@ "PipelinesV2" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -3519,9 +3540,6 @@ "PipelinesV2" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -6251,6 +6269,39 @@ ] } }, + "/customers/{id}/theme/reset": { + "post": { + "operationId": "ThemeController_resetTheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + }, + "201": { + "description": "" + } + }, + "tags": [ + "Theme" + ] + } + }, "/network-policy": { "get": { "operationId": "NetworkPolicyController_getNetworks", diff --git a/package-lock.json b/package-lock.json index b491a76..cc6dafc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.10", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -57,8 +57,8 @@ "swagger-ui-express": "^4.6.3" }, "devDependencies": { - "@types/cookie-parser": "^1.4.9", "@types/cache-manager": "^4.0.6", + "@types/cookie-parser": "^1.4.9", "@types/express": "^4.17.17", "@types/express-session": "^1.18.1", "@types/jest": "27.0.2", @@ -850,6 +850,7 @@ "integrity": "sha512-vMqyb7XCDMPvJFFOaT9kxtiRh42GwlZEg1/uIgtZshS5a/8OaduUfCi7kynKgc3Tw/6Uo2D+db9qBttghhmxwQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.26.2", @@ -1408,9 +1409,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.10", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz", - "integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-kIErOKMB4CvkhVPWx8GXtNicIlXpo7ShP+uEiZsVE7Y5UxIepkbmqP5vczFZPRPBlRwejizMyyn8wBsGuZTtjw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -2260,6 +2261,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-9.4.3.tgz", "integrity": "sha512-Gd6D4IaYj01o14Bwv81ukidn4w3bPHCblMUq+SmUmWLyosK+XQmInCS09SbDDZyL8jy86PngtBLTdhJ2bXSUig==", "license": "MIT", + "peer": true, "dependencies": { "iterare": "1.2.1", "tslib": "2.5.3", @@ -2338,6 +2340,7 @@ "integrity": "sha512-Qi63+wi55Jh4sDyaj5Hhx2jOpKqT386aeo+VOKsxnd+Ql9VvkO/FjmuwBGUyzkJt29ENYc+P0Sx/k5LtstNpPQ==", "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@nuxtjs/opencollective": "0.3.2", "fast-safe-stringify": "2.1.1", @@ -2475,6 +2478,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz", "integrity": "sha512-FpdczWoRSC0zz2dNL9u2AQLXKXRVtq4HgHklAhbL59X0uy+mcxhlSThG7DHzDMkoSnuuHY8ojDVf7mDxk+GtCw==", "license": "MIT", + "peer": true, "dependencies": { "body-parser": "1.20.2", "cors": "2.8.5", @@ -2838,6 +2842,7 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -2997,6 +3002,7 @@ "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", "license": "MIT", + "peer": true, "dependencies": { "cluster-key-slot": "1.1.2", "generic-pool": "3.9.0", @@ -3793,6 +3799,7 @@ "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -3873,6 +3880,7 @@ "integrity": "sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "jest-diff": "^27.0.0", "pretty-format": "^27.0.0" @@ -3936,7 +3944,8 @@ "version": "16.18.126", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz", "integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/oauth": { "version": "0.9.6", @@ -4166,6 +4175,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -4512,6 +4522,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4608,6 +4619,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -5236,6 +5248,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001688", "electron-to-chromium": "^1.5.73", @@ -5342,6 +5355,7 @@ "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz", "integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==", "license": "MIT", + "peer": true, "dependencies": { "eventemitter3": "^5.0.1", "lodash.clonedeep": "^4.5.0", @@ -5565,13 +5579,15 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/class-validator": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.1.tgz", "integrity": "sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ==", "license": "MIT", + "peer": true, "dependencies": { "@types/validator": "^13.11.8", "libphonenumber-js": "^1.10.53", @@ -6235,7 +6251,8 @@ "version": "0.0.1425554", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz", "integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/dezalgo": { "version": "1.0.4", @@ -6512,7 +6529,6 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.8" } @@ -6700,6 +6716,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -7046,7 +7063,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -7093,7 +7109,6 @@ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "license": "MIT", - "peer": true, "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -7118,7 +7133,6 @@ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.6" } @@ -7128,7 +7142,6 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", - "peer": true, "dependencies": { "ms": "2.0.0" } @@ -7137,22 +7150,19 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/express/node_modules/path-to-regexp": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/express/node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "license": "BSD-3-Clause", - "peer": true, "dependencies": { "side-channel": "^1.0.6" }, @@ -7181,8 +7191,7 @@ "url": "https://feross.org/support" } ], - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/external-editor": { "version": "3.1.0", @@ -7417,7 +7426,6 @@ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "license": "MIT", - "peer": true, "dependencies": { "debug": "2.6.9", "encodeurl": "~2.0.0", @@ -7436,7 +7444,6 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", - "peer": true, "dependencies": { "ms": "2.0.0" } @@ -7445,8 +7452,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/find-up": { "version": "5.0.0", @@ -8240,6 +8246,7 @@ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==", "license": "MIT", + "peer": true, "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", @@ -8581,6 +8588,7 @@ "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -9870,7 +9878,6 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -10628,6 +10635,7 @@ "resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz", "integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==", "license": "MIT", + "peer": true, "dependencies": { "passport-strategy": "1.x.x", "pause": "0.0.1", @@ -11017,6 +11025,7 @@ "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin-prettier.js" }, @@ -11519,7 +11528,8 @@ "version": "0.1.14", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz", "integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==", - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/relative-microtime": { "version": "2.0.0", @@ -11706,6 +11716,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -11767,6 +11778,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -11810,7 +11822,6 @@ "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "license": "MIT", - "peer": true, "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -11835,7 +11846,6 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", - "peer": true, "dependencies": { "ms": "2.0.0" } @@ -11844,15 +11854,13 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/send/node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.8" } @@ -11861,8 +11869,7 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/serialize-javascript": { "version": "6.0.2", @@ -11878,7 +11885,6 @@ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "license": "MIT", - "peer": true, "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", @@ -12856,6 +12862,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -13085,6 +13092,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13412,7 +13420,6 @@ "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz", "integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==", "license": "MIT", - "peer": true, "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", diff --git a/package.json b/package.json index 755c52f..64e4034 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.10", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -78,8 +78,8 @@ "multer": "1.4.5-lts.1" }, "devDependencies": { - "@types/cookie-parser": "^1.4.9", "@types/cache-manager": "^4.0.6", + "@types/cookie-parser": "^1.4.9", "@types/express": "^4.17.17", "@types/express-session": "^1.18.1", "@types/jest": "27.0.2", diff --git a/src/main.ts b/src/main.ts index 70161e2..290cf0c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -22,6 +22,7 @@ async function bootstrap() { logger, cors: { origin: [ + 'http://localhost:4200', 'https://app.stg.dadosfera.ai', 'https://app.dadosfera.ai', 'https://unimed.dadosfera.ai', diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index ed8e3a0..79186d7 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -124,4 +124,24 @@ export class ThemeController { } } + @Post('/:id/theme/reset') + @ApiOkResponse({ type: CustomerThemeResponse }) + async resetTheme(@Param('id') id: string) { + this.logger.info('getCustomerTheme with id' + id); + + try { + await this.themeService.resetTheme(id); + + return { theme: null }; + }catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } + } diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts index b62338b..8673788 100644 --- a/src/modules/theme/theme.service.ts +++ b/src/modules/theme/theme.service.ts @@ -44,6 +44,18 @@ export class ThemeService implements OnModuleInit { ); } + async resetTheme(id: string) { + const { theme } = await firstValueFrom( + this.themeService.ResetCustomerTheme({ + id + }), + ); + + return { + theme + } + } + async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) { if (!id) { this.logger.error('Error - saveCustomertheme - not found id:' + id); From 19e4daeea49daa270f376de6f8002c092b59def3 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Tue, 25 Nov 2025 10:35:08 -0300 Subject: [PATCH 05/30] FIX: private ingress annotations --- deploy/helm-chart/templates/ingress-private.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/deploy/helm-chart/templates/ingress-private.yaml b/deploy/helm-chart/templates/ingress-private.yaml index 1ace950..8b73b4b 100644 --- a/deploy/helm-chart/templates/ingress-private.yaml +++ b/deploy/helm-chart/templates/ingress-private.yaml @@ -2,11 +2,19 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" nginx.ingress.kubernetes.io/server-snippet: | underscores_in_headers on; ignore_invalid_headers on; + nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" + nginx.ingress.kubernetes.io/proxy-buffers-number: "8" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k" + {{- if .Values.maestro.restricted_ip}} + nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }} + {{- end }} generation: 1 labels: From 3ed26e648fd9d8428cd2c2ff2d9469bd9feb67d0 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Tue, 25 Nov 2025 10:48:06 -0300 Subject: [PATCH 06/30] FIX: ci --- .github/workflows/k8s-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s-deploy.yml b/.github/workflows/k8s-deploy.yml index d3d469d..5ecee62 100644 --- a/.github/workflows/k8s-deploy.yml +++ b/.github/workflows/k8s-deploy.yml @@ -56,9 +56,9 @@ jobs: - name: Install Helmfile run: | - wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz tar -xzf helmfile_0.148.0_linux_amd64.tar.gz - mv helmfile /usr/local/bin/ + sudo mv helmfile /usr/local/bin/ helmfile --version - name: Install Helm Diff Plugin From 5e909506602dac6f5b78ee74300a470b20dee902 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Tue, 25 Nov 2025 10:59:06 -0300 Subject: [PATCH 07/30] FIX: restore whitelist annotation --- deploy/helm-chart/templates/ingress-private.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/helm-chart/templates/ingress-private.yaml b/deploy/helm-chart/templates/ingress-private.yaml index 8b73b4b..a5be256 100644 --- a/deploy/helm-chart/templates/ingress-private.yaml +++ b/deploy/helm-chart/templates/ingress-private.yaml @@ -2,6 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: + nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip nginx.ingress.kubernetes.io/proxy-body-size: "0" nginx.ingress.kubernetes.io/proxy-read-timeout: "300" nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" From 576fdecf89f6596a6d2691ade22cff6f645142db Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Tue, 25 Nov 2025 11:27:04 -0300 Subject: [PATCH 08/30] FIX: ci to oracle github worker --- .github/workflows/k8s-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-deploy.yml b/.github/workflows/k8s-deploy.yml index 5ecee62..ac09af8 100644 --- a/.github/workflows/k8s-deploy.yml +++ b/.github/workflows/k8s-deploy.yml @@ -105,7 +105,7 @@ jobs: - name: Install Helmfile run: | - wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz tar -xzf helmfile_0.148.0_linux_amd64.tar.gz sudo mv helmfile /usr/local/bin/ helmfile --version From 6d9ecc356883eec982e0a16687e0285dbfd5a903 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Thu, 27 Nov 2025 09:43:43 -0300 Subject: [PATCH 09/30] FIX: mixpanel events --- src/authentication/extract-user.ts | 20 +++++++++++++ src/modules/mixpanel/mixpanel.controller.ts | 33 ++++++++++++++++----- 2 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 src/authentication/extract-user.ts diff --git a/src/authentication/extract-user.ts b/src/authentication/extract-user.ts new file mode 100644 index 0000000..96a3602 --- /dev/null +++ b/src/authentication/extract-user.ts @@ -0,0 +1,20 @@ +import jwt, { JwtPayload } from 'jsonwebtoken'; + +export function extractUserFrom(aRawJwt: string) { + const decodedToken = jwt.decode(aRawJwt, { + complete: true, + }); + + const payload = decodedToken.payload as JwtPayload; + + return { + user_id: payload.user_id, + username: payload.username, + permissions: payload.permissions, + customer_id: payload.customer_id, + customer_name: payload.customer_name, + customer_tier: payload.customer_tier, + customer_modules: payload.customer_modules, + access_token: aRawJwt, + } +} diff --git a/src/modules/mixpanel/mixpanel.controller.ts b/src/modules/mixpanel/mixpanel.controller.ts index 477569b..2b2d314 100644 --- a/src/modules/mixpanel/mixpanel.controller.ts +++ b/src/modules/mixpanel/mixpanel.controller.ts @@ -1,29 +1,46 @@ import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common'; -import { init } from 'mixpanel'; -import { Authenticated } from 'src/decorators/authentication.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; -import { RequestUser, User } from 'src/decorators/user.decorator'; +import { RequestUser } from 'src/decorators/user.decorator'; import { MixpanelService } from './mixpanel.service'; +import { extractUserFrom } from 'src/authentication/extract-user'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; @ApiInternalOnlyController() @Controller('trackEvent') export class MixpanelController { + logger: DadosferaLogger; + constructor( - private mixpanelService: MixpanelService - ) {} + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private mixpanelService: MixpanelService, + ) { + this.logger = dadosferaLogger.logger; + } + @Post(':id') async trackEvent( @Param('id') id, @Body() body, - @User() user: RequestUser, @Req() request ) { + this.logger.info(`POST Track Event: ${id}`) delete body.info; + + const anonymousUser = { + username: "anonymous", + customer_name: "anonymous" + } as RequestUser + + const hasToken = request.headers['authorization']; + + const user = hasToken ? extractUserFrom(hasToken) : anonymousUser; + + this.logger.info(`Has user: ${typeof hasToken == "string"}`) await this.mixpanelService.track(id, user, request, body) + this.logger.info(`Event successful`) return { id, body, user: user.username }; } - - } From 8800ac273668c616b22bc6bb5c28e5ffbb95e291 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Fri, 28 Nov 2025 10:50:31 -0300 Subject: [PATCH 10/30] FIX: include private frontend url in cors origins list --- src/main.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.ts b/src/main.ts index f990f13..1085fb3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -26,6 +26,7 @@ async function bootstrap() { corsOrigins.push( 'https://app.stg.dadosfera.ai', 'https://app.dadosfera.ai', + 'https://private-frontend.stg.dadosfera.ai', 'https://unimed.dadosfera.ai', 'https://boston-scientific.dadosfera.ai', 'https://plataforma.dadosfera.ai' From 0f5ed50af98dbb97fc2b9d207b14a95e5bdfd951 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Fri, 28 Nov 2025 16:37:38 -0300 Subject: [PATCH 11/30] FIX: 404 in get public key --- src/modules/assign/assign.controller.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/assign/assign.controller.ts b/src/modules/assign/assign.controller.ts index 81830b4..7e7c40e 100644 --- a/src/modules/assign/assign.controller.ts +++ b/src/modules/assign/assign.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Post, Body, Put, Get} from '@nestjs/common'; +import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common'; import { AssignService } from './assign.service'; import { CreateAssignDto } from './dto/create-assign.dto'; import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator'; @@ -28,6 +28,10 @@ export class AssignController { @RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED) async get(@User() user: RequestUser) { const metadata = PackTheMetadata(user); - return await this.assignService.get(metadata); + try { + return await this.assignService.get(metadata); + } catch (error) { + throw new NotFoundException(error.message) + } } } From 24fce721e3ced5caaf2d4adbc4d486977516f20e Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Mon, 1 Dec 2025 09:32:26 -0300 Subject: [PATCH 12/30] FIX: api key catalog roles --- src/modules/catalog/catalog.service.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index e08cd0b..1848dcd 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -197,9 +197,11 @@ class CatalogService implements OnModuleInit { async getUserRolesIds(userId: string) { const result = await this.userService.findOneById(userId).catch(() => null); - const roles_ids = result.user.roles.map((role) => role.id); + if (result) { + return result.user.roles.map((role) => role.id); + } - return roles_ids; + return []; } async searchDataAssets( From 6d608a045735e969ae66fcf463ac6b60083ea8f7 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Mon, 1 Dec 2025 12:26:33 -0300 Subject: [PATCH 13/30] FEAT: include more user info in /me --- docsfera.json | 36 ++++++++++++----- src/modules/auth/auth.controller.ts | 38 ++--------------- src/modules/auth/auth.service.ts | 63 +++++++++++++++++++++++++---- src/modules/auth/dtos/login.ts | 13 ++++++ 4 files changed, 99 insertions(+), 51 deletions(-) diff --git a/docsfera.json b/docsfera.json index a18dad5..c46031c 100644 --- a/docsfera.json +++ b/docsfera.json @@ -42,6 +42,33 @@ ] } }, + "/auth/sign-out": { + "post": { + "operationId": "AuthController_signOut", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "" + } + }, + "tags": [ + "Auth" + ] + } + }, "/auth/refresh-access-token": { "post": { "operationId": "AuthController_refreshAccessToken", @@ -3415,9 +3442,6 @@ "PipelinesV2" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -3467,9 +3491,6 @@ "PipelinesV2" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } @@ -3519,9 +3540,6 @@ "PipelinesV2" ], "security": [ - { - "access-token": [] - }, { "access-token": [] } diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 648a36f..42eb27d 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -54,7 +54,6 @@ import jwt, { JwtPayload } from 'jsonwebtoken'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; -import { Cookie } from 'express-session'; type CookiesValues = { accessToken?: string; @@ -498,29 +497,10 @@ export class AuthController { const userId = req.cookies['ddf-user-id']; this.logger.info('Has cookie: ' + Boolean(accessToken)) - let payload: any; - let userInfo: any = {}; try { // Decodifica e valida o JWT de acesso - const decoded: any = accessToken && jwt.decode(accessToken, { complete: true }); - if (!decoded) throw new Error('Invalid token') - const { kid } = decoded.header; - // Busca a chave pública - const { keys } = await this.authClient.getPublicKeys(); - const pemValue = keys.find((k) => k.kid === kid)?.pem; - if (!pemValue) throw new Error('Public key not found'); - jwt.verify(accessToken, pemValue); - payload = decoded.payload; - userInfo = { - id: payload.user_id, - name: payload.username, - customer: { - id: payload.customer_id, - name: payload.customer_name, - tier: payload.customer_tier, - } - }; - return res.status(200).json(userInfo); + const userDto = await this.authClient.extractUserFrom(accessToken); + return res.status(200).json({userDto}); } catch (err) { this.logger.error(err.message); const refreshToken = req.cookies['ddf-refresh-auth']; @@ -545,18 +525,8 @@ export class AuthController { userId }); // Decodifica novo token - const decoded: any = jwt.decode(data.accessToken, { complete: true }); - const payload = decoded.payload; - userInfo = { - id: payload.user_id, - name: payload.username, - customer: { - id: payload.customer_id, - name: payload.customer_name, - tier: payload.customer_tier, - } - }; - return res.status(200).json(userInfo); + const userDto = await this.authClient.extractUserFrom(accessToken); + return res.status(200).json(userDto); } catch (refreshErr) { this.logger.error(refreshErr) return res.status(401).json({ error: 'Not authenticated' }); diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index c94a8d0..3a17327 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -4,7 +4,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { lastValueFrom } from 'rxjs'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; -import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService, UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; import { AuthSnowflakeSignInRequest, AuthSignInRequest, @@ -21,18 +21,18 @@ import { } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; -import { BulkEditResponse } from './dtos/login'; - +import { BulkEditResponse, UserDTO } from './dtos/login'; +import jwt from 'jsonwebtoken'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; @Injectable() export class AuthClientService implements OnModuleInit { - logger: DadosferaLogger; - private authService: AuthServiceInterface; - private identityProviderService: IdentityProviderProtoService; + private userService: UsersProtoService; + constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, @@ -46,8 +46,8 @@ export class AuthClientService implements OnModuleInit { ProtoServices.AuthProtoService, ); - this.identityProviderService = this.grpcClient.getService( - ProtoServices.IdentityProviderProtoService, + this.userService = this.grpcClient.getService( + ProtoServices.UsersProtoService, ); } @@ -283,4 +283,51 @@ export class AuthClientService implements OnModuleInit { throw error; } } + + private async getUser(id: string, metadata: Metadata) { + this.logger.info('getUser'); + return await lastValueFrom( + this.userService.UserFindOneById({ id }, metadata), + ); + } + + public async extractUserFrom(token: string) { + const decoded: any = token && jwt.decode(token, { complete: true }); + if (!decoded) throw new Error('Invalid token'); + + const { kid } = decoded.header; + // Busca a chave pública + const { keys } = await this.getPublicKeys(); + const pemValue = keys.find((k) => k.kid === kid)?.pem; + if (!pemValue) + throw new Error('Public key not found'); + jwt.verify(token, pemValue); + + const payload = decoded.payload; + const metadata = PackTheMetadata({ + customer_id: payload.customer_id + }) + + const { + user + } = await this.getUser( + payload.user_id, + metadata + ); + + const userDto: UserDTO = { + id: user.id, + name: user.username, + jobTitle: user?.jobTitle || null, + department: user?.department || null, + hierarchy: user?.hierarchy || null, + customer: { + id: payload.customer_id, + name: payload.customer_name, + tier: payload.customer_tier, + } + }; + + return userDto; + } } diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index dc57d54..3a484c8 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -140,3 +140,16 @@ export interface BulkEditResponse { successfulUsers: string[]; failedUsers: string[]; } + +export type UserDTO = { + id: string, + name: string, + jobTitle?: string, + department?: string, + hierarchy?: string, + customer: { + id: string, + name: string, + tier: string, + } +} From adf1b3b97ee64f272773da9d00747868636eef1a Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Mon, 1 Dec 2025 17:30:37 -0300 Subject: [PATCH 14/30] FIX: user response --- src/modules/auth/auth.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 42eb27d..9a4fb01 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -500,7 +500,7 @@ export class AuthController { try { // Decodifica e valida o JWT de acesso const userDto = await this.authClient.extractUserFrom(accessToken); - return res.status(200).json({userDto}); + return res.status(200).json(userDto); } catch (err) { this.logger.error(err.message); const refreshToken = req.cookies['ddf-refresh-auth']; From 7616b1e32c88fb40e9ec5731fd05d6b79a5d8c7c Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Wed, 3 Dec 2025 13:51:32 -0300 Subject: [PATCH 15/30] FEAT: add resource origin validate in me endpoint --- src/modules/auth/auth.controller.ts | 125 ++++---------- src/modules/auth/auth.service.ts | 247 ++++++++++++++++++++++------ 2 files changed, 226 insertions(+), 146 deletions(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 9a4fb01..c648a41 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -13,6 +13,7 @@ import { Req, Param, Res, + UnauthorizedException, } from '@nestjs/common'; import { ApiHeaders, @@ -102,14 +103,13 @@ export class AuthController { const data = await this.authClient.signIn({ username, password, totp }, metadata); if (data.tokens) { - this.addTokenInCookie(res, { + this.authClient.writeAuthSession(res, { accessToken: data.tokens.accessToken, refreshToken: data.tokens.refreshToken, userId: data.user.id }); } - return res.send(data); } catch (error) { this.logger.error('/auth - SignIn - ERROR', error); @@ -126,27 +126,8 @@ export class AuthController { ) { try { this.logger.info('/auth - SignOut'); - const exp = 1000 * 60 * 3; - - res.cookie('ddf-auth', '', { - domain: 'dadosfera.local', - maxAge: Date.now() - exp, - expires: new Date(), - httpOnly: true, - secure: true, - sameSite: 'none', // Necessário para cookies em requisições cross-site - }); - - res.cookie('ddf-refresh-auth', '', { - domain: 'dadosfera.local', - maxAge: Date.now() - exp, - expires: new Date(), - httpOnly: true, - secure: true, - sameSite: 'none', // Necessário para cookies em requisições cross-site - }); - - this.logger.info('Clean cookie sessions'); + + this.authClient.cleanUpAuthSession(res); return res.send(); } catch (error) { @@ -175,8 +156,9 @@ export class AuthController { const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata); - this.addTokenInCookie(res, { + this.authClient.writeAuthSession(res, { accessToken: data.accessToken, + refreshToken: data.refreshToken, userId }); @@ -492,85 +474,36 @@ export class AuthController { @Get('me') async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') - // Lê cookies + + // Get token and headers const accessToken = req.cookies['ddf-auth']; - const userId = req.cookies['ddf-user-id']; + const refreshToken = req.cookies['ddf-refresh-auth']; + const userId = req.cookies['ddf-user-id']; + const resourceOrigin = req.headers["origin"] + + const hasUserSession = Boolean(accessToken) && Boolean(userId); + this.logger.info('Has User Session: ' + hasUserSession); + + if (!hasUserSession) { + throw new UnauthorizedException() + } - this.logger.info('Has cookie: ' + Boolean(accessToken)) try { - // Decodifica e valida o JWT de acesso - const userDto = await this.authClient.extractUserFrom(accessToken); + const userDto = await this.authClient.validateUserSession(accessToken, resourceOrigin); return res.status(200).json(userDto); - } catch (err) { - this.logger.error(err.message); - const refreshToken = req.cookies['ddf-refresh-auth']; + } catch (error) { - this.logger.info('Token is invalid') - this.logger.info('Has Refresh Token: '+ Boolean(refreshToken)) - // Se access token inválido, tenta refresh - if (!refreshToken || !userId) { + if (!refreshToken) { this.logger.error('Invalid refresh token or customer name'); - return res.status(401).json({ error: 'Not authenticated' }); - } - try { - // Chama refreshAccessToken - const metadata = PackTheMetadata({ - }); - this.logger.info('Call Refresh Token') - const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata); - this.logger.info('Finish Refresh Token') - // Retorna novo access token e dados mínimos - this.addTokenInCookie(res, { - accessToken: data.accessToken, - userId - }); - // Decodifica novo token - const userDto = await this.authClient.extractUserFrom(accessToken); - return res.status(200).json(userDto); - } catch (refreshErr) { - this.logger.error(refreshErr) - return res.status(401).json({ error: 'Not authenticated' }); - } - } - } + throw new UnauthorizedException("Invalid refresh token or customer name"); + }; - private addTokenInCookie(res: Response, data: CookiesValues) { - let exp = 1000 * 60 * 5; // 5 minutes - - if (data.accessToken) { - const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload; - exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms - - this.logger.info('Set Cookie ddf-auth') - res.cookie('ddf-auth', data.accessToken, { - domain: '.dadosfera.ai', - maxAge: exp, - httpOnly: true, - secure: true, - sameSite: 'none', // Necessário para cookies em requisições cross-site - }); - } - - if (data.refreshToken) { - this.logger.info('Set Cookie ddf-refresh-auth') - res.cookie('ddf-refresh-auth', data.refreshToken, { - domain: '.dadosfera.ai', - maxAge: exp, - httpOnly: true, - secure: true, - sameSite: 'none', // Necessário para cookies em requisições cross-site - }); - } - - if (data.userId) { - this.logger.info('Set Cookie ddf-refresh-auth') - res.cookie('ddf-user-id', data.userId, { - domain: '.dadosfera.ai', - maxAge: exp, - httpOnly: true, - secure: true, - sameSite: 'none', // Necessário para cookies em requisições cross-site - }); + const { + authSession, + user + } = await this.authClient.refreshUserSession(refreshToken, userId, resourceOrigin); + this.authClient.writeAuthSession(res, authSession); + return res.status(200).json(user); } } } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 3a17327..3133462 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -1,10 +1,20 @@ -import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common'; +import { + OnModuleInit, + Inject, + Injectable, + ForbiddenException, + UnauthorizedException, +} from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { lastValueFrom } from 'rxjs'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; -import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService, UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { + AuthProtoService as AuthServiceInterface, + IdentityProviderProtoService, + UsersProtoService, +} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; import { AuthSnowflakeSignInRequest, AuthSignInRequest, @@ -22,12 +32,18 @@ import { import { DucClient } from '../duc/client.config'; import { Metadata } from '@grpc/grpc-js'; import { BulkEditResponse, UserDTO } from './dtos/login'; -import jwt from 'jsonwebtoken'; +import jwt, { JwtPayload } from 'jsonwebtoken'; import { PackTheMetadata } from 'src/utils/PackTheMetadata'; +import { Request, Response } from 'express'; + +type AuthSession = { + accessToken?: string; + refreshToken?: string; + userId?: string; +}; @Injectable() export class AuthClientService implements OnModuleInit { - logger: DadosferaLogger; private authService: AuthServiceInterface; @@ -63,11 +79,11 @@ export class AuthClientService implements OnModuleInit { return lastValueFrom(this.authService.AuthSnowflakeSignIn(input)); } - checkDedicatedProxy({ - customer - }: AuthSignInResponse) { + checkDedicatedProxy({ customer }: AuthSignInResponse) { const DEDICATED_PROXY = process.env.DEDICATED_PROXY || ''; - this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY); + this.logger.info( + 'SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY, + ); this.logger.info('Customer ID: ' + customer.id); if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) { @@ -75,7 +91,9 @@ export class AuthClientService implements OnModuleInit { } // Bloquear o customer de acesso o maestro publico - this.logger.info('Check if customer have network policy: ' + customer.modules); + this.logger.info( + 'Check if customer have network policy: ' + customer.modules, + ); const hasNetworkPolicyModule = customer.modules.includes('network-policy'); if (hasNetworkPolicyModule && DEDICATED_PROXY === '') { throw new ForbiddenException(); @@ -94,7 +112,6 @@ export class AuthClientService implements OnModuleInit { result = await lastValueFrom( this.authService.AuthSignIn({ username, password, totp }, metadata), ); - } catch (error) { this.logger.error('SignIn - Error during sign-in'); this.logger.error(error); @@ -105,7 +122,7 @@ export class AuthClientService implements OnModuleInit { this.checkDedicatedProxy(result); } - return result + return result; } async refreshAccessToken( @@ -115,7 +132,10 @@ export class AuthClientService implements OnModuleInit { this.logger.info('RefreshAccessToken'); return lastValueFrom( - this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata), + this.authService.AuthRefreshAccessToken( + { refreshToken, userId }, + metadata, + ), ); } @@ -284,50 +304,177 @@ export class AuthClientService implements OnModuleInit { } } - private async getUser(id: string, metadata: Metadata) { - this.logger.info('getUser'); - return await lastValueFrom( - this.userService.UserFindOneById({ id }, metadata), - ); + public async validateUserSession(accessToken: any, originHeader: string) { + const payload = await this.validateJwtToken(accessToken); + + const userDto = await this.getUserfromPayload(payload); + + this.validateResourceAccess(originHeader, userDto); + return userDto; } - public async extractUserFrom(token: string) { - const decoded: any = token && jwt.decode(token, { complete: true }); - if (!decoded) throw new Error('Invalid token'); + public async refreshUserSession( + refreshToken: string, + userId: string, + originHeader: string, + ): Promise<{ + user: UserDTO; + authSession: AuthSession; + }> { + const metadata = PackTheMetadata({}); - const { kid } = decoded.header; - // Busca a chave pública - const { keys } = await this.getPublicKeys(); - const pemValue = keys.find((k) => k.kid === kid)?.pem; - if (!pemValue) - throw new Error('Public key not found'); - jwt.verify(token, pemValue); + this.logger.info('Call Refresh Token'); + const refreshCredentials = await this.refreshAccessToken( + { refreshToken, userId }, + metadata, + ); + this.logger.info('Finish Refresh Token'); - const payload = decoded.payload; - const metadata = PackTheMetadata({ - customer_id: payload.customer_id - }) + const userDto = await this.validateUserSession( + refreshCredentials.accessToken, + originHeader, + ); + return { + user: userDto, + authSession: { + accessToken: refreshCredentials.accessToken, + refreshToken: refreshCredentials.refreshToken, + userId, + }, + }; + } - const { - user - } = await this.getUser( - payload.user_id, - metadata - ); + public writeAuthSession(res: Response, data: AuthSession) { + let exp = 1000 * 60 * 5; // 5 minutes - const userDto: UserDTO = { - id: user.id, - name: user.username, - jobTitle: user?.jobTitle || null, - department: user?.department || null, - hierarchy: user?.hierarchy || null, - customer: { - id: payload.customer_id, - name: payload.customer_name, - tier: payload.customer_tier, - } - }; + if (data.accessToken) { + const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload; + exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms - return userDto; + this.logger.info('Set Cookie ddf-auth'); + res.cookie('ddf-auth', data.accessToken, { + domain: '.dadosfera.ai', + maxAge: exp, + httpOnly: true, + secure: true, + sameSite: 'none', // Necessário para cookies em requisições cross-site + }); } + + if (data.refreshToken) { + this.logger.info('Set Cookie ddf-refresh-auth'); + res.cookie('ddf-refresh-auth', data.refreshToken, { + domain: '.dadosfera.ai', + maxAge: exp, + httpOnly: true, + secure: true, + sameSite: 'none', // Necessário para cookies em requisições cross-site + }); + } + + if (data.userId) { + this.logger.info('Set Cookie ddf-refresh-auth'); + res.cookie('ddf-user-id', data.userId, { + domain: '.dadosfera.ai', + maxAge: exp, + httpOnly: true, + secure: true, + sameSite: 'none', // Necessário para cookies em requisições cross-site + }); + } + } + + public cleanUpAuthSession(res: Response) { + const exp = 1000 * 60 * 3; + + res.cookie('ddf-auth', '', { + domain: 'dadosfera.ai', + maxAge: Date.now() - exp, + expires: new Date(), + httpOnly: true, + secure: true, + sameSite: 'none', // Necessário para cookies em requisições cross-site + }); + + res.cookie('ddf-refresh-auth', '', { + domain: 'dadosfera.ai', + maxAge: Date.now() - exp, + expires: new Date(), + httpOnly: true, + secure: true, + sameSite: 'none', // Necessário para cookies em requisições cross-site + }); + + this.logger.info('Clean cookie sessions'); + } + + private async validateJwtToken(token: string) { + const decoded: any = token && jwt.decode(token, { complete: true }); + if (!decoded) throw new Error('Invalid token'); + + const { kid } = decoded.header; + // Busca a chave pública + const { keys } = await this.getPublicKeys(); + const pemValue = keys.find((k) => k.kid === kid)?.pem; + if (!pemValue) throw new Error('Public key not found'); + jwt.verify(token, pemValue); + + return decoded.payload; + } + + private async getUserfromPayload(payload: JwtPayload): Promise { + this.logger.info('getUser'); + + const metadata = PackTheMetadata({ + customer_id: payload.customer_id, + }); + + const { user } = await lastValueFrom( + this.userService.UserFindOneById({ id: payload.user_id }, metadata), + ); + + const userDto: UserDTO = { + id: user.id, + name: user.username, + jobTitle: user?.jobTitle || null, + department: user?.department || null, + hierarchy: user?.hierarchy || null, + customer: { + id: payload.customer_id, + name: payload.customer_name, + tier: payload.customer_tier, + }, + }; + + return userDto; + } + + private validateResourceAccess(origin: string, user: UserDTO) { + this.logger.info( + "Validate whether the source URL is a resource belonging to the user's client", + ); + this.logger.info('Origin: ' + origin); + this.logger.info('Customer: ' + user.customer.name); + + const urlParts = origin.replace('https://', '').split('.'); + const domain = urlParts[0]; + const isResouceStg = urlParts[1] === 'stg'; + + const domainParts = domain.split('-'); + const customerInDomain = domainParts[domainParts.length - 1]; + + if (isResouceStg && process.env.ENV !== 'stg') { + throw new ForbiddenException( + `Customer ${user.customer.name} cannot access ${origin}`, + ); + } + + if (customerInDomain != user.customer.name) { + throw new ForbiddenException( + `Customer ${user.customer.name} cannot access ${origin}`, + ); + } + + return; + } } From 82290285d015bdacfeaa01614937e8f7abbb63f0 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Wed, 3 Dec 2025 14:41:51 -0300 Subject: [PATCH 16/30] FIX: using header host --- src/modules/auth/auth.controller.ts | 6 +++--- src/modules/auth/auth.service.ts | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index c648a41..3d86f14 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -479,7 +479,7 @@ export class AuthController { const accessToken = req.cookies['ddf-auth']; const refreshToken = req.cookies['ddf-refresh-auth']; const userId = req.cookies['ddf-user-id']; - const resourceOrigin = req.headers["origin"] + const resourceHost = req.headers["host"] const hasUserSession = Boolean(accessToken) && Boolean(userId); this.logger.info('Has User Session: ' + hasUserSession); @@ -489,7 +489,7 @@ export class AuthController { } try { - const userDto = await this.authClient.validateUserSession(accessToken, resourceOrigin); + const userDto = await this.authClient.validateUserSession(accessToken, resourceHost); return res.status(200).json(userDto); } catch (error) { @@ -501,7 +501,7 @@ export class AuthController { const { authSession, user - } = await this.authClient.refreshUserSession(refreshToken, userId, resourceOrigin); + } = await this.authClient.refreshUserSession(refreshToken, userId, resourceHost); this.authClient.writeAuthSession(res, authSession); return res.status(200).json(user); } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 3133462..3229752 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -449,16 +449,16 @@ export class AuthClientService implements OnModuleInit { return userDto; } - private validateResourceAccess(origin: string, user: UserDTO) { + private validateResourceAccess(host: string, user: UserDTO) { this.logger.info( "Validate whether the source URL is a resource belonging to the user's client", ); - this.logger.info('Origin: ' + origin); + this.logger.info('Host: ' + host); this.logger.info('Customer: ' + user.customer.name); - const urlParts = origin.replace('https://', '').split('.'); - const domain = urlParts[0]; - const isResouceStg = urlParts[1] === 'stg'; + const hostParts = host.split('.'); + const domain = hostParts[0]; + const isResouceStg = hostParts[1] === 'stg'; const domainParts = domain.split('-'); const customerInDomain = domainParts[domainParts.length - 1]; From bb671a90d61310cbc35e765c637ee0cec04ee1e1 Mon Sep 17 00:00:00 2001 From: Marcos Rodrigues Date: Wed, 3 Dec 2025 15:20:41 -0300 Subject: [PATCH 17/30] FIX: change origin to host --- src/modules/auth/auth.service.ts | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index 3229752..c1b85f6 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -3,7 +3,8 @@ import { Inject, Injectable, ForbiddenException, - UnauthorizedException, + HttpException, + HttpStatus, } from '@nestjs/common'; import { ClientGrpc } from '@nestjs/microservices'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; @@ -12,7 +13,6 @@ import { lastValueFrom } from 'rxjs'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; import { AuthProtoService as AuthServiceInterface, - IdentityProviderProtoService, UsersProtoService, } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; import { @@ -304,12 +304,12 @@ export class AuthClientService implements OnModuleInit { } } - public async validateUserSession(accessToken: any, originHeader: string) { + public async validateUserSession(accessToken: any, resourceHost: string) { const payload = await this.validateJwtToken(accessToken); const userDto = await this.getUserfromPayload(payload); - this.validateResourceAccess(originHeader, userDto); + this.validateResourceAccess(resourceHost, userDto); return userDto; } @@ -460,18 +460,30 @@ export class AuthClientService implements OnModuleInit { const domain = hostParts[0]; const isResouceStg = hostParts[1] === 'stg'; + const notFoundCustomerInDomain = !domain.includes('-') + + if (notFoundCustomerInDomain) { + this.logger.info(`Not found Customer Name in domain`); + return; + } + const domainParts = domain.split('-'); + const customerInDomain = domainParts[domainParts.length - 1]; if (isResouceStg && process.env.ENV !== 'stg') { - throw new ForbiddenException( - `Customer ${user.customer.name} cannot access ${origin}`, + this.logger.error(`Customer ${user.customer.name} cannot access ${host}`); + throw new HttpException( + `Customer ${user.customer.name} cannot access ${host}`, + HttpStatus.FORBIDDEN ); } if (customerInDomain != user.customer.name) { - throw new ForbiddenException( - `Customer ${user.customer.name} cannot access ${origin}`, + this.logger.error(`Customer ${user.customer.name} cannot access ${host}`); + throw new HttpException( + `Customer ${user.customer.name} cannot access ${host}`, + HttpStatus.FORBIDDEN ); } From 399d3492d38ccb34ba5aaa93be2f0aa3a4180861 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 9 Dec 2025 14:46:02 -0300 Subject: [PATCH 18/30] DOCS: update @dadosfera/protospack-v2 to version 3.38.0-beta.15 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e58ffab..8c9f50b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.14", + "@dadosfera/protospack-v2": "3.38.0-beta.15", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From c21d977f7c6fba15a502465dee1517fbb725e895 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 9 Dec 2025 14:51:25 -0300 Subject: [PATCH 19/30] FEAT: enhance catalog service to support any asset type in data docs creation --- src/modules/catalog/catalog.controller.ts | 27 +++++++++++--- src/modules/catalog/catalog.service.ts | 45 ++++++++++++++++++----- src/modules/catalog/dtos/index.ts | 7 ++++ 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index b87829b..13bcf42 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -439,7 +439,13 @@ export class CatalogController { language, }); - const docs = await this.catalogService.getDataDocs(id, metadata); + const { data_asset } = await this.catalogService.getOneDataAsset({ + customer_id, + id, + metadata, + }); + + const docs = await this.catalogService.getDataDocs(id, data_asset.data_asset_type, metadata); return { docs }; } @@ -487,21 +493,32 @@ export class CatalogController { @Headers() headers, @Param('id') table_id: string, @Body('docs') docs: string, + @Query('asset_type') asset_type: string, ) { - const { user_id, customer_name } = user; + const { user_id, customer_name, customer_id, username } = user; + + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + }); - this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { + this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, { user_id, customer_name, }); - const res = await this.catalogService.createDataDocs({ + const body = { table_id, docs, + asset_type, info: { customer: customer_name, }, - }); + } + + const res = await this.catalogService.createDataDocs(body, metadata); return res; } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 1848dcd..16cc4ab 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -24,9 +24,11 @@ import { CatalogClientConfiguration } from './catalog-client'; import { UsersService } from '../users/users.service'; import { RolesService } from '../roles/roles.service'; import { Metadata } from '@grpc/grpc-js'; +import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { AssetReporter, BatchRemoveRlsRulesRequest, + CreateDataDocsDTO, IUpdateDataRequest, TriggerCatalogReq, } from './dtos'; @@ -337,9 +339,11 @@ class CatalogService implements OnModuleInit { return { data_asset: asset[0] }; } - async getDataDocs(id: string, metadata: Metadata) { + async getDataDocs(id: string, assetType: string, metadata: Metadata) { + const requestType = assetType === 'dataset' ? undefined : assetType; + const { documentation } = await lastValueFrom( - this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), + this.catalogReadService.GetDatasetDoc({ id, type: requestType }, metadata), ); const docs = JSON.parse(documentation); @@ -368,13 +372,36 @@ class CatalogService implements OnModuleInit { return result; } - async createDataDocs(body) { - const nimbusUrl = this._getNimbusUrl(body); - const { data } = await axios.post( - `${nimbusUrl}/api/catalog/data-docs/`, - body, - ); - return data; + async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) { + if (body.asset_type === 'table' || body.asset_type === 'view') { + const nimbusUrl = this._getNimbusUrl(body); + const { data } = await axios.post( + `${nimbusUrl}/api/catalog/data-docs/`, + body, + ); + return data; + } + + try { + const response: any = await lastValueFrom( + this.catalogWriteService.UpdateDataAssetDoc( + { + id: body.table_id, + docs: body.docs, + }, + metadata, + ), + ); + + return response; + + } catch (error) { + this.logger.error('Error creating data asset docs:', error); + throw new HttpException( + 'Failed to create data asset documentation', + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } } async findAllTags(data, metadata) { diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index ef12e78..4989672 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -328,3 +328,10 @@ export type AssetReporter = { created_at: string; tags: string; } + +export type CreateDataDocsDTO = { + table_id: string; + docs: string; + asset_type: string; + +} \ No newline at end of file From a0a1303515fedee6c81eb5a5e1a525d43f5c6aad Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 9 Dec 2025 14:54:39 -0300 Subject: [PATCH 20/30] DOCS: add automatic asset_type query parameter to API schema and packages dependencies --- docsfera.json | 8 +++++ package-lock.json | 76 +++++++++++++++++++++-------------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/docsfera.json b/docsfera.json index ab3a2c1..591cea9 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4471,6 +4471,14 @@ "schema": { "type": "string" } + }, + { + "name": "asset_type", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { diff --git a/package-lock.json b/package-lock.json index 0a1aa0e..2764511 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.14", + "@dadosfera/protospack-v2": "3.38.0-beta.15", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -850,7 +850,6 @@ "integrity": "sha512-vMqyb7XCDMPvJFFOaT9kxtiRh42GwlZEg1/uIgtZshS5a/8OaduUfCi7kynKgc3Tw/6Uo2D+db9qBttghhmxwQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.26.2", @@ -1409,10 +1408,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.14", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.14.tgz", - "integrity": "sha512-BiE1fUIHuam3cJjAGIknsRNIlf1Z0JHJOV/VZ2pyAp5mIkGOSoNFBAMXJZb+VMTnd2xsp+vPxNnWVu7OJ/CvMQ==", - "license": "ISC", + "version": "3.38.0-beta.15", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.15.tgz", + "integrity": "sha512-WJ6wANzTt/fo0BrE7nviAqwN1COh66qIujoYfBlHW7FgrHkgu8CAUF38WmtjqjYCvpfcd/A1j1MESsuUg9QWJw==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -2261,7 +2259,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-9.4.3.tgz", "integrity": "sha512-Gd6D4IaYj01o14Bwv81ukidn4w3bPHCblMUq+SmUmWLyosK+XQmInCS09SbDDZyL8jy86PngtBLTdhJ2bXSUig==", "license": "MIT", - "peer": true, "dependencies": { "iterare": "1.2.1", "tslib": "2.5.3", @@ -2340,7 +2337,6 @@ "integrity": "sha512-Qi63+wi55Jh4sDyaj5Hhx2jOpKqT386aeo+VOKsxnd+Ql9VvkO/FjmuwBGUyzkJt29ENYc+P0Sx/k5LtstNpPQ==", "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@nuxtjs/opencollective": "0.3.2", "fast-safe-stringify": "2.1.1", @@ -2478,7 +2474,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz", "integrity": "sha512-FpdczWoRSC0zz2dNL9u2AQLXKXRVtq4HgHklAhbL59X0uy+mcxhlSThG7DHzDMkoSnuuHY8ojDVf7mDxk+GtCw==", "license": "MIT", - "peer": true, "dependencies": { "body-parser": "1.20.2", "cors": "2.8.5", @@ -2842,7 +2837,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -3002,7 +2996,6 @@ "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", "license": "MIT", - "peer": true, "dependencies": { "cluster-key-slot": "1.1.2", "generic-pool": "3.9.0", @@ -3799,7 +3792,6 @@ "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -3880,7 +3872,6 @@ "integrity": "sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "jest-diff": "^27.0.0", "pretty-format": "^27.0.0" @@ -3944,8 +3935,7 @@ "version": "16.18.126", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz", "integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/oauth": { "version": "0.9.6", @@ -4175,7 +4165,6 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -4522,7 +4511,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4619,7 +4607,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -5248,7 +5235,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001688", "electron-to-chromium": "^1.5.73", @@ -5355,7 +5341,6 @@ "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz", "integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==", "license": "MIT", - "peer": true, "dependencies": { "eventemitter3": "^5.0.1", "lodash.clonedeep": "^4.5.0", @@ -5579,15 +5564,13 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/class-validator": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.1.tgz", "integrity": "sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ==", "license": "MIT", - "peer": true, "dependencies": { "@types/validator": "^13.11.8", "libphonenumber-js": "^1.10.53", @@ -6251,8 +6234,7 @@ "version": "0.0.1425554", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz", "integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==", - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/dezalgo": { "version": "1.0.4", @@ -6529,6 +6511,7 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "license": "MIT", + "peer": true, "engines": { "node": ">= 0.8" } @@ -6716,7 +6699,6 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -7063,6 +7045,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -7109,6 +7092,7 @@ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "license": "MIT", + "peer": true, "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -7133,6 +7117,7 @@ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "license": "MIT", + "peer": true, "engines": { "node": ">= 0.6" } @@ -7142,6 +7127,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", + "peer": true, "dependencies": { "ms": "2.0.0" } @@ -7150,19 +7136,22 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/express/node_modules/path-to-regexp": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/express/node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "side-channel": "^1.0.6" }, @@ -7191,7 +7180,8 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/external-editor": { "version": "3.1.0", @@ -7426,6 +7416,7 @@ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "license": "MIT", + "peer": true, "dependencies": { "debug": "2.6.9", "encodeurl": "~2.0.0", @@ -7444,6 +7435,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", + "peer": true, "dependencies": { "ms": "2.0.0" } @@ -7452,7 +7444,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/find-up": { "version": "5.0.0", @@ -8246,7 +8239,6 @@ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==", "license": "MIT", - "peer": true, "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", @@ -8588,7 +8580,6 @@ "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -9878,6 +9869,7 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -10635,7 +10627,6 @@ "resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz", "integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==", "license": "MIT", - "peer": true, "dependencies": { "passport-strategy": "1.x.x", "pause": "0.0.1", @@ -11025,7 +11016,6 @@ "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin-prettier.js" }, @@ -11528,8 +11518,7 @@ "version": "0.1.14", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz", "integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/relative-microtime": { "version": "2.0.0", @@ -11716,7 +11705,6 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", - "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -11778,7 +11766,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -11822,6 +11809,7 @@ "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "license": "MIT", + "peer": true, "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -11846,6 +11834,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "license": "MIT", + "peer": true, "dependencies": { "ms": "2.0.0" } @@ -11854,13 +11843,15 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/send/node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", "license": "MIT", + "peer": true, "engines": { "node": ">= 0.8" } @@ -11869,7 +11860,8 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/serialize-javascript": { "version": "6.0.2", @@ -11885,6 +11877,7 @@ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "license": "MIT", + "peer": true, "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", @@ -12862,7 +12855,6 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -13092,7 +13084,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13420,6 +13411,7 @@ "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz", "integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==", "license": "MIT", + "peer": true, "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", From b1dc5673942a900736cf061b2367cf6f8846bdb3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 10 Dec 2025 11:12:13 -0300 Subject: [PATCH 21/30] DOCS: update @dadosfera/protospack-v2 to version 3.38.0-beta.16 --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2764511..5d30872 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.15", + "@dadosfera/protospack-v2": "3.38.0-beta.16", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1408,9 +1408,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.15", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.15.tgz", - "integrity": "sha512-WJ6wANzTt/fo0BrE7nviAqwN1COh66qIujoYfBlHW7FgrHkgu8CAUF38WmtjqjYCvpfcd/A1j1MESsuUg9QWJw==", + "version": "3.38.0-beta.16", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.16.tgz", + "integrity": "sha512-ijURcmETXdgVgXhIPxapK7w0z6begXm4WDZ6EJtUGg+oYlJd9QOxbwaD7akRzW8XTdccmIo0Vov065KB812J0A==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 8c9f50b..1da6c28 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.15", + "@dadosfera/protospack-v2": "3.38.0-beta.16", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 165b533172502ba2bf35a778581b98fb52cc8bec Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 10 Dec 2025 11:17:43 -0300 Subject: [PATCH 22/30] FIX: removing unnecessary type parameter from GetDatasetDoc call --- src/modules/catalog/share/share.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/catalog/share/share.service.ts b/src/modules/catalog/share/share.service.ts index 3e088a6..86b17b9 100644 --- a/src/modules/catalog/share/share.service.ts +++ b/src/modules/catalog/share/share.service.ts @@ -160,7 +160,7 @@ export class ShareService implements OnModuleInit { }); const { documentation } = await lastValueFrom( - this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), + this.catalogReadService.GetDatasetDoc({ id }, metadata), ); console.log(documentation); const docs = JSON.parse(documentation); From 09dced9fcdec7da65342f703b736cf571df7ee0b Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 10 Dec 2025 11:21:46 -0300 Subject: [PATCH 23/30] REFACTOR: add asset_type query parameter to getDataDocs and separate data docs methods --- src/modules/catalog/catalog.controller.ts | 9 ++------ src/modules/catalog/catalog.service.ts | 28 +++++++++++++++-------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 13bcf42..83c1386 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -423,6 +423,7 @@ export class CatalogController { @User() user: RequestUser, @Language() language: LanguageEnum, @Param('id') id: string, + @Query('asset_type') asset_type: string, ): Promise { const { customer_name, customer_id, user_id, username } = user; @@ -439,13 +440,7 @@ export class CatalogController { language, }); - const { data_asset } = await this.catalogService.getOneDataAsset({ - customer_id, - id, - metadata, - }); - - const docs = await this.catalogService.getDataDocs(id, data_asset.data_asset_type, metadata); + const docs = await this.catalogService.getDataDocs(id, asset_type, metadata); return { docs }; } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 16cc4ab..73c9111 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -340,10 +340,8 @@ class CatalogService implements OnModuleInit { } async getDataDocs(id: string, assetType: string, metadata: Metadata) { - const requestType = assetType === 'dataset' ? undefined : assetType; - const { documentation } = await lastValueFrom( - this.catalogReadService.GetDatasetDoc({ id, type: requestType }, metadata), + this.catalogReadService.GetDatasetDoc({ id }, metadata), ); const docs = JSON.parse(documentation); @@ -374,14 +372,24 @@ class CatalogService implements OnModuleInit { async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) { if (body.asset_type === 'table' || body.asset_type === 'view') { - const nimbusUrl = this._getNimbusUrl(body); - const { data } = await axios.post( - `${nimbusUrl}/api/catalog/data-docs/`, - body, - ); - return data; + return this.createDataDocsViaNimbus(body); } - + + return this.createDataDocsViaGrpc(body, metadata); + } + + private async createDataDocsViaNimbus(body: CreateDataDocsDTO) { + this.logger.info('Creating data docs via Nimbus for table/view'); + const nimbusUrl = this._getNimbusUrl(body); + const { data } = await axios.post( + `${nimbusUrl}/api/catalog/data-docs/`, + body, + ); + return data; + } + + private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) { + this.logger.info('Creating data docs via gRPC for other asset types'); try { const response: any = await lastValueFrom( this.catalogWriteService.UpdateDataAssetDoc( From 2f140d213a8bcf68f4181ca16304fdd31c014845 Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Wed, 10 Dec 2025 16:51:17 +0000 Subject: [PATCH 24/30] catalogo: alteracao nas pastas e acrescimo de filtros --- docsfera.json | 20 ++++++++ src/modules/catalog/catalog.service.ts | 54 ++++++++++++++++++++-- src/modules/catalog/dtos/index.ts | 6 +++ src/modules/catalog/share/share.service.ts | 4 +- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/docsfera.json b/docsfera.json index ab3a2c1..9cb09aa 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3877,6 +3877,16 @@ ], "type": "string" } + }, + { + "name": "owner", + "required": false, + "in": "query", + "description": "ID do usuário owner para filtrar data assets", + "example": "user-id-1,user-id-2", + "schema": { + "type": "string" + } } ], "responses": { @@ -4006,6 +4016,16 @@ ], "type": "string" } + }, + { + "name": "owner", + "required": false, + "in": "query", + "description": "ID do usuário owner para filtrar data assets", + "example": "user-id-1,user-id-2", + "schema": { + "type": "string" + } } ], "responses": { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 1848dcd..4fd9619 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -209,10 +209,58 @@ class CatalogService implements OnModuleInit { metadata: Metadata, customer_id: string, ) { - this.logger.info('CatalogService - searchDataAssets'); + this.logger.info('CatalogService - searchDataAssets', { query }); const { search, page, size, sort_by, order, ...filters } = query; + this.logger.debug('Extracted filters:', { filters }); + + if (filters.owner) { + const { users: customer_users } = + await this.userService.findAllUsersByCustomerId(customer_id); + + this.logger.info('Available users in database count:', { count: customer_users.length }); + this.logger.info('First 5 users:', { + users: customer_users.slice(0, 5).map(u => ({ id: u.id, email: u.email, name: u.name })) + }); + + const ownerValues = Array.isArray(filters.owner) + ? filters.owner + : (typeof filters.owner === 'string' && filters.owner.includes(',') + ? filters.owner.split(',').map((o: string) => o.trim()) + : [filters.owner]); + + this.logger.info('Owner values to convert:', { ownerValues, ownerFiltersOriginal: filters.owner }); + + const ownerIds = ownerValues + .map((ownerValue: string) => { + const normalizedOwner = ownerValue.replace(/\s/g, '+'); + const user = customer_users.find(u => { + const isIdMatch = u.id === ownerValue; + const isEmailMatch = u.email === ownerValue || u.email === normalizedOwner; + const isNameMatch = u.name === ownerValue || u.name === normalizedOwner; + this.logger.info('Comparing:', { + userId: u.id, + userEmail: u.email, + userName: u.name, + filterValue: ownerValue, + normalizedFilter: normalizedOwner, + idMatch: isIdMatch, + emailMatch: isEmailMatch, + nameMatch: isNameMatch, + }); + return isIdMatch || isEmailMatch || isNameMatch; + }); + this.logger.info('Looking for owner result:', { ownerValue, found: !!user, userId: user?.id }); + return user?.id || ownerValue; + }) + .filter((id: string) => id); + + if (ownerIds.length > 0) { + filters.owner = ownerIds; + } + } + const { data_assets, total } = await lastValueFrom( this.catalogReadService.GetAllDataAssets( { @@ -404,7 +452,7 @@ class CatalogService implements OnModuleInit { return data_assets.map((data_asset) => { const owner = customer_users.find( (u) => u.id === data_asset.owner, - )?.username; + )?.email; const roles = []; const users = []; @@ -417,7 +465,7 @@ class CatalogService implements OnModuleInit { const data_asset_users = data_asset?.users || [] for (const user_id of data_asset_users) { const user = customer_users.find((r) => r.id === user_id); - if (user) users.push({ id: user.id, username: user.username }); + if (user) users.push({ id: user.id, email: user.email }); } return { ...data_asset, diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index ef12e78..59de404 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -147,6 +147,12 @@ export class ICatalogAllRequest { description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ', }) order?: OrderEnum; + + @ApiPropertyOptional({ + description: 'ID do usuário owner para filtrar data assets', + example: 'user-id-1,user-id-2', + }) + owner?: string; } export class ICatalogAllResponse { diff --git a/src/modules/catalog/share/share.service.ts b/src/modules/catalog/share/share.service.ts index 3e088a6..48f2d5c 100644 --- a/src/modules/catalog/share/share.service.ts +++ b/src/modules/catalog/share/share.service.ts @@ -180,7 +180,7 @@ export class ShareService implements OnModuleInit { return data_assets.map((data_asset) => { const owner = customer_users.find( (u) => u.id === data_asset.owner, - )?.username; + )?.email; const roles = []; const users = []; @@ -190,7 +190,7 @@ export class ShareService implements OnModuleInit { } for (const user_id of data_asset.users) { const user = customer_users.find((r) => r.id === user_id); - if (user) users.push({ id: user.id, username: user.username }); + if (user) users.push({ id: user.id, email: user.email }); } return { ...data_asset, From 00304262bae0fe3b0429cd7478bc1b6107d67ae3 Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Thu, 11 Dec 2025 13:52:58 +0000 Subject: [PATCH 25/30] FIX: wget change in helmfile --- .github/workflows/validate-k8s.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 4d23e66..7f4255c 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -66,7 +66,7 @@ jobs: - name: Install Helmfile run: | - wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz tar -xzf helmfile_0.148.0_linux_amd64.tar.gz sudo mv helmfile /usr/local/bin/ helmfile --version From 5c29e0745046141478ba15ec4fee4beb99e50144 Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Thu, 11 Dec 2025 14:07:04 +0000 Subject: [PATCH 26/30] FIX: helmfile --- .github/workflows/validate-k8s.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 7f4255c..89dba0b 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -71,8 +71,10 @@ jobs: sudo mv helmfile /usr/local/bin/ helmfile --version - - name: Install Helm Diff Plugin - run: helm plugin install https://github.com/databus23/helm-diff || true + - name: Fix Helm Diff Plugin + run: | + helm plugin rm diff || true + helm plugin install https://github.com/databus23/helm-diff - name: Authenticate with OKE cluster env: From 6919a2a8d0a8b77c58513a85e73e8134b04d157c Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Thu, 11 Dec 2025 14:14:02 +0000 Subject: [PATCH 27/30] FIX: helmfile diff plugin --- .github/workflows/validate-k8s.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 89dba0b..3ad7408 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -72,9 +72,16 @@ jobs: helmfile --version - name: Fix Helm Diff Plugin + env: + HELM_PLUGINS: /home/runner/.local/share/helm/plugins run: | - helm plugin rm diff || true + rm -rf "$HELM_PLUGINS/diff" "$HELM_PLUGINS/helm-diff" || true + + mkdir -p "$HELM_PLUGINS" + ls -R "$HELM_PLUGINS" || true + helm plugin install https://github.com/databus23/helm-diff + helm plugin list - name: Authenticate with OKE cluster env: From dd699614aeb3d1b067e0817a4c02e5f8b2759d63 Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Thu, 11 Dec 2025 16:47:28 +0000 Subject: [PATCH 28/30] FIX: Maestro helm fix --- .github/workflows/validate-k8s.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml index 3ad7408..8a67a9c 100644 --- a/.github/workflows/validate-k8s.yml +++ b/.github/workflows/validate-k8s.yml @@ -71,17 +71,11 @@ jobs: sudo mv helmfile /usr/local/bin/ helmfile --version - - name: Fix Helm Diff Plugin - env: - HELM_PLUGINS: /home/runner/.local/share/helm/plugins + - name: Debug Helm env run: | - rm -rf "$HELM_PLUGINS/diff" "$HELM_PLUGINS/helm-diff" || true - - mkdir -p "$HELM_PLUGINS" - ls -R "$HELM_PLUGINS" || true - - helm plugin install https://github.com/databus23/helm-diff - helm plugin list + helm env + echo "HOME=$HOME" + ls -R $HOME/.local/share/helm || true - name: Authenticate with OKE cluster env: From df3f2489f986823fe2dcd1b80b9f80e22470d612 Mon Sep 17 00:00:00 2001 From: Rafael Moraes Date: Thu, 11 Dec 2025 22:10:32 +0000 Subject: [PATCH 29/30] FIX: Calendar Fix --- docsfera.json | 40 +++++++++++ src/modules/catalog/catalog.service.ts | 96 +++++++++++++++++++++++++- src/modules/catalog/dtos/index.ts | 12 ++++ 3 files changed, 147 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 9cb09aa..d27ab0a 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3887,6 +3887,26 @@ "schema": { "type": "string" } + }, + { + "name": "catalog_date_from", + "required": false, + "in": "query", + "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-01-01", + "schema": { + "type": "string" + } + }, + { + "name": "catalog_date_to", + "required": false, + "in": "query", + "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-12-31", + "schema": { + "type": "string" + } } ], "responses": { @@ -4026,6 +4046,26 @@ "schema": { "type": "string" } + }, + { + "name": "catalog_date_from", + "required": false, + "in": "query", + "description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-01-01", + "schema": { + "type": "string" + } + }, + { + "name": "catalog_date_to", + "required": false, + "in": "query", + "description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)", + "example": "2025-12-31", + "schema": { + "type": "string" + } } ], "responses": { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 4fd9619..8d87d9c 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -39,6 +39,7 @@ import { import { TypeParser } from 'src/utils/FileParser/parser-types'; import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; + class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; catalogWriteService: WriteService.CatalogWriteServices; @@ -55,6 +56,7 @@ class CatalogService implements OnModuleInit { this.logger = dadosferaLogger.logger; } + onModuleInit() { this.catalogReadService = this.grpcClient.getService( @@ -70,20 +72,24 @@ class CatalogService implements OnModuleInit { ); } + _getNimbusUrl(body) { this.logger.debug(`Body: ${JSON.stringify(body)}`); const customer = body.info.customer.toLowerCase(); + if (process.env.ENV === 'prd') { return `https://nimbus-${customer}.dadosfera.ai`; } + return `https://nimbus-${customer}.${process.env.ENV.replace( 'local', 'stg', )}.dadosfera.ai`; } + async getPiiReporter(metadata: Metadata, type: TypeParser) { this.logger.info('getPiiReporter: ' + type) try { @@ -94,8 +100,10 @@ class CatalogService implements OnModuleInit { ) this.logger.info("Finish grpc call") + const parser = ParserBuilder.build(type); + this.logger.info('parser file to: ' + type) const file = await parser.parse(data) this.logger.info('finish parser') @@ -105,9 +113,11 @@ class CatalogService implements OnModuleInit { 'pdf': 'application/pdf' } + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `relatorio-pii-${timestamp}.${type}`; + return { file, filename: filename, @@ -118,20 +128,25 @@ class CatalogService implements OnModuleInit { throw error; } + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; + return lastValueFrom( this.catalogWriteService.CreateDataAsset(data, metadata), ); } + async managePermissions(data: Messages.ManagePermissionRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.ManagePermission(data, metadata), ).catch((err) => { @@ -142,9 +157,11 @@ class CatalogService implements OnModuleInit { }); } + async revokePermissions(data: Messages.RevokePermissionRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.RevokePermission(data, metadata), ).catch((err) => { @@ -155,9 +172,11 @@ class CatalogService implements OnModuleInit { }); } + async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.MakeAComment(data, metadata), ).catch((err) => { @@ -168,9 +187,11 @@ class CatalogService implements OnModuleInit { }); } + async deleteComment(data: Messages.UpdateACommentRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.UpdateAComment(data, metadata), ).catch((err) => { @@ -181,9 +202,11 @@ class CatalogService implements OnModuleInit { }); } + async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); + return lastValueFrom( this.catalogWriteService.DeleteDataAsset(data, metadata), ).catch((err) => { @@ -194,16 +217,20 @@ class CatalogService implements OnModuleInit { }); } + async getUserRolesIds(userId: string) { const result = await this.userService.findOneById(userId).catch(() => null); + if (result) { return result.user.roles.map((role) => role.id); } + return []; } + async searchDataAssets( query: Record, metadata: Metadata, @@ -211,10 +238,19 @@ class CatalogService implements OnModuleInit { ) { this.logger.info('CatalogService - searchDataAssets', { query }); + const { search, page, size, sort_by, order, ...filters } = query; + this.logger.debug('Extracted filters:', { filters }); + + console.log('MAESTRO VAI CHAMAR PI-FACTORY COM:', { + search, page, size, sort_by, order, + filters: filters + }); + + if (filters.owner) { const { users: customer_users } = await this.userService.findAllUsersByCustomerId(customer_id); @@ -224,6 +260,7 @@ class CatalogService implements OnModuleInit { users: customer_users.slice(0, 5).map(u => ({ id: u.id, email: u.email, name: u.name })) }); + const ownerValues = Array.isArray(filters.owner) ? filters.owner : (typeof filters.owner === 'string' && filters.owner.includes(',') @@ -232,6 +269,7 @@ class CatalogService implements OnModuleInit { this.logger.info('Owner values to convert:', { ownerValues, ownerFiltersOriginal: filters.owner }); + const ownerIds = ownerValues .map((ownerValue: string) => { const normalizedOwner = ownerValue.replace(/\s/g, '+'); @@ -256,11 +294,13 @@ class CatalogService implements OnModuleInit { }) .filter((id: string) => id); + if (ownerIds.length > 0) { filters.owner = ownerIds; } } + const { data_assets, total } = await lastValueFrom( this.catalogReadService.GetAllDataAssets( { @@ -275,16 +315,25 @@ class CatalogService implements OnModuleInit { ), ); + + + console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY'); + + + const result = JSON.parse(data_assets); + const response = await this.getAssetsUsersAndRoles( result.data_assets, customer_id, ); + return { data_assets: response, total }; } + async downloadAssets( query: Record, metadata: Metadata, @@ -292,6 +341,7 @@ class CatalogService implements OnModuleInit { ) { const data = await this.searchDataAssets(query, metadata, customer_id); + const formatData = data.data_assets.map(asset => ({ id: asset.id, display_name: asset.display_name, @@ -300,19 +350,24 @@ class CatalogService implements OnModuleInit { tags: '[' + asset.tags.join(', ') + ']' })) + const parser = ParserBuilder.build('csv'); + const file = await parser.parse(formatData); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `dadosfera_assets_${timestamp}.csv`; + return { file, filename } } + async getOneDataAsset(data: { id: string; customer_id: string; @@ -333,9 +388,11 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); + return { data_asset: asset[0] }; } + async getOneDataAssetByPipelineAndObject(data: { customer_id: string; pipeline: string; @@ -357,9 +414,11 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); + return { data_asset: asset[0] }; } + async updateOneDataAsset(data: { data_asset_id: string; customer_id: string; @@ -368,6 +427,7 @@ class CatalogService implements OnModuleInit { }) { const { body, customer_id, data_asset_id, metadata } = data; + const { data_asset } = await lastValueFrom( this.catalogWriteService.UpdateDataAsset( { id: data_asset_id, changes: JSON.stringify(body) }, @@ -382,9 +442,11 @@ class CatalogService implements OnModuleInit { }; asset = await this.getAssetsUsersAndRoles([asset], customer_id); + return { data_asset: asset[0] }; } + async getDataDocs(id: string, metadata: Metadata) { const { documentation } = await lastValueFrom( this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata), @@ -394,6 +456,7 @@ class CatalogService implements OnModuleInit { return docs; } + async getDatasetPreview(id: string, metadata: Metadata) { const { preview } = await lastValueFrom( this.catalogReadService.GetDatasetPreview( @@ -405,6 +468,7 @@ class CatalogService implements OnModuleInit { return result; } + async getDatasetColumnsMetadata(id: string, metadata: Metadata) { const { columns_metadata } = await lastValueFrom( this.catalogReadService.GetDatasetColumnsMetadata( @@ -416,6 +480,7 @@ class CatalogService implements OnModuleInit { return result; } + async createDataDocs(body) { const nimbusUrl = this._getNimbusUrl(body); const { data } = await axios.post( @@ -425,9 +490,11 @@ class CatalogService implements OnModuleInit { return data; } + async findAllTags(data, metadata) { this.logger.info('CatalogService - findAllCustomerTags'); + const response = await lastValueFrom( this.catalogReadService.GetCustomerTags(data, metadata), ) @@ -440,6 +507,7 @@ class CatalogService implements OnModuleInit { throw new Error(err); }); + return response; } async getAssetsUsersAndRoles(data_assets: Array, customer_id: string) { @@ -454,6 +522,7 @@ class CatalogService implements OnModuleInit { (u) => u.id === data_asset.owner, )?.email; + const roles = []; const users = []; const data_asset_roles = data_asset?.roles || [] @@ -462,6 +531,7 @@ class CatalogService implements OnModuleInit { if (role) roles.push({ id: role.id, name: role.name }); } + const data_asset_users = data_asset?.users || [] for (const user_id of data_asset_users) { const user = customer_users.find((r) => r.id === user_id); @@ -476,6 +546,7 @@ class CatalogService implements OnModuleInit { }); } + async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) { const { session } = await lastValueFrom( this.catalogWriteService.TriggerDatasetCataloging(data, metadata), @@ -489,6 +560,7 @@ class CatalogService implements OnModuleInit { return res; } + async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) { const res = await lastValueFrom( this.catalogWriteService.AddRlsRule(data, metadata), @@ -496,6 +568,7 @@ class CatalogService implements OnModuleInit { return res; } + async removeRlsRule(id: number, metadata: Metadata) { const res = await lastValueFrom( this.catalogWriteService.RemoveRlsRule({ id }, metadata), @@ -503,12 +576,14 @@ class CatalogService implements OnModuleInit { return res; } + async batchRemoveRlsRule( query: BatchRemoveRlsRulesRequest, metadata: Metadata, ) { const { id_rls, nimbus_dashboard_id } = query; + if (id_rls && nimbus_dashboard_id) { throw new BadRequestException( "You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'", @@ -529,6 +604,7 @@ class CatalogService implements OnModuleInit { return 'OK'; } + async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) { const res = await lastValueFrom( this.catalogReadService.GetRlsRules(data, metadata), @@ -536,6 +612,7 @@ class CatalogService implements OnModuleInit { return res.rls_rules; } + async getOneRlsRule(id: number, metadata: Metadata) { const res = await lastValueFrom( this.catalogReadService.GetOneRlsRule({ id }, metadata), @@ -543,6 +620,7 @@ class CatalogService implements OnModuleInit { return res.rls_rule; } + async getNimbusDashboards( data: GetNimbusDashboardsRequest, metadata: Metadata, @@ -553,19 +631,24 @@ class CatalogService implements OnModuleInit { return res.dashboards; } + async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); + const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; + this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); + try { const { data, status } = await axios.post(endpoint, {...body.table_metadata}); + this.logger.info( `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, {...body.logMetadata}, @@ -580,19 +663,23 @@ class CatalogService implements OnModuleInit { } } + async createColumnMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; + + try { this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); const { data, status } = await axios.post(endpoint, body.column_metadata); + this.logger.info( `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, {...body.logMetadata}, @@ -607,18 +694,22 @@ class CatalogService implements OnModuleInit { } } + async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; + this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); + try { const { data, status } = await axios.post(endpoint, body.data_preview); + this.logger.info( `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, {...body.logMetadata}, @@ -635,9 +726,11 @@ class CatalogService implements OnModuleInit { } } + async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); + const customer_name = customer_name_raw?.[0]?.toString(); if (!customer_name) { throw new BadRequestException('Customer name not found in metadata'); @@ -660,4 +753,5 @@ class CatalogService implements OnModuleInit { } } -export { CatalogService }; \ No newline at end of file + +export { CatalogService }; diff --git a/src/modules/catalog/dtos/index.ts b/src/modules/catalog/dtos/index.ts index 59de404..39d2708 100644 --- a/src/modules/catalog/dtos/index.ts +++ b/src/modules/catalog/dtos/index.ts @@ -153,6 +153,18 @@ export class ICatalogAllRequest { example: 'user-id-1,user-id-2', }) owner?: string; + + @ApiPropertyOptional({ + description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)', + example: '2025-01-01', + }) + catalog_date_from?: string; + + @ApiPropertyOptional({ + description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)', + example: '2025-12-31', + }) + catalog_date_to?: string; } export class ICatalogAllResponse { From 1ff5589a2e567bb6ec506f347e2648ac43608cd7 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 12 Dec 2025 15:03:43 -0300 Subject: [PATCH 30/30] FIX: Simplify searchDataAssets by owner logging and remove log code --- src/modules/catalog/catalog.service.ts | 67 +------------------------- 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 27c158f..5929849 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -238,71 +238,11 @@ class CatalogService implements OnModuleInit { metadata: Metadata, customer_id: string, ) { - this.logger.info('CatalogService - searchDataAssets', { query }); + this.logger.info('CatalogService - searchDataAssets'); const { search, page, size, sort_by, order, ...filters } = query; - - this.logger.debug('Extracted filters:', { filters }); - - - console.log('MAESTRO VAI CHAMAR PI-FACTORY COM:', { - search, page, size, sort_by, order, - filters: filters - }); - - - if (filters.owner) { - const { users: customer_users } = - await this.userService.findAllUsersByCustomerId(customer_id); - - this.logger.info('Available users in database count:', { count: customer_users.length }); - this.logger.info('First 5 users:', { - users: customer_users.slice(0, 5).map(u => ({ id: u.id, email: u.email, name: u.name })) - }); - - - const ownerValues = Array.isArray(filters.owner) - ? filters.owner - : (typeof filters.owner === 'string' && filters.owner.includes(',') - ? filters.owner.split(',').map((o: string) => o.trim()) - : [filters.owner]); - - this.logger.info('Owner values to convert:', { ownerValues, ownerFiltersOriginal: filters.owner }); - - - const ownerIds = ownerValues - .map((ownerValue: string) => { - const normalizedOwner = ownerValue.replace(/\s/g, '+'); - const user = customer_users.find(u => { - const isIdMatch = u.id === ownerValue; - const isEmailMatch = u.email === ownerValue || u.email === normalizedOwner; - const isNameMatch = u.name === ownerValue || u.name === normalizedOwner; - this.logger.info('Comparing:', { - userId: u.id, - userEmail: u.email, - userName: u.name, - filterValue: ownerValue, - normalizedFilter: normalizedOwner, - idMatch: isIdMatch, - emailMatch: isEmailMatch, - nameMatch: isNameMatch, - }); - return isIdMatch || isEmailMatch || isNameMatch; - }); - this.logger.info('Looking for owner result:', { ownerValue, found: !!user, userId: user?.id }); - return user?.id || ownerValue; - }) - .filter((id: string) => id); - - - if (ownerIds.length > 0) { - filters.owner = ownerIds; - } - } - - const { data_assets, total } = await lastValueFrom( this.catalogReadService.GetAllDataAssets( { @@ -318,11 +258,6 @@ class CatalogService implements OnModuleInit { ); - - console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY'); - - - const result = JSON.parse(data_assets);