Compare commits

..
Author SHA1 Message Date
Marcos Rodrigues Silva 8eddb9e1bf Merge pull request #428 from dadosfera/hotfix/user-patch
Hotfix/user patch
2025-12-26 17:58:28 -03:00
Marcos Rodrigues 76485f929d FIX: user patch route 2025-12-26 17:53:02 -03:00
vinicius gadea 33ebc91826 Merge pull request #426 from dadosfera/feature/catalog-update
FEAT: Add optional 'docs' field to assets
2025-12-19 08:24:54 -03:00
vinicius gadea 6948156693 Merge branch 'beta' into feature/catalog-update 2025-12-19 08:21:52 -03:00
viniciusgadea 0aaa4384c3 DOCS: Add 'docs' field to schemas in docsfera.json and update protospack-v2 to version 3.38.0-beta.19 in package-lock.json 2025-12-18 17:43:54 -03:00
Marcos Rodrigues Silva 1f9d0c29ec Merge pull request #425 from dadosfera/beta
Beta
2025-12-18 17:42:10 -03:00
viniciusgadea 039c652b28 DOC: Update @dadosfera/protospack-v2 to version 3.38.0-beta.19 in package.json and package-lock.json 2025-12-18 17:11:18 -03:00
viniciusgadea 653d4f53b5 FEAT: Add optional 'docs' field to IDataAsset, IUpdateDataRequest, and ICreateDataAsset 2025-12-18 16:16:00 -03:00
Marcos Rodrigues Silva 31f8c2c1a6 Merge pull request #424 from dadosfera/feature/user-form
FEAT: protected update route with admin seqid or if user request to e…
2025-12-18 12:33:29 -03:00
Marcos Rodrigues 8b93d4e97b FEAT: protected update route with admin seqid or if user request to equal a jwt user id claim 2025-12-18 12:32:53 -03:00
Rafael Santana adeb022818 Merge pull request #423 from dadosfera/fix/reference-column-object-format
FIX: inject customer_id in pipeline execute/pause/unpause routes
2025-12-18 10:53:31 -03:00
Marcos Rodrigues Silva e326cab44d Merge pull request #422 from dadosfera/feature/user-form
FIX: send new user info
2025-12-18 10:29:09 -03:00
Marcos Rodrigues 61109f8ae9 FIX: send new user info 2025-12-18 10:24:29 -03:00
Marcos Rodrigues Silva 3f8dc5cabe Merge pull request #421 from dadosfera/feature/user-form
Feature/user form
2025-12-18 09:53:12 -03:00
Marcos Rodrigues 7f5d157739 FEAT: update protospack version 2025-12-18 09:52:11 -03:00
Marcos Rodrigues bce73fb11f FEAT: update user route 2025-12-17 17:24:03 -03:00
Rafael Santana 9473e65deb Merge pull request #419 from dadosfera/fix/reference-column-object-format
FIX: store reference_column as object with name and type
2025-12-17 13:48:38 -03:00
Rafael Santana 77b9acd2d0 Merge pull request #418 from dadosfera/fix/reference-column-object-format
fix: store reference_column as object with name and type
2025-12-17 13:46:05 -03:00
RafaelandClaude Opus 4.5 e99306adba fix: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:45:18 -03:00
Rafael Santana 96b947ebdc Merge pull request #417 from dadosfera/fix/redis-tls-configurable
FIX: pass type field to Elasticsearch createPipeline
2025-12-16 21:10:19 -03:00
Rafael Santana 6f7436f33f Merge pull request #416 from dadosfera/fix/redis-tls-configurable
fix: pass type field to Elasticsearch createPipeline
2025-12-16 21:05:55 -03:00
RafaelandClaude Opus 4.5 865140e681 fix: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:05:21 -03:00
Rafael Santana c4a664572a Merge pull request #415 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-16 19:17:08 -03:00
Rafael Santana 7a10f88113 Merge pull request #414 from dadosfera/fix/redis-tls-configurable
fix: normalize IDs in pipeline run routes before calling Platform API
2025-12-16 19:11:09 -03:00
Rafael Santana 466f8fb8cc Merge pull request #413 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-16 18:23:44 -03:00
Rafael Santana 5989822263 Merge pull request #412 from dadosfera/fix/redis-tls-configurable
feat: add Elasticsearch access to Maestro deployment
2025-12-16 18:15:08 -03:00
vinicius gadea 64a3e2652e Merge pull request #410 from dadosfera/feature/catalog-update
Feature/catalog update
2025-12-16 18:01:47 -03:00
Rafael Moraes ea44a1cbb6 FEAT: Manual/Auto filter and UX Adjusments 2025-12-16 20:52:28 +00:00
Rafael Moraes fc9c0b0991 WIP: Filtro Source - backend 2025-12-16 20:13:20 +00:00
Rafael Moraes a4b5a44e44 Merge branch 'feature/catalog-update' of github.com:dadosfera/maestro into feature/catalog-update 2025-12-16 20:01:19 +00:00
Rafael Moraes d99a6aa322 Merge branch 'beta' of github.com:dadosfera/maestro into feature/catalog-update 2025-12-16 19:58:48 +00:00
Rafael Santana 3f910f851a Merge pull request #411 from dadosfera/fix/redis-tls-configurable
Fix/redis tls configurable
2025-12-16 16:58:41 -03:00
Rafael Moraes bd231382eb WIP: adiciona parâmetro asset_type no docsfera.json 2025-12-16 19:58:38 +00:00
Rafael Santana 9c1979e17a Merge pull request #405 from dadosfera/fix/redis-tls-configurable
Fix/redis tls configurable
2025-12-16 16:41:04 -03:00
viniciusgadea fcf7fb054e FIX: Update searchDataAssets to handle owner filtering and convert owner names to IDs 2025-12-16 09:55:24 -03:00
viniciusgadea 288aaeabc0 FIX: Enhance filter handling in BuildDataAssetsFilterTerms to support empty owner condition 2025-12-15 12:00:50 -03:00
viniciusgadea 918c3d7416 FIX: Enhance searchDataAssets to convert owner names to IDs and log the conversion 2025-12-15 09:08:08 -03:00
Marcos Rodrigues Silva e7f410831f Merge pull request #395 from dadosfera/beta
Beta
2025-12-01 17:47:24 -03:00
9 changed files with 329 additions and 214 deletions
+62 -30
View File
@@ -1321,9 +1321,6 @@
"Users"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4483,6 +4480,14 @@
"schema": {
"type": "string"
}
},
{
"name": "asset_type",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -7217,33 +7222,6 @@
]
}
},
"/platform/pipeline/pipeline_run": {
"post": {
"operationId": "PlatformApiController_createPipelineRun",
"summary": "Create a pipeline run",
"parameters": [],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run": {
"get": {
"operationId": "PlatformApiController_getPipelineRuns",
@@ -8185,6 +8163,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companyName": {
"type": "string"
},
"companySite": {
"type": "string"
},
"mfaStatus": {
"type": "string"
},
@@ -8215,6 +8202,9 @@
"jobTitle",
"department",
"hierarchy",
"bio",
"companyName",
"companySite",
"mfaStatus",
"createdAt",
"updatedAt",
@@ -8348,6 +8338,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companyName": {
"type": "string"
},
"companySite": {
"type": "string"
},
"customer": {
"$ref": "#/components/schemas/Customer"
},
@@ -8375,6 +8374,9 @@
"jobTitle",
"department",
"hierarchy",
"bio",
"companyName",
"companySite",
"mfaStatus",
"createdAt",
"updatedAt",
@@ -8515,6 +8517,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companySite": {
"type": "string"
},
"companyName": {
"type": "string"
},
"roleNames": {
"type": "array",
"items": {
@@ -8555,6 +8566,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companyName": {
"type": "string"
},
"companySite": {
"type": "string"
},
"mfaStatus": {
"type": "string"
},
@@ -8578,6 +8598,9 @@
"jobTitle",
"department",
"hierarchy",
"bio",
"companyName",
"companySite",
"mfaStatus",
"createdAt",
"updatedAt",
@@ -9626,6 +9649,9 @@
"public",
"private"
]
},
"docs": {
"type": "string"
}
},
"required": [
@@ -9880,6 +9906,9 @@
"public",
"private"
]
},
"docs": {
"type": "string"
}
},
"required": [
@@ -9919,6 +9948,9 @@
},
"embed": {
"$ref": "#/components/schemas/EmbedObject"
},
"docs": {
"type": "string"
}
},
"required": [
+42 -34
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@dadosfera/protospack-v2": "3.38.0-beta.19",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -377,6 +377,7 @@
"resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.946.0.tgz",
"integrity": "sha512-sMmJRa2Y+jQ+qMzx7V3SBKsTSQzLBvSuVbMNM/YGSZkr849a4a6Frl4U8HfFj4O+o71dvQ6h5PP6vlc/fW672g==",
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"@aws-crypto/sha256-browser": "5.2.0",
"@aws-crypto/sha256-js": "5.2.0",
@@ -2412,6 +2413,7 @@
"integrity": "sha512-vMqyb7XCDMPvJFFOaT9kxtiRh42GwlZEg1/uIgtZshS5a/8OaduUfCi7kynKgc3Tw/6Uo2D+db9qBttghhmxwQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@ampproject/remapping": "^2.2.0",
"@babel/code-frame": "^7.26.2",
@@ -2970,9 +2972,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.16",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.16.tgz",
"integrity": "sha512-ijURcmETXdgVgXhIPxapK7w0z6begXm4WDZ6EJtUGg+oYlJd9QOxbwaD7akRzW8XTdccmIo0Vov065KB812J0A==",
"version": "3.38.0-beta.19",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.19.tgz",
"integrity": "sha512-qevGunl1IKv4POa2hhXNaDpqlh/bvnq5Hxo6534OHYPZ//S+XcRZgTrxFnylK3+uoROM8epF3Mey5GwynxY7Tw==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
@@ -3821,6 +3823,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/common/-/common-9.4.3.tgz",
"integrity": "sha512-Gd6D4IaYj01o14Bwv81ukidn4w3bPHCblMUq+SmUmWLyosK+XQmInCS09SbDDZyL8jy86PngtBLTdhJ2bXSUig==",
"license": "MIT",
"peer": true,
"dependencies": {
"iterare": "1.2.1",
"tslib": "2.5.3",
@@ -3899,6 +3902,7 @@
"integrity": "sha512-Qi63+wi55Jh4sDyaj5Hhx2jOpKqT386aeo+VOKsxnd+Ql9VvkO/FjmuwBGUyzkJt29ENYc+P0Sx/k5LtstNpPQ==",
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@nuxtjs/opencollective": "0.3.2",
"fast-safe-stringify": "2.1.1",
@@ -4036,6 +4040,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",
"integrity": "sha512-FpdczWoRSC0zz2dNL9u2AQLXKXRVtq4HgHklAhbL59X0uy+mcxhlSThG7DHzDMkoSnuuHY8ojDVf7mDxk+GtCw==",
"license": "MIT",
"peer": true,
"dependencies": {
"body-parser": "1.20.2",
"cors": "2.8.5",
@@ -4399,6 +4404,7 @@
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
"license": "Apache-2.0",
"peer": true,
"engines": {
"node": ">=8.0.0"
}
@@ -4558,6 +4564,7 @@
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"peer": true,
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
@@ -5383,6 +5390,7 @@
"integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.33",
@@ -5463,6 +5471,7 @@
"integrity": "sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"jest-diff": "^27.0.0",
"pretty-format": "^27.0.0"
@@ -5526,7 +5535,8 @@
"version": "16.18.126",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz",
"integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/@types/oauth": {
"version": "0.9.6",
@@ -5756,6 +5766,7 @@
"integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.62.0",
"@typescript-eslint/types": "5.62.0",
@@ -6102,6 +6113,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz",
"integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -6198,6 +6210,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
"integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
@@ -6826,6 +6839,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"caniuse-lite": "^1.0.30001688",
"electron-to-chromium": "^1.5.73",
@@ -6932,6 +6946,7 @@
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"peer": true,
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
@@ -7155,13 +7170,15 @@
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz",
"integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/class-validator": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.1.tgz",
"integrity": "sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/validator": "^13.11.8",
"libphonenumber-js": "^1.10.53",
@@ -7825,7 +7842,8 @@
"version": "0.0.1425554",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz",
"integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==",
"license": "BSD-3-Clause"
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/dezalgo": {
"version": "1.0.4",
@@ -8102,7 +8120,6 @@
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -8290,6 +8307,7 @@
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -8636,7 +8654,6 @@
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"license": "MIT",
"peer": true,
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -8683,7 +8700,6 @@
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
"integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"license": "MIT",
"peer": true,
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.5",
@@ -8708,7 +8724,6 @@
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz",
"integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.6"
}
@@ -8718,7 +8733,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -8727,22 +8741,19 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/path-to-regexp": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
"integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/qs": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
"integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"side-channel": "^1.0.6"
},
@@ -8771,8 +8782,7 @@
"url": "https://feross.org/support"
}
],
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/external-editor": {
"version": "3.1.0",
@@ -9007,7 +9017,6 @@
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
"integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"encodeurl": "~2.0.0",
@@ -9026,7 +9035,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -9035,8 +9043,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/find-up": {
"version": "5.0.0",
@@ -9830,6 +9837,7 @@
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
@@ -10171,6 +10179,7 @@
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@jest/core": "^27.5.1",
"import-local": "^3.0.2",
@@ -11460,7 +11469,6 @@
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
"integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
@@ -12233,6 +12241,7 @@
"resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz",
"integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==",
"license": "MIT",
"peer": true,
"dependencies": {
"passport-strategy": "1.x.x",
"pause": "0.0.1",
@@ -12622,6 +12631,7 @@
"integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin-prettier.js"
},
@@ -13124,7 +13134,8 @@
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
"integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==",
"license": "Apache-2.0"
"license": "Apache-2.0",
"peer": true
},
"node_modules/relative-microtime": {
"version": "2.0.0",
@@ -13311,6 +13322,7 @@
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"tslib": "^2.1.0"
}
@@ -13372,6 +13384,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@@ -13415,7 +13428,6 @@
"resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
"integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
@@ -13440,7 +13452,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -13449,15 +13460,13 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/send/node_modules/encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -13466,8 +13475,7 @@
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/serialize-javascript": {
"version": "6.0.2",
@@ -13483,7 +13491,6 @@
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
"integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"license": "MIT",
"peer": true,
"dependencies": {
"encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
@@ -14461,6 +14468,7 @@
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
@@ -14690,6 +14698,7 @@
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -15017,7 +15026,6 @@
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz",
"integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/json-schema": "^7.0.9",
"ajv": "^8.9.0",
+2 -2
View File
@@ -30,12 +30,12 @@
"@aws-crypto/sha256-js": "^5.2.0",
"@aws-sdk/client-dynamodb": "^3.414.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/credential-provider-node": "^3.940.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@dadosfera/protospack-v2": "3.38.0-beta.19",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+161 -124
View File
@@ -41,7 +41,6 @@ import {
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
catalogWriteService: WriteService.CatalogWriteServices;
@@ -58,7 +57,6 @@ class CatalogService implements OnModuleInit {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
@@ -74,81 +72,65 @@ class CatalogService implements OnModuleInit {
);
}
_getNimbusUrl(body) {
this.logger.debug(`Body: ${JSON.stringify(body)}`);
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
return `https://nimbus-${customer}.dadosfera.ai`;
}
return `https://nimbus-${customer}.${process.env.ENV.replace(
'local',
'stg',
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
this.logger.info('getPiiReporter: ' + type);
try {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const { data } = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata),
);
this.logger.info('Finish grpc call');
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
this.logger.info('parser file to: ' + type);
const file = await parser.parse(data);
this.logger.info('finish parser');
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
csv: 'text/csv',
html: 'text/html',
pdf: 'application/pdf',
};
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
type: mimeTypes[type],
};
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
return lastValueFrom(
this.catalogWriteService.CreateDataAsset(data, metadata),
);
}
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.ManagePermission(data, metadata),
).catch((err) => {
@@ -159,11 +141,9 @@ class CatalogService implements OnModuleInit {
});
}
async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.RevokePermission(data, metadata),
).catch((err) => {
@@ -174,11 +154,9 @@ class CatalogService implements OnModuleInit {
});
}
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.MakeAComment(data, metadata),
).catch((err) => {
@@ -189,11 +167,9 @@ class CatalogService implements OnModuleInit {
});
}
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.UpdateAComment(data, metadata),
).catch((err) => {
@@ -204,11 +180,9 @@ class CatalogService implements OnModuleInit {
});
}
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.DeleteDataAsset(data, metadata),
).catch((err) => {
@@ -219,30 +193,114 @@ class CatalogService implements OnModuleInit {
});
}
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
if (result) {
return result.user.roles.map((role) => role.id);
}
return [];
}
async searchDataAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
this.logger.info('CatalogService - searchDataAssets');
this.logger.info('CatalogService - searchDataAssets', { query });
const { search, page, size, sort_by, order, ...filters } = query;
this.logger.debug('Extracted filters:', { filters });
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (
filters.manually !== undefined &&
filters.manually !== null &&
filters.manually !== ''
) {
filters.manually = Number(filters.manually); // 1 ou 0
} else {
delete filters.manually;
}
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (filters.owner) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
this.logger.info('Available users in database count:', {
count: customer_users.length,
});
this.logger.info('First 5 users:', {
users: customer_users
.slice(0, 5)
.map((u) => ({ id: u.id, email: u.email, name: u.name })),
});
const ownerValues = Array.isArray(filters.owner)
? filters.owner
: typeof filters.owner === 'string' && filters.owner.includes(',')
? filters.owner.split(',').map((o: string) => o.trim())
: [filters.owner];
this.logger.info('Owner values to convert:', {
ownerValues,
ownerFiltersOriginal: filters.owner,
});
const ownerIds = ownerValues
.map((ownerValue: string) => {
const normalizedOwner = ownerValue.replace(/\s/g, '+');
const user = customer_users.find((u) => {
const isIdMatch = u.id === ownerValue;
const isEmailMatch =
u.email === ownerValue || u.email === normalizedOwner;
const isNameMatch =
u.name === ownerValue || u.name === normalizedOwner;
this.logger.info('Comparing:', {
userId: u.id,
userEmail: u.email,
userName: u.name,
filterValue: ownerValue,
normalizedFilter: normalizedOwner,
idMatch: isIdMatch,
emailMatch: isEmailMatch,
nameMatch: isNameMatch,
});
return isIdMatch || isEmailMatch || isNameMatch;
});
this.logger.info('Looking for owner result:', {
ownerValue,
found: !!user,
userId: user?.id,
});
return user?.id || ownerValue;
})
.filter((id: string) => id);
if (ownerIds.length > 0) {
filters.owner = ownerIds;
}
}
const { data_assets, total } = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(
{
@@ -257,20 +315,18 @@ class CatalogService implements OnModuleInit {
),
);
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
const result = JSON.parse(data_assets);
const response = await this.getAssetsUsersAndRoles(
result.data_assets,
customer_id,
);
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
@@ -278,33 +334,27 @@ class CatalogService implements OnModuleInit {
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
const formatData = data.data_assets.map((asset) => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
tags: '[' + asset.tags.join(', ') + ']',
}));
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
filename,
};
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -325,11 +375,9 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getOneDataAssetByPipelineAndObject(data: {
customer_id: string;
pipeline: string;
@@ -351,11 +399,9 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
@@ -364,7 +410,6 @@ class CatalogService implements OnModuleInit {
}) {
const { body, customer_id, data_asset_id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: data_asset_id, changes: JSON.stringify(body) },
@@ -379,7 +424,6 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
@@ -387,12 +431,11 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
const docs = JSON.parse(documentation);
return docs;
}
async getDatasetPreview(id: string, metadata: Metadata) {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
@@ -404,7 +447,6 @@ class CatalogService implements OnModuleInit {
return result;
}
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
@@ -420,7 +462,7 @@ class CatalogService implements OnModuleInit {
if (body.asset_type === 'table' || body.asset_type === 'view') {
return this.createDataDocsViaNimbus(body);
}
return this.createDataDocsViaGrpc(body, metadata);
}
@@ -434,7 +476,6 @@ class CatalogService implements OnModuleInit {
return data;
}
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
this.logger.info('Creating data docs via gRPC for other asset types');
try {
@@ -447,9 +488,8 @@ class CatalogService implements OnModuleInit {
metadata,
),
);
return response;
} catch (error) {
this.logger.error('Error creating data asset docs:', error);
throw new HttpException(
@@ -462,7 +502,6 @@ class CatalogService implements OnModuleInit {
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
const response = await lastValueFrom(
this.catalogReadService.GetCustomerTags(data, metadata),
)
@@ -475,9 +514,9 @@ class CatalogService implements OnModuleInit {
throw new Error(err);
});
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
@@ -490,17 +529,15 @@ class CatalogService implements OnModuleInit {
(u) => u.id === data_asset.owner,
)?.email;
const roles = [];
const users = [];
const data_asset_roles = data_asset?.roles || []
const data_asset_roles = data_asset?.roles || [];
for (const role_id of data_asset_roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
const data_asset_users = data_asset?.users || []
const data_asset_users = data_asset?.users || [];
for (const user_id of data_asset_users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, email: user.email });
@@ -514,7 +551,6 @@ class CatalogService implements OnModuleInit {
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -528,7 +564,6 @@ class CatalogService implements OnModuleInit {
return res;
}
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.AddRlsRule(data, metadata),
@@ -536,7 +571,6 @@ class CatalogService implements OnModuleInit {
return res;
}
async removeRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
@@ -544,14 +578,12 @@ class CatalogService implements OnModuleInit {
return res;
}
async batchRemoveRlsRule(
query: BatchRemoveRlsRulesRequest,
metadata: Metadata,
) {
const { id_rls, nimbus_dashboard_id } = query;
if (id_rls && nimbus_dashboard_id) {
throw new BadRequestException(
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
@@ -572,7 +604,6 @@ class CatalogService implements OnModuleInit {
return 'OK';
}
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetRlsRules(data, metadata),
@@ -580,7 +611,6 @@ class CatalogService implements OnModuleInit {
return res.rls_rules;
}
async getOneRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetOneRlsRule({ id }, metadata),
@@ -588,7 +618,6 @@ class CatalogService implements OnModuleInit {
return res.rls_rule;
}
async getNimbusDashboards(
data: GetNimbusDashboardsRequest,
metadata: Metadata,
@@ -599,88 +628,99 @@ class CatalogService implements OnModuleInit {
return res.dashboards;
}
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
this.logger.info(
`Creating table metadata for table ${body.table_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
...body.logMetadata,
});
try {
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
const { data, status } = await axios.post(endpoint, {
...body.table_metadata,
});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
} because of ${JSON.stringify(error.response?.data) || error.message}`,
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createColumnMetadata(body: any): Promise<number[]> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Creating column metadata for table ${body.column_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.column_metadata)}`,
{ ...body.logMetadata },
);
const { data, status } = await axios.post(
endpoint,
body.column_metadata,
);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
} because of ${error.response?.data || error.message}`,
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
this.logger.info(
`Creating data preview for table ${body.data_preview.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.data_preview)}`,
{ ...body.logMetadata },
);
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.id;
} catch (error) {
@@ -688,17 +728,15 @@ class CatalogService implements OnModuleInit {
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
const customer_name = customer_name_raw?.[0]?.toString();
if (!customer_name) {
throw new BadRequestException('Customer name not found in metadata');
@@ -721,5 +759,4 @@ class CatalogService implements OnModuleInit {
}
}
export { CatalogService };
+6
View File
@@ -98,6 +98,8 @@ export class IDataAsset {
embed?: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class IOneDataAsset {
@@ -200,6 +202,8 @@ export class IUpdateDataRequest {
embed: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
@@ -214,6 +218,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
location: string;
@ApiPropertyOptional()
embed: EmbedObject;
@ApiPropertyOptional()
docs: string;
}
export class IPreview {
+22 -16
View File
@@ -229,27 +229,33 @@ export class RolesService {
const [roleTreated] = this.getRolesPermissionsName([role.role]);
return { role: roleTreated };
}
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
const newRoles: RoleDto[] = [];
for (const role of roles) {
const allPermissions = this.permissionsService.getAllPermissions(
this.language,
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
const newRole: RoleDto = {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};
const newRole: RoleDto = this.formatRole(role);
newRoles.push(newRole);
}
return newRoles;
}
private formatRole(role: GetRolesPermissionsName): RoleDto {
const allPermissions = this.permissionsService.getAllPermissions(
this.language
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
return {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};;
}
}
+12
View File
@@ -38,6 +38,12 @@ export class User {
department?: string;
@ApiProperty()
hierarchy?: string;
@ApiProperty()
bio?: string;
@ApiProperty()
companyName?: string;
@ApiProperty()
companySite?: string;
@ApiPropertyOptional()
customer?: Customer;
@ApiProperty()
@@ -109,6 +115,12 @@ export class UpdateUserReq {
@ApiPropertyOptional()
hierarchy?: string;
@ApiPropertyOptional()
bio?: string;
@ApiPropertyOptional()
companySite?: string;
@ApiPropertyOptional()
companyName?: string;
@ApiPropertyOptional()
roleNames?: string[];
}
+12 -1
View File
@@ -266,7 +266,6 @@ export class UsersController {
}
@Patch(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: UpdateUserRes })
async updateUser(
@User() user: RequestUser,
@@ -274,6 +273,18 @@ export class UsersController {
@Param('id') id: string,
@Language() language: LanguageEnum,
) {
const isSameUser = user.user_id === id;
const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
if (!isSameUser && !isSuperAdmin) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
if (isSameUser && !isSuperAdmin && body.roleNames) {
// Prevent users from updating their own roles
delete body.roleNames;
}
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
return await this.userService.updateUser(body, id, user.customer_id);
+10 -7
View File
@@ -148,20 +148,23 @@ export class UsersService implements OnModuleInit {
}
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
if (roleNames) {
const { roleNames, ...updateUserDTO } = req;
if (roleNames && roleNames.length > 0) {
await this.setRoles({ roleNames, userId: id }, customerId);
}
const { user } = await lastValueFrom(
this.usersClientService.UserUpdate({
name,
department: updateUserDTO.department,
email: updateUserDTO.email,
hierarchy: updateUserDTO.hierarchy,
jobTitle: updateUserDTO.jobTitle,
name: updateUserDTO.name,
bio: updateUserDTO.bio,
companyName: updateUserDTO.companyName,
companySite: updateUserDTO.companySite,
customerId,
id,
department,
hierarchy,
jobTitle,
email,
metabaseUserId: undefined,
}),
);