Compare commits

..
Author SHA1 Message Date
Marcos Rodrigues 76485f929d FIX: user patch route 2025-12-26 17:53:02 -03:00
Marcos Rodrigues Silva 1f9d0c29ec Merge pull request #425 from dadosfera/beta
Beta
2025-12-18 17:42:10 -03:00
Marcos Rodrigues Silva 31f8c2c1a6 Merge pull request #424 from dadosfera/feature/user-form
FEAT: protected update route with admin seqid or if user request to e…
2025-12-18 12:33:29 -03:00
Marcos Rodrigues 8b93d4e97b FEAT: protected update route with admin seqid or if user request to equal a jwt user id claim 2025-12-18 12:32:53 -03:00
Rafael Santana adeb022818 Merge pull request #423 from dadosfera/fix/reference-column-object-format
FIX: inject customer_id in pipeline execute/pause/unpause routes
2025-12-18 10:53:31 -03:00
RafaelandClaude Opus 4.5 f61c241dde FIX: inject customer_id in pipeline execute/pause/unpause routes
Users no longer need to provide customer_id in the request body for
execute, pause, and unpause pipeline operations - it's now automatically
injected from the authenticated user's session.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-18 10:30:09 -03:00
Marcos Rodrigues Silva e326cab44d Merge pull request #422 from dadosfera/feature/user-form
FIX: send new user info
2025-12-18 10:29:09 -03:00
Marcos Rodrigues 61109f8ae9 FIX: send new user info 2025-12-18 10:24:29 -03:00
Marcos Rodrigues Silva 3f8dc5cabe Merge pull request #421 from dadosfera/feature/user-form
Feature/user form
2025-12-18 09:53:12 -03:00
Marcos Rodrigues 7f5d157739 FEAT: update protospack version 2025-12-18 09:52:11 -03:00
Marcos Rodrigues bce73fb11f FEAT: update user route 2025-12-17 17:24:03 -03:00
Rafael Santana 9473e65deb Merge pull request #419 from dadosfera/fix/reference-column-object-format
FIX: store reference_column as object with name and type
2025-12-17 13:48:38 -03:00
RafaelandClaude Opus 4.5 9c55c22230 FIX: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:47:57 -03:00
Rafael Santana 77b9acd2d0 Merge pull request #418 from dadosfera/fix/reference-column-object-format
fix: store reference_column as object with name and type
2025-12-17 13:46:05 -03:00
RafaelandClaude Opus 4.5 e99306adba fix: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:45:18 -03:00
Rafael Santana 96b947ebdc Merge pull request #417 from dadosfera/fix/redis-tls-configurable
FIX: pass type field to Elasticsearch createPipeline
2025-12-16 21:10:19 -03:00
RafaelandClaude Opus 4.5 fb521f53cd FIX: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:09:07 -03:00
Rafael Santana 6f7436f33f Merge pull request #416 from dadosfera/fix/redis-tls-configurable
fix: pass type field to Elasticsearch createPipeline
2025-12-16 21:05:55 -03:00
RafaelandClaude Opus 4.5 865140e681 fix: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:05:21 -03:00
Marcos Rodrigues Silva e7f410831f Merge pull request #395 from dadosfera/beta
Beta
2025-12-01 17:47:24 -03:00
9 changed files with 200 additions and 108 deletions
+45 -30
View File
@@ -1321,9 +1321,6 @@
"Users"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7225,33 +7222,6 @@
]
}
},
"/platform/pipeline/pipeline_run": {
"post": {
"operationId": "PlatformApiController_createPipelineRun",
"summary": "Create a pipeline run",
"parameters": [],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run": {
"get": {
"operationId": "PlatformApiController_getPipelineRuns",
@@ -8193,6 +8163,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companyName": {
"type": "string"
},
"companySite": {
"type": "string"
},
"mfaStatus": {
"type": "string"
},
@@ -8223,6 +8202,9 @@
"jobTitle",
"department",
"hierarchy",
"bio",
"companyName",
"companySite",
"mfaStatus",
"createdAt",
"updatedAt",
@@ -8356,6 +8338,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companyName": {
"type": "string"
},
"companySite": {
"type": "string"
},
"customer": {
"$ref": "#/components/schemas/Customer"
},
@@ -8383,6 +8374,9 @@
"jobTitle",
"department",
"hierarchy",
"bio",
"companyName",
"companySite",
"mfaStatus",
"createdAt",
"updatedAt",
@@ -8523,6 +8517,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companySite": {
"type": "string"
},
"companyName": {
"type": "string"
},
"roleNames": {
"type": "array",
"items": {
@@ -8563,6 +8566,15 @@
"hierarchy": {
"type": "string"
},
"bio": {
"type": "string"
},
"companyName": {
"type": "string"
},
"companySite": {
"type": "string"
},
"mfaStatus": {
"type": "string"
},
@@ -8586,6 +8598,9 @@
"jobTitle",
"department",
"hierarchy",
"bio",
"companyName",
"companySite",
"mfaStatus",
"createdAt",
"updatedAt",
+43 -34
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@dadosfera/protospack-v2": "3.38.0-beta.18",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -377,6 +377,7 @@
"resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.946.0.tgz",
"integrity": "sha512-sMmJRa2Y+jQ+qMzx7V3SBKsTSQzLBvSuVbMNM/YGSZkr849a4a6Frl4U8HfFj4O+o71dvQ6h5PP6vlc/fW672g==",
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"@aws-crypto/sha256-browser": "5.2.0",
"@aws-crypto/sha256-js": "5.2.0",
@@ -2412,6 +2413,7 @@
"integrity": "sha512-vMqyb7XCDMPvJFFOaT9kxtiRh42GwlZEg1/uIgtZshS5a/8OaduUfCi7kynKgc3Tw/6Uo2D+db9qBttghhmxwQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@ampproject/remapping": "^2.2.0",
"@babel/code-frame": "^7.26.2",
@@ -2970,9 +2972,10 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.16",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.16.tgz",
"integrity": "sha512-ijURcmETXdgVgXhIPxapK7w0z6begXm4WDZ6EJtUGg+oYlJd9QOxbwaD7akRzW8XTdccmIo0Vov065KB812J0A==",
"version": "3.38.0-beta.18",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.18.tgz",
"integrity": "sha512-rD2UOKa0P17A6YXIpclXF7F9b86mhRNE7fImTrbFamOcHpGmXBBVU8U53mCfLzzOA+MdDZ4/1uDNurCyQCjsLQ==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
@@ -3821,6 +3824,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/common/-/common-9.4.3.tgz",
"integrity": "sha512-Gd6D4IaYj01o14Bwv81ukidn4w3bPHCblMUq+SmUmWLyosK+XQmInCS09SbDDZyL8jy86PngtBLTdhJ2bXSUig==",
"license": "MIT",
"peer": true,
"dependencies": {
"iterare": "1.2.1",
"tslib": "2.5.3",
@@ -3899,6 +3903,7 @@
"integrity": "sha512-Qi63+wi55Jh4sDyaj5Hhx2jOpKqT386aeo+VOKsxnd+Ql9VvkO/FjmuwBGUyzkJt29ENYc+P0Sx/k5LtstNpPQ==",
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@nuxtjs/opencollective": "0.3.2",
"fast-safe-stringify": "2.1.1",
@@ -4036,6 +4041,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",
"integrity": "sha512-FpdczWoRSC0zz2dNL9u2AQLXKXRVtq4HgHklAhbL59X0uy+mcxhlSThG7DHzDMkoSnuuHY8ojDVf7mDxk+GtCw==",
"license": "MIT",
"peer": true,
"dependencies": {
"body-parser": "1.20.2",
"cors": "2.8.5",
@@ -4399,6 +4405,7 @@
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
"license": "Apache-2.0",
"peer": true,
"engines": {
"node": ">=8.0.0"
}
@@ -4558,6 +4565,7 @@
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"peer": true,
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
@@ -5383,6 +5391,7 @@
"integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.33",
@@ -5463,6 +5472,7 @@
"integrity": "sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"jest-diff": "^27.0.0",
"pretty-format": "^27.0.0"
@@ -5526,7 +5536,8 @@
"version": "16.18.126",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz",
"integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/@types/oauth": {
"version": "0.9.6",
@@ -5756,6 +5767,7 @@
"integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.62.0",
"@typescript-eslint/types": "5.62.0",
@@ -6102,6 +6114,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz",
"integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -6198,6 +6211,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
"integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
@@ -6826,6 +6840,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"caniuse-lite": "^1.0.30001688",
"electron-to-chromium": "^1.5.73",
@@ -6932,6 +6947,7 @@
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"peer": true,
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
@@ -7155,13 +7171,15 @@
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz",
"integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/class-validator": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.1.tgz",
"integrity": "sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/validator": "^13.11.8",
"libphonenumber-js": "^1.10.53",
@@ -7825,7 +7843,8 @@
"version": "0.0.1425554",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz",
"integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==",
"license": "BSD-3-Clause"
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/dezalgo": {
"version": "1.0.4",
@@ -8102,7 +8121,6 @@
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -8290,6 +8308,7 @@
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -8636,7 +8655,6 @@
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"license": "MIT",
"peer": true,
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -8683,7 +8701,6 @@
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
"integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"license": "MIT",
"peer": true,
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.5",
@@ -8708,7 +8725,6 @@
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz",
"integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.6"
}
@@ -8718,7 +8734,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -8727,22 +8742,19 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/path-to-regexp": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
"integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/qs": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
"integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"side-channel": "^1.0.6"
},
@@ -8771,8 +8783,7 @@
"url": "https://feross.org/support"
}
],
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/external-editor": {
"version": "3.1.0",
@@ -9007,7 +9018,6 @@
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
"integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"encodeurl": "~2.0.0",
@@ -9026,7 +9036,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -9035,8 +9044,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/find-up": {
"version": "5.0.0",
@@ -9830,6 +9838,7 @@
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
@@ -10171,6 +10180,7 @@
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@jest/core": "^27.5.1",
"import-local": "^3.0.2",
@@ -11460,7 +11470,6 @@
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
"integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
@@ -12233,6 +12242,7 @@
"resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz",
"integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==",
"license": "MIT",
"peer": true,
"dependencies": {
"passport-strategy": "1.x.x",
"pause": "0.0.1",
@@ -12622,6 +12632,7 @@
"integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin-prettier.js"
},
@@ -13124,7 +13135,8 @@
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
"integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==",
"license": "Apache-2.0"
"license": "Apache-2.0",
"peer": true
},
"node_modules/relative-microtime": {
"version": "2.0.0",
@@ -13311,6 +13323,7 @@
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"tslib": "^2.1.0"
}
@@ -13372,6 +13385,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@@ -13415,7 +13429,6 @@
"resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
"integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
@@ -13440,7 +13453,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -13449,15 +13461,13 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/send/node_modules/encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -13466,8 +13476,7 @@
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/serialize-javascript": {
"version": "6.0.2",
@@ -13483,7 +13492,6 @@
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
"integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"license": "MIT",
"peer": true,
"dependencies": {
"encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
@@ -14461,6 +14469,7 @@
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
@@ -14690,6 +14699,7 @@
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -15017,7 +15027,6 @@
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz",
"integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/json-schema": "^7.0.9",
"ajv": "^8.9.0",
+2 -2
View File
@@ -30,12 +30,12 @@
"@aws-crypto/sha256-js": "^5.2.0",
"@aws-sdk/client-dynamodb": "^3.414.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/credential-provider-node": "^3.940.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@dadosfera/protospack-v2": "3.38.0-beta.18",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -22,7 +22,7 @@ import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService } from '../../services/dynamodb';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
@@ -155,7 +155,7 @@ export class PlatformApiController {
* Extract and transform tables from jobs for DynamoDB input.
* Maps connector-specific fields to a common table format.
*
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name (reference_column)
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name/type (reference_column object)
* - Singer: type maps replication_method (FULL_TABLE -> full_load, INCREMENTAL -> incremental), no columns
* - S3: same mapping as Singer, no columns
*/
@@ -163,7 +163,7 @@ export class PlatformApiController {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
}> {
if (!jobs || jobs.length === 0) return [];
@@ -171,7 +171,7 @@ export class PlatformApiController {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
}> = [];
for (const job of jobs) {
@@ -179,12 +179,12 @@ export class PlatformApiController {
if (!input) continue;
if (connector === 'jdbc') {
// JDBC: table_name, load_type, column_include_list, incremental_column_name
// JDBC: table_name, load_type, column_include_list, incremental_column_name/type
const table: {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
} = {
name: input.table_name || '',
type: input.load_type || 'full_load',
@@ -193,8 +193,11 @@ export class PlatformApiController {
table.columns = input.column_include_list;
}
if (input.incremental_column_name) {
// reference_column is stored as a string (column name)
table.reference_column = input.incremental_column_name;
// reference_column is stored as an object with name and type
table.reference_column = {
name: input.incremental_column_name,
type: input.incremental_column_type || 'unknown',
};
}
tables.push(table);
} else if (connector === 'singer' || connector === 's3') {
@@ -317,11 +320,11 @@ export class PlatformApiController {
}
// Build changes for DynamoDB table entry
// reference_column is stored as a string (column name), not an object
// reference_column is stored as an object with name and type
const changes: {
type?: string;
columns?: string[];
reference_column?: string | null;
reference_column?: ReferenceColumn | null;
} = {};
@@ -332,8 +335,15 @@ export class PlatformApiController {
changes.columns = body.column_include_list;
}
if ('incremental_column_name' in body) {
// reference_column is just the column name as a string
changes.reference_column = body.incremental_column_name || null;
// reference_column is stored as an object with name and type
if (body.incremental_column_name) {
changes.reference_column = {
name: body.incremental_column_name,
type: body.incremental_column_type || 'unknown',
};
} else {
changes.reference_column = null;
}
}
// Update DynamoDB if there are changes
@@ -466,11 +476,13 @@ export class PlatformApiController {
});
}
const pipelineType = this.mapConnectorToDynamoType(connectorType);
this.logger.info('Syncing pipeline to Elasticsearch', {
customerName: user.customer_name,
pipelineId: body.id,
plugin,
connector: connectorType,
type: pipelineType,
properties,
inputId,
});
@@ -490,6 +502,7 @@ export class PlatformApiController {
cron: body.cron,
tables: inputId,
properties,
type: pipelineType,
},
connector,
);
@@ -617,21 +630,39 @@ export class PlatformApiController {
@ApiOperation({ summary: 'Execute a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async executePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/execute', user, body);
// Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
}
@Post('pipeline/pause')
@ApiOperation({ summary: 'Pause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pausePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/pause', user, body);
// Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
}
@Post('pipeline/unpause')
@ApiOperation({ summary: 'Unpause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, body);
// Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
}
@Put('pipeline/:pipelineId/memory')
+22 -16
View File
@@ -229,27 +229,33 @@ export class RolesService {
const [roleTreated] = this.getRolesPermissionsName([role.role]);
return { role: roleTreated };
}
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
const newRoles: RoleDto[] = [];
for (const role of roles) {
const allPermissions = this.permissionsService.getAllPermissions(
this.language,
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
const newRole: RoleDto = {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};
const newRole: RoleDto = this.formatRole(role);
newRoles.push(newRole);
}
return newRoles;
}
private formatRole(role: GetRolesPermissionsName): RoleDto {
const allPermissions = this.permissionsService.getAllPermissions(
this.language
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
return {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};;
}
}
+12
View File
@@ -38,6 +38,12 @@ export class User {
department?: string;
@ApiProperty()
hierarchy?: string;
@ApiProperty()
bio?: string;
@ApiProperty()
companyName?: string;
@ApiProperty()
companySite?: string;
@ApiPropertyOptional()
customer?: Customer;
@ApiProperty()
@@ -109,6 +115,12 @@ export class UpdateUserReq {
@ApiPropertyOptional()
hierarchy?: string;
@ApiPropertyOptional()
bio?: string;
@ApiPropertyOptional()
companySite?: string;
@ApiPropertyOptional()
companyName?: string;
@ApiPropertyOptional()
roleNames?: string[];
}
+12 -1
View File
@@ -266,7 +266,6 @@ export class UsersController {
}
@Patch(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: UpdateUserRes })
async updateUser(
@User() user: RequestUser,
@@ -274,6 +273,18 @@ export class UsersController {
@Param('id') id: string,
@Language() language: LanguageEnum,
) {
const isSameUser = user.user_id === id;
const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
if (!isSameUser && !isSuperAdmin) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
if (isSameUser && !isSuperAdmin && body.roleNames) {
// Prevent users from updating their own roles
delete body.roleNames;
}
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
return await this.userService.updateUser(body, id, user.customer_id);
+10 -7
View File
@@ -148,20 +148,23 @@ export class UsersService implements OnModuleInit {
}
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
if (roleNames) {
const { roleNames, ...updateUserDTO } = req;
if (roleNames && roleNames.length > 0) {
await this.setRoles({ roleNames, userId: id }, customerId);
}
const { user } = await lastValueFrom(
this.usersClientService.UserUpdate({
name,
department: updateUserDTO.department,
email: updateUserDTO.email,
hierarchy: updateUserDTO.hierarchy,
jobTitle: updateUserDTO.jobTitle,
name: updateUserDTO.name,
bio: updateUserDTO.bio,
companyName: updateUserDTO.companyName,
companySite: updateUserDTO.companySite,
customerId,
id,
department,
hierarchy,
jobTitle,
email,
metabaseUserId: undefined,
}),
);
+8 -3
View File
@@ -11,6 +11,11 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { v4 as uuid } from 'uuid';
import { DYNAMODB_CONFIG } from './dynamodb.config';
export interface ReferenceColumn {
name: string;
type: string;
}
export interface InputDocument {
id: string;
client_id: string;
@@ -24,7 +29,7 @@ export interface InputDocument {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
}>;
credentials?: Record<string, any>;
}
@@ -62,7 +67,7 @@ export class DynamoDBService {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
}>;
},
): Promise<InputDocument> {
@@ -157,7 +162,7 @@ export class DynamoDBService {
changes: {
type?: string;
columns?: string[];
reference_column?: string | null;
reference_column?: ReferenceColumn | null;
},
): Promise<void> {
const dynamoTableName = DYNAMODB_CONFIG.inputsTable();