Compare commits

...
Author SHA1 Message Date
Marcos Rodrigues Silva 71b3f278a5 Merge pull request #469 from dadosfera/feat/override-menu2
Feat/override menu2
2026-03-25 18:02:14 -03:00
viniciusgadea c92542ed90 FEAT: update protospack-v2 version to 3.39.0 in package.json and package-lock.json 2026-03-25 15:53:02 -03:00
viniciusgadeaandClaude Sonnet 4.6 69a9d78642 FIX: add multer@2.0.2 and update package-lock.json — missing transitive dep of protospack-v2@3.38.0-beta.30
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-25 15:41:02 -03:00
viniciusgadea bf4f3cfd8c FEAT: update CustomerSidebarSection and related DTOs to use object type for title; update protospack-v2 version; update from id to customerId 2026-03-25 15:40:57 -03:00
viniciusgadeaandClaude Sonnet 4.6 01afa69fcb FIX: remove duplicate identifier_columns declaration in TableColumns DTO
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-25 15:37:15 -03:00
viniciusgadea 80f3913fd2 FEAT: update CustomerSidebarSection to support both menu and link items 2026-03-25 15:24:59 -03:00
viniciusgadea 10293ad6a9 FEAT: update customer links handling and DTOs for improved structure sidebar. update protospack-v2 2026-03-25 15:24:59 -03:00
Marcos Rodrigues Silva c26194554c Merge pull request #455 from dadosfera/fix/header-validation
Fix/header validation
2026-03-11 11:58:23 -03:00
marcos-silva-rodrigues 4ddd5edcfd FIX: types 2026-03-11 10:26:33 -03:00
marcos-silva-rodrigues 67d47a9642 FIX: correct header 2026-03-10 17:35:05 -03:00
Marcos Rodrigues Silva 1a2153d62f Merge pull request #448 from dadosfera/fix/proxy-urls
Fix/proxy urls
2026-02-20 16:03:05 -03:00
marcos-silva-rodrigues d4ba45fd03 FIX: storage url 2026-02-20 14:28:55 -03:00
marcos-silva-rodrigues 82f1035a5e FIX: update storage port 2026-02-20 14:28:36 -03:00
marcos-silva-rodrigues 9a217dff57 FIX: storage service dns 2026-02-02 16:37:01 -03:00
marcos-silva-rodrigues cd4382c1ff FIX: send token by storage 2026-02-02 15:37:13 -03:00
marcos-silva-rodrigues 0e038d0b12 FIX: storage api 2026-02-02 14:41:53 -03:00
marcos-silva-rodrigues c97a02cb17 FIX: remove / 2026-02-02 14:27:08 -03:00
marcos-silva-rodrigues 878ec977b8 FIX: maestro proxy urls 2026-02-02 14:25:13 -03:00
Marcos Rodrigues Silva 8006867bc2 Merge pull request #436 from dadosfera/beta
Beta
2026-01-26 18:03:42 -03:00
Marcos Rodrigues Silva 784b0ef090 Merge pull request #435 from dadosfera/fix/validation-pipe
FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
2026-01-21 14:46:57 -03:00
RafaelandClaude b736cddf07 FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
This fix addresses a breaking change introduced in class-validator 0.14.0
where the default for forbidUnknownValues changed from false to true.

Issue:
- POST /connections was returning 400 "an unknown value was passed to the
  validate function" errors
- This occurred because CreateConnectionDto and UpdateConnectionDto have no
  validation decorators, causing class-validator 0.14.0+ to treat them as
  "unknown values"
- Extra fields (like connector_version) in request payloads would fail
  validation

Solution:
- Set forbidUnknownValues: false to allow DTOs without validation decorators
- Set whitelist: true to automatically strip extra properties not defined
  in DTOs
- This maintains backward compatibility while adding security by removing
  unexpected fields

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-21 14:43:43 -03:00
Rafael Santana 3d6328fb0b Merge pull request #434 from dadosfera/fix/class-validator-0.14-compatibility
UPDATE: updating /me to get api key
2026-01-20 17:53:09 -03:00
Rafael 4246c7495e UPDATE: updating /me to get api key 2026-01-20 17:52:34 -03:00
vinicius gadea 3d40746ccb Merge pull request #433 from dadosfera/feat/storage-explorer-api-integration
FEAT: integrate Storage Explorer API with new endpoints and permissions
2026-01-14 09:37:53 -03:00
Rafael Santana 121e30ce45 Merge pull request #432 from dadosfera/fix/class-validator-0.14-compatibility
FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
2026-01-12 16:30:41 -03:00
RafaelandClaude 85c8a4937d FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
This fix addresses a breaking change introduced in class-validator 0.14.0
where the default for forbidUnknownValues changed from false to true.

Issue:
- POST /connections was returning 400 "an unknown value was passed to the
  validate function" errors
- This occurred because CreateConnectionDto and UpdateConnectionDto have no
  validation decorators, causing class-validator 0.14.0+ to treat them as
  "unknown values"
- Extra fields (like connector_version) in request payloads would fail
  validation

Solution:
- Set forbidUnknownValues: false to allow DTOs without validation decorators
- Set whitelist: true to automatically strip extra properties not defined
  in DTOs
- This maintains backward compatibility while adding security by removing
  unexpected fields

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-12 16:22:25 -03:00
Marcos Rodrigues Silva 007f3911ff Merge pull request #429 from dadosfera/beta
Beta
2026-01-05 17:43:34 -03:00
16 changed files with 808 additions and 348 deletions
+1
View File
@@ -9,6 +9,7 @@ maestro:
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
hostname: maestro.stg.dadosfera.ai
+1 -1
View File
@@ -47,7 +47,7 @@ maestro:
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api"
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
+519 -307
View File
@@ -715,6 +715,158 @@
]
}
},
"/api-key": {
"post": {
"operationId": "ApiKeyController_create",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyDto"
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyResponseDto"
}
}
}
},
"default": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyResponseDto"
}
}
}
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
}
]
},
"get": {
"operationId": "ApiKeyController_findAll",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyBaseResponseDto"
}
}
}
}
},
"default": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyBaseResponseDto"
}
}
}
}
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
}
]
}
},
"/api-key/{id}": {
"delete": {
"operationId": "ApiKeyController_remove",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
}
]
}
},
"/connections": {
"post": {
"operationId": "ConnectionController_createConnection",
@@ -6584,158 +6736,6 @@
]
}
},
"/api-key": {
"post": {
"operationId": "ApiKeyController_create",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyDto"
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyResponseDto"
}
}
}
},
"default": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateApiKeyResponseDto"
}
}
}
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
}
]
},
"get": {
"operationId": "ApiKeyController_findAll",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyBaseResponseDto"
}
}
}
}
},
"default": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyBaseResponseDto"
}
}
}
}
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
}
]
}
},
"/api-key/{id}": {
"delete": {
"operationId": "ApiKeyController_remove",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"ApiKey"
],
"security": [
{
"access-token": []
}
]
}
},
"/identity-providers": {
"post": {
"operationId": "IdentityProviderController_addIdentityProvider",
@@ -7275,6 +7275,100 @@
]
}
},
"/platform/pipelines/catalog/tables": {
"get": {
"operationId": "PlatformApiController_getAvailableTables",
"summary": "Get all tables available",
"parameters": [],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipelines/catalog/schemas": {
"get": {
"operationId": "PlatformApiController_getAvailableSchemas",
"summary": "Get all schemas available",
"parameters": [],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipelines/catalog/tables/validate": {
"post": {
"operationId": "PlatformApiController_validateTableAndSchema",
"summary": "Validate tables and schemas",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run": {
"get": {
"operationId": "PlatformApiController_getPipelineRuns",
@@ -8440,36 +8534,6 @@
]
}
},
"/storage-explorer/storage/link-to-table": {
"post": {
"operationId": "StorageExplorerController_linkFileToTable",
"summary": "Link an existing file in storage to a table",
"parameters": [],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Storage Explorer"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/storage-explorer/storage/metadata": {
"get": {
"operationId": "StorageExplorerController_getFileMetadata",
@@ -8593,6 +8657,104 @@
"accessToken"
]
},
"CreateApiKeyDto": {
"type": "object",
"properties": {
"permissions": {
"description": "Array of permission IDs",
"type": "array",
"items": {
"type": "number"
}
}
},
"required": [
"permissions"
]
},
"PermissionDto": {
"type": "object",
"properties": {
"id": {
"type": "number"
},
"name": {
"type": "string"
}
},
"required": [
"id",
"name"
]
},
"CreateApiKeyResponseDto": {
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uuid"
},
"key_mask": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PermissionDto"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
},
"key": {
"type": "string"
}
},
"required": [
"id",
"key_mask",
"permissions",
"created_at",
"created_by",
"key"
]
},
"ApiKeyBaseResponseDto": {
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uuid"
},
"key_mask": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PermissionDto"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
}
},
"required": [
"id",
"key_mask",
"permissions",
"created_at",
"created_by"
]
},
"CreateConnectionDto": {
"type": "object",
"properties": {
@@ -8816,37 +8978,6 @@
"message"
]
},
"PermissionDto": {
"type": "object",
"properties": {
"id": {
"type": "number"
},
"seqid": {
"type": "number"
},
"name": {
"type": "string"
},
"claim": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"updatedAt": {
"type": "string"
}
},
"required": [
"id",
"seqid",
"name",
"claim",
"createdAt",
"updatedAt"
]
},
"Customer": {
"type": "object",
"properties": {
@@ -9969,6 +10100,21 @@
"entities"
]
},
"Column": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string"
}
},
"required": [
"name",
"type"
]
},
"TableColumns": {
"type": "object",
"properties": {
@@ -9984,17 +10130,25 @@
"references": {
"type": "array",
"items": {
"type": "string"
"$ref": "#/components/schemas/Column"
}
},
"type": {
"type": "string"
},
"identifier_columns": {
"type": "array",
"items": {
"type": "string"
}
},
"reference_column": {
"$ref": "#/components/schemas/Column"
}
},
"required": [
"name",
"columns",
"references",
"type"
]
},
@@ -10274,6 +10428,114 @@
"engine"
]
},
"ValidationTableDTO": {
"type": "object",
"properties": {
"tables": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"tables"
]
},
"EnforceMfa": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean"
}
},
"required": [
"enabled"
]
},
"CustomerLinkItem": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerSidebarSection": {
"type": "object",
"properties": {
"title": {
"type": "object"
},
"items": {
"type": "array",
"items": {
"oneOf": [
{
"$ref": "#/components/schemas/CustomerSidebarMenuItem"
},
{
"$ref": "#/components/schemas/CustomerSidebarLinkItem"
}
]
}
}
},
"required": [
"title",
"items"
]
},
"CustomerLinksConfig": {
"type": "object",
"properties": {
"home": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLinkItem"
}
},
"sidebar": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerSidebarSection"
}
}
}
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"$ref": "#/components/schemas/CustomerLinksConfig"
}
}
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"$ref": "#/components/schemas/CustomerLinksConfig"
}
},
"required": [
"links"
]
},
"IRole": {
"type": "object",
"properties": {
@@ -10874,7 +11136,7 @@
"enabled"
]
},
"CustomerLink": {
"CustomerLinkItem": {
"type": "object",
"properties": {
"href": {
@@ -10896,28 +11158,61 @@
"description"
]
},
"CustomerLinksResponse": {
"CustomerSidebarSection": {
"type": "object",
"properties": {
"links": {
"title": {
"type": "string"
},
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
"oneOf": [
{
"$ref": "#/components/schemas/CustomerSidebarMenuItem"
},
{
"$ref": "#/components/schemas/CustomerSidebarLinkItem"
}
]
}
}
},
"required": [
"links"
"title",
"items"
]
},
"CustomerLinksConfig": {
"type": "object",
"properties": {
"home": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLinkItem"
}
},
"sidebar": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerSidebarSection"
}
}
}
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"$ref": "#/components/schemas/CustomerLinksConfig"
}
}
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
"$ref": "#/components/schemas/CustomerLinksConfig"
}
},
"required": [
@@ -11330,89 +11625,6 @@
"publicKey"
]
},
"CreateApiKeyDto": {
"type": "object",
"properties": {
"permissions": {
"description": "Array of permission IDs",
"type": "array",
"items": {
"type": "number"
}
}
},
"required": [
"permissions"
]
},
"CreateApiKeyResponseDto": {
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uuid"
},
"key_mask": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PermissionDto"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
},
"key": {
"type": "string"
}
},
"required": [
"id",
"key_mask",
"permissions",
"created_at",
"created_by",
"key"
]
},
"ApiKeyBaseResponseDto": {
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uuid"
},
"key_mask": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PermissionDto"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
}
},
"required": [
"id",
"key_mask",
"permissions",
"created_at",
"created_by"
]
},
"CreateIdentityProvider": {
"type": "object",
"properties": {
+79 -6
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@dadosfera/protospack-v2": "^3.39.0",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -47,6 +47,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
@@ -1744,9 +1745,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.26",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
"integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
"version": "3.39.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.39.0.tgz",
"integrity": "sha512-z6q+ZRpbDT6apfmly1+X818DgAFUCilFO0ATP7xR5+SH+GAUAJLY1dJRQDEkq5vFIfXnC2Nmm9aODZAs+9/5vw==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
@@ -2927,6 +2928,20 @@
"node": ">= 0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"engines": [
"node >= 0.8"
],
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3096,6 +3111,24 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/@nestjs/platform-express/node_modules/multer": {
"version": "1.4.4-lts.1",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.0.0",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.4",
"object-assign": "^4.1.1",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3120,6 +3153,25 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3194,6 +3246,19 @@
"node": ">= 0.8"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/tslib": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -6508,7 +6573,6 @@
"engines": [
"node >= 6.0"
],
"license": "MIT",
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
@@ -9127,6 +9191,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10724,7 +10793,6 @@
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
"license": "MIT",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.6.0",
@@ -11727,6 +11795,11 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/process-warning": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
+2 -2
View File
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@dadosfera/protospack-v2": "^3.39.0",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -65,6 +65,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
@@ -80,7 +81,6 @@
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "2.0.2",
"form-data": "^4.0.4",
"body-parser": "^1.20.3",
"cross-spawn": "^7.0.5",
+24 -3
View File
@@ -55,6 +55,7 @@ import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
type CookiesValues = {
accessToken?: string;
@@ -74,6 +75,7 @@ export class AuthController {
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
@@ -477,11 +479,30 @@ export class AuthController {
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Check for API key header first
const apiKey = req.get('X-Api-key');
if (apiKey) {
this.logger.info('Authenticating via X-Api-key header');
const { api_key } = await this.apiKeyService.get(apiKey);
const userDto = {
id: api_key.user_id,
name: api_key.username,
customer: {
id: api_key.customer_id,
name: api_key.customer_name,
tier: api_key.customer_tier,
}
};
return res.status(200).json(userDto);
}
// Get token and headers
const accessToken = req.cookies['ddf-auth'];
const refreshToken = req.cookies['ddf-refresh-auth'];
const userId = req.cookies['ddf-user-id'];
const resourceHost = req.headers["host"]
const refreshToken = req.cookies['ddf-refresh-auth'];
const userId = req.cookies['ddf-user-id'];
const resourceHost = req.headers["x-original-url"] as string || "" ;
const hasUserSession = Boolean(accessToken) && Boolean(userId);
this.logger.info('Has User Session: ' + hasUserSession);
+2 -1
View File
@@ -8,10 +8,11 @@ import { AuthClientService } from './auth.service';
import { DucClient } from '../duc/client.config';
import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
import { getOauthSecrets } from 'src/utils/OauthSecrets';
import { ApiKeyModule } from '../api-key/api-key.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule],
controllers: [AuthController],
providers: [
AuthClientService,
+10 -10
View File
@@ -9,12 +9,12 @@ import {
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { CustomerLinksConfig } from './dtos/customers';
import ErrorCodes from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import {
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
)
}
async getLinks(customerId: string) {
async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
try {
const result = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId }),
this.customerService.CustomerGetLinks({ customerId }),
);
return result.customer?.links || [];
return (result.links as CustomerLinksConfig) || null;
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
}
}
async setLinks(customerId: string, links: Link[]) {
async setLinks(customerId: string, links: CustomerLinksConfig) {
if (!customerId || !links) {
throw new HttpException(null, HttpStatus.BAD_REQUEST);
}
try {
return await firstValueFrom(
this.customerService.CustomerUpdate({
id: customerId,
links,
} as CustomerUpdateRequest),
this.customerService.CustomerSetLinks({
customerId,
links: links as CustomerSetLinksRequest['links'],
}),
);
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+52 -9
View File
@@ -1,7 +1,6 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CustomerLink implements Link {
export class CustomerLinkItem {
@ApiProperty()
href: string;
@ApiProperty()
@@ -9,15 +8,59 @@ export class CustomerLink implements Link {
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
iconSrc?: string;
}
export class CustomerSidebarLinkItem {
@ApiProperty()
type: 'link';
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiProperty()
link: string;
@ApiPropertyOptional()
icon?: string;
}
export class CustomerSidebarMenuItem {
@ApiProperty()
type: 'menu';
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiPropertyOptional()
icon?: string;
@ApiProperty({ type: [CustomerSidebarLinkItem] })
items: CustomerSidebarLinkItem[];
}
export class CustomerSidebarSection {
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiProperty({
type: 'array',
items: {
oneOf: [
{ $ref: '#/components/schemas/CustomerSidebarMenuItem' },
{ $ref: '#/components/schemas/CustomerSidebarLinkItem' },
],
},
})
items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
}
export class CustomerLinksConfig {
@ApiPropertyOptional({ type: [CustomerLinkItem] })
home?: CustomerLinkItem[];
@ApiPropertyOptional({ type: [CustomerSidebarSection] })
sidebar?: CustomerSidebarSection[];
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
@ApiProperty({ type: CustomerLinksConfig })
links: CustomerLinksConfig;
}
export class CustomerLinksResponse {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
@ApiPropertyOptional({ type: CustomerLinksConfig })
links?: CustomerLinksConfig;
}
+10 -1
View File
@@ -11,10 +11,19 @@ export class TableColumns {
name: string;
@ApiProperty()
columns: string[];
@ApiProperty()
@ApiPropertyOptional({ type: [Column] })
references: Column[];
@ApiProperty()
destination: Record<'raw' | 'qualify', {
table_name: string;
table_schema: string;
}> | null;
@ApiProperty()
type: string;
@ApiPropertyOptional({ type: [String] })
identifier_columns?: string[];
@ApiPropertyOptional({ type: Column })
reference_column?: Column;
}
export class AvailableEntity {
@ApiProperty()
+5
View File
@@ -164,6 +164,11 @@ export class InputsService {
const inputCreateGenericRequest: InputCreateGenericRequest = {
input: {
...body,
tables: (body.tables || []).map((table) => ({
...table,
identifier_columns: table.identifier_columns || [],
reference_column: table.reference_column || table.references?.[0],
})),
},
info,
};
@@ -26,6 +26,14 @@ import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
type ValidateTablesDTO = {
tables: Array<{
table_schema: string,
table_name: string
}>
}
@ApiTags('Platform API')
@Controller('platform')
export class PlatformApiController {
@@ -720,6 +728,57 @@ export class PlatformApiController {
);
}
// ==================== Catalog ROUTES ====================
@Get('pipelines/catalog/tables')
@ApiOperation({ summary: 'Get all tables available' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getAvailableTables(
@User() user: RequestUser,
@Query() query: Record<string, string>,
) {
return this.platformApiService.proxy(
'GET',
'/catalog/tables',
user,
undefined,
query,
);
}
@Get('pipelines/catalog/schemas')
@ApiOperation({ summary: 'Get all schemas available' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getAvailableSchemas(
@User() user: RequestUser,
@Query() query: Record<string, string>,
) {
return this.platformApiService.proxy(
'GET',
'/catalog/schemas',
user,
undefined,
query,
);
}
@Post('pipelines/catalog/tables/validate')
@ApiOperation({ summary: 'Validate tables and schemas' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async validateTableAndSchema(
@User() user: RequestUser,
@Query() query: Record<string, string>,
@Body() validateTablesDto: ValidateTablesDTO[]
) {
return this.platformApiService.proxy(
'POST',
'/catalog/tables/validate',
user,
validateTablesDto,
query,
);
}
// ==================== PIPELINE RUN ROUTES ====================
@Get('pipeline/:pipelineId/pipeline_run')
@@ -1,13 +1,13 @@
export const STORAGE_EXPLORER_CONFIG = {
getUrl: (customerId: string): string => {
getUrl: (customerName: string): string => {
const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
if (!urlTemplate) {
throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
}
// Replace {customer_id} placeholder with actual customer ID
// For local: http://172.17.0.1:8000/api (no placeholder)
// For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api
return urlTemplate.replace('{customer_id}', customerId);
// For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
return urlTemplate.replace('{customer}', customerName);
},
timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
};
@@ -3,13 +3,13 @@ import {
Get,
Post,
Put,
Delete,
Param,
Body,
Query,
Inject,
UseInterceptors,
UploadedFiles,
Headers,
} from '@nestjs/common';
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
import { FilesInterceptor } from '@nestjs/platform-express';
@@ -47,11 +47,13 @@ export class StorageExplorerController {
async validateTableName(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/validate-name',
user,
body,
);
}
@@ -62,11 +64,13 @@ export class StorageExplorerController {
async createTable(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/',
user,
body,
);
}
@@ -77,11 +81,13 @@ export class StorageExplorerController {
async listTables(
@Query('page') page: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/tables/',
user,
undefined,
{ page },
);
@@ -93,6 +99,7 @@ export class StorageExplorerController {
async getTable(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
@@ -108,11 +115,13 @@ export class StorageExplorerController {
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/datasets/${datasetId}`,
user,
);
}
@@ -122,11 +131,13 @@ export class StorageExplorerController {
async getTableDatasets(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/datasets`,
user,
);
}
@@ -136,11 +147,13 @@ export class StorageExplorerController {
async getTableSchema(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/schema`,
user,
);
}
@@ -156,6 +169,7 @@ export class StorageExplorerController {
'POST',
`/tables/${tableId}/validate-compatibility/${datasetId}`,
user,
);
}
@@ -170,11 +184,13 @@ export class StorageExplorerController {
@Param('datasetId') datasetId: string,
@Query('limit') limit: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/preview`,
user,
undefined,
{ limit },
);
@@ -187,11 +203,13 @@ export class StorageExplorerController {
@Param('datasetId') datasetId: string,
@Query('force_refresh') forceRefresh: boolean,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/schema`,
user,
undefined,
{ force_refresh: forceRefresh },
);
@@ -203,11 +221,13 @@ export class StorageExplorerController {
async listDatasetsByUpload(
@Param('uploadId') uploadId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/upload/${uploadId}`,
user,
);
}
@@ -218,11 +238,13 @@ export class StorageExplorerController {
@Param('datasetId') datasetId: string,
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'PUT',
`/datasets/${datasetId}/refresh-schema`,
user,
body,
);
}
@@ -239,11 +261,13 @@ export class StorageExplorerController {
@Query('limit') limit: number,
@Query('folder_path') folderPath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/uploads/history',
user,
undefined,
{ page, limit, folder_path: folderPath },
);
@@ -255,11 +279,13 @@ export class StorageExplorerController {
async browseStorage(
@Query('path') path: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/browse',
user,
undefined,
{ path },
);
@@ -274,6 +300,7 @@ export class StorageExplorerController {
@UploadedFiles() files: Array<Express.Multer.File>,
@Body('folder_path') folderPath: string,
@User() user: RequestUser,
) {
// Create FormData to forward files to storage-explorer API
const formData = new FormData();
@@ -297,6 +324,7 @@ export class StorageExplorerController {
'POST',
'/storage/upload/batch',
user,
formData,
);
}
@@ -307,11 +335,13 @@ export class StorageExplorerController {
async createFolder(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/storage/folder/create',
user,
body,
);
}
@@ -322,11 +352,13 @@ export class StorageExplorerController {
async downloadFile(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/download',
user,
undefined,
{ file_path: filePath },
);
@@ -338,6 +370,7 @@ export class StorageExplorerController {
async getFileMetadata(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
@@ -20,7 +20,7 @@ export class StorageExplorerService {
path: string,
user: RequestUser,
body?: any,
query?: Record<string, any>,
query?: Record<string, any>
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
@@ -28,7 +28,7 @@ export class StorageExplorerService {
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
const url = new URL(`${baseUrl}${path}`);
// Add query params
@@ -113,7 +113,7 @@ export class StorageExplorerService {
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
const url = new URL(`${baseUrl}${path}`);
// Add query params
+4 -1
View File
@@ -14,7 +14,10 @@ export class ValidationPipe implements PipeTransform<any> {
return value;
}
const object = plainToInstance(metatype, value);
const errors = await validate(object);
const errors = await validate(object, {
forbidUnknownValues: false,
whitelist: true,
});
if (errors.length > 0) {
const errorMessages = errors.map((err) => err.constraints);
throw new BadRequestException(errorMessages);