Compare commits

..
Author SHA1 Message Date
marcos-silva-rodrigues 4ddd5edcfd FIX: types 2026-03-11 10:26:33 -03:00
marcos-silva-rodrigues 67d47a9642 FIX: correct header 2026-03-10 17:35:05 -03:00
Marcos Rodrigues Silva 1a2153d62f Merge pull request #448 from dadosfera/fix/proxy-urls
Fix/proxy urls
2026-02-20 16:03:05 -03:00
marcos-silva-rodrigues d4ba45fd03 FIX: storage url 2026-02-20 14:28:55 -03:00
marcos-silva-rodrigues 82f1035a5e FIX: update storage port 2026-02-20 14:28:36 -03:00
marcos-silva-rodrigues 9a217dff57 FIX: storage service dns 2026-02-02 16:37:01 -03:00
marcos-silva-rodrigues cd4382c1ff FIX: send token by storage 2026-02-02 15:37:13 -03:00
marcos-silva-rodrigues 0e038d0b12 FIX: storage api 2026-02-02 14:41:53 -03:00
marcos-silva-rodrigues c97a02cb17 FIX: remove / 2026-02-02 14:27:08 -03:00
marcos-silva-rodrigues 878ec977b8 FIX: maestro proxy urls 2026-02-02 14:25:13 -03:00
Marcos Rodrigues Silva 8006867bc2 Merge pull request #436 from dadosfera/beta
Beta
2026-01-26 18:03:42 -03:00
Marcos Rodrigues Silva 784b0ef090 Merge pull request #435 from dadosfera/fix/validation-pipe
FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
2026-01-21 14:46:57 -03:00
RafaelandClaude b736cddf07 FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
This fix addresses a breaking change introduced in class-validator 0.14.0
where the default for forbidUnknownValues changed from false to true.

Issue:
- POST /connections was returning 400 "an unknown value was passed to the
  validate function" errors
- This occurred because CreateConnectionDto and UpdateConnectionDto have no
  validation decorators, causing class-validator 0.14.0+ to treat them as
  "unknown values"
- Extra fields (like connector_version) in request payloads would fail
  validation

Solution:
- Set forbidUnknownValues: false to allow DTOs without validation decorators
- Set whitelist: true to automatically strip extra properties not defined
  in DTOs
- This maintains backward compatibility while adding security by removing
  unexpected fields

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-21 14:43:43 -03:00
Rafael Santana 3d6328fb0b Merge pull request #434 from dadosfera/fix/class-validator-0.14-compatibility
UPDATE: updating /me to get api key
2026-01-20 17:53:09 -03:00
Rafael 4246c7495e UPDATE: updating /me to get api key 2026-01-20 17:52:34 -03:00
vinicius gadea 3d40746ccb Merge pull request #433 from dadosfera/feat/storage-explorer-api-integration
FEAT: integrate Storage Explorer API with new endpoints and permissions
2026-01-14 09:37:53 -03:00
viniciusgadea 3ab2f8f27d CHORE: remove linkFileToTable endpoint from Storage Explorer controller 2026-01-14 08:51:45 -03:00
viniciusgadea b44d23552c CHORE: remove unecessary header for request 2026-01-14 06:40:52 -03:00
viniciusgadea 8e63757738 FEAT: add STORAGE_EXPLORER_API_URL to deployment and values configuration 2026-01-14 06:26:21 -03:00
viniciusgadea 3b84409003 CHORE: add unique authentication decorator to Storage Explorer controller methods 2026-01-14 06:25:58 -03:00
viniciusgadea 0ce3822300 CHORE: update new endpoint link-table and remove /delete from storage-explorer api 2026-01-13 14:36:43 -03:00
viniciusgadea a2c7ce00db FEAT: integrate Storage Explorer API with new endpoints and permissions 2026-01-13 10:29:38 -03:00
Rafael Santana 121e30ce45 Merge pull request #432 from dadosfera/fix/class-validator-0.14-compatibility
FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
2026-01-12 16:30:41 -03:00
RafaelandClaude 85c8a4937d FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility
This fix addresses a breaking change introduced in class-validator 0.14.0
where the default for forbidUnknownValues changed from false to true.

Issue:
- POST /connections was returning 400 "an unknown value was passed to the
  validate function" errors
- This occurred because CreateConnectionDto and UpdateConnectionDto have no
  validation decorators, causing class-validator 0.14.0+ to treat them as
  "unknown values"
- Extra fields (like connector_version) in request payloads would fail
  validation

Solution:
- Set forbidUnknownValues: false to allow DTOs without validation decorators
- Set whitelist: true to automatically strip extra properties not defined
  in DTOs
- This maintains backward compatibility while adding security by removing
  unexpected fields

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-12 16:22:25 -03:00
Rafael Santana 53920f2f3f Merge pull request #431 from dadosfera/feature/mgc-keycloak-migration
Feature/mgc keycloak migration
2026-01-08 18:21:10 -03:00
RafaelandClaude Opus 4.5 8c34914806 FEAT: add AUTH_PROVIDER config to Helm chart
- Add auth_provider to values.yaml (cognito/keycloak)
- Add AUTH_PROVIDER env var to deployment template
- Note: maestro only needs to know which provider is used, duc handles connection

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-08 12:22:47 -03:00
RafaelandClaude Opus 4.5 1881d07c4a FEAT: add dual auth provider support to Helm chart
- Add auth_provider config to values.yaml (cognito/keycloak)
- Add AUTH_PROVIDER env var to deployment template
- Add conditional Keycloak env vars (URL, realm, client ID, secret)
- Enables parallel deployments with different auth providers

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-08 12:10:17 -03:00
RafaelandClaude Opus 4.5 3b8310fdca CHORE: simplify Dockerfile.local to use npm ci only
Remove protospack tarball fallback logic

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-07 19:04:07 -03:00
RafaelandClaude Opus 4.5 52bda8ebe2 FEAT: add authProvider field to user response
- Add authProvider to user entity DTO
- Update user service to include authProvider
- Update auth controller response

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-07 18:55:36 -03:00
Rafael 275a53dbd1 Merge remote-tracking branch 'origin/beta' into feature/mgc-keycloak-migration 2026-01-06 12:44:05 -03:00
vinicius gadea 9cefdb226d Merge pull request #430 from dadosfera/feat/organization-form
Feat/organization form
2026-01-06 10:52:33 -03:00
Marcos Rodrigues Silva 007f3911ff Merge pull request #429 from dadosfera/beta
Beta
2026-01-05 17:43:34 -03:00
viniciusgadea 8ac0a8a79f REFACTOR: rename 'companySite' to 'personalSite' in user DTOs and related services 2026-01-05 15:43:35 -03:00
viniciusgadea 285de97375 FIX: update organization info field from 'name' to 'companyName' in updateOrganizationInfo method 2026-01-05 12:01:04 -03:00
viniciusgadea e2a7d2b92b FIX: update organization info field from 'name' to 'companyName' 2026-01-05 11:47:20 -03:00
viniciusgadea da23ad76db DOCS: update docs to organization forms 2026-01-02 09:22:27 -03:00
viniciusgadea 55b0961b82 FEAT: add organization info endpoints and DTOs for update and retrieval new organization forms 2026-01-02 09:21:46 -03:00
Rafael Santana d3c5c0fa63 Merge pull request #427 from dadosfera/fix/security-vulnerabilities-high
FIX: resolve HIGH security vulnerabilities (18 → 0)
2025-12-29 11:47:51 -03:00
RafaelandClaude Opus 4.5 5dbc644d1d fix: security vulnerability - upgrade multer to 2.0.2
- Update multer override from 1.4.5-lts.1 to 2.0.2 to fix HIGH vulnerabilities:
  - CVE-2025-47935 (DoS via memory leaks)
  - CVE-2025-47944
  - CVE-2025-48997
  - CVE-2025-7338

Reduces HIGH vulnerabilities from 4 to 0

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-29 11:42:04 -03:00
Marcos Rodrigues Silva 8eddb9e1bf Merge pull request #428 from dadosfera/hotfix/user-patch
Hotfix/user patch
2025-12-26 17:58:28 -03:00
Marcos Rodrigues 76485f929d FIX: user patch route 2025-12-26 17:53:02 -03:00
RafaelandClaude Opus 4.5 8e0182aa50 FEAT: add TOTP support for change password and local build support
- Pass TOTP code to DUC for Keycloak users with MFA
- Add Dockerfile.local for local protospack builds
- Add .dockerignore to exclude node_modules from Docker context

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-24 19:03:50 -03:00
Rafael Santana a18bdccc09 Merge branch 'beta' into fix/security-vulnerabilities-high 2025-12-23 13:51:25 -03:00
RafaelandClaude b27298501d FIX: resolve HIGH security vulnerabilities (18 → 0)
Upgraded Node.js and dependencies to address critical vulnerabilities:

**Infrastructure:**
- Upgrade Node.js from 18.17-alpine to 20-alpine (Alpine 3.18 → 3.23)
- Upgrade npm to 11.7.0 (includes glob 13.0.0 fix)
- Replace npm ci with npm install, then back to npm ci with npm 11 lock

**Dependencies:**
- Update axios: 0.27.2 → 0.30.2 (fixes 2 CVEs)
- Update body-parser: 1.20.1/1.20.2 → 1.20.3
- Update cross-spawn: 7.0.3 → 7.0.6 (hoisted)
- Update glob: 10.2.4 → 10.5.0 (hoisted)
- Update path-to-regexp: 3.2.0 → 3.3.0 (maintains @nestjs/swagger compatibility)
- Update semver: 7.5.1 → 7.5.2

**Results:**
- HIGH vulnerabilities: 18 → 0 (100% reduction)
- Alpine OS vulnerabilities: 6 → 0 (fixed by Alpine 3.23)
- Node.js vulnerabilities: 12 → 0
- All tests passing
- Production verified working

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-12-23 13:48:02 -03:00
vinicius gadea 33ebc91826 Merge pull request #426 from dadosfera/feature/catalog-update
FEAT: Add optional 'docs' field to assets
2025-12-19 08:24:54 -03:00
vinicius gadea 6948156693 Merge branch 'beta' into feature/catalog-update 2025-12-19 08:21:52 -03:00
viniciusgadea 0aaa4384c3 DOCS: Add 'docs' field to schemas in docsfera.json and update protospack-v2 to version 3.38.0-beta.19 in package-lock.json 2025-12-18 17:43:54 -03:00
Marcos Rodrigues Silva 1f9d0c29ec Merge pull request #425 from dadosfera/beta
Beta
2025-12-18 17:42:10 -03:00
viniciusgadea 039c652b28 DOC: Update @dadosfera/protospack-v2 to version 3.38.0-beta.19 in package.json and package-lock.json 2025-12-18 17:11:18 -03:00
viniciusgadea 653d4f53b5 FEAT: Add optional 'docs' field to IDataAsset, IUpdateDataRequest, and ICreateDataAsset 2025-12-18 16:16:00 -03:00
Marcos Rodrigues Silva 31f8c2c1a6 Merge pull request #424 from dadosfera/feature/user-form
FEAT: protected update route with admin seqid or if user request to e…
2025-12-18 12:33:29 -03:00
Marcos Rodrigues 8b93d4e97b FEAT: protected update route with admin seqid or if user request to equal a jwt user id claim 2025-12-18 12:32:53 -03:00
Rafael Santana adeb022818 Merge pull request #423 from dadosfera/fix/reference-column-object-format
FIX: inject customer_id in pipeline execute/pause/unpause routes
2025-12-18 10:53:31 -03:00
RafaelandClaude Opus 4.5 f61c241dde FIX: inject customer_id in pipeline execute/pause/unpause routes
Users no longer need to provide customer_id in the request body for
execute, pause, and unpause pipeline operations - it's now automatically
injected from the authenticated user's session.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-18 10:30:09 -03:00
Marcos Rodrigues Silva e326cab44d Merge pull request #422 from dadosfera/feature/user-form
FIX: send new user info
2025-12-18 10:29:09 -03:00
Marcos Rodrigues 61109f8ae9 FIX: send new user info 2025-12-18 10:24:29 -03:00
Marcos Rodrigues Silva 3f8dc5cabe Merge pull request #421 from dadosfera/feature/user-form
Feature/user form
2025-12-18 09:53:12 -03:00
Marcos Rodrigues 7f5d157739 FEAT: update protospack version 2025-12-18 09:52:11 -03:00
Marcos Rodrigues bce73fb11f FEAT: update user route 2025-12-17 17:24:03 -03:00
Rafael Santana 9473e65deb Merge pull request #419 from dadosfera/fix/reference-column-object-format
FIX: store reference_column as object with name and type
2025-12-17 13:48:38 -03:00
RafaelandClaude Opus 4.5 9c55c22230 FIX: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:47:57 -03:00
Rafael Santana 77b9acd2d0 Merge pull request #418 from dadosfera/fix/reference-column-object-format
fix: store reference_column as object with name and type
2025-12-17 13:46:05 -03:00
RafaelandClaude Opus 4.5 e99306adba fix: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:45:18 -03:00
Rafael Santana 96b947ebdc Merge pull request #417 from dadosfera/fix/redis-tls-configurable
FIX: pass type field to Elasticsearch createPipeline
2025-12-16 21:10:19 -03:00
RafaelandClaude Opus 4.5 fb521f53cd FIX: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:09:07 -03:00
Rafael Santana 6f7436f33f Merge pull request #416 from dadosfera/fix/redis-tls-configurable
fix: pass type field to Elasticsearch createPipeline
2025-12-16 21:05:55 -03:00
RafaelandClaude Opus 4.5 865140e681 fix: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:05:21 -03:00
Rafael Santana c4a664572a Merge pull request #415 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-16 19:17:08 -03:00
Rafael acb631e33d UPDATE: force deployment 2025-12-16 19:16:35 -03:00
Rafael Santana 7a10f88113 Merge pull request #414 from dadosfera/fix/redis-tls-configurable
fix: normalize IDs in pipeline run routes before calling Platform API
2025-12-16 19:11:09 -03:00
RafaelandClaude Opus 4.5 e616061c21 fix: normalize IDs in pipeline run routes before calling Platform API
- Add normalization for pipelineId and runId in getPipelineRuns, getPipelineRun, and getPipelineRunLogs
- Remove unused createPipelineRun route

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 19:10:23 -03:00
Rafael Santana 466f8fb8cc Merge pull request #413 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-16 18:23:44 -03:00
Rafael e03b9e7a14 UPDATE: force deployment 2025-12-16 18:23:19 -03:00
Rafael Santana 5989822263 Merge pull request #412 from dadosfera/fix/redis-tls-configurable
feat: add Elasticsearch access to Maestro deployment
2025-12-16 18:15:08 -03:00
RafaelandClaude Opus 4.5 2cc8f46418 feat: add Elasticsearch access to Maestro deployment
- Add ELASTICSEARCH_URL and ELASTICSEARCH_API_KEY env vars
- Pull credentials from {env}/microservices/elasticsearch secret

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 18:10:38 -03:00
vinicius gadea 64a3e2652e Merge pull request #410 from dadosfera/feature/catalog-update
Feature/catalog update
2025-12-16 18:01:47 -03:00
Rafael Moraes ea44a1cbb6 FEAT: Manual/Auto filter and UX Adjusments 2025-12-16 20:52:28 +00:00
Rafael Moraes fc9c0b0991 WIP: Filtro Source - backend 2025-12-16 20:13:20 +00:00
Rafael Moraes a4b5a44e44 Merge branch 'feature/catalog-update' of github.com:dadosfera/maestro into feature/catalog-update 2025-12-16 20:01:19 +00:00
Rafael Moraes d99a6aa322 Merge branch 'beta' of github.com:dadosfera/maestro into feature/catalog-update 2025-12-16 19:58:48 +00:00
Rafael Santana 3f910f851a Merge pull request #411 from dadosfera/fix/redis-tls-configurable
Fix/redis tls configurable
2025-12-16 16:58:41 -03:00
Rafael Moraes bd231382eb WIP: adiciona parâmetro asset_type no docsfera.json 2025-12-16 19:58:38 +00:00
Rafael 295f1f86ca Merge branch 'fix/redis-tls-configurable' of github.com:dadosfera/maestro into fix/redis-tls-configurable 2025-12-16 16:57:59 -03:00
Rafael 0a5e8001f9 UPDATE: adding platform_api_url 2025-12-16 16:57:25 -03:00
Rafael Santana 9c1979e17a Merge pull request #405 from dadosfera/fix/redis-tls-configurable
Fix/redis tls configurable
2025-12-16 16:41:04 -03:00
Rafael Santana b2700d4bb0 Merge branch 'beta' into fix/redis-tls-configurable 2025-12-16 16:39:11 -03:00
RafaelandClaude Opus 4.5 01c1087e07 FEAT: add schedule limit validation and improve ES update
- Add schedule limit validation against customer's scheduleLimit from DUC
- Improve ES update to only update provided fields
- Mark GET /platform/pipeline/:pipelineId/pipeline_run as READY

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 16:37:33 -03:00
viniciusgadea fcf7fb054e FIX: Update searchDataAssets to handle owner filtering and convert owner names to IDs 2025-12-16 09:55:24 -03:00
viniciusgadea 288aaeabc0 FIX: Enhance filter handling in BuildDataAssetsFilterTerms to support empty owner condition 2025-12-15 12:00:50 -03:00
viniciusgadea 918c3d7416 FIX: Enhance searchDataAssets to convert owner names to IDs and log the conversion 2025-12-15 09:08:08 -03:00
vinicius gadea 7dedb3bd33 Merge pull request #409 from dadosfera/feature/catalog-update
FIX: Simplify searchDataAssets by owner logging and remove log code
2025-12-12 15:51:17 -03:00
viniciusgadea 1ff5589a2e FIX: Simplify searchDataAssets by owner logging and remove log code 2025-12-12 15:03:43 -03:00
Rafael Moraes d24e9a1d80 Merge pull request #408 from dadosfera/feature/catalog-update
FIX: Calendar Fix
2025-12-11 22:26:25 +00:00
Rafael Moraes 7d3ef1ef92 FIX: Conflit resolution 2025-12-11 22:25:42 +00:00
Rafael Moraes df3f2489f9 FIX: Calendar Fix 2025-12-11 22:10:32 +00:00
Rafael Moraes 53245b0067 Merge pull request #407 from dadosfera/feature/catalog-update
FIX: correção no validate k8s
2025-12-11 16:50:11 +00:00
Rafael Moraes dd699614ae FIX: Maestro helm fix 2025-12-11 16:47:28 +00:00
Rafael Moraes 6919a2a8d0 FIX: helmfile diff plugin 2025-12-11 14:14:02 +00:00
Rafael Moraes 5c29e07450 FIX: helmfile 2025-12-11 14:07:04 +00:00
Rafael Moraes 00304262ba FIX: wget change in helmfile 2025-12-11 13:52:58 +00:00
vinicius gadea bc931c6dd8 Merge pull request #406 from dadosfera/feature/catalog-update
catalogo: alteracao nas pastas e acrescimo de filtros
2025-12-11 09:18:57 -03:00
Rafael Moraes c444d6e956 Merge pull request #404 from dadosfera/feat/add-docs-assets
Feat/add docs assets
2025-12-10 19:42:42 +00:00
RafaelandClaude Opus 4.5 f7efb757bf FEAT: add schedule limit validation and improve ES update
- Add cron-validation utility with Airflow presets support (@once, @daily, etc.)
- Validate cron expressions against customer scheduleLimit from DUC
- Fetch scheduleLimit via gRPC to ensure up-to-date configuration
- Improve ES updatePipeline to fetch current doc and merge changes
- Only update fields explicitly provided in PATCH request body
- Preserve config.tables when updating config.cron

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-10 13:56:49 -03:00
Rafael Moraes 2f140d213a catalogo: alteracao nas pastas e acrescimo de filtros 2025-12-10 16:51:28 +00:00
RafaelandClaude Opus 4.5 df674dd441 FEAT: add Platform API proxy with Elasticsearch and DynamoDB sync
- Add Platform API module to proxy requests to Platform API service
- Add Elasticsearch service for pipeline catalog sync (CRUD operations)
- Add DynamoDB service for storing pipeline tables/inputs configuration
- Sync pipeline creation/update/delete to Elasticsearch index
- Extract tables from jobs with connector-specific mappings:
  - JDBC: table_name, load_type, column_include_list, incremental_column_name
  - Singer: replication_method -> full_load/incremental
  - S3: same as Singer
- Map connector types to DynamoDB types (jdbc->database, singer->application, s3->file)
- Validate connector type is provided in job input
- Normalize pipeline IDs for Platform API (replace - with _), keep UUIDs for ES

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-10 12:42:18 -03:00
viniciusgadea 09dced9fcd REFACTOR: add asset_type query parameter to getDataDocs and separate data docs methods 2025-12-10 11:21:46 -03:00
viniciusgadea 165b533172 FIX: removing unnecessary type parameter from GetDatasetDoc call 2025-12-10 11:17:43 -03:00
viniciusgadea b1dc567394 DOCS: update @dadosfera/protospack-v2 to version 3.38.0-beta.16 2025-12-10 11:12:13 -03:00
viniciusgadea a0a1303515 DOCS: add automatic asset_type query parameter to API schema and packages dependencies 2025-12-09 14:56:51 -03:00
viniciusgadea c21d977f7c FEAT: enhance catalog service to support any asset type in data docs creation 2025-12-09 14:56:51 -03:00
viniciusgadea 399d3492d3 DOCS: update @dadosfera/protospack-v2 to version 3.38.0-beta.15 2025-12-09 14:56:51 -03:00
Marcos Rodrigues Silva b0b557246e Merge pull request #403 from dadosfera/feat/user-properties
FIX:  change origin to host
2025-12-03 15:26:10 -03:00
Marcos Rodrigues bb671a90d6 FIX: change origin to host 2025-12-03 15:20:41 -03:00
Marcos Rodrigues Silva f6ababbe7a Merge pull request #402 from dadosfera/feat/user-properties
FIX: using header host
2025-12-03 14:46:14 -03:00
Marcos Rodrigues 82290285d0 FIX: using header host 2025-12-03 14:41:51 -03:00
Marcos Rodrigues Silva c490814a98 Merge pull request #401 from dadosfera/feat/user-properties
FEAT: add resource origin validate in me endpoint
2025-12-03 13:53:33 -03:00
Marcos Rodrigues 7616b1e32c FEAT: add resource origin validate in me endpoint 2025-12-03 13:51:32 -03:00
Rafael Santana 3ba2c91893 Merge pull request #400 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-02 15:30:03 -03:00
Rafael e789076ed4 UPDATE: force deployment 2025-12-02 15:29:36 -03:00
Rafael Santana 3b6ddaaee6 Merge pull request #399 from dadosfera/fix/redis-tls-configurable
fix: make Redis TLS configurable via REDIS_TLS env var
2025-12-02 15:18:05 -03:00
RafaelandClaude 8073194604 fix: make Redis TLS configurable via REDIS_TLS env var
- Change TLS check from ENV !== 'local' to REDIS_TLS === 'true'
- Add REDIS_TLS env var to Helm deployment template
- Default to true for OCI Redis, can override for self-hosted K8s

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-12-02 15:10:31 -03:00
Marcos Rodrigues Silva e7f410831f Merge pull request #395 from dadosfera/beta
Beta
2025-12-01 17:47:24 -03:00
Marcos Rodrigues Silva cd2c53b5c5 Merge pull request #398 from dadosfera/feat/user-properties
FIX: user response
2025-12-01 17:36:56 -03:00
Marcos Rodrigues adf1b3b97e FIX: user response 2025-12-01 17:30:37 -03:00
Marcos Rodrigues Silva d84b5e184b Merge pull request #397 from dadosfera/feat/user-properties
FEAT: include more user info in /me
2025-12-01 12:36:16 -03:00
Marcos Rodrigues 6d608a0457 FEAT: include more user info in /me 2025-12-01 12:26:33 -03:00
Marcos Rodrigues Silva b8be2c7803 Merge pull request #396 from dadosfera/fix/api-key-catalog-roles
FIX: api key catalog roles
2025-12-01 09:34:21 -03:00
Marcos Rodrigues 24fce721e3 FIX: api key catalog roles 2025-12-01 09:32:26 -03:00
Marcos Rodrigues Silva 23a9a27db1 Merge pull request #394 from dadosfera/fix/private-front-cors
FIX: 404 in  get public key
2025-11-28 16:44:54 -03:00
Marcos Rodrigues 0f5ed50af9 FIX: 404 in get public key 2025-11-28 16:37:38 -03:00
Marcos Rodrigues Silva 5f8f6a64ab Merge pull request #393 from dadosfera/fix/private-front-cors
FIX: include private frontend url in cors origins list
2025-11-28 10:53:45 -03:00
Marcos Rodrigues 8800ac2736 FIX: include private frontend url in cors origins list 2025-11-28 10:50:31 -03:00
Marcos Rodrigues Silva bdb82c2ce4 Merge pull request #392 from dadosfera/fix/authenticated-track-events
Fix/authenticated track events
2025-11-27 09:45:46 -03:00
Marcos Rodrigues 6d9ecc3568 FIX: mixpanel events 2025-11-27 09:43:43 -03:00
Marcos Rodrigues Silva 7764447adc Merge pull request #391 from dadosfera/fix/private-ingress-annotations
FIX: ci to oracle github worker
2025-11-25 11:37:29 -03:00
Marcos Rodrigues 576fdecf89 FIX: ci to oracle github worker 2025-11-25 11:27:04 -03:00
Marcos Rodrigues Silva fff3523152 Merge pull request #389 from dadosfera/fix/private-ingress-annotations
FIX: private ingress annotations
2025-11-25 10:38:52 -03:00
Marcos Rodrigues Silva 8624d3f016 Merge pull request #388 from dadosfera/fix/reset-theme
Fix/reset theme
2025-11-21 11:01:48 -03:00
Marcos Rodrigues 4f4da5bebe FIX: reset theme 2025-11-21 10:51:31 -03:00
Marcos Rodrigues f00d2bf41d FIX: reset cstumer theme 2025-11-21 09:55:39 -03:00
Marcos Rodrigues Silva dc1d1400d8 Merge pull request #387 from dadosfera/feat/lineage
FIX: change wget to curl
2025-11-13 18:19:20 -03:00
Marcos Rodrigues d4451153a3 FIX: change wget to curl 2025-11-13 18:14:47 -03:00
Marcos Rodrigues Silva 06ba759ea3 Merge pull request #386 from dadosfera/feat/lineage
Feat/lineage
2025-11-13 17:58:33 -03:00
Marcos Rodrigues dd0d08ad6e FEAT: add permissions to lineage module 2025-11-13 17:48:55 -03:00
Marcos Rodrigues ec81082877 FIX: resolve cors from localhost 2025-11-13 17:48:38 -03:00
56 changed files with 8178 additions and 2330 deletions
+12
View File
@@ -0,0 +1,12 @@
node_modules
dist
.git
*.log
npm-debug.log*
.DS_Store
.env
.env.*
coverage
.nyc_output
*.tgz
!protospack.tgz
+1 -1
View File
@@ -105,7 +105,7 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
+6 -3
View File
@@ -66,13 +66,16 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Debug Helm env
run: |
helm env
echo "HOME=$HOME"
ls -R $HOME/.local/share/helm || true
- name: Authenticate with OKE cluster
env:
+2 -1
View File
@@ -1,4 +1,5 @@
FROM node:18.17-alpine AS base_image
FROM node:20-alpine AS base_image
RUN npm install -g npm@latest
FROM base_image AS build_base
WORKDIR /app
+47
View File
@@ -0,0 +1,47 @@
FROM node:22-alpine AS base_image
RUN npm install -g npm@latest
FROM base_image AS build_base
WORKDIR /app
RUN apk update
RUN apk add --no-cache \
aws-cli \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY package*.json ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Local build with secrets
FROM build_base AS build
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
RUN npm ci
COPY . .
RUN npm run build
FROM base_image
WORKDIR /app
COPY --from=build /app/dist ./dist
COPY --from=build /app/node_modules ./node_modules
COPY --from=build /app/package*.json ./
RUN apk update
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT ["npm", "run", "start:prod"]
-1
View File
@@ -2,7 +2,6 @@
<image src="./assets/maestro.svg" style="width:10rem">
</p>
# Maestro
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
@@ -48,6 +48,9 @@ spec:
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
# Auth Provider Configuration (cognito or keycloak)
- name: AUTH_PROVIDER
value: {{ .Values.maestro.auth_provider | default "cognito" | quote }}
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
- name: AWS_REGION
@@ -104,6 +107,12 @@ spec:
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: REDIS_TLS
value: "{{ .Values.maestro.redis_tls }}"
- name: PLATFORM_API_URL
value: {{ .Values.maestro.platform_api_url }}
- name: STORAGE_EXPLORER_API_URL
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -124,3 +133,14 @@ spec:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
# Elasticsearch
- name: ELASTICSEARCH_URL
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: ELASTICSEARCH_URL
- name: ELASTICSEARCH_API_KEY
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: ELASTICSEARCH_API_KEY
+12
View File
@@ -38,3 +38,15 @@ spec:
version: "AWSCURRENT"
property: token
- secretKey: ELASTICSEARCH_URL
remoteRef:
key: {{ .Values.maestro.env }}/microservices/elasticsearch
version: "AWSCURRENT"
property: ELASTICSEARCH_URL
- secretKey: ELASTICSEARCH_API_KEY
remoteRef:
key: {{ .Values.maestro.env }}/microservices/elasticsearch
version: "AWSCURRENT"
property: ELASTICSEARCH_API_KEY
+2
View File
@@ -8,6 +8,8 @@ maestro:
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
hostname: maestro.stg.dadosfera.ai
+6
View File
@@ -27,6 +27,9 @@ resources:
cpu: 2000m
memory: 2Gi
maestro:
# Auth provider: "cognito" (default) or "keycloak"
# Note: maestro doesn't connect to Keycloak directly, only duc does
auth_provider: "cognito"
aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
duc_url: duc.dadosfera.ai
in_factory_url: in-factory.dadosfera.ai
@@ -43,11 +46,14 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
redis_tls: "true"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
+2140 -286
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -16,6 +16,7 @@ declare global {
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
REDIS_TLS?: string;
}
}
}
+2525 -1780
View File
File diff suppressed because it is too large Load Diff
+15 -4
View File
@@ -27,10 +27,15 @@
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@aws-crypto/sha256-js": "^5.2.0",
"@aws-sdk/client-dynamodb": "^3.414.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@aws-sdk/credential-provider-node": "^3.940.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -44,7 +49,7 @@
"@nestjs/schematics": "^9.2.0",
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"axios": "^0.30.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
@@ -75,11 +80,17 @@
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "1.4.5-lts.1"
"multer": "2.0.2",
"form-data": "^4.0.4",
"body-parser": "^1.20.3",
"cross-spawn": "^7.0.5",
"glob": "^10.5.0",
"path-to-regexp": "^3.3.0",
"semver": "^7.5.2"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/cookie-parser": "^1.4.9",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
+4
View File
@@ -33,6 +33,8 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
@Module({
providers: [
@@ -73,6 +75,8 @@ import { ApiKeyModule } from './modules/api-key/api-key.module';
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
PlatformApiModule,
StorageExplorerModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
],
+20
View File
@@ -0,0 +1,20 @@
import jwt, { JwtPayload } from 'jsonwebtoken';
export function extractUserFrom(aRawJwt: string) {
const decodedToken = jwt.decode(aRawJwt, {
complete: true,
});
const payload = decodedToken.payload as JwtPayload;
return {
user_id: payload.user_id,
username: payload.username,
permissions: payload.permissions,
customer_id: payload.customer_id,
customer_name: payload.customer_name,
customer_tier: payload.customer_tier,
customer_modules: payload.customer_modules,
access_token: aRawJwt,
}
}
+48
View File
@@ -390,6 +390,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
LINEAGE: {
title: {
'pt-br': 'Explorar | Linhagem',
'en-us': 'Explore | Lineage',
'es-es': 'Explorar | Linaje',
},
permissions: {
VIEW: {
seqid: 50,
claim: 'lineage:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar ao módulo de Linhagem',
'en-us': 'Access to Lineage module',
'es-es': 'Acceda al módulo de Linaje',
},
}
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
@@ -649,6 +668,35 @@ export const PERMISSIONS_GROUPS = {
},
},
},
STORAGE_EXPLORER: {
title: {
'pt-br': 'Storage Explorer',
'en-us': 'Storage Explorer',
'es-es': 'Storage Explorer',
},
permissions: {
READ: {
seqid: 51,
claim: 'storage-explorer:read',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Ler dados do Storage Explorer',
'en-us': 'Read Storage Explorer data',
'es-es': 'Leer datos del Storage Explorer',
},
},
WRITE: {
seqid: 52,
claim: 'storage-explorer:write',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Escrever dados no Storage Explorer',
'en-us': 'Write Storage Explorer data',
'es-es': 'Escribir datos en Storage Explorer',
},
},
},
},
};
export interface DadosferaModule {
name: string;
+19 -8
View File
@@ -18,25 +18,36 @@ async function bootstrap() {
});
const logger = new DadosferaLogger();
const corsOrigins = [];
if (process.env.ENV === 'local') {
corsOrigins.push('http://localhost:4200');
} else {
corsOrigins.push(
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://private-frontend.stg.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
);
}
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: [
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
],
origin: corsOrigins,
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true,
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
if (process.env.ENV !== 'local') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use(
'/catalog/register-dataset',
+6 -2
View File
@@ -1,4 +1,4 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
@@ -28,6 +28,10 @@ export class AssignController {
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
try {
return await this.assignService.get(metadata);
} catch (error) {
throw new NotFoundException(error.message)
}
}
}
+54 -128
View File
@@ -13,6 +13,7 @@ import {
Req,
Param,
Res,
UnauthorizedException,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -54,7 +55,7 @@ import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
type CookiesValues = {
accessToken?: string;
@@ -74,6 +75,7 @@ export class AuthController {
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
@@ -103,14 +105,13 @@ export class AuthController {
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
this.authClient.writeAuthSession(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
@@ -127,27 +128,8 @@ export class AuthController {
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
this.authClient.cleanUpAuthSession(res);
return res.send();
} catch (error) {
@@ -176,8 +158,9 @@ export class AuthController {
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
this.authClient.writeAuthSession(res, {
accessToken: data.accessToken,
refreshToken: data.refreshToken,
userId
});
@@ -209,13 +192,14 @@ export class AuthController {
) {
this.logger.info('/auth - change-password');
const { oldPassword, newPassword } = body;
const { oldPassword, newPassword, totpCode } = body;
const { authorization: accessToken } = headers;
return this.authClient.changePassword({
accessToken,
oldPassword,
newPassword,
totpCode,
});
}
@@ -233,7 +217,8 @@ export class AuthController {
const { username } = body;
return this.authClient.resetPassword({ username }, metadata);
await this.authClient.resetPassword({ username }, metadata);
return { authProvider: process.env.AUTH_PROVIDER || 'cognito' };
}
@ApiInternalOnlyEndpoint()
@@ -493,114 +478,55 @@ export class AuthController {
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
// Check for API key header first
const apiKey = req.get('X-Api-key');
if (apiKey) {
this.logger.info('Authenticating via X-Api-key header');
const { api_key } = await this.apiKeyService.get(apiKey);
const userDto = {
id: api_key.user_id,
name: api_key.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
id: api_key.customer_id,
name: api_key.customer_name,
tier: api_key.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
return res.status(200).json(userDto);
}
// Get token and headers
const accessToken = req.cookies['ddf-auth'];
const refreshToken = req.cookies['ddf-refresh-auth'];
const userId = req.cookies['ddf-user-id'];
const resourceHost = req.headers["x-original-url"] as string || "" ;
const hasUserSession = Boolean(accessToken) && Boolean(userId);
this.logger.info('Has User Session: ' + hasUserSession);
if (!hasUserSession) {
throw new UnauthorizedException()
}
try {
const userDto = await this.authClient.validateUserSession(accessToken, resourceHost);
return res.status(200).json(userDto);
} catch (error) {
if (!refreshToken) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
throw new UnauthorizedException("Invalid refresh token or customer name");
};
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
const {
authSession,
user
} = await this.authClient.refreshUserSession(refreshToken, userId, resourceHost);
this.authClient.writeAuthSession(res, authSession);
return res.status(200).json(user);
}
}
}
+2 -1
View File
@@ -8,10 +8,11 @@ import { AuthClientService } from './auth.service';
import { DucClient } from '../duc/client.config';
import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
import { getOauthSecrets } from 'src/utils/OauthSecrets';
import { ApiKeyModule } from '../api-key/api-key.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule],
controllers: [AuthController],
providers: [
AuthClientService,
+225 -17
View File
@@ -1,10 +1,20 @@
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import {
OnModuleInit,
Inject,
Injectable,
ForbiddenException,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthProtoService as AuthServiceInterface,
UsersProtoService,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -21,18 +31,24 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
import { BulkEditResponse, UserDTO } from './dtos/login';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { Request, Response } from 'express';
type AuthSession = {
accessToken?: string;
refreshToken?: string;
userId?: string;
};
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
private userService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -46,8 +62,8 @@ export class AuthClientService implements OnModuleInit {
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
this.userService = this.grpcClient.getService<UsersProtoService>(
ProtoServices.UsersProtoService,
);
}
@@ -63,11 +79,11 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
checkDedicatedProxy({ customer }: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info(
'SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY,
);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
@@ -75,7 +91,9 @@ export class AuthClientService implements OnModuleInit {
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
this.logger.info(
'Check if customer have network policy: ' + customer.modules,
);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
@@ -94,7 +112,6 @@ export class AuthClientService implements OnModuleInit {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
@@ -105,7 +122,7 @@ export class AuthClientService implements OnModuleInit {
this.checkDedicatedProxy(result);
}
return result
return result;
}
async refreshAccessToken(
@@ -115,7 +132,10 @@ export class AuthClientService implements OnModuleInit {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
this.authService.AuthRefreshAccessToken(
{ refreshToken, userId },
metadata,
),
);
}
@@ -123,6 +143,7 @@ export class AuthClientService implements OnModuleInit {
accessToken,
oldPassword,
newPassword,
totpCode,
}: AuthChangePasswordRequest) {
this.logger.info('ChangePassword');
@@ -131,6 +152,7 @@ export class AuthClientService implements OnModuleInit {
accessToken,
oldPassword,
newPassword,
totpCode,
}),
);
}
@@ -283,4 +305,190 @@ export class AuthClientService implements OnModuleInit {
throw error;
}
}
public async validateUserSession(accessToken: any, resourceHost: string) {
const payload = await this.validateJwtToken(accessToken);
const userDto = await this.getUserfromPayload(payload);
this.validateResourceAccess(resourceHost, userDto);
return userDto;
}
public async refreshUserSession(
refreshToken: string,
userId: string,
originHeader: string,
): Promise<{
user: UserDTO;
authSession: AuthSession;
}> {
const metadata = PackTheMetadata({});
this.logger.info('Call Refresh Token');
const refreshCredentials = await this.refreshAccessToken(
{ refreshToken, userId },
metadata,
);
this.logger.info('Finish Refresh Token');
const userDto = await this.validateUserSession(
refreshCredentials.accessToken,
originHeader,
);
return {
user: userDto,
authSession: {
accessToken: refreshCredentials.accessToken,
refreshToken: refreshCredentials.refreshToken,
userId,
},
};
}
public writeAuthSession(res: Response, data: AuthSession) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth');
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth');
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth');
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
public cleanUpAuthSession(res: Response) {
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.ai',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.ai',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
}
private async validateJwtToken(token: string) {
const decoded: any = token && jwt.decode(token, { complete: true });
if (!decoded) throw new Error('Invalid token');
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(token, pemValue);
return decoded.payload;
}
private async getUserfromPayload(payload: JwtPayload): Promise<UserDTO> {
this.logger.info('getUser');
const metadata = PackTheMetadata({
customer_id: payload.customer_id,
});
const { user } = await lastValueFrom(
this.userService.UserFindOneById({ id: payload.user_id }, metadata),
);
const userDto: UserDTO = {
id: user.id,
name: user.username,
jobTitle: user?.jobTitle || null,
department: user?.department || null,
hierarchy: user?.hierarchy || null,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
},
};
return userDto;
}
private validateResourceAccess(host: string, user: UserDTO) {
this.logger.info(
"Validate whether the source URL is a resource belonging to the user's client",
);
this.logger.info('Host: ' + host);
this.logger.info('Customer: ' + user.customer.name);
const hostParts = host.split('.');
const domain = hostParts[0];
const isResouceStg = hostParts[1] === 'stg';
const notFoundCustomerInDomain = !domain.includes('-')
if (notFoundCustomerInDomain) {
this.logger.info(`Not found Customer Name in domain`);
return;
}
const domainParts = domain.split('-');
const customerInDomain = domainParts[domainParts.length - 1];
if (isResouceStg && process.env.ENV !== 'stg') {
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
throw new HttpException(
`Customer ${user.customer.name} cannot access ${host}`,
HttpStatus.FORBIDDEN
);
}
if (customerInDomain != user.customer.name) {
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
throw new HttpException(
`Customer ${user.customer.name} cannot access ${host}`,
HttpStatus.FORBIDDEN
);
}
return;
}
}
+13
View File
@@ -140,3 +140,16 @@ export interface BulkEditResponse {
successfulUsers: string[];
failedUsers: string[];
}
export type UserDTO = {
id: string,
name: string,
jobTitle?: string,
department?: string,
hierarchy?: string,
customer: {
id: string,
name: string,
tier: string,
}
}
+17 -5
View File
@@ -423,6 +423,7 @@ export class CatalogController {
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('asset_type') asset_type: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -439,7 +440,7 @@ export class CatalogController {
language,
});
const docs = await this.catalogService.getDataDocs(id, metadata);
const docs = await this.catalogService.getDataDocs(id, asset_type, metadata);
return { docs };
}
@@ -487,21 +488,32 @@ export class CatalogController {
@Headers() headers,
@Param('id') table_id: string,
@Body('docs') docs: string,
@Query('asset_type') asset_type: string,
) {
const { user_id, customer_name } = user;
const { user_id, customer_name, customer_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const res = await this.catalogService.createDataDocs({
const body = {
table_id,
docs,
asset_type,
info: {
customer: customer_name,
},
});
}
const res = await this.catalogService.createDataDocs(body, metadata);
return res;
}
+204 -55
View File
@@ -24,9 +24,11 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
CreateDataDocsDTO,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
@@ -85,25 +87,23 @@ class CatalogService implements OnModuleInit {
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
this.logger.info('getPiiReporter: ' + type);
try {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const { data } = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata),
);
this.logger.info('Finish grpc call');
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
this.logger.info('parser file to: ' + type);
const file = await parser.parse(data);
this.logger.info('finish parser');
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
csv: 'text/csv',
html: 'text/html',
pdf: 'application/pdf',
};
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
@@ -111,13 +111,12 @@ class CatalogService implements OnModuleInit {
return {
file,
filename: filename,
type: mimeTypes[type]
}
type: mimeTypes[type],
};
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
@@ -197,9 +196,11 @@ class CatalogService implements OnModuleInit {
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
const roles_ids = result.user.roles.map((role) => role.id);
if (result) {
return result.user.roles.map((role) => role.id);
}
return roles_ids;
return [];
}
async searchDataAssets(
@@ -207,10 +208,99 @@ class CatalogService implements OnModuleInit {
metadata: Metadata,
customer_id: string,
) {
this.logger.info('CatalogService - searchDataAssets');
this.logger.info('CatalogService - searchDataAssets', { query });
const { search, page, size, sort_by, order, ...filters } = query;
this.logger.debug('Extracted filters:', { filters });
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (
filters.manually !== undefined &&
filters.manually !== null &&
filters.manually !== ''
) {
filters.manually = Number(filters.manually); // 1 ou 0
} else {
delete filters.manually;
}
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (filters.owner) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
this.logger.info('Available users in database count:', {
count: customer_users.length,
});
this.logger.info('First 5 users:', {
users: customer_users
.slice(0, 5)
.map((u) => ({ id: u.id, email: u.email, name: u.name })),
});
const ownerValues = Array.isArray(filters.owner)
? filters.owner
: typeof filters.owner === 'string' && filters.owner.includes(',')
? filters.owner.split(',').map((o: string) => o.trim())
: [filters.owner];
this.logger.info('Owner values to convert:', {
ownerValues,
ownerFiltersOriginal: filters.owner,
});
const ownerIds = ownerValues
.map((ownerValue: string) => {
const normalizedOwner = ownerValue.replace(/\s/g, '+');
const user = customer_users.find((u) => {
const isIdMatch = u.id === ownerValue;
const isEmailMatch =
u.email === ownerValue || u.email === normalizedOwner;
const isNameMatch =
u.name === ownerValue || u.name === normalizedOwner;
this.logger.info('Comparing:', {
userId: u.id,
userEmail: u.email,
userName: u.name,
filterValue: ownerValue,
normalizedFilter: normalizedOwner,
idMatch: isIdMatch,
emailMatch: isEmailMatch,
nameMatch: isNameMatch,
});
return isIdMatch || isEmailMatch || isNameMatch;
});
this.logger.info('Looking for owner result:', {
ownerValue,
found: !!user,
userId: user?.id,
});
return user?.id || ownerValue;
})
.filter((id: string) => id);
if (ownerIds.length > 0) {
filters.owner = ownerIds;
}
}
const { data_assets, total } = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(
{
@@ -225,6 +315,8 @@ class CatalogService implements OnModuleInit {
),
);
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
const result = JSON.parse(data_assets);
const response = await this.getAssetsUsersAndRoles(
@@ -242,13 +334,13 @@ class CatalogService implements OnModuleInit {
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
const formatData = data.data_assets.map((asset) => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
tags: '[' + asset.tags.join(', ') + ']',
}));
const parser = ParserBuilder.build<AssetReporter>('csv');
@@ -259,8 +351,8 @@ class CatalogService implements OnModuleInit {
return {
file,
filename
}
filename,
};
}
async getOneDataAsset(data: {
@@ -335,11 +427,11 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] };
}
async getDataDocs(id: string, metadata: Metadata) {
async getDataDocs(id: string, assetType: string, metadata: Metadata) {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
const docs = JSON.parse(documentation);
return docs;
}
@@ -366,7 +458,16 @@ class CatalogService implements OnModuleInit {
return result;
}
async createDataDocs(body) {
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
if (body.asset_type === 'table' || body.asset_type === 'view') {
return this.createDataDocsViaNimbus(body);
}
return this.createDataDocsViaGrpc(body, metadata);
}
private async createDataDocsViaNimbus(body: CreateDataDocsDTO) {
this.logger.info('Creating data docs via Nimbus for table/view');
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-docs/`,
@@ -375,6 +476,29 @@ class CatalogService implements OnModuleInit {
return data;
}
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
this.logger.info('Creating data docs via gRPC for other asset types');
try {
const response: any = await lastValueFrom(
this.catalogWriteService.UpdateDataAssetDoc(
{
id: body.table_id,
docs: body.docs,
},
metadata,
),
);
return response;
} catch (error) {
this.logger.error('Error creating data asset docs:', error);
throw new HttpException(
'Failed to create data asset documentation',
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
@@ -392,6 +516,7 @@ class CatalogService implements OnModuleInit {
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
@@ -402,20 +527,20 @@ class CatalogService implements OnModuleInit {
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
)?.email;
const roles = [];
const users = [];
const data_asset_roles = data_asset?.roles || []
const data_asset_roles = data_asset?.roles || [];
for (const role_id of data_asset_roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
const data_asset_users = data_asset?.users || []
const data_asset_users = data_asset?.users || [];
for (const user_id of data_asset_users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
if (user) users.push({ id: user.id, email: user.email });
}
return {
...data_asset,
@@ -505,27 +630,36 @@ class CatalogService implements OnModuleInit {
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
this.logger.info(
`Creating table metadata for table ${body.table_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
...body.logMetadata,
});
try {
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
const { data, status } = await axios.post(endpoint, {
...body.table_metadata,
});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
} because of ${JSON.stringify(error.response?.data) || error.message}`,
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
@@ -535,43 +669,58 @@ class CatalogService implements OnModuleInit {
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Creating column metadata for table ${body.column_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.column_metadata)}`,
{ ...body.logMetadata },
);
const { data, status } = await axios.post(
endpoint,
body.column_metadata,
);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
} because of ${error.response?.data || error.message}`,
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
this.logger.info(
`Creating data preview for table ${body.data_preview.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.data_preview)}`,
{ ...body.logMetadata },
);
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.id;
} catch (error) {
@@ -579,7 +728,7 @@ class CatalogService implements OnModuleInit {
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
@@ -610,4 +759,4 @@ class CatalogService implements OnModuleInit {
}
}
export { CatalogService };
export { CatalogService };
+31
View File
@@ -98,6 +98,8 @@ export class IDataAsset {
embed?: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class IOneDataAsset {
@@ -147,6 +149,24 @@ export class ICatalogAllRequest {
description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
})
order?: OrderEnum;
@ApiPropertyOptional({
description: 'ID do usuário owner para filtrar data assets',
example: 'user-id-1,user-id-2',
})
owner?: string;
@ApiPropertyOptional({
description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
example: '2025-01-01',
})
catalog_date_from?: string;
@ApiPropertyOptional({
description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
example: '2025-12-31',
})
catalog_date_to?: string;
}
export class ICatalogAllResponse {
@@ -182,6 +202,8 @@ export class IUpdateDataRequest {
embed: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
@@ -196,6 +218,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
location: string;
@ApiPropertyOptional()
embed: EmbedObject;
@ApiPropertyOptional()
docs: string;
}
export class IPreview {
@@ -328,3 +352,10 @@ export type AssetReporter = {
created_at: string;
tags: string;
}
export type CreateDataDocsDTO = {
table_id: string;
docs: string;
asset_type: string;
}
+3 -3
View File
@@ -160,7 +160,7 @@ export class ShareService implements OnModuleInit {
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
@@ -180,7 +180,7 @@ export class ShareService implements OnModuleInit {
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
)?.email;
const roles = [];
const users = [];
@@ -190,7 +190,7 @@ export class ShareService implements OnModuleInit {
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
if (user) users.push({ id: user.id, email: user.email });
}
return {
...data_asset,
@@ -136,4 +136,32 @@ export class CustomersController {
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
return result;
}
@Get(':id/organization-info')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ description: 'Organization information' })
async getOrganizationInfo(@Param('id') id: string) {
this.logger.info('getOrganizationInfo', { id });
return this.customersService.getOrganizationInfo(id);
}
@Put(':id/organization-info')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ description: 'Organization information updated' })
async updateOrganizationInfo(
@Param('id') id: string,
@Body() body: {
companyName: string;
companySite: string;
domain: string;
cnpj: string;
description: string;
},
) {
return this.customersService.updateOrganizationInfo(id, body);
}
}
+54 -1
View File
@@ -223,4 +223,57 @@ export class CustomersService implements OnModuleInit {
})
)
}
}
async updateOrganizationInfo(
customerId: string,
data: {
companyName: string;
companySite: string;
domain: string;
cnpj: string;
description: string;
},
) {
try {
const result = await lastValueFrom(
this.customerService.OrganizationUpdate({
customerId,
companyName: data.companyName || '',
companySite: data.companySite || '',
domain: data.domain || '',
cnpj: data.cnpj || '',
description: data.description || '',
}),
);
return result;
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
async getOrganizationInfo(customerId: string) {
try {
const customerResponse = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId })
);
const customer = customerResponse.customer;
return {
companyName: customer.companyName || '',
companySite: customer.companySite || '',
domain: customer.domain || '',
cnpj: customer.cnpj || '',
description: customer.description || ''
};
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
}
@@ -0,0 +1,27 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class OrganizationUpdateRequest {
@ApiProperty()
name: string;
@ApiPropertyOptional()
companySite: string;
@ApiProperty()
domain: string;
@ApiPropertyOptional()
info: string;
@ApiPropertyOptional()
cnpj: string;
}
export class OrganizationResponse {
@ApiProperty()
name: string;
@ApiPropertyOptional()
companySite: string;
@ApiProperty()
domain: string;
@ApiPropertyOptional()
info: string;
@ApiPropertyOptional()
cnpj: string;
}
+25 -8
View File
@@ -1,29 +1,46 @@
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
import { extractUserFrom } from 'src/authentication/extract-user';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@ApiInternalOnlyController()
@Controller('trackEvent')
export class MixpanelController {
logger: DadosferaLogger;
constructor(
private mixpanelService: MixpanelService
) {}
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private mixpanelService: MixpanelService,
) {
this.logger = dadosferaLogger.logger;
}
@Post(':id')
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
this.logger.info(`POST Track Event: ${id}`)
delete body.info;
const anonymousUser = {
username: "anonymous",
customer_name: "anonymous"
} as RequestUser
const hasToken = request.headers['authorization'];
const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
this.logger.info(`Has user: ${typeof hasToken == "string"}`)
await this.mixpanelService.track(id, user, request, body)
this.logger.info(`Event successful`)
return { id, body, user: user.username };
}
}
@@ -0,0 +1,11 @@
export const PLATFORM_API_CONFIG = {
getUrl: (): string => {
const url = process.env.PLATFORM_API_URL;
if (!url) {
throw new Error('PLATFORM_API_URL environment variable is not set');
}
return url;
},
region: process.env.AWS_REGION || 'us-east-1',
timeout: parseInt(process.env.PLATFORM_API_TIMEOUT || '30000', 10),
};
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,17 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PlatformApiController } from './platform-api.controller';
import { PlatformApiService } from './platform-api.service';
import { ElasticsearchModule } from '../../services/elasticsearch';
import { DynamoDBModule } from '../../services/dynamodb';
import { CustomersModule } from '../customers/customers.module';
@Module({
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
controllers: [PlatformApiController],
providers: [PlatformApiService, DadosferaLogger],
exports: [PlatformApiService],
})
export class PlatformApiModule {}
@@ -0,0 +1,123 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import { SignatureV4 } from '@aws-sdk/signature-v4';
import { Sha256 } from '@aws-crypto/sha256-js';
import { defaultProvider } from '@aws-sdk/credential-provider-node';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { PLATFORM_API_CONFIG } from './platform-api.config';
@Injectable()
export class PlatformApiService {
private signer: SignatureV4;
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
this.signer = new SignatureV4({
service: 'execute-api',
region: PLATFORM_API_CONFIG.region,
credentials: defaultProvider(),
sha256: Sha256,
});
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, string>,
): Promise<any> {
const baseUrl = PLATFORM_API_CONFIG.getUrl();
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
host: url.hostname,
'content-type': 'application/json',
// Forward user context headers
// Note: platform-api expects customer_name in the 'customer_id' header (contract inconsistency)
'customer_id': user.customer_name || '',
'customer_name': user.customer_name || '',
'x-user-id': user.user_id || '',
'x-username': user.username || '',
'x-customer-tier': user.customer_tier || '',
'x-customer-id': user.customer_id || '',
};
const requestToSign = {
method: method.toUpperCase(),
protocol: url.protocol,
hostname: url.hostname,
port: url.port ? parseInt(url.port, 10) : undefined,
path: url.pathname + url.search,
headers,
body: body ? JSON.stringify(body) : undefined,
};
this.logger.info('Proxying request to platform-api', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
user_id: user.user_id,
});
try {
// Sign with IAM v4
const signedRequest = await this.signer.sign(requestToSign);
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers: signedRequest.headers as Record<string, string>,
data: body,
timeout: PLATFORM_API_CONFIG.timeout,
validateStatus: () => true, // Don't throw on non-2xx
});
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Platform API proxy error', {
error: error.message,
status: error.response?.status,
path,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Platform API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Platform API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
}
+22 -16
View File
@@ -229,27 +229,33 @@ export class RolesService {
const [roleTreated] = this.getRolesPermissionsName([role.role]);
return { role: roleTreated };
}
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
const newRoles: RoleDto[] = [];
for (const role of roles) {
const allPermissions = this.permissionsService.getAllPermissions(
this.language,
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
const newRole: RoleDto = {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};
const newRole: RoleDto = this.formatRole(role);
newRoles.push(newRole);
}
return newRoles;
}
private formatRole(role: GetRolesPermissionsName): RoleDto {
const allPermissions = this.permissionsService.getAllPermissions(
this.language
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
return {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};;
}
}
@@ -0,0 +1,13 @@
export const STORAGE_EXPLORER_CONFIG = {
getUrl: (customerName: string): string => {
const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
if (!urlTemplate) {
throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
}
// Replace {customer_id} placeholder with actual customer ID
// For local: http://172.17.0.1:8000/api (no placeholder)
// For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
return urlTemplate.replace('{customer}', customerName);
},
timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
};
@@ -0,0 +1,383 @@
import {
Controller,
Get,
Post,
Put,
Param,
Body,
Query,
Inject,
UseInterceptors,
UploadedFiles,
Headers,
} from '@nestjs/common';
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
import { FilesInterceptor } from '@nestjs/platform-express';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import FormData from 'form-data';
import {
Authenticated,
RequireAllPermissions,
} from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator';
import { StorageExplorerService } from './storage-explorer.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
@ApiTags('Storage Explorer')
@Controller('storage-explorer')
@Authenticated()
export class StorageExplorerController {
private logger: any;
constructor(
private readonly storageExplorerService: StorageExplorerService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
// ============================================
// TABLE OPERATIONS
// ============================================
@ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' })
@Post('tables/validate-name')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async validateTableName(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/validate-name',
user,
body,
);
}
@ApiOperation({ summary: 'Create a new table' })
@Post('tables')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async createTable(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/',
user,
body,
);
}
@ApiOperation({ summary: 'List all tables with pagination' })
@Get('tables')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listTables(
@Query('page') page: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/tables/',
user,
undefined,
{ page },
);
}
@ApiOperation({ summary: 'Get table details by ID' })
@Get('tables/:tableId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTable(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}`,
user,
);
}
@ApiOperation({ summary: 'Link a dataset to a table' })
@Post('tables/:tableId/datasets/:datasetId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async linkDatasetToTable(
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/datasets/${datasetId}`,
user,
);
}
@ApiOperation({ summary: 'Get all datasets linked to a table' })
@Get('tables/:tableId/datasets')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTableDatasets(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/datasets`,
user,
);
}
@ApiOperation({ summary: 'Get table schema' })
@Get('tables/:tableId/schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTableSchema(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/schema`,
user,
);
}
@ApiOperation({ summary: 'Validate schema compatibility between table and dataset' })
@Post('tables/:tableId/validate-compatibility/:datasetId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async validateSchemaCompatibility(
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/validate-compatibility/${datasetId}`,
user,
);
}
// ============================================
// DATASET OPERATIONS
// ============================================
@ApiOperation({ summary: 'Get dataset preview data' })
@Get('datasets/:datasetId/preview')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getDatasetPreview(
@Param('datasetId') datasetId: string,
@Query('limit') limit: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/preview`,
user,
undefined,
{ limit },
);
}
@ApiOperation({ summary: 'Get dataset schema information' })
@Get('datasets/:datasetId/schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getDatasetSchema(
@Param('datasetId') datasetId: string,
@Query('force_refresh') forceRefresh: boolean,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/schema`,
user,
undefined,
{ force_refresh: forceRefresh },
);
}
@ApiOperation({ summary: 'List all datasets for a specific upload' })
@Get('datasets/upload/:uploadId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listDatasetsByUpload(
@Param('uploadId') uploadId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/upload/${uploadId}`,
user,
);
}
@ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' })
@Put('datasets/:datasetId/refresh-schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async refreshDatasetSchema(
@Param('datasetId') datasetId: string,
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'PUT',
`/datasets/${datasetId}/refresh-schema`,
user,
body,
);
}
// ============================================
// STORAGE OPERATIONS
// ============================================
@ApiOperation({ summary: 'List file explorer uploads with pagination' })
@Get('storage/uploads/history')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listFileExplorerUploads(
@Query('page') page: number,
@Query('limit') limit: number,
@Query('folder_path') folderPath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/uploads/history',
user,
undefined,
{ page, limit, folder_path: folderPath },
);
}
@ApiOperation({ summary: 'Browse folders and files in storage' })
@Get('storage/browse')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async browseStorage(
@Query('path') path: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/browse',
user,
undefined,
{ path },
);
}
@ApiOperation({ summary: 'Upload multiple files to storage' })
@Post('storage/upload/batch')
@ApiConsumes('multipart/form-data')
@UseInterceptors(FilesInterceptor('files'))
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async batchUpload(
@UploadedFiles() files: Array<Express.Multer.File>,
@Body('folder_path') folderPath: string,
@User() user: RequestUser,
) {
// Create FormData to forward files to storage-explorer API
const formData = new FormData();
// Add files
if (files && files.length > 0) {
files.forEach((file) => {
formData.append('files', file.buffer, {
filename: file.originalname,
contentType: file.mimetype,
});
});
}
// Add folder_path
if (folderPath) {
formData.append('folder_path', folderPath);
}
return this.storageExplorerService.proxyFormData(
'POST',
'/storage/upload/batch',
user,
formData,
);
}
@ApiOperation({ summary: 'Create a new folder in storage' })
@Post('storage/folder/create')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async createFolder(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/storage/folder/create',
user,
body,
);
}
@ApiOperation({ summary: 'Download a file from storage' })
@Get('storage/download')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async downloadFile(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/download',
user,
undefined,
{ file_path: filePath },
);
}
@ApiOperation({ summary: 'Get detailed file metadata' })
@Get('storage/metadata')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getFileMetadata(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/metadata',
user,
undefined,
{ file_path: filePath },
);
}
}
@@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { StorageExplorerController } from './storage-explorer.controller';
import { StorageExplorerService } from './storage-explorer.service';
@Module({
imports: [],
controllers: [StorageExplorerController],
providers: [StorageExplorerService, DadosferaLogger],
exports: [StorageExplorerService],
})
export class StorageExplorerModule {}
@@ -0,0 +1,178 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { STORAGE_EXPLORER_CONFIG } from './storage-explorer.config';
@Injectable()
export class StorageExplorerService {
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, any>
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
throw new HttpException('Customer ID is required for storage operations', 400);
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
'content-type': 'application/json',
};
this.logger.info('Proxying request to storage-explorer', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
storage_url: baseUrl,
user_id: user.user_id,
});
try {
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers,
data: body,
timeout: STORAGE_EXPLORER_CONFIG.timeout,
validateStatus: () => true, // Don't throw on non-2xx
});
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Storage Explorer API proxy error', {
error: error.message,
status: error.response?.status,
path,
storage_url: baseUrl,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Storage Explorer API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Storage Explorer API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
/**
* Proxy with file upload support (multipart/form-data)
*/
async proxyFormData(
method: string,
path: string,
user: RequestUser,
formData: any,
query?: Record<string, any>,
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
throw new HttpException('Customer ID is required for storage operations', 400);
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
// Let axios set Content-Type for multipart/form-data with boundary
...formData.getHeaders?.(),
};
this.logger.info('Proxying form data request to storage-explorer', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
storage_url: baseUrl,
user_id: user.user_id,
});
try {
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers,
data: formData,
timeout: STORAGE_EXPLORER_CONFIG.timeout,
maxContentLength: Infinity,
maxBodyLength: Infinity,
validateStatus: () => true,
});
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Storage Explorer API form data proxy error', {
error: error.message,
status: error.response?.status,
path,
storage_url: baseUrl,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
throw new HttpException('Internal server error', 500);
}
}
}
+20
View File
@@ -124,4 +124,24 @@ export class ThemeController {
}
}
@Post('/:id/theme/reset')
@ApiOkResponse({ type: CustomerThemeResponse })
async resetTheme(@Param('id') id: string) {
this.logger.info('getCustomerTheme with id' + id);
try {
await this.themeService.resetTheme(id);
return { theme: null };
}catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
} else {
this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message);
return { theme: null };
};
}
}
}
+12
View File
@@ -44,6 +44,18 @@ export class ThemeService implements OnModuleInit {
);
}
async resetTheme(id: string) {
const { theme } = await firstValueFrom(
this.themeService.ResetCustomerTheme({
id
}),
);
return {
theme
}
}
async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) {
if (!id) {
this.logger.error('Error - saveCustomertheme - not found id:' + id);
+14
View File
@@ -38,6 +38,12 @@ export class User {
department?: string;
@ApiProperty()
hierarchy?: string;
@ApiProperty()
bio?: string;
@ApiProperty()
companyName?: string;
@ApiProperty()
personalSite?: string;
@ApiPropertyOptional()
customer?: Customer;
@ApiProperty()
@@ -58,6 +64,8 @@ export class UserNoRolesAndCustomer extends OmitType(UserNoRoles, [
export class IUserByCustomer extends OmitType(User, ['customer']) {
@ApiPropertyOptional()
permissions?: string[];
@ApiPropertyOptional()
authProvider?: string;
}
export class CreateUserReq {
@@ -109,6 +117,12 @@ export class UpdateUserReq {
@ApiPropertyOptional()
hierarchy?: string;
@ApiPropertyOptional()
bio?: string;
@ApiPropertyOptional()
personalSite?: string;
@ApiPropertyOptional()
companyName?: string;
@ApiPropertyOptional()
roleNames?: string[];
}
+12 -1
View File
@@ -266,7 +266,6 @@ export class UsersController {
}
@Patch(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: UpdateUserRes })
async updateUser(
@User() user: RequestUser,
@@ -274,6 +273,18 @@ export class UsersController {
@Param('id') id: string,
@Language() language: LanguageEnum,
) {
const isSameUser = user.user_id === id;
const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
if (!isSameUser && !isSuperAdmin) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
if (isSameUser && !isSuperAdmin && body.roleNames) {
// Prevent users from updating their own roles
delete body.roleNames;
}
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
return await this.userService.updateUser(body, id, user.customer_id);
+11 -7
View File
@@ -125,6 +125,7 @@ export class UsersService implements OnModuleInit {
return { permissions };
});
res.user.permissions = permissions;
res.user.authProvider = process.env.AUTH_PROVIDER || 'cognito';
return res;
}
@@ -148,20 +149,23 @@ export class UsersService implements OnModuleInit {
}
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
if (roleNames) {
const { roleNames, ...updateUserDTO } = req;
if (roleNames && roleNames.length > 0) {
await this.setRoles({ roleNames, userId: id }, customerId);
}
const { user } = await lastValueFrom(
this.usersClientService.UserUpdate({
name,
department: updateUserDTO.department,
email: updateUserDTO.email,
hierarchy: updateUserDTO.hierarchy,
jobTitle: updateUserDTO.jobTitle,
name: updateUserDTO.name,
bio: updateUserDTO.bio,
companyName: updateUserDTO.companyName,
personalSite: updateUserDTO.personalSite,
customerId,
id,
department,
hierarchy,
jobTitle,
email,
metabaseUserId: undefined,
}),
);
+4 -1
View File
@@ -14,7 +14,10 @@ export class ValidationPipe implements PipeTransform<any> {
return value;
}
const object = plainToInstance(metatype, value);
const errors = await validate(object);
const errors = await validate(object, {
forbidUnknownValues: false,
whitelist: true,
});
if (errors.length > 0) {
const errorMessages = errors.map((err) => err.constraints);
throw new BadRequestException(errorMessages);
+4
View File
@@ -0,0 +1,4 @@
export const DYNAMODB_CONFIG = {
region: () => process.env.AWS_REGION || 'us-east-1',
inputsTable: () => process.env.INPUTS_DB || 'dadosfera-inputs-prd',
};
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { DynamoDBService } from './dynamodb.service';
@Module({
providers: [DynamoDBService, DadosferaLogger],
exports: [DynamoDBService],
})
export class DynamoDBModule {}
+234
View File
@@ -0,0 +1,234 @@
import { Injectable, Inject } from '@nestjs/common';
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
import {
DynamoDBDocumentClient,
GetCommand,
PutCommand,
DeleteCommand,
TranslateConfig,
} from '@aws-sdk/lib-dynamodb';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { v4 as uuid } from 'uuid';
import { DYNAMODB_CONFIG } from './dynamodb.config';
export interface ReferenceColumn {
name: string;
type: string;
}
export interface InputDocument {
id: string;
client_id: string;
user_id: string;
created_at: string;
name: string;
description?: string;
plugin: string;
type: string;
tables?: Array<{
name: string;
type: string;
columns?: string[];
reference_column?: ReferenceColumn;
}>;
credentials?: Record<string, any>;
}
@Injectable()
export class DynamoDBService {
private documentClient: DynamoDBDocumentClient;
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
const dynamoConfig = { region: DYNAMODB_CONFIG.region() };
const marshallOptions: TranslateConfig = {
marshallOptions: {
removeUndefinedValues: true,
},
};
const dynamoDb = new DynamoDBClient(dynamoConfig);
this.documentClient = DynamoDBDocumentClient.from(dynamoDb, marshallOptions);
}
async createInput(
clientId: string,
userId: string,
data: {
name: string;
description?: string;
plugin: string;
type: string;
tables?: Array<{
name: string;
type: string;
columns?: string[];
reference_column?: ReferenceColumn;
}>;
},
): Promise<InputDocument> {
const tableName = DYNAMODB_CONFIG.inputsTable();
const id = uuid();
const created_at = new Date().toISOString();
const item: InputDocument = {
id,
client_id: clientId,
user_id: userId,
created_at,
name: data.name,
description: data.description,
plugin: data.plugin,
type: data.type,
tables: data.tables,
};
this.logger.info('DynamoDB: Creating input', {
tableName,
inputId: id,
plugin: data.plugin,
});
const putCommand = new PutCommand({
TableName: tableName,
Item: item,
});
try {
await this.documentClient.send(putCommand);
this.logger.info('DynamoDB: Input created successfully', { inputId: id });
return item;
} catch (error) {
this.logger.error('DynamoDB: Failed to create input', {
tableName,
inputId: id,
region: DYNAMODB_CONFIG.region(),
error: error.message,
errorName: error.name,
});
throw error;
}
}
async findInput(clientId: string, inputId: string): Promise<InputDocument | null> {
const tableName = DYNAMODB_CONFIG.inputsTable();
const getCommand = new GetCommand({
TableName: tableName,
Key: {
id: inputId,
client_id: clientId,
},
});
const { Item } = await this.documentClient.send(getCommand);
return Item as InputDocument | null;
}
async deleteInput(clientId: string, inputId: string): Promise<void> {
const tableName = DYNAMODB_CONFIG.inputsTable();
this.logger.info('DynamoDB: Deleting input', {
tableName,
inputId,
});
const deleteCommand = new DeleteCommand({
TableName: tableName,
Key: {
id: inputId,
client_id: clientId,
},
});
await this.documentClient.send(deleteCommand);
this.logger.info('DynamoDB: Input deleted successfully', { inputId });
}
/**
* Update a specific table entry in the input document.
* Fetches the current document, updates the matching table, and saves.
*/
async updateInputTable(
clientId: string,
inputId: string,
tableName: string,
changes: {
type?: string;
columns?: string[];
reference_column?: ReferenceColumn | null;
},
): Promise<void> {
const dynamoTableName = DYNAMODB_CONFIG.inputsTable();
this.logger.info('DynamoDB: Updating input table', {
inputId,
tableName,
changes: Object.keys(changes),
});
// Get current document
const current = await this.findInput(clientId, inputId);
if (!current) {
this.logger.warn('DynamoDB: Input not found for update', { inputId });
return;
}
// Find and update the matching table
const tables = current.tables || [];
const tableIndex = tables.findIndex((t) => t.name === tableName);
if (tableIndex === -1) {
this.logger.warn('DynamoDB: Table not found in input', {
inputId,
tableName,
});
return;
}
// Merge changes into the table entry
const updatedTable = { ...tables[tableIndex] };
if ('type' in changes) updatedTable.type = changes.type;
if ('columns' in changes) updatedTable.columns = changes.columns;
if ('reference_column' in changes) {
if (changes.reference_column === null) {
delete updatedTable.reference_column;
} else {
updatedTable.reference_column = changes.reference_column;
}
}
tables[tableIndex] = updatedTable;
// Save updated document
const putCommand = new PutCommand({
TableName: dynamoTableName,
Item: {
...current,
tables,
updated_at: new Date().toISOString(),
},
});
try {
await this.documentClient.send(putCommand);
this.logger.info('DynamoDB: Input table updated successfully', {
inputId,
tableName,
});
} catch (error) {
this.logger.error('DynamoDB: Failed to update input table', {
inputId,
tableName,
error: error.message,
});
throw error;
}
}
}
+3
View File
@@ -0,0 +1,3 @@
export * from './dynamodb.service';
export * from './dynamodb.module';
export * from './dynamodb.config';
@@ -0,0 +1,5 @@
export const ELASTICSEARCH_CONFIG = {
getUrl: () => process.env.ELASTICSEARCH_URL || 'http://localhost:9200',
getApiKey: () => process.env.ELASTICSEARCH_API_KEY || '',
timeout: 10000, // 10 seconds
};
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ElasticsearchService } from './elasticsearch.service';
@Module({
providers: [ElasticsearchService, DadosferaLogger],
exports: [ElasticsearchService],
})
export class ElasticsearchModule {}
@@ -0,0 +1,382 @@
import { Injectable, Inject } from '@nestjs/common';
import axios, { AxiosInstance, AxiosError } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ELASTICSEARCH_CONFIG } from './elasticsearch.config';
interface MultiLang {
'en-us': string;
'pt-br': string;
'es-es': string;
}
interface MultiLangArray {
'en-us': string[];
'pt-br': string[];
'es-es': string[];
}
interface ConnectorInfo {
plugin: string;
name: MultiLang;
image: string;
version: string;
tags: string[];
}
interface PipelineDocument {
id: string;
name: MultiLang;
description: MultiLang;
customer_id: string;
user_id: string;
username: string;
status: string;
created_at: string;
updated_at: string;
last_status_updated: string;
tags: string[];
// Connector metadata
connection_id: string;
connector_name: string;
connector_plugin: string;
connector_version: string;
image_url: string;
// Config
config: {
cron: string;
tables?: string;
};
properties: string;
type: string;
in_use: number;
keywords: MultiLangArray;
}
@Injectable()
export class ElasticsearchService {
private client: AxiosInstance;
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
this.client = axios.create({
baseURL: ELASTICSEARCH_CONFIG.getUrl(),
headers: {
Authorization: `ApiKey ${ELASTICSEARCH_CONFIG.getApiKey()}`,
'Content-Type': 'application/json',
},
timeout: ELASTICSEARCH_CONFIG.timeout,
});
}
private getIndex(customerName: string): string {
return `${customerName}_pipelines`;
}
private formatMultiLang(value: string): MultiLang {
return {
'en-us': value,
'pt-br': value,
'es-es': value,
};
}
private formatMultiLangArray(value: string[] = []): MultiLangArray {
return {
'en-us': value,
'pt-br': value,
'es-es': value,
};
}
buildPipelineDocument(
pipelineId: string,
data: {
name: string;
description?: string;
user_id: string;
username: string;
customer_id: string;
plugin: string;
connection_id: string;
cron?: string;
tables?: string;
properties?: Record<string, any>;
type?: string;
status?: string;
created_at?: string;
keywords?: string[];
},
connector: ConnectorInfo | null,
): PipelineDocument {
const now = new Date().toISOString();
return {
id: pipelineId,
connection_id: data.connection_id,
connector_name: connector?.name?.['en-us'] || '',
connector_plugin: connector?.plugin || data.plugin,
connector_version: connector?.version || '1.0.0',
created_at: data.created_at || now,
updated_at: now,
customer_id: data.customer_id,
description: this.formatMultiLang(data.description || ''),
image_url: connector?.image || '',
keywords: this.formatMultiLangArray(data.keywords),
name: this.formatMultiLang(data.name || ''),
status: data.status || 'CREATED',
user_id: data.user_id,
username: data.username,
config: {
cron: data.cron,
tables: data.tables,
},
properties: data.properties ? JSON.stringify(data.properties) : '{}',
type: data.type,
in_use: 1,
last_status_updated: now,
tags: connector?.tags || [],
};
}
async getConnectorByPlugin(plugin: string): Promise<ConnectorInfo | null> {
this.logger.info('Elasticsearch: Looking up connector', { plugin });
try {
const response = await this.client.post('/connectors/_search', {
query: {
term: { plugin: plugin },
},
size: 1,
});
const hits = response.data.hits?.hits || [];
if (hits.length === 0) {
this.logger.warn('Elasticsearch: Connector not found', { plugin });
return null;
}
const source = hits[0]._source;
return {
plugin: source.plugin,
name: source.name,
image: source.image,
version: source.version,
tags: source.tags || [],
};
} catch (error) {
this.handleError('getConnectorByPlugin', error, { plugin });
return null;
}
}
async createPipeline(
customerName: string,
pipelineId: string,
data: {
name: string;
description?: string;
user_id: string;
username: string;
customer_id: string;
status?: string;
created_at?: string;
plugin: string;
connection_id: string;
cron?: string;
tables?: string;
properties?: Record<string, any>;
type?: string;
},
connector: ConnectorInfo | null,
): Promise<any> {
const index = this.getIndex(customerName);
const document = this.buildPipelineDocument(pipelineId, data, connector);
this.logger.info('Elasticsearch: Creating pipeline', {
index,
pipelineId,
plugin: document.connector_plugin,
});
try {
const response = await this.client.post(
`${index}/_doc/${pipelineId}`,
document,
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Pipeline created successfully', {
pipelineId,
result: response.data.result,
});
return response.data;
} catch (error) {
this.handleError('createPipeline', error, { pipelineId, index });
throw error;
}
}
async updatePipeline(
customerName: string,
pipelineId: string,
changes: {
name?: string;
description?: string;
cron?: string;
status?: string;
tags?: string[];
},
): Promise<any> {
const index = this.getIndex(customerName);
const now = new Date().toISOString();
this.logger.info('Elasticsearch: Updating pipeline', {
index,
pipelineId,
fields: Object.keys(changes),
});
try {
// Fetch current document
const currentDoc = await this.client.get(`${index}/_doc/${pipelineId}`);
const current = currentDoc.data._source;
// Build updated document, preserving existing values
const updated: Record<string, any> = {
...current,
updated_at: now,
};
if ('name' in changes) {
updated.name = this.formatMultiLang(changes.name);
}
if ('description' in changes) {
updated.description = this.formatMultiLang(changes.description);
}
if ('cron' in changes) {
updated.config = {
...current.config,
cron: changes.cron,
};
}
if ('status' in changes) {
updated.status = changes.status;
updated.last_status_updated = now;
}
if ('tags' in changes) {
updated.tags = changes.tags;
}
const response = await this.client.post(
`${index}/_doc/${pipelineId}`,
updated,
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Pipeline updated successfully', {
pipelineId,
result: response.data.result,
});
return response.data;
} catch (error) {
this.handleError('updatePipeline', error, { pipelineId, index });
throw error;
}
}
async getPipeline(
customerName: string,
pipelineId: string,
): Promise<PipelineDocument | null> {
const index = this.getIndex(customerName);
this.logger.info('Elasticsearch: Getting pipeline', {
index,
pipelineId,
});
try {
const response = await this.client.get(`${index}/_doc/${pipelineId}`);
return response.data._source as PipelineDocument;
} catch (error) {
if (error instanceof AxiosError && error.response?.status === 404) {
this.logger.warn('Elasticsearch: Pipeline not found', {
pipelineId,
index,
});
return null;
}
this.handleError('getPipeline', error, { pipelineId, index });
throw error;
}
}
async deletePipeline(
customerName: string,
pipelineId: string,
): Promise<any> {
const index = this.getIndex(customerName);
this.logger.info('Elasticsearch: Deleting pipeline', {
index,
pipelineId,
});
try {
const response = await this.client.delete(
`${index}/_doc/${pipelineId}`,
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Pipeline deleted successfully', {
pipelineId,
result: response.data.result,
});
return response.data;
} catch (error) {
// If document not found, log warning but don't throw
if (error instanceof AxiosError && error.response?.status === 404) {
this.logger.warn('Elasticsearch: Pipeline not found for deletion', {
pipelineId,
index,
});
return { result: 'not_found' };
}
this.handleError('deletePipeline', error, { pipelineId, index });
throw error;
}
}
private handleError(
operation: string,
error: any,
context: Record<string, any>,
): void {
if (error instanceof AxiosError) {
this.logger.error(`Elasticsearch: ${operation} failed`, {
...context,
status: error.response?.status,
statusText: error.response?.statusText,
errorData: error.response?.data,
message: error.message,
});
} else {
this.logger.error(`Elasticsearch: ${operation} failed`, {
...context,
message: error.message,
stack: error.stack,
});
}
}
}
+3
View File
@@ -0,0 +1,3 @@
export * from './elasticsearch.module';
export * from './elasticsearch.service';
export * from './elasticsearch.config';
+1 -1
View File
@@ -14,7 +14,7 @@ import { redisStore } from 'cache-manager-ioredis-yet';
keyPrefix: 'maestro:sso',
}
if (process.env.ENV !== 'local') {
if (process.env.REDIS_TLS === 'true') {
baseRedisConfig['tls'] = {
servername: process.env.REDIS_HOST,
}
+90
View File
@@ -0,0 +1,90 @@
import CronParser from 'cron-parser';
export enum ScheduleLimits {
MINUTE = 'minute',
HOUR = 'hour',
DAY = 'day',
UNLIMITED = 'unlimited',
}
const SECONDS_IN_MINUTE = 60;
const SECONDS_IN_HOUR = 3600;
const SECONDS_IN_DAY = 86400;
/**
* Airflow preset schedules mapped to cron expressions.
* @once is special - it means run only once (no recurring schedule).
*/
const AIRFLOW_PRESETS: Record<string, string | null> = {
'@once': null, // No recurring schedule - always valid
'@hourly': '0 * * * *', // Every hour
'@daily': '0 0 * * *', // Every day at midnight
'@weekly': '0 0 * * 0', // Every week on Sunday
'@monthly': '0 0 1 * *', // First day of every month
'@yearly': '0 0 1 1 *', // First day of every year
'@annually': '0 0 1 1 *', // Same as @yearly
};
/**
* Convert Airflow preset to cron expression.
* Returns null for @once (no recurring schedule).
* Returns original string if not an Airflow preset.
*/
export function convertAirflowPresetToCron(schedule: string): string | null {
const preset = AIRFLOW_PRESETS[schedule.toLowerCase()];
if (preset !== undefined) {
return preset;
}
return schedule;
}
export function getMinimumIntervalSeconds(scheduleLimit: string): number {
switch (scheduleLimit) {
case ScheduleLimits.MINUTE:
return SECONDS_IN_MINUTE;
case ScheduleLimits.HOUR:
return SECONDS_IN_HOUR;
case ScheduleLimits.DAY:
return SECONDS_IN_DAY;
case ScheduleLimits.UNLIMITED:
default:
return 0;
}
}
export function getCronIntervalSeconds(cron: string): number {
const interval = CronParser.parseExpression(cron);
const nextDate = interval.next().toDate();
const afterNextDate = interval.next().toDate();
return Math.floor((afterNextDate.getTime() - nextDate.getTime()) / 1000);
}
export function validateCronAgainstScheduleLimit(
cron: string,
scheduleLimit: string,
): { valid: boolean; message?: string } {
if (!cron) return { valid: true };
// Convert Airflow presets to cron expressions
const cronExpression = convertAirflowPresetToCron(cron);
// @once returns null - no recurring schedule, always valid
if (cronExpression === null) {
return { valid: true };
}
try {
const cronInterval = getCronIntervalSeconds(cronExpression);
const minInterval = getMinimumIntervalSeconds(scheduleLimit);
if (cronInterval < minInterval) {
return {
valid: false,
message: `Schedule interval (${cronInterval}s) is below customer limit (${scheduleLimit}: ${minInterval}s minimum)`,
};
}
return { valid: true };
} catch (error) {
return { valid: false, message: `Invalid cron expression: ${error.message}` };
}
}