Compare commits

..
Author SHA1 Message Date
ruanwebdev 8d04c1d495 FEAT: add logic to native documentation with AI 2025-09-23 16:28:38 -03:00
Marcos Rodrigues Silva cd55dc0dc4 Merge pull request #378 from dadosfera/hotfix/uptime-ip
UPDATE: uptime internal ip
2025-09-15 18:04:14 -03:00
marcos-silva-rodrigues f1d56e4c2c UPDATE: uptime internal ip 2025-09-15 10:34:36 -03:00
Marcos Rodrigues Silva bc49f79cb3 Merge pull request #373 from dadosfera/hotfix/oracle
Network policies
2025-09-09 18:08:28 -03:00
marcos-silva-rodrigues 00163ad894 UPDATE: network policies 2025-09-09 11:17:45 -03:00
marcos-silva-rodrigues cb8798d871 FIX: enable restricted ips 2025-09-05 10:21:11 -03:00
marcos-silva-rodrigues be70f4da09 FIX: disabled network policies 2025-09-04 10:02:20 -03:00
Marcos Rodrigues Silva b436d7de7a Merge pull request #372 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-03 16:49:31 -03:00
marcos-silva-rodrigues 2a3ab4228f FIX: infinite loading when login is incorrect 2025-09-03 16:25:46 -03:00
marcos-silva-rodrigues 8e605aa361 FIX: affinity and tls config local 2025-09-03 12:19:30 -03:00
Marcos Rodrigues Silva fa4267f54a Merge pull request #369 from dadosfera/hotfix/oracle
UPDATE: oracle vpn
2025-08-29 10:16:45 -03:00
marcos-silva-rodrigues 47d8ca2760 UPDATE: oracle vpn 2025-08-29 10:12:58 -03:00
Marcos Rodrigues Silva f3c51e5328 Merge pull request #368 from dadosfera/beta
UPDATE: fixed cloud oracle
2025-08-28 18:54:02 -03:00
marcos-silva-rodrigues dcf70fdaae UPDATE: fixed cloud oracle 2025-08-28 18:40:43 -03:00
Marcos Rodrigues Silva 70f663374d Merge pull request #366 from dadosfera/beta
MIgração para oracle
2025-08-28 18:32:59 -03:00
Marcos Rodrigues Silva 38f2317bb6 Merge pull request #367 from dadosfera/chore/oracle
UPDATE: affinity and log
2025-08-28 17:05:32 -03:00
marcos-silva-rodrigues 31e0ca6c91 UPDATE: affinity and log 2025-08-28 16:52:52 -03:00
Marcos Rodrigues Silva dc063cbf3f Merge pull request #365 from dadosfera/chore/oracle
Chore/oracle
2025-08-27 17:45:34 -03:00
marcos-silva-rodrigues a4c82ae4a8 UPDATE:migration to oracle 2025-08-27 17:38:26 -03:00
Marcos Rodrigues Silva 11a65e11d1 Merge pull request #364 from dadosfera/chore/oracle
UPDATE: redis host
2025-08-27 14:15:27 -03:00
marcos-silva-rodrigues 73d47f0ff5 UPDATE: merge 2025-08-27 14:13:17 -03:00
marcos-silva-rodrigues d36e532c8e UPDATE: redis host 2025-08-27 13:58:31 -03:00
Marcos Rodrigues Silva b2a24e3a49 Merge pull request #363 from dadosfera/feature/auth-session
UPDATE: test subdomain
2025-08-14 17:51:36 -03:00
marcos-silva-rodrigues 301b6e98ec UPDATE: test subdomain 2025-08-14 17:42:42 -03:00
Marcos Rodrigues Silva 381a401ebb Merge pull request #362 from dadosfera/feature/auth-session
CI: add cookie secret
2025-08-14 17:27:09 -03:00
marcos-silva-rodrigues ba53934068 CI: add cookie secret 2025-08-14 17:23:40 -03:00
Marcos Rodrigues Silva 1b6846532f Merge pull request #361 from dadosfera/feature/auth-session
CHORE: update nginx ingress annotations
2025-08-14 16:07:31 -03:00
marcos-silva-rodrigues 216349f303 CHORE: update nginx ingress annotations 2025-08-14 16:03:28 -03:00
Marcos Rodrigues Silva 878c9cf450 Merge pull request #360 from dadosfera/feature/auth-session
FIX: set cookies if exists tokens
2025-08-14 15:39:57 -03:00
marcos-silva-rodrigues a654baef13 FIX: set cookies if exists tokens 2025-08-14 15:37:17 -03:00
Marcos Rodrigues Silva df852449b2 Merge pull request #359 from dadosfera/feature/auth-session
Feature/auth session
2025-08-14 15:28:23 -03:00
marcos-silva-rodrigues 04d761ca14 Merge branch 'beta' into feature/auth-session 2025-08-14 15:26:15 -03:00
Marcos Rodrigues Silva 9b168612a4 Merge pull request #358 from dadosfera/beta
Beta
2025-08-08 11:23:02 -03:00
Marcos Rodrigues Silva a772804dcd Merge pull request #357 from dadosfera/chore/oracle
CI: fix multi cloud deploy and maestro env
2025-08-07 18:04:10 -03:00
marcos-silva-rodrigues 8a399fbbcf CI: fix multi cloud deploy and maestro env 2025-08-07 18:03:22 -03:00
Marcos Rodrigues Silva 94049d6e8e Merge pull request #356 from dadosfera/chore/oracle
CI: oracle
2025-08-07 12:27:08 -03:00
marcos-silva-rodrigues c28cc58ac7 CI: oracle 2025-08-07 12:24:44 -03:00
Marcos Rodrigues Silva 56a655bb90 Merge pull request #355 from dadosfera/chore/oracle
CI: fix env home
2025-08-07 12:16:02 -03:00
marcos-silva-rodrigues b3d4a6b028 CI: fix env home 2025-08-07 12:13:19 -03:00
Marcos Rodrigues Silva c7bd1f761a Merge pull request #354 from dadosfera/chore/oracle
CI: remove k8s-setup need
2025-08-07 12:01:43 -03:00
marcos-silva-rodrigues 09c045455e CI: remove k8s-setup need 2025-08-07 11:59:37 -03:00
Marcos Rodrigues Silva 52d651126e Merge pull request #353 from dadosfera/chore/oracle
CI: revert k8s-setup workflow for the step in job helmfile-deploy
2025-08-07 11:57:38 -03:00
marcos-silva-rodrigues 2f11b1e78f CI: revert k8s-setup workflow for the step in job helmfile-deploy 2025-08-07 11:56:54 -03:00
Marcos Rodrigues Silva 885e9b71bb Merge pull request #352 from dadosfera/fix/tracker-sso
Fix/tracker sso
2025-08-07 11:45:44 -03:00
marcos-silva-rodrigues 193031dfdd UPDATE: increase redis ttl 2025-08-07 10:41:48 -03:00
marcos-silva-rodrigues f626a9bb5e UPDATE: add more logger 2025-08-07 10:32:22 -03:00
Marcos Rodrigues Silva ac26ad9f44 Merge pull request #351 from dadosfera/chore/oracle
FIX: correct name
2025-08-01 18:06:27 -03:00
marcos-silva-rodrigues fbe00af6dc FIX: correct name 2025-08-01 18:05:19 -03:00
Marcos Rodrigues Silva bfa9929b69 Merge pull request #350 from dadosfera/chore/oracle
Chore/oracle
2025-08-01 18:01:38 -03:00
marcos-silva-rodrigues bada7000f6 CHORE: change ci to oracle cloud 2025-08-01 18:01:10 -03:00
Marcos Rodrigues Silva 2c908d8d95 Merge pull request #349 from dadosfera/beta
Beta
2025-07-31 15:24:28 -03:00
Marcos Rodrigues Silva da35633d42 Merge pull request #348 from dadosfera/hotfix/module-assigned
Hotfix/module assigned
2025-07-31 12:04:02 -03:00
marcos-silva-rodrigues cc7e06013f FIX: change module permission 2025-07-31 11:59:19 -03:00
Marcos Rodrigues Silva 5ec36a05b5 Merge pull request #344 from dadosfera/beta
Beta
2025-07-30 11:56:22 -03:00
Marcos Rodrigues Silva f61e46c7c1 Merge pull request #347 from dadosfera/release/07-29
FEAT: add permission embed access
2025-07-30 11:15:28 -03:00
marcos-silva-rodrigues b9013b1493 FEAT: add permission embed access 2025-07-30 11:12:08 -03:00
Marcos Rodrigues Silva 5267fa6d2f Merge pull request #346 from dadosfera/release/07-29
FIX: comment test
2025-07-29 18:05:26 -03:00
marcos-silva-rodrigues 7f40cbdeed FIX: comment test 2025-07-29 18:03:20 -03:00
Marcos Rodrigues Silva b590c9402d Merge pull request #345 from dadosfera/release/07-29
Release/07 29
2025-07-29 15:19:52 -03:00
marcos-silva-rodrigues 9546feda11 FIX: merge 2025-07-29 15:12:25 -03:00
Marcos Rodrigues Silva 3587f82976 Merge pull request #343 from dadosfera/bugfix/public-link
Bugfix/public link
2025-07-29 12:31:34 -03:00
marcos-silva-rodrigues 94a3962590 FIX: add needs steps 2025-07-29 12:26:21 -03:00
marcos-silva-rodrigues 3b2d79a451 CHORE: remove needs step 2025-07-29 12:06:33 -03:00
marcos-silva-rodrigues da56a645af CHORE: update affinity to azure 2025-07-29 11:57:10 -03:00
Marcos Rodrigues Silva af4e55af1d Merge pull request #342 from dadosfera/bugfix/public-link
CHORE: remove sudo
2025-07-29 11:35:09 -03:00
marcos-silva-rodrigues d6a08a8f82 CHORE: remove sudo 2025-07-29 11:32:02 -03:00
Marcos Rodrigues Silva c73eb8869f Merge pull request #341 from dadosfera/bugfix/public-link
CHORE: install kubectl
2025-07-29 11:15:24 -03:00
marcos-silva-rodrigues 929fc97ac6 CHORE: install kubectl 2025-07-29 11:12:47 -03:00
Marcos Rodrigues Silva 7594a75bea Merge pull request #340 from dadosfera/bugfix/public-link
CHORE: update ci to wait extract_environment job
2025-07-29 11:02:14 -03:00
marcos-silva-rodrigues ec36a056b2 CHORE: update ci to wait extract_environment job 2025-07-29 10:59:35 -03:00
Marcos Rodrigues Silva 8d30f6ef15 Merge pull request #339 from dadosfera/bugfix/public-link
CHORE: multi cloud deployment
2025-07-29 10:57:26 -03:00
marcos-silva-rodrigues 08d1e3164f CHORE: multi cloud deployment 2025-07-29 10:53:41 -03:00
Marcos Rodrigues Silva a5f3ef20fe Merge pull request #338 from dadosfera/bugfix/public-link
FIX: add user by bearer token if it exists
2025-07-29 09:51:17 -03:00
marcos-silva-rodrigues 467445fe05 FIX: add user by bearer token if it exists 2025-07-29 09:47:56 -03:00
Marcos Rodrigues Silva d0448f844a Merge pull request #337 from dadosfera/bugfix/google-oauth
UPDATE: more logs
2025-07-21 16:06:22 -03:00
marcos-silva-rodrigues b2299a5f2f UPDATE: more logs 2025-07-21 15:51:56 -03:00
Marcos Rodrigues Silva 9b42754c69 Merge pull request #336 from dadosfera/bugfix/google-oauth
Bugfix/google oauth
2025-07-21 15:06:49 -03:00
marcos-silva-rodrigues aee33334ac UPDATE: add log to debug 2025-07-21 15:01:41 -03:00
Marcos Rodrigues Silva cc530fb12d Merge pull request #335 from dadosfera/release/sso
Release/sso
2025-07-18 17:12:51 -03:00
marcos-silva-rodrigues 19877f1b29 FIX: merge with main 2025-07-18 16:46:19 -03:00
marcos-silva-rodrigues e9880bcf68 FIX: add logger in header 2025-07-18 15:56:23 -03:00
marcos-silva-rodrigues 4c7111dcb6 FIX: using referer headaer 2025-07-18 15:56:10 -03:00
marcos-silva-rodrigues e81cfdcce1 FIX: using only header origin 2025-07-18 15:55:56 -03:00
marcos-silva-rodrigues 3eba3f414f FIX: send redirect uri 2025-07-18 15:55:45 -03:00
marcos-silva-rodrigues fbeb8dfd91 FIX: redis env 2025-07-18 15:55:29 -03:00
marcos-silva-rodrigues dda3e8baaf FIX: add logger 2025-07-18 15:54:27 -03:00
marcos-silva-rodrigues c1e4f0b18a FIX: remove express session 2025-07-18 15:54:24 -03:00
Marcos Rodrigues Silva b0d9f3a468 Merge pull request #334 from dadosfera/chore/stg-env
FIX: revert env
2025-07-18 10:27:01 -03:00
marcos-silva-rodrigues 5b117acaa9 FIX: revert env 2025-07-18 10:23:10 -03:00
Marcos Rodrigues Silva a5cb607650 Merge pull request #333 from dadosfera/feature/sso-openid
FIX: change env to stg
2025-07-17 16:12:51 -03:00
marcos-silva-rodrigues fe626155c5 FIX: change env to stg 2025-07-17 16:10:42 -03:00
Marcos Rodrigues Silva debbe1c1b1 Merge pull request #332 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 18:50:04 -03:00
marcos-silva-rodrigues f937a6efe2 FIX: add logger in header 2025-07-16 18:49:27 -03:00
marcos-silva-rodrigues 30cb3c87eb FIX: using referer headaer 2025-07-16 18:45:47 -03:00
Marcos Rodrigues Silva fa46294154 Merge pull request #331 from dadosfera/feature/sso-openid
FIX: using only header origin
2025-07-16 18:37:53 -03:00
marcos-silva-rodrigues 72554fb27f FIX: using only header origin 2025-07-16 18:36:36 -03:00
Marcos Rodrigues Silva 4e6985b572 Merge pull request #330 from dadosfera/feature/sso-openid
FIX: send redirect uri
2025-07-16 18:16:27 -03:00
marcos-silva-rodrigues 4477647613 FIX: send redirect uri 2025-07-16 18:15:47 -03:00
Marcos Rodrigues Silva 2b3066abc9 Merge pull request #329 from dadosfera/feature/sso-openid
FIX: redis env
2025-07-16 17:57:26 -03:00
marcos-silva-rodrigues da1bcb38da FIX: redis env 2025-07-16 17:56:43 -03:00
Marcos Rodrigues Silva 4a7524422d Merge pull request #328 from dadosfera/feature/sso-openid
FIX: add logger
2025-07-16 17:41:40 -03:00
marcos-silva-rodrigues 275af03195 FIX: add logger 2025-07-16 17:39:15 -03:00
Marcos Rodrigues Silva 0c75cf4619 Merge pull request #327 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 12:33:44 -03:00
marcos-silva-rodrigues 80de277529 FIX: merge problems 2025-07-16 12:27:33 -03:00
marcos-silva-rodrigues 98f669db66 FIX: remove express session 2025-07-16 11:48:15 -03:00
marcos-silva-rodrigues 2c8dab5927 FIX: merge 2025-07-16 11:46:05 -03:00
marcos-silva-rodrigues cb66d4f6c0 FEAT: dynamic callback and issuer urls 2025-07-16 11:35:47 -03:00
marcos-silva-rodrigues 42a39ee891 FEAT: send user id to refresh token endpoint 2025-07-15 16:21:44 -03:00
marcos-silva-rodrigues c9bfe68ea5 FEAT: list identity providers link for sso 2025-07-14 15:16:19 -03:00
marcos-silva-rodrigues 8cb7144ff4 FIX: send accessToken in refresh token endpoint 2025-07-14 15:14:25 -03:00
Marcos Rodrigues Silva 68cc58e2fe Merge pull request #326 from dadosfera/release/11-07
Release/11 07
2025-07-14 10:54:14 -03:00
marcos-silva-rodrigues cbe5556ac9 FIX: auth callback 2025-07-14 10:51:36 -03:00
marcos-silva-rodrigues eb80967608 FIX: tests 2025-07-11 11:36:23 -03:00
marcos-silva-rodrigues 66fea0722d FEAT: enabled network check in app.dadosfera 2025-07-11 11:12:35 -03:00
marcos-silva-rodrigues c917061b97 Merge branch 'feature/export-users-and-assets' into release/11-07 2025-07-11 11:04:31 -03:00
Rafael Santana da39a07f25 Merge pull request #325 from dadosfera/feat/oracle-migration-and-affinity-standardization
FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
2025-07-09 13:02:48 -03:00
Rafael f2f65285de resolve: merge conflicts with beta branch
- Resolve ingress.yaml timeout annotation conflicts by keeping both configurations
- Resolve stg.yaml conflicts by merging affinity/resources config with restricted_ip
- Fix chart references in unimed-maestro and private-maestro sections
- Update all chart paths from ../maestro to ../helm-chart for consistency

This merge brings in the latest beta changes while preserving our Oracle migration
and affinity standardization configuration.
2025-07-09 13:00:52 -03:00
Rafael cac8f93330 CI: update validate-k8s.yml to use beta branch for staging
- Change trigger branches from main/stg to main/beta
- Update environment extraction to map beta → stg environment
- Align with standard branch naming convention (beta for staging)
- Consistent with deploy-manually.yml workflow configuration
2025-07-09 12:58:11 -03:00
Rafael bb03b18f4a FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
Oracle Migration Changes:
- Create deploy/helm-chart/ directory structure (moved from maestro/)
- Create deploy/helmfiles/ directory structure (moved from helmfiles/)
- Update chart references in helmfiles from ../maestro to ../helm-chart
- Update deploy-manually.yml from Azure AKS to Oracle OKE
- Update validate-k8s.yml from Azure AKS to Oracle OKE
- Replace Azure CLI with OCI CLI installation and configuration
- Replace Azure authentication with OCI authentication using secrets
- Replace az aks get-credentials with oci ce cluster create-kubeconfig
- Update helmfile paths from helmfiles/ to deploy/helmfiles/
- Remove DockerHub integration (push_to_dockerhub input and related steps)
- Change runner from [self-hosted, prd-azure] to [self-hosted, prd-oracle]
- Add environment field for proper Oracle deployment environment handling
- Add HOME environment variable for OCI CLI

Affinity/Resources Standardization:
- Add affinity configuration with name=general node selector in values.yaml
- Update deployment template to use conditional affinity and resources blocks
- Configure staging to disable both affinity and resources (null values)
- Enable production to use affinity targeting name=general nodes
- Enable production to use standard resource limits (100m-2000m CPU, 1500Mi-2Gi memory)

This brings maestro into full compliance with Oracle migration requirements and
implements the standardized affinity/resources pattern consistent with in-factory and duc.
2025-07-09 12:55:49 -03:00
Marcos Rodrigues Silva ab280ee152 Merge pull request #324 from dadosfera/chore/update-unimed-ips
CHORE: Update unimed ips
2025-07-08 09:22:26 -03:00
marcos-silva-rodrigues 16c8137160 CHORE: Update unimed ips 2025-07-07 17:26:26 -03:00
Marcos Rodrigues Silva b8b903c8cd Merge pull request #323 from dadosfera/feature/export-users-and-assets
Feature/export users and assets
2025-07-05 11:27:10 -03:00
marcos-silva-rodrigues c98a5ff896 FEAT: endpoint to dowload assets list 2025-07-05 11:21:05 -03:00
marcos-silva-rodrigues d68e436a8b FEAT: export users in csv 2025-07-05 09:38:16 -03:00
marcos-silva-rodrigues 8efa6790d6 FEAT: sign out endpoint 2025-07-04 18:03:08 -03:00
marcos-silva-rodrigues a381f50e88 FEAT: set cookie after user login 2025-07-03 15:52:55 -03:00
marcos-silva-rodrigues d3040cd5a8 FEAT: update and delete identity provider 2025-07-03 10:59:43 -03:00
marcos-silva-rodrigues f726a9883f FEAT: create user from identity provider 2025-06-27 13:21:59 -03:00
Marcos Rodrigues Silva c18a7baa37 Merge pull request #322 from dadosfera/chore/add-dadosfera-vpn-unimed-chart
FIX: add dadosfera vpn ip in unimed chart
2025-06-25 11:12:33 -03:00
marcos-silva-rodrigues d638d518f8 FIX: change unimed chart name 2025-06-25 11:06:41 -03:00
marcos-silva-rodrigues bdd371ffe8 FIX: add dadosfera vpn ip in unimed chart 2025-06-25 10:58:18 -03:00
marcos-silva-rodrigues 57e6e49c7c FEAT: init url callback 2025-06-25 10:28:26 -03:00
marcos-silva-rodrigues a5c8de496d FEAT: map permission to identity provider 2025-06-24 16:20:03 -03:00
Marcos Rodrigues Silva d478f32760 Merge pull request #321 from dadosfera/hotfix/nginx-timeout
FIX: add nginx proxy timeout 5 min
2025-06-24 11:02:29 -03:00
marcos-silva-rodrigues 5d8e908339 FIX: add nginx proxy timeout 5 min 2025-06-24 10:26:50 -03:00
Marcos Rodrigues Silva 606bae016b Merge pull request #320 from dadosfera/hotfix/maesto-unimed-hostname
FIX: correct helm value
2025-06-23 18:43:58 -03:00
marcos-silva-rodrigues 6910a2f04c FIX: correct helm value 2025-06-23 18:38:20 -03:00
Marcos Rodrigues Silva 8bee1788c1 Merge pull request #319 from dadosfera/hotfix/maesto-unimed-hostname
FIX: maestro custom hostname to unimed
2025-06-23 18:04:43 -03:00
marcos-silva-rodrigues b10459cb15 FIX: maestro custom hostname to unimed 2025-06-23 17:51:14 -03:00
Marcos Rodrigues Silva 3b48dd1613 Merge pull request #317 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 17:07:55 -03:00
Marcos Rodrigues Silva e5cef90a64 Merge pull request #318 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 16:04:59 -03:00
marcos-silva-rodrigues bac74c9577 FIX: default value and throw correct Error Class 2025-06-23 15:36:26 -03:00
marcos-silva-rodrigues 19d748ae09 CHORE: add restricted ip and unimed chart 2025-06-23 14:47:39 -03:00
marcos-silva-rodrigues 4db865371f Merge branch 'bugfix/update-pupperter-timeout' into release/2025-06 2025-06-23 14:12:25 -03:00
marcos-silva-rodrigues 75e8b3bf4b FIX: correct dto name 2025-06-17 14:58:35 -03:00
Marcos Rodrigues Silva 194cdc66b7 Merge pull request #316 from dadosfera/feature/public-and-assign-embed
Feature/public and assign embed
2025-06-17 08:57:53 -03:00
marcos-silva-rodrigues 53059c0a0d FIX: import 2025-06-16 18:09:25 -03:00
marcos-silva-rodrigues ed4ffc8a86 FIX: merge with beta 2025-06-16 18:01:53 -03:00
Marcos Rodrigues Silva 9967c172da Merge pull request #315 from dadosfera/feature/dedicated-maestro
FEAT: block user from login when maestro is dedicated
2025-06-16 10:37:47 -03:00
marcos-silva-rodrigues 0aa9c065a5 FEAT: block user from login when maestro is dedicated 2025-06-16 10:34:11 -03:00
Marcos Rodrigues Silva d0c2abd38d Merge pull request #314 from dadosfera/bugfix/update-pupperter-timeout
FIX: update timeout
2025-06-13 15:10:08 -03:00
Marcos Rodrigues Silva f446e0d5b1 Merge pull request #313 from dadosfera/feature/dedicated-maestro
FIX: send env DEDICATED_PROXY
2025-06-12 14:47:54 -03:00
marcos-silva-rodrigues 17de31f1a6 FIX: send env DEDICATED_PROXY 2025-06-12 14:45:13 -03:00
Marcos Rodrigues Silva d81a466089 Merge pull request #312 from dadosfera/feature/dedicated-maestro
FIX: open data ingress route
2025-06-12 14:08:13 -03:00
marcos-silva-rodrigues 9e597fadba FIX: open data ingress route 2025-06-12 14:04:45 -03:00
Marcos Rodrigues Silva 1e6aa0f4c2 Merge pull request #311 from dadosfera/feature/dedicated-maestro
FIX: chart private maestro
2025-06-12 13:59:11 -03:00
marcos-silva-rodrigues 41842f7c6a FIX: chart private maestro 2025-06-12 13:56:57 -03:00
Marcos Rodrigues Silva 0d5564990d Merge pull request #309 from dadosfera/feature/dedicated-maestro
Feature/dedicated maestro
2025-06-12 12:31:26 -03:00
marcos-silva-rodrigues 7f9755493a FIX: experimental hostname 2025-06-12 12:29:51 -03:00
marcos-silva-rodrigues fea587ad36 FEAT: control login by dedicated customer id injected 2025-06-12 11:51:56 -03:00
Marcos Rodrigues Silva 57f99fe7af Merge pull request #307 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 14:25:23 -03:00
Marcos Rodrigues Silva e66f9a7244 Merge pull request #306 from dadosfera/feature/api-key
FEAT: add api key endpoint
2025-06-06 15:55:53 -03:00
marcos-silva-rodrigues 54cb3f8e7a FEAT: add api key endpoint 2025-06-06 15:53:12 -03:00
Marcos Rodrigues Silva 4c861cf5e4 Merge pull request #305 from dadosfera/fix/maestro-stg-url
Fix/maestro stg url
2025-05-30 10:15:10 -03:00
marcos-silva-rodrigues a89f57e690 FIX: return pi_factory_url and tr_factory_url to stg url 2025-05-30 10:13:37 -03:00
marcos-silva-rodrigues d5141e5b6a REFACTOR: rename openid to identity provider 2025-05-26 13:37:25 -03:00
marcos-silva-rodrigues a4b7573ce7 FIX: sync with main 2025-05-23 14:07:05 -03:00
marcos-silva-rodrigues 6fc8c0b4a8 FEAT: add mixpanel service to track share access 2025-05-20 16:50:04 -03:00
marcos-silva-rodrigues ced2a97547 FIX: add logger and return info 2025-05-19 16:36:54 -03:00
marcos-silva-rodrigues d15cc4642a FEAT: assign embed and public key changes 2025-05-13 09:21:44 -03:00
marcos-silva-rodrigues 3c4f45678a CHORE: update protospack 2025-05-12 12:25:45 -03:00
marcos-silva-rodrigues b312849fff FEAT: catalog share endpoint for embeds 2025-05-12 12:15:16 -03:00
marcos-silva-rodrigues 10580dfa03 FEAT: share endpoint 2025-05-12 12:13:39 -03:00
marcos-silva-rodrigues 62d0e8b041 FEAT: assign pubic key endpoint 2025-05-12 12:12:58 -03:00
marcos-silva-rodrigues d94c5dee4e FEAT: init public routes to get data asset by public embed 2025-05-06 14:44:09 -03:00
marcos-silva-rodrigues 4c94a51a9d REFACTOR: openid module 2025-04-03 07:46:16 -03:00
marcos-silva-rodrigues f1345472b5 UPDATE: add routes to login with idp 2025-03-25 09:18:27 -03:00
marcos-silva-rodrigues ea9c0502b7 UPDATE: new route to add identity provider 2025-03-24 07:58:26 -03:00
87 changed files with 5124 additions and 937 deletions
+7 -72
View File
@@ -13,11 +13,6 @@ on:
options:
- stg
- prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs:
extract_environment:
@@ -118,23 +113,6 @@ jobs:
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2
with:
username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker compose -f build.docker-compose.dockerhub.yml build
docker compose -f build.docker-compose.dockerhub.yml push
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -164,54 +142,11 @@ jobs:
docker system prune --volumes -a -f
docker system df
helmfile-deploy:
k8s-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: 'oracle'
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
+137
View File
@@ -0,0 +1,137 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+45 -22
View File
@@ -4,7 +4,7 @@ on:
pull_request:
branches:
- main
- stg
- beta
jobs:
extract_environment:
@@ -21,17 +21,13 @@ jobs:
fi
id: extract_environment
helmfile-deploy:
helmfile-check:
env:
HOME: /home/runner
needs: [extract_environment]
runs-on: [self-hosted, "prd-azure"]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
steps:
- name: Summary
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
- name: Checkout code
uses: actions/checkout@v3
@@ -40,13 +36,28 @@ jobs:
with:
version: 'v3.9.0'
- name: Install Azure ClI
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
@@ -57,23 +68,35 @@ jobs:
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f helmfiles/${ENV}.yaml diff
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
Binary file not shown.
@@ -1,16 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: maestro
app: {{ .Values.app_name }}
strategy:
rollingUpdate:
@@ -20,22 +20,17 @@ spec:
template:
metadata:
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: application
operator: In
values:
- general
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
@@ -48,13 +43,10 @@ spec:
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
requests:
cpu: {{ .Values.resources.requests.cpu }}
memory: {{ .Values.resources.requests.memory }}
limits:
cpu: {{ .Values.resources.limits.cpu }}
memory: {{ .Values.resources.limits.memory }}
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
@@ -82,6 +74,8 @@ spec:
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: LOGGER_CONSOLE_EXTRA
value: "true"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
@@ -100,6 +94,16 @@ spec:
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -108,15 +112,15 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
@@ -10,8 +10,12 @@ metadata:
generation: 1
labels:
app: maestro
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
@@ -21,7 +25,7 @@ spec:
paths:
- backend:
service:
name: maestro
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
+36
View File
@@ -0,0 +1,36 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
@@ -1,17 +1,17 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-maestro
name: prd-{{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-maestro
name: prd-{{ .Values.app_name }}
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
@@ -1,18 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
type: ClusterIP
ports:
- name: maestro
- name: {{ .Values.app_name }}
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: maestro
app: {{ .Values.app_name }}
+16
View File
@@ -0,0 +1,16 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity: null
@@ -9,6 +9,7 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
@@ -42,9 +43,25 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- product
+56
View File
@@ -0,0 +1,56 @@
releases:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
# 10.70.0.0/16 internal network
# 137.131.167.254/32 loadbalancer
# 159.112.184.81/32 cluster ip for the uptime request ingest
- name: maestro.restricted_ip
value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+30
View File
@@ -0,0 +1,30 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
- name: private-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
set:
- name: app_name
value: maestro-private
- name: hostname
value: private-maestro.stg.dadosfera.ai
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: affinity
value: null
- name: resources
value: null
- name: maestro.restricted_ip
value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+1479 -510
View File
File diff suppressed because it is too large Load Diff
+10
View File
@@ -14,6 +14,16 @@ declare global {
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
// Autodrive Configuration
AUTODRIVE_USERNAME?: string;
AUTODRIVE_PASSWORD?: string;
AUTODRIVE_BASE_URL?: string;
AUTODRIVE_MODEL?: string;
AUTODRIVE_KEY?: string;
AUTO_DRIVE_KEY?: string;
}
}
}
-22
View File
@@ -1,22 +0,0 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
-22
View File
@@ -1,22 +0,0 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
-23
View File
@@ -1,23 +0,0 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
-27
View File
@@ -1,27 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: maestro
name: maestro
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
+331 -4
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.23",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -27,8 +27,11 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -39,6 +42,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -46,13 +50,17 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.6.3"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
@@ -1400,9 +1408,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.37.0-beta.23",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz",
"integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==",
"version": "3.38.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz",
"integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -1600,6 +1608,12 @@
"dev": true,
"license": "BSD-3-Clause"
},
"node_modules/@ioredis/commands": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
"license": "MIT"
},
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
@@ -2969,6 +2983,65 @@
"node": ">=18"
}
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz",
"integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz",
"integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz",
"integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@sinonjs/commons": {
"version": "1.8.6",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz",
@@ -3654,6 +3727,13 @@
"@types/node": "*"
}
},
"node_modules/@types/cache-manager": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.6.tgz",
"integrity": "sha512-8qL93MF05/xrzFm/LSPtzNEOE1eQF3VwGHAcQEylgp5hDSTe41jtFwbSYAPfyYcVa28y1vYSjIt0c1fLLUiC/Q==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/connect": {
"version": "3.4.38",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
@@ -3664,6 +3744,16 @@
"@types/node": "*"
}
},
"node_modules/@types/cookie-parser": {
"version": "1.4.9",
"resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.9.tgz",
"integrity": "sha512-tGZiZ2Gtc4m3wIdLkZ8mkj1T6CEHb35+VApbL2T14Dew8HA7c+04dmKqsKRNC+8RJPm16JEK0tFSwdZqubfc4g==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"@types/express": "*"
}
},
"node_modules/@types/cookiejar": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
@@ -3723,6 +3813,16 @@
"@types/send": "*"
}
},
"node_modules/@types/express-session": {
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.18.1.tgz",
"integrity": "sha512-S6TkD/lljxDlQ2u/4A70luD8/ZxZcrU5pQwI1rVXCiaVIywoFgbA+PIUNDjPhQpPdK0dGleLtYc/y7XWBfclBg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/express": "*"
}
},
"node_modules/@types/graceful-fs": {
"version": "4.1.9",
"resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz",
@@ -5237,6 +5337,41 @@
"node": ">= 0.8"
}
},
"node_modules/cache-manager": {
"version": "5.7.6",
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
"lru-cache": "^10.2.2",
"promise-coalesce": "^1.1.2"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/cache-manager-ioredis-yet": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cache-manager-ioredis-yet/-/cache-manager-ioredis-yet-1.2.2.tgz",
"integrity": "sha512-o03N/tQxfFONZ1XLGgIxOFHuQQpjpRdnSAL1THG1YWZIVp1JMUfjU3ElSAjFN1LjbJXa55IpC8waG+VEoLUCUw==",
"deprecated": "With cache-manager v6 we now are using Keyv",
"license": "MIT",
"dependencies": {
"cache-manager": "^5.2.3",
"ioredis": "^5.3.2"
},
"engines": {
"node": ">= 16.17.0"
}
},
"node_modules/cache-manager/node_modules/lru-cache": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
"node_modules/call-bind": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz",
@@ -5514,6 +5649,15 @@
"node": ">=0.8"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -5740,6 +5884,28 @@
"node": ">= 0.6"
}
},
"node_modules/cookie-parser": {
"version": "1.4.7",
"resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz",
"integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==",
"license": "MIT",
"dependencies": {
"cookie": "0.7.2",
"cookie-signature": "1.0.6"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/cookie-parser/node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
@@ -6027,6 +6193,15 @@
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -6802,6 +6977,12 @@
"node": ">= 0.6"
}
},
"node_modules/eventemitter3": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
"integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==",
"license": "MIT"
},
"node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -7470,6 +7651,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -8045,6 +8235,30 @@
"node": ">= 0.10"
}
},
"node_modules/ioredis": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ip-address": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
@@ -9044,6 +9258,15 @@
"url": "https://github.com/chalk/supports-color?sponsor=1"
}
},
"node_modules/jose": {
"version": "4.15.9",
"resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz",
"integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -9366,12 +9589,30 @@
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
"license": "MIT"
},
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==",
"license": "MIT"
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
"license": "MIT"
},
"node_modules/lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
"license": "MIT"
},
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
"license": "MIT"
},
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -10023,6 +10264,15 @@
"integrity": "sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==",
"license": "MIT"
},
"node_modules/object-hash": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz",
"integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==",
"license": "MIT",
"engines": {
"node": ">= 6"
}
},
"node_modules/object-identity-map": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/object-identity-map/-/object-identity-map-1.0.2.tgz",
@@ -10068,6 +10318,15 @@
"node": ">= 0.4"
}
},
"node_modules/oidc-token-hash": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.1.0.tgz",
"integrity": "sha512-y0W+X7Ppo7oZX6eovsRkuzcSM40Bicg2JEJkDJ4irIt1wsYAP5MLSNv+QAogO8xivMffw/9OvV3um1pxXgt1uA==",
"license": "MIT",
"engines": {
"node": "^10.13.0 || >=12.0.0"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -10113,6 +10372,21 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/openid-client": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.7.1.tgz",
"integrity": "sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==",
"license": "MIT",
"dependencies": {
"jose": "^4.15.9",
"lru-cache": "^6.0.0",
"object-hash": "^2.2.0",
"oidc-token-hash": "^5.0.3"
},
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/optional-js": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/optional-js/-/optional-js-2.3.0.tgz",
@@ -10815,6 +11089,15 @@
"node": ">=0.4.0"
}
},
"node_modules/promise-coalesce": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/promise-coalesce/-/promise-coalesce-1.1.2.tgz",
"integrity": "sha512-zLaJ9b8hnC564fnJH6NFSOGZYYdzrAJn2JUUIwzoQb32fG2QAakpDNM+CZo1km6keXkRXRM+hml1BFAPVnPkxg==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=16"
}
},
"node_modules/prompts": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
@@ -11194,6 +11477,44 @@
"node": ">= 0.10"
}
},
"node_modules/redis": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz",
"integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==",
"license": "MIT",
"workspaces": [
"./packages/*"
],
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.6.1",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.7",
"@redis/search": "1.2.0",
"@redis/time-series": "1.1.0"
}
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"license": "MIT",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/reflect-metadata": {
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
@@ -11881,6 +12202,12 @@
"integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==",
"license": "MIT"
},
"node_modules/standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
"license": "MIT"
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+9 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-beta.23",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -45,8 +45,11 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -57,6 +60,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -64,6 +68,7 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
@@ -73,7 +78,10 @@
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
+11 -1
View File
@@ -28,7 +28,11 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
providers: [
@@ -62,9 +66,15 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule
],
})
export class AppModule {}
+20 -12
View File
@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
+39 -4
View File
@@ -4,6 +4,7 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -17,6 +18,7 @@ import {
import { RequestUser } from '../decorators/user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
@Injectable()
export class AuthenticationGuard
@@ -31,6 +33,7 @@ export class AuthenticationGuard
dadosferaLogger: DadosferaLogger,
private reflector: Reflector,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService
) {
this.pems = new Map();
this.logger = dadosferaLogger.logger;
@@ -48,20 +51,40 @@ export class AuthenticationGuard
});
}
canActivate(ctx: ExecutionContext): boolean {
async canActivate(ctx: ExecutionContext): Promise<boolean> {
const authFunctions = this.reflector.getAllAndMerge<
AuthenticationFunction[]
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
const mustBeAuthenticated = authFunctions.length > 0;
const request = ctx.switchToHttp().getRequest();
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!mustBeAuthenticated) {
// no need to be authenticated
return true;
}
const request = ctx.switchToHttp().getRequest();
const apiKey = request.get('X-api-key');
if (apiKey) {
const {
api_key
} = await this.apiKeyService.get(apiKey);
request.user = {
user_id: api_key.user_id,
username: api_key.username,
permissions: api_key.permissions,
customer_id: api_key.customer_id,
customer_name: api_key.customer_name,
customer_tier: api_key.customer_tier,
customer_modules: api_key.customer_modules,
access_token: apiKey,
};
return true;
}
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!accessToken) {
// couldn't load valid token
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
@@ -113,6 +136,18 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
+22 -11
View File
@@ -340,16 +340,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -362,6 +352,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -614,7 +623,9 @@ export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii'
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+9 -1
View File
@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -52,6 +53,7 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
UserTest(token);
// UserTest(token);
});
+9 -2
View File
@@ -9,6 +9,8 @@ import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -23,11 +25,16 @@ async function bootstrap() {
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true
},
});
app.use(helmet());
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
}
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
+68
View File
@@ -0,0 +1,68 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@Controller('api-key')
@Authenticated()
@ApiTags('ApiKey')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
export class ApiKeyController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
method: 'create'
});
const result = await this.apiKeyService.create(createApiKeyDto, user);
this.logger.info('POST /api-key success', {
id: result.id,
method: 'create'
});
return result;
}
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
});
const result = await this.apiKeyService.findAll(user);
this.logger.info('GET /api-key success', {
count: result.length,
method: 'findAll'
});
return result;
}
@Delete(':id')
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
method: 'remove'
});
await this.apiKeyService.remove(id, user);
this.logger.info('DELETE /api-key/:id success', {
id,
method: 'remove'
});
}
}
+18
View File
@@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { ApiKeyController } from './api-key.controller';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([ducClient.providerOptions])
],
controllers: [ApiKeyController],
providers: [ApiKeyService, DadosferaLogger],
exports: [ApiKeyService]
})
export class ApiKeyModule {}
+50
View File
@@ -0,0 +1,50 @@
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc,
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
}
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
return lastValueFrom(this.apiKeyService.CreateApiKey({
permissions: createApiKeyDto.permissions
}, metadata));
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata)
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
return data.api_keys;
}
async remove(id: string, user: RequestUser) {
const metadata = PackTheMetadata(user);
await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
}
async get(key: string) {
const metadata = PackTheMetadata({});
return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
}
}
+39
View File
@@ -0,0 +1,39 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsArray, IsNumber } from 'class-validator';
export class PermissionDto {
@ApiProperty({ type: Number })
id: number;
@ApiProperty({ type: String })
name: string;
}
export class ApiKeyBaseResponseDto {
@ApiProperty({ type: String, format: 'uuid' })
id: string;
@ApiProperty({ type: String })
key_mask: string;
@ApiProperty({ type: [PermissionDto] })
permissions: PermissionDto[];
@ApiProperty({ type: String, format: 'date-time' })
created_at: string;
@ApiProperty({ type: String })
created_by: string;
}
export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
@ApiProperty({ type: String })
key: string;
}
export class CreateApiKeyDto {
@ApiProperty({ type: [Number], description: 'Array of permission IDs' })
@IsArray()
@IsNumber({}, { each: true })
permissions: number[];
}
+33
View File
@@ -0,0 +1,33 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
}
}
+15
View File
@@ -0,0 +1,15 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
+38
View File
@@ -0,0 +1,38 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -0,0 +1,3 @@
export class CreateAssignDto {
publicKey: string;
}
+197 -14
View File
@@ -12,6 +12,7 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -26,7 +27,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthVerifyTotpMfaRequest
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -45,14 +46,21 @@ import {
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request } from 'express';
import { Request, Response } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -86,13 +94,66 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Headers('origin') origin = '',
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ language, custom_host: frontHost });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
throw error;
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('refresh-access-token')
@@ -102,17 +163,25 @@ export class AuthController {
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
language,
customer_name,
custom_host: frontHost,
});
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
}
@ApiInternalOnlyEndpoint()
@@ -420,4 +489,118 @@ export class AuthController {
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
}
+51 -7
View File
@@ -1,10 +1,10 @@
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -17,16 +17,22 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -39,6 +45,10 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async getPublicKeys() {
@@ -53,25 +63,59 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
}
async refreshAccessToken(
{ refreshToken }: AuthRefreshAccessTokenRequest,
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
);
}
+1 -1
View File
@@ -122,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
customerName: string;
userId: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
@@ -26,6 +26,7 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
+153 -48
View File
@@ -22,6 +22,9 @@ import {
ApiHeaders,
ApiOkResponse,
ApiTags,
ApiOperation,
ApiParam,
ApiResponse,
} from '@nestjs/swagger';
import {
Authenticated,
@@ -32,7 +35,7 @@ import {
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
@@ -88,10 +91,6 @@ export class CatalogController {
@Query() query: ICatalogAllRequest,
): Promise<ICatalogAllResponse> {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
@@ -117,15 +116,53 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query;
this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
username,
customer_name,
});
if (!pipeline || !object) {
throw new BadRequestException('Query params not provided');
@@ -178,10 +215,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { user_id, customer, customer_id } = body.info;
const metadata = PackTheMetadata({
@@ -207,10 +240,6 @@ export class CatalogController {
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`GET /data-asset/${id}`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
@@ -263,10 +292,6 @@ export class CatalogController {
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
@@ -318,10 +343,6 @@ export class CatalogController {
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - columns-metadata`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
@@ -349,10 +370,6 @@ export class CatalogController {
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username, customer_modules } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
@@ -378,24 +395,31 @@ export class CatalogController {
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
try {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
id,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const docs = await this.catalogService.getDataDocs(id, metadata);
const docs = await this.catalogService.getDataDocs(id, metadata);
return { docs };
return { docs };
} catch (error) {
this.logger.error(`Error in getDataAssetDocs for id ${id}: ${error.message}`);
this.logger.error(`Error details: ${JSON.stringify(error)}`);
throw error;
}
}
@Put('data-asset/:id')
@@ -444,10 +468,6 @@ export class CatalogController {
) {
const { user_id, customer_name } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const res = await this.catalogService.createDataDocs({
table_id,
@@ -460,6 +480,7 @@ export class CatalogController {
return res;
}
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
async manageDataAssetPermissions(
@@ -885,4 +906,88 @@ export class CatalogController {
this.logger.error(error.message);
}
}
}
@Post('/data-asset/:nimbus_id/docs/ai')
@ApiOperation({
summary: 'Save documentation for data asset',
description: 'Saves documentation content for a data asset',
})
@ApiParam({
name: 'nimbus_id',
description: 'Nimbus ID of the data asset',
type: 'string',
})
@ApiResponse({
status: 201,
description: 'Documentation saved successfully',
})
async saveDocumentation(
@Param('nimbus_id') nimbusId: string,
@Body() body: { docs: string },
@User() user: RequestUser,
) {
const metadata = PackTheMetadata(user);
try {
await this.catalogService.updateDataAssetDocumentation(nimbusId, body.docs, metadata);
return {
message: 'Documentation saved successfully',
};
} catch (error) {
this.logger.error(`Error saving documentation for ${nimbusId}: ${error.message}`);
throw error;
}
}
@Post('/data-asset/:nimbus_id/docs/generate-ai')
@ApiOperation({
summary: 'Generate AI documentation for data asset',
description: 'Generates comprehensive documentation for a data asset using AI (Autodrive)',
})
@ApiParam({
name: 'nimbus_id',
description: 'Nimbus ID of the data asset',
type: 'string',
})
@ApiResponse({
status: 201,
description: 'AI documentation generated successfully',
schema: {
type: 'object',
properties: {
message: { type: 'string' },
documentation: { type: 'string' },
},
},
})
@ApiResponse({
status: 400,
description: 'Bad request - invalid nimbus_id or missing data',
})
@ApiResponse({
status: 404,
description: 'Data asset not found',
})
@ApiResponse({
status: 500,
description: 'Internal server error during AI generation',
})
async generateAiDocumentation(
@Param('nimbus_id') dataAssetId: string,
@User() user: RequestUser,
) {
const metadata = PackTheMetadata(user);
try {
const result = await this.catalogService.generateAiDocumentation(dataAssetId, metadata, user);
return {
message: 'AI documentation generated successfully',
documentation: result,
};
} catch (error) {
this.logger.error(`Error generating AI documentation for ${dataAssetId}: ${error.message}`);
throw error;
}
}
}
+6 -1
View File
@@ -3,12 +3,15 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -18,6 +21,8 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+611 -9
View File
@@ -1,4 +1,5 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import FormData from 'form-data';
import {
WriteService,
ReadService,
@@ -25,6 +26,7 @@ import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
IUpdateDataRequest,
TriggerCatalogReq,
@@ -35,14 +37,22 @@ import {
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
import { AI_DOCUMENTATION_PROMPT, AI_DOCUMENTATION_CONFIG } from './prompts/ai-documentation.prompt';
import { AUTODRIVE_CONSTANTS, GEOGRAPHIC_KEYS, COMMON_COUNTRIES } from './constants/autodrive.constants';
import {
AutodriveCredentials,
AutodriveAskPayload,
AutodriveAskResponse,
AutodriveAnswerResponse,
AutodriveUploadResponse,
DatasetStatusResponse,
ColumnData,
ColumnsMetadata,
DataPreview,
FormattedDataForAI
} from './types/ai-documentation.types';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -240,6 +250,34 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -313,15 +351,44 @@ class CatalogService implements OnModuleInit {
}
async getDataDocs(id: string, metadata: Metadata) {
const { documentation } = await lastValueFrom(
let documentation: string;
try {
const response = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const doc = response.documentation;
documentation = doc;
if (!documentation) {
this.logger.warn(`No documentation found for id: ${id}`);
return null;
}
// Verificar se a documentação é um JSON válido
if (documentation.trim().startsWith('{') || documentation.trim().startsWith('[')) {
const docs = JSON.parse(documentation);
return docs;
} else {
this.logger.warn(`Documentation is not JSON format, returning as raw text for id: ${id}`);
return { raw_documentation: documentation };
}
} catch (error) {
this.logger.error(`getDataDocs failed for id ${id}: ${error.message}`);
// Se for erro de JSON parsing, tentar retornar a documentação como string
if (error.message.includes('JSON') || error.message.includes('parse')) {
this.logger.warn(`JSON parsing failed, returning raw documentation for id: ${id}`);
return { raw_documentation: documentation || 'No documentation available' };
}
throw error;
}
}
async getDatasetPreview(id: string, metadata: Metadata) {
try {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id, type: undefined },
@@ -330,19 +397,35 @@ class CatalogService implements OnModuleInit {
);
const result = JSON.parse(preview);
return result;
} catch (error) {
this.logger.error(`getDatasetPreview failed for id ${id}: ${error.message}`);
throw error;
}
}
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
try {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id, type: undefined },
metadata,
),
);
if (!columns_metadata || columns_metadata.length <= 2) {
this.logger.warn(`Empty or minimal response from gRPC: "${columns_metadata}"`);
throw new Error('Empty response from gRPC service');
}
const result = JSON.parse(columns_metadata);
return result;
} catch (error) {
this.logger.error(`getDatasetColumnsMetadata failed for id ${id}: ${error.message}`);
throw error;
}
}
async createDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
@@ -352,6 +435,7 @@ class CatalogService implements OnModuleInit {
return data;
}
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
@@ -582,6 +666,524 @@ class CatalogService implements OnModuleInit {
);
return res;
}
private capitalizeFirst(str: string): string {
return str.charAt(0).toUpperCase() + str.slice(1);
}
async updateDataAssetDocumentation(nimbusId: string, documentation: string, metadata: Metadata): Promise<void> {
try {
if (!documentation) {
this.logger.warn(`Documentation is undefined for ${nimbusId}`);
return;
}
const changes = {
documentation: documentation
};
await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: nimbusId, changes: JSON.stringify(changes) },
metadata,
),
);
this.logger.info(`Documentation updated successfully for ${nimbusId}`);
} catch (error) {
this.logger.error(`Error updating documentation for ${nimbusId}: ${error.message}`);
throw error;
}
}
async generateAiDocumentation(dataAssetId: string, metadata: Metadata, user?: any): Promise<string> {
this.logger.info(`Generating AI documentation for data asset: ${dataAssetId}`);
try {
const customer_id = metadata.get('customer_id')?.[0]?.toString();
// 1. Buscar metadados do data asset
const dataAsset = await this.getOneDataAsset({
id: dataAssetId,
customer_id,
metadata
});
if (!dataAsset || !dataAsset.data_asset || !dataAsset.data_asset.nimbus_id) {
throw new Error("Nimbus ID not found in data asset metadata");
}
const nimbusId = dataAsset.data_asset.nimbus_id.toString();
// 2. Buscar informações das colunas
let columnsData = null;
try {
columnsData = await this.getDatasetColumnsMetadata(dataAssetId, metadata);
} catch (error) {
this.logger.warn(`Failed to get column metadata: ${error.message}`);
columnsData = null;
}
// 2.1. Fallback para dados das colunas
const dataAssetWithColumns = dataAsset as any;
if (!columnsData && dataAssetWithColumns.columns && Array.isArray(dataAssetWithColumns.columns)) {
columnsData = { columns: dataAssetWithColumns.columns };
} else if (!columnsData) {
const numColumns = parseInt(dataAsset.data_asset.num_columns) || 0;
if (numColumns > 0) {
columnsData = {
columns: Array.from({ length: numColumns }, (_, i) => ({
name: `COLUMN_${i + 1}`,
type: 'UNKNOWN',
description: 'Column information not available',
nullable: 'UNKNOWN'
}))
};
}
}
// 3. Buscar preview dos dados (opcional)
let dataPreview = [];
try {
dataPreview = await this.getDatasetPreview(nimbusId, metadata);
} catch (error) {
this.logger.warn(`Failed to get data preview: ${error.message}`);
dataPreview = [];
}
// 4. Formatar dados e chamar Autodrive
const combinedText = this.formatDataForAI(dataAsset, dataPreview, columnsData);
const documentation = await this.callAutodriveForDocumentation(combinedText, user?.access_token);
// 5. Salvar a documentação
const createDataDocsPayload = {
table_id: dataAsset.data_asset.nimbus_id,
docs: documentation,
info: {
customer: user.customer_name,
},
};
await this.createDataDocs(createDataDocsPayload);
this.logger.info(`AI documentation generated and saved successfully for ${dataAssetId}, length: ${documentation.length} chars`);
return documentation;
} catch (error) {
this.logger.error(`Error generating AI documentation for ${dataAssetId}: ${error.message}`);
let errorMessage = error.message;
if (error.response) {
errorMessage = `API Error ${error.response.status}: ${error.response.data?.detail || error.message}`;
}
throw new HttpException(
`Failed to generate AI documentation: ${errorMessage}`,
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
private formatDataForAI(dataAsset: any, dataPreview: any[], columnsData?: any): string {
let combinedText = '';
// Seguir exatamente o padrão do Python: chamar json_to_free_text para cada objeto separadamente
// 1. metadata_data (dataAsset)
combinedText += this.jsonToFreeText(dataAsset);
// 2. info_data (columns) - usar columnsData se disponível, senão tentar dataAsset.columns
if (columnsData) {
// Adicionar aviso se os dados das colunas são limitados
if (columnsData.columns && columnsData.columns.some((col: any) => col.name?.startsWith('COLUMN_'))) {
combinedText += 'WARNING: Column information is limited or unavailable. Generated column names are placeholders.\n\n';
}
combinedText += this.jsonToFreeText(columnsData);
} else if (dataAsset.columns && Array.isArray(dataAsset.columns)) {
const columnsDataFromAsset = { columns: dataAsset.columns };
combinedText += this.jsonToFreeText(columnsDataFromAsset);
} else {
// Se não há dados das colunas, informar explicitamente
combinedText += 'WARNING: No column metadata available. Cannot generate detailed table structure.\n\n';
}
// 3. preview_data
if (dataPreview && dataPreview.length > 0) {
const previewData = { preview: dataPreview };
combinedText += this.jsonToFreeText(previewData);
}
return combinedText;
}
private jsonToFreeText(jsonObj: any, indentLevel: number = 0): string {
if (!jsonObj || typeof jsonObj !== 'object') {
return '';
}
let text = '';
const indent = ' '.repeat(indentLevel);
// Processa primeiro os dados geográficos se existirem (igual ao Python)
const geoKeys = GEOGRAPHIC_KEYS;
// Adiciona uma seção especial para dados de preview se existirem (igual ao Python)
if ('preview' in jsonObj) {
text += "=== PREVIEW DATA START ===\n";
if (Array.isArray(jsonObj['preview'])) {
// Primeiro, vamos procurar por colunas geográficas
const geoColumns = [];
if (jsonObj['preview'].length > 0 && typeof jsonObj['preview'][0] === 'object') {
for (const key of Object.keys(jsonObj['preview'][0])) {
const keyLower = key.toLowerCase();
if (geoKeys.some(geoTerm => keyLower.includes(geoTerm))) {
geoColumns.push(key);
}
}
}
// Se encontramos colunas geográficas, vamos destacá-las
if (geoColumns.length > 0) {
text += "GEOGRAPHIC DATA FOUND IN COLUMNS:\n";
for (const col of geoColumns) {
text += `=== Column: ${col} ===\n`;
const values = jsonObj['preview'].map(row => String(row[col] || '')).filter(v => v);
text += "Values: " + values.join(", ") + "\n\n";
}
}
// Agora processamos todos os dados normalmente
for (const row of jsonObj['preview']) {
text += this.jsonToFreeText(row, indentLevel + 1);
}
} else {
text += this.jsonToFreeText(jsonObj['preview'], indentLevel + 1);
}
text += "=== PREVIEW DATA END ===\n\n";
}
// Processa o resto dos dados (igual ao Python)
for (const [key, value] of Object.entries(jsonObj)) {
if (key === 'preview') { // Skip preview here since we handled it above
continue;
}
if (typeof value === 'object' && value !== null) {
if (Array.isArray(value)) {
text += `${indent}${this.capitalize(key)}:\n`;
for (const item of value) {
if (typeof item === 'object' && item !== null) {
text += this.jsonToFreeText(item, indentLevel + 1);
} else {
text += `${indent} - ${item}\n`;
}
}
} else {
text += `${indent}${this.capitalize(key)}:\n`;
text += this.jsonToFreeText(value, indentLevel + 1);
}
} else {
// Destaca campos geográficos
if (geoKeys.includes(key.toLowerCase() as any)) {
text += `${indent}!!! GEOGRAPHIC DATA !!! ${this.capitalize(key)}: ${value}\n`;
} else {
text += `${indent}${this.capitalize(key)}: ${value}\n`;
}
}
}
text += '\n';
return text;
}
private capitalize(str: string): string {
return str.charAt(0).toUpperCase() + str.slice(1);
}
private async callAutodriveForDocumentation(combinedText: string, userAccessToken?: string): Promise<string> {
const credentials = this.getAutodriveCredentials();
try {
// Criar dataset temporário com os dados reais
const datasetId = await this.createTemporaryDataset(combinedText, userAccessToken);
// Fazer pergunta ao dataset
const documentation = await this.askQuestionToDataset(datasetId, credentials);
return documentation;
} catch (error) {
this.logger.error(`Error calling Autodrive API: ${error.message}`);
throw new Error(`Autodrive API call failed: ${error.message}`);
}
}
private getAutodriveCredentials(): AutodriveCredentials {
const fallbackCredentials = process.env.AUTODRIVE_KEY || process.env.AUTO_DRIVE_KEY;
// Verificar se as credenciais principais estão disponíveis
if (AUTODRIVE_CONSTANTS.USERNAME && AUTODRIVE_CONSTANTS.PASSWORD) {
return {
username: AUTODRIVE_CONSTANTS.USERNAME,
password: AUTODRIVE_CONSTANTS.PASSWORD,
baseUrl: AUTODRIVE_CONSTANTS.BASE_URL,
model: AUTODRIVE_CONSTANTS.DEFAULT_MODEL,
};
}
// Fallback para credenciais alternativas
if (fallbackCredentials) {
const cleanKey = fallbackCredentials.startsWith("'") && fallbackCredentials.endsWith("'")
? fallbackCredentials.slice(1, -1)
: fallbackCredentials;
return {
username: '',
password: '',
baseUrl: AUTODRIVE_CONSTANTS.BASE_URL,
model: AUTODRIVE_CONSTANTS.DEFAULT_MODEL,
authHeader: `Basic ${cleanKey}`,
};
}
// Fallback de emergência removido por segurança
// Configure as variáveis de ambiente necessárias
throw new Error('No authentication credentials available. Please configure AUTODRIVE_USERNAME and AUTODRIVE_PASSWORD in your .env file');
}
private async askQuestionToDataset(datasetId: string, credentials: AutodriveCredentials): Promise<string> {
const authHeader = credentials.authHeader || this.createAuthHeader(credentials);
const askPayload: AutodriveAskPayload = {
question: AI_DOCUMENTATION_PROMPT.trim(),
fetch_k: AI_DOCUMENTATION_CONFIG.FETCH_K,
k: AI_DOCUMENTATION_CONFIG.K,
model: credentials.model
};
const askHeaders = {
'Authorization': authHeader,
...AUTODRIVE_CONSTANTS.HEADERS
};
const askResponse = await axios.post<AutodriveAskResponse>(
`${credentials.baseUrl}/dataset/${datasetId}/ai_question`,
askPayload,
{
headers: askHeaders,
timeout: AUTODRIVE_CONSTANTS.ASK_TIMEOUT
}
);
// Verificar se a resposta contém a documentação diretamente
if (askResponse.data?.answer) {
const documentation = askResponse.data.answer;
this.logger.info(`Documentation generated successfully - Length: ${documentation.length} characters`);
return documentation;
}
// Se não contém a resposta diretamente, verificar se tem question_id para buscar
if (askResponse.data?.question_id) {
const questionId = askResponse.data.question_id;
// Buscar a resposta da pergunta
const answerResponse = await axios.get<AutodriveAnswerResponse>(
`${credentials.baseUrl}/dataset/${datasetId}/ai_question/${questionId}`,
{
headers: { 'Authorization': authHeader },
timeout: AUTODRIVE_CONSTANTS.ANSWER_TIMEOUT
}
);
// Se a resposta ainda está sendo processada, fazer polling contínuo
if (answerResponse.data?.status === 'started' || !answerResponse.data?.answer) {
let attempts = 0;
const maxAttempts = 12;
const pollInterval = 10000;
while (attempts < maxAttempts) {
attempts++;
await new Promise(resolve => setTimeout(resolve, pollInterval));
try {
const pollResponse = await axios.get<AutodriveAnswerResponse>(
`${credentials.baseUrl}/dataset/${datasetId}/ai_question/${questionId}`,
{
headers: { 'Authorization': authHeader },
timeout: AUTODRIVE_CONSTANTS.ANSWER_TIMEOUT
}
);
// Se a resposta está pronta, retornar
if (pollResponse.data?.answer && pollResponse.data?.status !== 'started') {
const documentation = pollResponse.data.answer;
this.logger.info(`Documentation generated successfully after ${attempts} attempts - Length: ${documentation.length} characters`);
return documentation;
}
// Se ainda está processando, continuar o loop
if (pollResponse.data?.status === 'started') {
continue;
}
// Se falhou, mostrar erro e quebrar
if (pollResponse.data?.status === 'failed') {
throw new Error(`Answer processing failed: ${pollResponse.data.status_reason || 'Unknown error'}`);
}
// Se houve erro ou status inesperado, quebrar o loop
break;
} catch (pollError) {
this.logger.error(`Polling attempt ${attempts} failed: ${pollError.message}`);
continue;
}
}
throw new Error(`Answer still not ready after ${maxAttempts} polling attempts`);
}
if (!answerResponse.data || !answerResponse.data.answer) {
throw new Error(`Answer response invalid: ${JSON.stringify(answerResponse.data)}`);
}
const documentation = answerResponse.data.answer;
this.logger.info(`Documentation generated successfully - Length: ${documentation.length} characters`);
return documentation;
}
throw new Error(`Autodrive API response invalid: ${JSON.stringify(askResponse.data)}`);
}
private async createTemporaryDataset(combinedText: string, userAccessToken?: string): Promise<string> {
if (!AUTODRIVE_CONSTANTS.BASE_URL) {
throw new Error('AUTODRIVE_BASE_URL not configured');
}
if (!combinedText || combinedText.trim().length === 0) {
throw new Error('combinedText is empty - cannot create dataset');
}
const credentials = this.getAutodriveCredentials();
const authHeader = credentials.authHeader || this.createAuthHeader(credentials);
const tempFilePath = await this.createTempFile(combinedText);
try {
const uploadResponse = await this.uploadDataset(tempFilePath, authHeader);
const datasetId = uploadResponse.dataset_id;
await this.waitForDatasetReady(datasetId, authHeader, credentials.baseUrl);
return datasetId;
} catch (error) {
this.logger.error(`Error creating temporary dataset: ${error.message}`);
throw error;
} finally {
await this.cleanupTempFile(tempFilePath);
}
}
private async createTempFile(combinedText: string): Promise<string> {
const tempFileName = `temp_data_${Date.now()}.txt`;
const tempFilePath = `/tmp/${tempFileName}`;
const fs = await import('fs');
fs.writeFileSync(tempFilePath, combinedText, 'utf8');
return tempFilePath;
}
private async uploadDataset(tempFilePath: string, authHeader: string): Promise<AutodriveUploadResponse> {
const tempFileName = tempFilePath.split('/').pop() || 'temp_data.txt';
const fs = await import('fs');
const formData = new FormData();
formData.append('files', fs.createReadStream(tempFilePath), tempFileName);
formData.append('name', tempFileName);
const uploadResponse = await axios.post<AutodriveUploadResponse>(
`${AUTODRIVE_CONSTANTS.BASE_URL}/upload`,
formData,
{
headers: {
'Authorization': authHeader,
...formData.getHeaders(),
},
timeout: 30000, // 30 segundos
}
);
if (!uploadResponse.data?.dataset_id) {
throw new Error('No dataset_id in upload response');
}
return uploadResponse.data;
}
private async waitForDatasetReady(datasetId: string, authHeader: string, autodriveBaseUrl: string): Promise<void> {
const maxAttempts = 12;
const pollInterval = 10000; // 10 segundos
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const statusResponse = await axios.get<DatasetStatusResponse>(
`${autodriveBaseUrl}/dataset/${datasetId}`,
{
headers: { 'Authorization': authHeader },
timeout: 30000,
}
);
if (statusResponse.data?.status === 'success') {
return;
}
if (statusResponse.data?.status === 'failed') {
throw new Error(`Dataset processing failed: ${statusResponse.data.status_reason || 'Unknown error'}`);
}
if (statusResponse.data?.status === 'processing') {
if (attempt < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, pollInterval));
continue;
}
}
if (attempt === maxAttempts) {
throw new Error(`Dataset still not ready after ${maxAttempts} attempts`);
}
} catch (statusError) {
this.logger.error(`Status check error on attempt ${attempt}: ${statusError.message}`);
if (attempt === maxAttempts) {
throw statusError;
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
}
}
private async cleanupTempFile(tempFilePath: string | null): Promise<void> {
if (!tempFilePath) return;
try {
const fs = await import('fs');
if (fs.existsSync(tempFilePath)) {
fs.unlinkSync(tempFilePath);
}
} catch (error) {
this.logger.warn(`Failed to remove temporary file: ${error.message}`);
}
}
private createAuthHeader(credentials: AutodriveCredentials): string {
return `Basic ${Buffer.from(`${credentials.username}:${credentials.password}`).toString('base64')}`;
}
}
export { CatalogService };
export { CatalogService };
@@ -0,0 +1,42 @@
/**
* Constantes relacionadas ao Autodrive
*
*/
export const AUTODRIVE_CONSTANTS = {
// URLs e endpoints (apenas do ENV)
BASE_URL: process.env.BASE_URL_AUTODRIVE || process.env.AUTODRIVE_BASE_URL,
// Credenciais (apenas do ENV, sem fallback para segurança)
USERNAME: process.env.AUTODRIVE_USERNAME,
PASSWORD: process.env.AUTODRIVE_PASSWORD,
// Modelo padrão
DEFAULT_MODEL: process.env.AUTODRIVE_MODEL || "gpt-4o",
// Timeouts (em milissegundos)
ASK_TIMEOUT: 120000,
ANSWER_TIMEOUT: 180000,
// Headers
HEADERS: {
'Content-Type': 'application/json',
},
} as const;
/**
* Chaves geográficas para detecção de dados de localização
*/
export const GEOGRAPHIC_KEYS = [
'country', 'countries', 'city', 'cities',
'region', 'regions', 'location', 'state',
'states', 'address'
] as const;
/**
* Países comuns para detecção automática
*/
export const COMMON_COUNTRIES = [
'brazil', 'brasil', 'usa', 'united states',
'canada', 'mexico', 'argentina', 'chile',
'colombia'
] as const;
+8
View File
@@ -320,3 +320,11 @@ export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
@@ -0,0 +1,88 @@
/**
*/
export const AI_DOCUMENTATION_PROMPT = `crie uma documentação em Portugues, Ingles e Espanhol seguindo essas instruções
1. Persona: como profissional de governança e engenharia de dados
2. Tarefa: ao receber as informações da tabela criar uma documentação com o seguinte escopo
**A primeira linha do documento tem que conter a seguinte informação: ## Document languages: EN / BR / ES
**A segunda linha tem que obrigatoriamente conter a escrita Table: nome da tabela
**A terceira linha tem que obrigatoriamente conter a escrita Table Schema: nome do table schema
**DIRETRIZ CRUCIAL DE CONSISTÊNCIA E COMPLETUDE DE SCHEMA:**
**1. Fonte Exclusiva de Metadados:** O 'Table Schema' definido na linha acima é a ÚNICA fonte de verdade para o schema dos dados a serem documentados. TODAS as informações subsequentes, especialmente na seção 'Estrutura da Tabela' (incluindo a lista de colunas, seus nomes, tipos de dados, descrições e exemplos) DEVEM ser extraídas EXCLUSIVAMENTE de metadados que correspondem a ESTE 'Table Schema'. Se os dados de entrada que você recebeu contiverem informações para a mesma tabela ou colunas mas de schemas diferentes (ex: um schema 'bronze' e um 'silver'), você DEVE IGNORAR TOTALMENTE as informações dos schemas divergentes para esta tarefa de documentação e utilizar APENAS as do 'Table Schema' aqui especificado.
**2. Listagem Completa de Colunas:** Sua principal tarefa na seção 'Estrutura da Tabela' é identificar e listar TODAS as colunas que pertencem ao 'Table Schema' especificado. Verifique nos dados de entrada fornecidos se há uma indicação explícita do número total de colunas para esta tabela neste schema (por exemplo, um campo como 'Num_columns' ou similar nos metadados da tabela). Você deve se esforçar para listar exatamente essa quantidade de colunas. Se essa contagem não estiver disponível, liste todas as colunas que você puder identificar como pertencentes exclusivamente a este 'Table Schema'. A completude em relação ao schema especificado é essencial.
**Depois de "Estrutura da tablea", incluir a mensagem "Este documento foi gerado por IA", traduzida corretamente para cada idioma.**
**Obrigatoriamente:Após finalizar a versão em Inglês, começar a versão em Português** **Após finalizar a versão em Português, começar a versão em Espanhol** **Antes de começar cada versão, colocar um título como:** - \`## English Version\` (para inglês)
- \`## Versão em Português\` (para português)
- \`## Versión en Español\` (para espanhol)
- Descrição: fornece uma visão geral do ativo de dados,
destacando seu propósito e principal funcionalidade.
Esta sessão resume o conteúdo e o objetivo do ativo, ajudando os usuários a entender rapidamente o que o ativo representa
e como pode ser utilizado em suas análises e decisões.
- Sugestão de Domínio de Dados:
Analise cuidadosamente os dados da tabela e sugira o domínio mais apropriado. Inclua:
- Domínio Sugerido: [Nome do domínio]
- Motivo: [Explicação breve sobre porque a tabela pertence a este domínio]
- Observações: [Qualquer observação adicional relevante]
Exemplos de Domínios de Dados para referência:
- Financeiro: Dados sobre transações, receitas, despesas, etc.
- Recursos Humanos: Dados sobre funcionários, cargos, salários, etc.
- Produtos: Dados sobre produtos, categorias, preços, etc.
- Fornecedores: Dados sobre fornecedores, produtos fornecidos, localizações, etc.
- Marketing: Dados sobre campanhas, leads, conversões, etc.
- Vendas: Dados sobre vendas, clientes, produtos vendidos, etc.
- Operações: Dados sobre processos, logística, produção, etc.
- Clientes: Dados sobre clientes, interações, histórico, etc.
-Tags Sugeridas:
A IA deve gerar tags relevantes **com base nos dados da tabela**.
- **IMPORTANTE: Analise cuidadosamente os dados de preview da tabela (PREVIEW DATA) para encontrar países. Procure em todas as colunas por nomes de países, cidades ou regiões.**
- **Garanta que as tags estejam separadas por espaços vazios, todas na mesma linha, exemplo: #marketing #sales #australia #canada, limitar até 3 países que mais aparecem** - **Os países DEVEM ser extraídos dos dados de preview da tabela. Procure em colunas como City, Country, Region, Location, etc.** - Por que esta tabela é interessante:
Nesta sessão, é destacada a importância do ativo, explicando como ele pode ser útil para os usuários.
São abordadas as formas como o ativo pode melhorar a tomada de decisões, identificar padrões relevantes ou fornecer insights valiosos.
O objetivo é ressaltar a utilidade prática e o impacto positivo que o ativo pode ter em suas atividades.
- Análises potencialmente úteis feitas com esses dados:
Aqui são listadas algumas das análises que podem ser realizadas com o ativo de dados. Inclui sugestões de dashboards,
relatórios ou outros tipos de análises que aproveitam as informações fornecidas pelo ativo.
O objetivo é oferecer maneiras de utilizar os dados para obter insights valiosos e apoiar a tomada de decisões informadas.
- Links Úteis:
Os Links Úteis oferecem recursos adicionais relacionados ao ativo de dados, incluindo guias,
artigos ou outras fontes de informação que podem ajudar os usuários a compreender melhor o ativo e suas aplicações. Além disso,
inclui um link rápido dentro da Dadosfera para ativos relacionados diretamente com o ativo em questão, facilitando a navegação entre os ativos.
- Estrutura da Tabela:
A Estrutura da Tabela detalha TODAS as colunas e os dados disponíveis no ativo, conforme pertencentes ao 'Table Schema' principal definido no início deste documento.
**Instrução Detalhada para Estrutura da Tabela:**
Siga rigorosamente estes passos:
1. Identifique nos dados de entrada (metadados da tabela e das colunas) todas as colunas que pertencem EXCLUSIVAMENTE ao 'Table Schema' especificado no cabeçalho deste documento. Se houver uma contagem de colunas (ex: 'Num_columns') para este schema específico, assegure-se de listar essa quantidade.
2. Para CADA uma dessas colunas identificadas, formate a saída da seguinte maneira, **SEM utilizar NENHUM marcador de lista (como traços ou asteriscos) no início de cada entrada de coluna**. Cada coluna deve ser apresentada como um bloco de texto. Inclua uma linha em branco entre a documentação de cada coluna para separação visual.
- Apresente o NOME_DA_COLUNA em maiúsculas, seguido pelo (TIPO_DE_DADO_EXTRAÍDO_DOS_METADADOS_DO_SCHEMA_CORRETO) entre parênteses.
- O **NOME_DA_COLUNA (TIPO_DE_DADO_EXTRAÍDO_DOS_METADADOS_DO_SCHEMA_CORRETO)** deve estar na primeira linha do bloco da coluna e **inteiramente em negrito**.
- Na linha seguinte, a etiqueta "**Descrição:**" deve estar **em negrito**, seguida pelo texto da descrição da coluna.
- Na linha seguinte à descrição, a etiqueta "**Exemplo:**" deve estar **em negrito**, seguida pelo valor do exemplo. Se o exemplo for um valor literal ou código, formate-o entre crases (\`) se apropriado.
- Se houver informações adicionais relevantes (como "Valores Possíveis:", "Observações:", etc.), coloque a etiqueta correspondente **em negrito** em uma nova linha, seguida pelo seu texto.
Este documento foi gerado por IA.
NOME_COLUNA_1 (TIPO_DADO_SCHEMA_CORRETO_1):
Descrição: [Descrição da coluna 1, do schema correto]
Exemplo: \`[Exemplo de valor para coluna 1, do schema correto]\`
NOME_COLUNA_2 (TIPO_DADO_SCHEMA_CORRETO_2):
Descrição: [Descrição da coluna 2, do schema correto]
Exemplo: \`[Exemplo de valor para coluna 2, do schema correto]\`
(continue este formato com início de cada coluna para TODAS as colunas do 'Table Schema' especificado, garanta com que NUNCA tenha TRAÇO OU PONTO no inicio)
3. Contexto : O usuário ira cadastrar um ativo de dados na nossa plataforma e para ter um bom catalogo ele ira querer gerar a documentação padronizada mas explicativa e
automática
4. Restrições : A documentação deve seguir obrigatoriamente o mesmo padrão principalmente na parte de estrutura de dados
5. Objetivo: O principal objetivo é gerar uma documentação acessível, clara,
automática e padronizada para os usuários que desejem cadastrar um ativo de dados na plataforma`;
/**
* Configurações para a geração de documentação com IA
*/
export const AI_DOCUMENTATION_CONFIG = {
FETCH_K: 250,
K: 100,
} as const;
@@ -0,0 +1,89 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
+30
View File
@@ -0,0 +1,30 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
+275
View File
@@ -0,0 +1,275 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -0,0 +1,60 @@
/**
* Tipos relacionados à geração de documentação com IA
*/
export interface AutodriveCredentials {
username: string;
password: string;
baseUrl: string;
model: string;
authHeader?: string;
}
export interface AutodriveAskPayload {
question: string;
fetch_k: number;
k: number;
model: string;
}
export interface AutodriveAskResponse {
answer?: string;
question_id?: string;
dataset_id?: string;
}
export interface AutodriveAnswerResponse {
status: 'started' | 'success' | 'failed';
answer?: string;
status_reason?: string;
}
export interface AutodriveUploadResponse {
dataset_id: string;
}
export interface DatasetStatusResponse {
status: 'processing' | 'success' | 'failed';
status_reason?: string;
}
export interface ColumnData {
name: string;
type: string;
description?: string;
nullable?: string;
}
export interface ColumnsMetadata {
columns: ColumnData[];
}
export interface DataPreview {
preview: any[];
}
export interface FormattedDataForAI {
dataAsset: any;
dataPreview: any[];
columnsData?: ColumnsMetadata;
}
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ConnectionTestService {
@@ -28,7 +28,7 @@ import {
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -25,7 +25,7 @@ import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiTags('Customers')
@@ -59,6 +59,14 @@ export class CustomersService implements OnModuleInit {
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
@@ -141,6 +149,7 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
@@ -158,6 +167,7 @@ export class CustomersService implements OnModuleInit {
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
@@ -0,0 +1,47 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -0,0 +1,10 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -0,0 +1,246 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -0,0 +1,185 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+4 -76
View File
@@ -3,14 +3,13 @@ import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
private mixpanelService: MixpanelService
) {}
@Post(':id')
async trackEvent(
@@ -20,82 +19,11 @@ export class MixpanelController {
@Req() request
) {
delete body.info;
const mixpanel = init(this.mixpanelToken);
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
await this.mixpanelService.track(id, user, request, body)
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+6 -1
View File
@@ -1,6 +1,8 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`${process.env.ENV}/root/mixpanel_token`,
`prd/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
+118
View File
@@ -0,0 +1,118 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -8,7 +8,7 @@ import {
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@@ -32,7 +32,7 @@ import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
@@ -205,13 +205,10 @@ export class PipelinesController {
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const response = await this.oldPipelinesService.getPipelineStatus(body);
@@ -256,6 +253,7 @@ export class PipelinesController {
});
return res;
});
return result;
}
@@ -0,0 +1,10 @@
export class CreateShareMetadataDto {
assetId: string;
proposedId: string;
type: string;
}
export class UpdateShareMetadataDto {
shareId: string;
type: string;
}
@@ -0,0 +1,31 @@
import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Controller('share')
@Authenticated()
export class ShareMetadataController {
constructor(private readonly shareMetadataService: ShareMetadataService) {}
@Put('/')
findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.create(data, metadata);
}
@Patch('/')
updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.updateShare(data, metadata);
}
@Get('/:id')
get(@Param('id') id: string, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.get(id, metadata);
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { ShareMetadataController } from './share-metadata.controller';
import { DucClient } from '../duc/client.config';
import { ClientsModule } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [ShareMetadataController],
providers: [ShareMetadataService, DadosferaLogger],
exports: [ShareMetadataService]
})
export class ShareMetadataModule {}
@@ -0,0 +1,62 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Metadata } from '@grpc/grpc-js';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Injectable()
export class ShareMetadataService implements OnModuleInit {
ducService: ShareMetadataProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<ShareMetadataProtoService>(
ProtoServices.ShareMetadataProtoService,
);
}
create(createShareDto: CreateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.FindOrCreateShareMetadata(
{
assetId: createShareDto.assetId,
proposedId: createShareDto.proposedId,
type: createShareDto.type
},
metadata,
),
);
}
updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.ChangeShareMetadataType(
updateShare,
metadata,
),
);
}
get(id: string, metadata: Metadata) {
return lastValueFrom(
this.ducService.GetShareMetadata(
{
id,
},
metadata,
),
);
}
}
+11
View File
@@ -174,3 +174,14 @@ export class GetAllDepartmentsRes {
@ApiProperty()
departments: string[];
}
export interface UserReporter {
name: string;
email: string;
mfaStatus: string;
status: string;
lastLogin: string;
createdAt: string;
updatedAt: string;
}
@@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
+24 -1
View File
@@ -12,6 +12,7 @@ import {
Post,
Put,
Query,
Res,
UseFilters,
} from '@nestjs/common';
import {
@@ -24,12 +25,13 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -52,6 +54,7 @@ import {
UpdateUserRes,
} from './dtos/entities';
import { UsersService } from './users.service';
import { Response } from 'express';
@ApiInternalOnlyController()
@ApiTags('Users')
@@ -80,6 +83,26 @@ export class UsersController {
return await this.userService.findAllUsersByCustomerId(user.customer_id);
}
@Get('/download')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async downloadUsersInCsv(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Res() res: Response
) {
this.logger.info('downloadUsersInCsv');
this.userService.setLanguage(language);
const {
file,
filename
} = await this.userService.downloadUsersInCsv(user.customer_id);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@Get('hierarchies')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: GetAllHierarchiesRes })
+11
View File
@@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
error: (...args) => args,
+30
View File
@@ -21,6 +21,7 @@ import {
IUserByCustomer,
SetUserRolesReq,
UpdateUserReq,
UserReporter,
} from './dtos/entities';
import { RolesService } from '../roles/roles.service';
import { HIERARCHIES } from './hierarchies';
@@ -29,6 +30,7 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces
import { EnrichErrorCode } from 'src/utils/ErrorBuilder';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@Injectable()
export class UsersService implements OnModuleInit {
@@ -78,6 +80,34 @@ export class UsersService implements OnModuleInit {
};
}
async downloadUsersInCsv(customerId: string) {
const { users } = await lastValueFrom(
this.usersClientService.UserFindAllByCustomerId({ customerId }),
);
const formatUsers: UserReporter[] = users.map(user => ({
createdAt: user.createdAt,
email: user.email,
lastLogin: user.lastLogin,
mfaStatus: user.mfaStatus,
name: user.name,
status: user.status,
updatedAt: user.updatedAt
}))
const parser = ParserBuilder.build<UserReporter>('csv');
const file = await parser.parse(formatUsers);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_users_${timestamp}.csv`;
return {
file,
filename
}
}
async findOneById(id: string): Promise<{ user: IUserByCustomer }> {
const { user } = await lastValueFrom(
this.usersClientService.UserFindOneById({ id }),
+23
View File
@@ -0,0 +1,23 @@
import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';
@Injectable()
export class CacheService<T> {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
) {}
async get(key: string): Promise<T | null> {
return await this.cacheManager.get<T>(key);
}
async set(key: string, value: T): Promise<void> {
await this.cacheManager.set(key, value);
}
async delete(key: string) {
await this.cacheManager.del(key);
}
}
+32
View File
@@ -0,0 +1,32 @@
import { CacheModule, Module } from '@nestjs/common';
import { CacheService } from './cache.service';
import { redisStore } from 'cache-manager-ioredis-yet';
@Module({
imports: [
CacheModule.registerAsync({
useFactory: async () => {
const baseRedisConfig = {
ttl: 5 * 1000 * 60, // 5 minute
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT && Number(process.env.REDIS_PORT),
db: process.env.REDIS_DATABASE && Number(process.env.REDIS_DATABASE),
keyPrefix: 'maestro:sso',
}
if (process.env.ENV !== 'local') {
baseRedisConfig['tls'] = {
servername: process.env.REDIS_HOST,
}
}
return {
store: await redisStore(baseRedisConfig),
}
},
}),
],
providers: [CacheService],
exports: [CacheService],
})
export class ServicesModule {}
+7 -1
View File
@@ -24,15 +24,21 @@ export async function getOauthSecrets() {
const path = process.env.SM_OAUTH_PATH;
const secretsManagerClient = new SecretsManagerClient({});
for (const key in secrets) {
console.log(`Fetching secret for ${key} from path ${path}/${key}`);
const getSecretComand = new GetSecretValueCommand({
SecretId: `${path}/${key}`,
});
const res = await secretsManagerClient
.send(getSecretComand)
.catch(() => null);
.catch((err) => {
console.error(`Error fetching secret for ${key}:`, err);
return null;
});
if (res) {
secrets[key] = JSON.parse(res.SecretString);
console.log(JSON.parse(res.SecretString).client_id)
} else {
console.log("No secret found for", key);
secrets[key] = {
client_id: 'id',
client_secret: 'id',
+5
View File
@@ -75,6 +75,10 @@ const CATALOG = {
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
};
const IDENTITY_PROVIDER = {
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',
INVALID_HEADER: 'IDENTITY_PROVIDER.INVALID_HEADER',
}
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
@@ -91,6 +95,7 @@ const ErrorCodes = {
TRANSFORMATION,
CONNECTION,
CATALOG,
IDENTITY_PROVIDER
};
export default ErrorCodes;