Compare commits

...
Author SHA1 Message Date
marcos-silva-rodrigues 301b6e98ec UPDATE: test subdomain 2025-08-14 17:42:42 -03:00
marcos-silva-rodrigues ba53934068 CI: add cookie secret 2025-08-14 17:23:40 -03:00
marcos-silva-rodrigues 216349f303 CHORE: update nginx ingress annotations 2025-08-14 16:03:28 -03:00
marcos-silva-rodrigues a654baef13 FIX: set cookies if exists tokens 2025-08-14 15:37:17 -03:00
marcos-silva-rodrigues 04d761ca14 Merge branch 'beta' into feature/auth-session 2025-08-14 15:26:15 -03:00
Marcos Rodrigues Silva a772804dcd Merge pull request #357 from dadosfera/chore/oracle
CI: fix multi cloud deploy and maestro env
2025-08-07 18:04:10 -03:00
marcos-silva-rodrigues 8a399fbbcf CI: fix multi cloud deploy and maestro env 2025-08-07 18:03:22 -03:00
Marcos Rodrigues Silva 94049d6e8e Merge pull request #356 from dadosfera/chore/oracle
CI: oracle
2025-08-07 12:27:08 -03:00
marcos-silva-rodrigues c28cc58ac7 CI: oracle 2025-08-07 12:24:44 -03:00
Marcos Rodrigues Silva 56a655bb90 Merge pull request #355 from dadosfera/chore/oracle
CI: fix env home
2025-08-07 12:16:02 -03:00
marcos-silva-rodrigues b3d4a6b028 CI: fix env home 2025-08-07 12:13:19 -03:00
Marcos Rodrigues Silva c7bd1f761a Merge pull request #354 from dadosfera/chore/oracle
CI: remove k8s-setup need
2025-08-07 12:01:43 -03:00
marcos-silva-rodrigues 09c045455e CI: remove k8s-setup need 2025-08-07 11:59:37 -03:00
Marcos Rodrigues Silva 52d651126e Merge pull request #353 from dadosfera/chore/oracle
CI: revert k8s-setup workflow for the step in job helmfile-deploy
2025-08-07 11:57:38 -03:00
marcos-silva-rodrigues 2f11b1e78f CI: revert k8s-setup workflow for the step in job helmfile-deploy 2025-08-07 11:56:54 -03:00
Marcos Rodrigues Silva 885e9b71bb Merge pull request #352 from dadosfera/fix/tracker-sso
Fix/tracker sso
2025-08-07 11:45:44 -03:00
marcos-silva-rodrigues 193031dfdd UPDATE: increase redis ttl 2025-08-07 10:41:48 -03:00
marcos-silva-rodrigues f626a9bb5e UPDATE: add more logger 2025-08-07 10:32:22 -03:00
Marcos Rodrigues Silva ac26ad9f44 Merge pull request #351 from dadosfera/chore/oracle
FIX: correct name
2025-08-01 18:06:27 -03:00
marcos-silva-rodrigues fbe00af6dc FIX: correct name 2025-08-01 18:05:19 -03:00
Marcos Rodrigues Silva bfa9929b69 Merge pull request #350 from dadosfera/chore/oracle
Chore/oracle
2025-08-01 18:01:38 -03:00
marcos-silva-rodrigues bada7000f6 CHORE: change ci to oracle cloud 2025-08-01 18:01:10 -03:00
Marcos Rodrigues Silva 2c908d8d95 Merge pull request #349 from dadosfera/beta
Beta
2025-07-31 15:24:28 -03:00
Marcos Rodrigues Silva da35633d42 Merge pull request #348 from dadosfera/hotfix/module-assigned
Hotfix/module assigned
2025-07-31 12:04:02 -03:00
marcos-silva-rodrigues cc7e06013f FIX: change module permission 2025-07-31 11:59:19 -03:00
Marcos Rodrigues Silva 5ec36a05b5 Merge pull request #344 from dadosfera/beta
Beta
2025-07-30 11:56:22 -03:00
Marcos Rodrigues Silva f61e46c7c1 Merge pull request #347 from dadosfera/release/07-29
FEAT: add permission embed access
2025-07-30 11:15:28 -03:00
marcos-silva-rodrigues b9013b1493 FEAT: add permission embed access 2025-07-30 11:12:08 -03:00
Marcos Rodrigues Silva 5267fa6d2f Merge pull request #346 from dadosfera/release/07-29
FIX: comment test
2025-07-29 18:05:26 -03:00
marcos-silva-rodrigues 7f40cbdeed FIX: comment test 2025-07-29 18:03:20 -03:00
Marcos Rodrigues Silva b590c9402d Merge pull request #345 from dadosfera/release/07-29
Release/07 29
2025-07-29 15:19:52 -03:00
marcos-silva-rodrigues 9546feda11 FIX: merge 2025-07-29 15:12:25 -03:00
Marcos Rodrigues Silva 3587f82976 Merge pull request #343 from dadosfera/bugfix/public-link
Bugfix/public link
2025-07-29 12:31:34 -03:00
marcos-silva-rodrigues 94a3962590 FIX: add needs steps 2025-07-29 12:26:21 -03:00
marcos-silva-rodrigues 3b2d79a451 CHORE: remove needs step 2025-07-29 12:06:33 -03:00
marcos-silva-rodrigues da56a645af CHORE: update affinity to azure 2025-07-29 11:57:10 -03:00
Marcos Rodrigues Silva af4e55af1d Merge pull request #342 from dadosfera/bugfix/public-link
CHORE: remove sudo
2025-07-29 11:35:09 -03:00
marcos-silva-rodrigues d6a08a8f82 CHORE: remove sudo 2025-07-29 11:32:02 -03:00
Marcos Rodrigues Silva c73eb8869f Merge pull request #341 from dadosfera/bugfix/public-link
CHORE: install kubectl
2025-07-29 11:15:24 -03:00
marcos-silva-rodrigues 929fc97ac6 CHORE: install kubectl 2025-07-29 11:12:47 -03:00
Marcos Rodrigues Silva 7594a75bea Merge pull request #340 from dadosfera/bugfix/public-link
CHORE: update ci to wait extract_environment job
2025-07-29 11:02:14 -03:00
marcos-silva-rodrigues ec36a056b2 CHORE: update ci to wait extract_environment job 2025-07-29 10:59:35 -03:00
Marcos Rodrigues Silva 8d30f6ef15 Merge pull request #339 from dadosfera/bugfix/public-link
CHORE: multi cloud deployment
2025-07-29 10:57:26 -03:00
marcos-silva-rodrigues 08d1e3164f CHORE: multi cloud deployment 2025-07-29 10:53:41 -03:00
Marcos Rodrigues Silva a5f3ef20fe Merge pull request #338 from dadosfera/bugfix/public-link
FIX: add user by bearer token if it exists
2025-07-29 09:51:17 -03:00
marcos-silva-rodrigues 467445fe05 FIX: add user by bearer token if it exists 2025-07-29 09:47:56 -03:00
Marcos Rodrigues Silva d0448f844a Merge pull request #337 from dadosfera/bugfix/google-oauth
UPDATE: more logs
2025-07-21 16:06:22 -03:00
marcos-silva-rodrigues b2299a5f2f UPDATE: more logs 2025-07-21 15:51:56 -03:00
Marcos Rodrigues Silva 9b42754c69 Merge pull request #336 from dadosfera/bugfix/google-oauth
Bugfix/google oauth
2025-07-21 15:06:49 -03:00
marcos-silva-rodrigues aee33334ac UPDATE: add log to debug 2025-07-21 15:01:41 -03:00
Marcos Rodrigues Silva cc530fb12d Merge pull request #335 from dadosfera/release/sso
Release/sso
2025-07-18 17:12:51 -03:00
marcos-silva-rodrigues 19877f1b29 FIX: merge with main 2025-07-18 16:46:19 -03:00
marcos-silva-rodrigues e9880bcf68 FIX: add logger in header 2025-07-18 15:56:23 -03:00
marcos-silva-rodrigues 4c7111dcb6 FIX: using referer headaer 2025-07-18 15:56:10 -03:00
marcos-silva-rodrigues e81cfdcce1 FIX: using only header origin 2025-07-18 15:55:56 -03:00
marcos-silva-rodrigues 3eba3f414f FIX: send redirect uri 2025-07-18 15:55:45 -03:00
marcos-silva-rodrigues fbeb8dfd91 FIX: redis env 2025-07-18 15:55:29 -03:00
marcos-silva-rodrigues dda3e8baaf FIX: add logger 2025-07-18 15:54:27 -03:00
marcos-silva-rodrigues c1e4f0b18a FIX: remove express session 2025-07-18 15:54:24 -03:00
Marcos Rodrigues Silva b0d9f3a468 Merge pull request #334 from dadosfera/chore/stg-env
FIX: revert env
2025-07-18 10:27:01 -03:00
marcos-silva-rodrigues 5b117acaa9 FIX: revert env 2025-07-18 10:23:10 -03:00
Marcos Rodrigues Silva a5cb607650 Merge pull request #333 from dadosfera/feature/sso-openid
FIX: change env to stg
2025-07-17 16:12:51 -03:00
marcos-silva-rodrigues fe626155c5 FIX: change env to stg 2025-07-17 16:10:42 -03:00
Marcos Rodrigues Silva debbe1c1b1 Merge pull request #332 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 18:50:04 -03:00
marcos-silva-rodrigues f937a6efe2 FIX: add logger in header 2025-07-16 18:49:27 -03:00
marcos-silva-rodrigues 30cb3c87eb FIX: using referer headaer 2025-07-16 18:45:47 -03:00
Marcos Rodrigues Silva fa46294154 Merge pull request #331 from dadosfera/feature/sso-openid
FIX: using only header origin
2025-07-16 18:37:53 -03:00
marcos-silva-rodrigues 72554fb27f FIX: using only header origin 2025-07-16 18:36:36 -03:00
Marcos Rodrigues Silva 4e6985b572 Merge pull request #330 from dadosfera/feature/sso-openid
FIX: send redirect uri
2025-07-16 18:16:27 -03:00
marcos-silva-rodrigues 4477647613 FIX: send redirect uri 2025-07-16 18:15:47 -03:00
Marcos Rodrigues Silva 2b3066abc9 Merge pull request #329 from dadosfera/feature/sso-openid
FIX: redis env
2025-07-16 17:57:26 -03:00
marcos-silva-rodrigues da1bcb38da FIX: redis env 2025-07-16 17:56:43 -03:00
Marcos Rodrigues Silva 4a7524422d Merge pull request #328 from dadosfera/feature/sso-openid
FIX: add logger
2025-07-16 17:41:40 -03:00
marcos-silva-rodrigues 275af03195 FIX: add logger 2025-07-16 17:39:15 -03:00
Marcos Rodrigues Silva 0c75cf4619 Merge pull request #327 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 12:33:44 -03:00
marcos-silva-rodrigues 80de277529 FIX: merge problems 2025-07-16 12:27:33 -03:00
marcos-silva-rodrigues 98f669db66 FIX: remove express session 2025-07-16 11:48:15 -03:00
marcos-silva-rodrigues 2c8dab5927 FIX: merge 2025-07-16 11:46:05 -03:00
marcos-silva-rodrigues cb66d4f6c0 FEAT: dynamic callback and issuer urls 2025-07-16 11:35:47 -03:00
marcos-silva-rodrigues 42a39ee891 FEAT: send user id to refresh token endpoint 2025-07-15 16:21:44 -03:00
marcos-silva-rodrigues c9bfe68ea5 FEAT: list identity providers link for sso 2025-07-14 15:16:19 -03:00
marcos-silva-rodrigues 8cb7144ff4 FIX: send accessToken in refresh token endpoint 2025-07-14 15:14:25 -03:00
Marcos Rodrigues Silva 68cc58e2fe Merge pull request #326 from dadosfera/release/11-07
Release/11 07
2025-07-14 10:54:14 -03:00
marcos-silva-rodrigues cbe5556ac9 FIX: auth callback 2025-07-14 10:51:36 -03:00
marcos-silva-rodrigues eb80967608 FIX: tests 2025-07-11 11:36:23 -03:00
marcos-silva-rodrigues 66fea0722d FEAT: enabled network check in app.dadosfera 2025-07-11 11:12:35 -03:00
marcos-silva-rodrigues c917061b97 Merge branch 'feature/export-users-and-assets' into release/11-07 2025-07-11 11:04:31 -03:00
Rafael Santana da39a07f25 Merge pull request #325 from dadosfera/feat/oracle-migration-and-affinity-standardization
FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
2025-07-09 13:02:48 -03:00
Rafael f2f65285de resolve: merge conflicts with beta branch
- Resolve ingress.yaml timeout annotation conflicts by keeping both configurations
- Resolve stg.yaml conflicts by merging affinity/resources config with restricted_ip
- Fix chart references in unimed-maestro and private-maestro sections
- Update all chart paths from ../maestro to ../helm-chart for consistency

This merge brings in the latest beta changes while preserving our Oracle migration
and affinity standardization configuration.
2025-07-09 13:00:52 -03:00
Rafael cac8f93330 CI: update validate-k8s.yml to use beta branch for staging
- Change trigger branches from main/stg to main/beta
- Update environment extraction to map beta → stg environment
- Align with standard branch naming convention (beta for staging)
- Consistent with deploy-manually.yml workflow configuration
2025-07-09 12:58:11 -03:00
Rafael bb03b18f4a FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
Oracle Migration Changes:
- Create deploy/helm-chart/ directory structure (moved from maestro/)
- Create deploy/helmfiles/ directory structure (moved from helmfiles/)
- Update chart references in helmfiles from ../maestro to ../helm-chart
- Update deploy-manually.yml from Azure AKS to Oracle OKE
- Update validate-k8s.yml from Azure AKS to Oracle OKE
- Replace Azure CLI with OCI CLI installation and configuration
- Replace Azure authentication with OCI authentication using secrets
- Replace az aks get-credentials with oci ce cluster create-kubeconfig
- Update helmfile paths from helmfiles/ to deploy/helmfiles/
- Remove DockerHub integration (push_to_dockerhub input and related steps)
- Change runner from [self-hosted, prd-azure] to [self-hosted, prd-oracle]
- Add environment field for proper Oracle deployment environment handling
- Add HOME environment variable for OCI CLI

Affinity/Resources Standardization:
- Add affinity configuration with name=general node selector in values.yaml
- Update deployment template to use conditional affinity and resources blocks
- Configure staging to disable both affinity and resources (null values)
- Enable production to use affinity targeting name=general nodes
- Enable production to use standard resource limits (100m-2000m CPU, 1500Mi-2Gi memory)

This brings maestro into full compliance with Oracle migration requirements and
implements the standardized affinity/resources pattern consistent with in-factory and duc.
2025-07-09 12:55:49 -03:00
Marcos Rodrigues Silva ab280ee152 Merge pull request #324 from dadosfera/chore/update-unimed-ips
CHORE: Update unimed ips
2025-07-08 09:22:26 -03:00
marcos-silva-rodrigues 16c8137160 CHORE: Update unimed ips 2025-07-07 17:26:26 -03:00
Marcos Rodrigues Silva b8b903c8cd Merge pull request #323 from dadosfera/feature/export-users-and-assets
Feature/export users and assets
2025-07-05 11:27:10 -03:00
marcos-silva-rodrigues c98a5ff896 FEAT: endpoint to dowload assets list 2025-07-05 11:21:05 -03:00
marcos-silva-rodrigues d68e436a8b FEAT: export users in csv 2025-07-05 09:38:16 -03:00
marcos-silva-rodrigues 8efa6790d6 FEAT: sign out endpoint 2025-07-04 18:03:08 -03:00
marcos-silva-rodrigues a381f50e88 FEAT: set cookie after user login 2025-07-03 15:52:55 -03:00
marcos-silva-rodrigues d3040cd5a8 FEAT: update and delete identity provider 2025-07-03 10:59:43 -03:00
marcos-silva-rodrigues f726a9883f FEAT: create user from identity provider 2025-06-27 13:21:59 -03:00
Marcos Rodrigues Silva c18a7baa37 Merge pull request #322 from dadosfera/chore/add-dadosfera-vpn-unimed-chart
FIX: add dadosfera vpn ip in unimed chart
2025-06-25 11:12:33 -03:00
marcos-silva-rodrigues d638d518f8 FIX: change unimed chart name 2025-06-25 11:06:41 -03:00
marcos-silva-rodrigues bdd371ffe8 FIX: add dadosfera vpn ip in unimed chart 2025-06-25 10:58:18 -03:00
marcos-silva-rodrigues 57e6e49c7c FEAT: init url callback 2025-06-25 10:28:26 -03:00
marcos-silva-rodrigues a5c8de496d FEAT: map permission to identity provider 2025-06-24 16:20:03 -03:00
Marcos Rodrigues Silva d478f32760 Merge pull request #321 from dadosfera/hotfix/nginx-timeout
FIX: add nginx proxy timeout 5 min
2025-06-24 11:02:29 -03:00
marcos-silva-rodrigues 5d8e908339 FIX: add nginx proxy timeout 5 min 2025-06-24 10:26:50 -03:00
Marcos Rodrigues Silva 606bae016b Merge pull request #320 from dadosfera/hotfix/maesto-unimed-hostname
FIX: correct helm value
2025-06-23 18:43:58 -03:00
marcos-silva-rodrigues 6910a2f04c FIX: correct helm value 2025-06-23 18:38:20 -03:00
Marcos Rodrigues Silva 8bee1788c1 Merge pull request #319 from dadosfera/hotfix/maesto-unimed-hostname
FIX: maestro custom hostname to unimed
2025-06-23 18:04:43 -03:00
marcos-silva-rodrigues b10459cb15 FIX: maestro custom hostname to unimed 2025-06-23 17:51:14 -03:00
Marcos Rodrigues Silva 3b48dd1613 Merge pull request #317 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 17:07:55 -03:00
Marcos Rodrigues Silva e5cef90a64 Merge pull request #318 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 16:04:59 -03:00
marcos-silva-rodrigues bac74c9577 FIX: default value and throw correct Error Class 2025-06-23 15:36:26 -03:00
marcos-silva-rodrigues 19d748ae09 CHORE: add restricted ip and unimed chart 2025-06-23 14:47:39 -03:00
marcos-silva-rodrigues 4db865371f Merge branch 'bugfix/update-pupperter-timeout' into release/2025-06 2025-06-23 14:12:25 -03:00
marcos-silva-rodrigues 75e8b3bf4b FIX: correct dto name 2025-06-17 14:58:35 -03:00
Marcos Rodrigues Silva 194cdc66b7 Merge pull request #316 from dadosfera/feature/public-and-assign-embed
Feature/public and assign embed
2025-06-17 08:57:53 -03:00
marcos-silva-rodrigues 53059c0a0d FIX: import 2025-06-16 18:09:25 -03:00
marcos-silva-rodrigues ed4ffc8a86 FIX: merge with beta 2025-06-16 18:01:53 -03:00
Marcos Rodrigues Silva 9967c172da Merge pull request #315 from dadosfera/feature/dedicated-maestro
FEAT: block user from login when maestro is dedicated
2025-06-16 10:37:47 -03:00
marcos-silva-rodrigues 0aa9c065a5 FEAT: block user from login when maestro is dedicated 2025-06-16 10:34:11 -03:00
Marcos Rodrigues Silva d0c2abd38d Merge pull request #314 from dadosfera/bugfix/update-pupperter-timeout
FIX: update timeout
2025-06-13 15:10:08 -03:00
marcos-silva-rodrigues cb98099a91 FIX: update timeout 2025-06-13 15:07:50 -03:00
Marcos Rodrigues Silva f446e0d5b1 Merge pull request #313 from dadosfera/feature/dedicated-maestro
FIX: send env DEDICATED_PROXY
2025-06-12 14:47:54 -03:00
marcos-silva-rodrigues 17de31f1a6 FIX: send env DEDICATED_PROXY 2025-06-12 14:45:13 -03:00
Marcos Rodrigues Silva d81a466089 Merge pull request #312 from dadosfera/feature/dedicated-maestro
FIX: open data ingress route
2025-06-12 14:08:13 -03:00
marcos-silva-rodrigues 9e597fadba FIX: open data ingress route 2025-06-12 14:04:45 -03:00
Marcos Rodrigues Silva 1e6aa0f4c2 Merge pull request #311 from dadosfera/feature/dedicated-maestro
FIX: chart private maestro
2025-06-12 13:59:11 -03:00
marcos-silva-rodrigues 41842f7c6a FIX: chart private maestro 2025-06-12 13:56:57 -03:00
Marcos Rodrigues Silva 0d5564990d Merge pull request #309 from dadosfera/feature/dedicated-maestro
Feature/dedicated maestro
2025-06-12 12:31:26 -03:00
marcos-silva-rodrigues 7f9755493a FIX: experimental hostname 2025-06-12 12:29:51 -03:00
marcos-silva-rodrigues fea587ad36 FEAT: control login by dedicated customer id injected 2025-06-12 11:51:56 -03:00
Marcos Rodrigues Silva b2b7537fee Merge pull request #308 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 15:25:27 -03:00
Marcos Rodrigues Silva 57f99fe7af Merge pull request #307 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 14:25:23 -03:00
marcos-silva-rodrigues 01263017e7 FIX: comment block list 2025-06-10 14:24:26 -03:00
Marcos Rodrigues Silva e66f9a7244 Merge pull request #306 from dadosfera/feature/api-key
FEAT: add api key endpoint
2025-06-06 15:55:53 -03:00
marcos-silva-rodrigues 54cb3f8e7a FEAT: add api key endpoint 2025-06-06 15:53:12 -03:00
Marcos Rodrigues Silva 4c861cf5e4 Merge pull request #305 from dadosfera/fix/maestro-stg-url
Fix/maestro stg url
2025-05-30 10:15:10 -03:00
marcos-silva-rodrigues a89f57e690 FIX: return pi_factory_url and tr_factory_url to stg url 2025-05-30 10:13:37 -03:00
marcos-silva-rodrigues d5141e5b6a REFACTOR: rename openid to identity provider 2025-05-26 13:37:25 -03:00
Rafael Santana 8aded98a11 Merge pull request #304 from dadosfera/beta
Beta
2025-05-26 10:51:13 -03:00
Rafael Santana 0b5a0d0d7e Merge pull request #303 from dadosfera/adjusting-nestjs-payload
Adjusting nestjs payload
2025-05-26 10:40:43 -03:00
Rafael 5a36762544 UPDATE: route /catalog/register-dataset will have an limit of 10mb 2025-05-26 10:24:48 -03:00
marcos-silva-rodrigues a4b7573ce7 FIX: sync with main 2025-05-23 14:07:05 -03:00
marcos-silva-rodrigues 6fc8c0b4a8 FEAT: add mixpanel service to track share access 2025-05-20 16:50:04 -03:00
marcos-silva-rodrigues ced2a97547 FIX: add logger and return info 2025-05-19 16:36:54 -03:00
Marcos Rodrigues Silva f579171024 Merge pull request #302 from dadosfera/beta
Default value to header origin
2025-05-15 11:18:17 -03:00
Marcos Rodrigues Silva 885e5f4568 Merge pull request #301 from dadosfera/fix/remove-default-host
Fix/remove default host
2025-05-15 11:00:45 -03:00
marcos-silva-rodrigues 81d8cd8a07 FIX: default value to header origin 2025-05-15 11:00:07 -03:00
Marcos Rodrigues Silva f153f41357 Merge pull request #300 from dadosfera/beta
Beta
2025-05-14 18:30:54 -03:00
marcos-silva-rodrigues 2b9e3ff3bc Merge branch 'main' into beta 2025-05-13 14:32:34 -03:00
marcos-silva-rodrigues d15cc4642a FEAT: assign embed and public key changes 2025-05-13 09:21:44 -03:00
marcos-silva-rodrigues 3c4f45678a CHORE: update protospack 2025-05-12 12:25:45 -03:00
marcos-silva-rodrigues b312849fff FEAT: catalog share endpoint for embeds 2025-05-12 12:15:16 -03:00
marcos-silva-rodrigues 10580dfa03 FEAT: share endpoint 2025-05-12 12:13:39 -03:00
marcos-silva-rodrigues 62d0e8b041 FEAT: assign pubic key endpoint 2025-05-12 12:12:58 -03:00
Marcos Rodrigues Silva 17cdaac380 Merge pull request #299 from dadosfera/fix/stg-envs
FIX: stg to prd for sbm environment
2025-05-09 11:07:22 -03:00
marcos-silva-rodrigues e4eb388e18 FIX: stg to prd for sbm environment 2025-05-09 11:06:23 -03:00
marcos-silva-rodrigues d94c5dee4e FEAT: init public routes to get data asset by public embed 2025-05-06 14:44:09 -03:00
Marcos Rodrigues Silva 4bea0816b6 Merge pull request #298 from dadosfera/feature/pii-rules
FIX: install chromiun in base_image
2025-04-29 16:22:22 -03:00
marcos-silva-rodrigues fb491328d6 FIX: install chromiun in base_image 2025-04-29 16:20:37 -03:00
Marcos Rodrigues Silva 4e043d93ce Merge pull request #297 from dadosfera/feature/pii-rules
REFACTOR: move template to assets folder
2025-04-29 15:41:16 -03:00
marcos-silva-rodrigues 0b525b835f REFACTOR: move template to assets folder 2025-04-29 15:40:37 -03:00
Marcos Rodrigues Silva 3c8c39ec26 Merge pull request #296 from dadosfera/feature/pii-rules
REFACTOR: create builder for parser reporter and fix pdf
2025-04-29 15:20:46 -03:00
marcos-silva-rodrigues 36997403e1 REFACTOR: create builder for parser reporterand fix pdf 2025-04-29 15:18:46 -03:00
Marcos Rodrigues Silva 1175591521 Merge pull request #295 from dadosfera/feature/pii-rules
Feature/pii rules
2025-04-29 14:04:06 -03:00
marcos-silva-rodrigues 04accb6689 FIX: correct stg urls 2025-04-29 14:02:24 -03:00
marcos-silva-rodrigues 58e3c7be94 FEAT: pii 2025-04-29 14:00:23 -03:00
marcos-silva-rodrigues 7a6db36554 FEAT: generate pii reporter 2025-04-27 13:00:49 -03:00
Marcos Rodrigues Silva 80ebd98f6f Merge pull request #294 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-23 17:46:18 -03:00
Marcos Rodrigues Silva 9e0495fc26 Merge pull request #293 from dadosfera/feat/enforce-mfa
Feat/enforce mfa
2025-04-22 14:50:39 -03:00
marcos-silva-rodrigues f2437a8b3c FIX: add decorator to require danger zone module 2025-04-22 14:46:39 -03:00
marcos-silva-rodrigues 9e87562e23 FEAT: update protospack 2025-04-22 09:43:34 -03:00
marcos-silva-rodrigues 8f5cb13cc3 FEAT: enforce mfa 2025-04-21 18:53:34 -03:00
Marcos Rodrigues Silva e0a0ec9911 Merge pull request #292 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-17 18:12:28 -03:00
marcos-silva-rodrigues 419c063c6c FEAT: add access dashboard 2025-04-17 18:04:18 -03:00
marcos-silva-rodrigues 2aed66d371 FEAT: add decorator to block endpoint if not found module 2025-04-17 15:20:27 -03:00
Marcos Rodrigues Silva 1f45436c13 Merge pull request #291 from dadosfera/fix/network-policy
FIX: send origin header
2025-04-16 17:13:55 -03:00
marcos-silva-rodrigues df8520afd3 FIX: send origin header 2025-04-15 14:24:41 -03:00
Marcos Rodrigues Silva 7044d6606e Merge pull request #278 from dadosfera/beta
Beta
2025-04-08 18:04:41 -03:00
marcos-silva-rodrigues 4c94a51a9d REFACTOR: openid module 2025-04-03 07:46:16 -03:00
marcos-silva-rodrigues f1345472b5 UPDATE: add routes to login with idp 2025-03-25 09:18:27 -03:00
marcos-silva-rodrigues ea9c0502b7 UPDATE: new route to add identity provider 2025-03-24 07:58:26 -03:00
90 changed files with 5461 additions and 868 deletions
+7 -72
View File
@@ -13,11 +13,6 @@ on:
options:
- stg
- prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs:
extract_environment:
@@ -118,23 +113,6 @@ jobs:
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2
with:
username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker compose -f build.docker-compose.dockerhub.yml build
docker compose -f build.docker-compose.dockerhub.yml push
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -164,54 +142,11 @@ jobs:
docker system prune --volumes -a -f
docker system df
helmfile-deploy:
k8s-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: ${{ (needs.extract_environment.outputs.environment == 'prd' && 'azure' ) || 'oracle' }}
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
+137
View File
@@ -0,0 +1,137 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+45 -22
View File
@@ -4,7 +4,7 @@ on:
pull_request:
branches:
- main
- stg
- beta
jobs:
extract_environment:
@@ -21,17 +21,13 @@ jobs:
fi
id: extract_environment
helmfile-deploy:
helmfile-check:
env:
HOME: /home/runner
needs: [extract_environment]
runs-on: [self-hosted, "prd-azure"]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
steps:
- name: Summary
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
- name: Checkout code
uses: actions/checkout@v3
@@ -40,13 +36,28 @@ jobs:
with:
version: 'v3.9.0'
- name: Install Azure ClI
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
@@ -57,23 +68,35 @@ jobs:
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f helmfiles/${ENV}.yaml diff
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+24 -1
View File
@@ -4,9 +4,19 @@ FROM base_image AS build_base
WORKDIR /app
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache aws-cli
RUN apk add --no-cache \
aws-cli \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY package*.json ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# run aws cli without mounting secret, because CI already has AWS credentials
FROM build_base AS ci_image
@@ -40,4 +50,17 @@ WORKDIR /app
COPY --from=prod_build /app/dist ./dist
COPY --from=prod_build /app/node_modules ./node_modules
COPY --from=prod_build /app/package*.json ./
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT npm run start:prod
@@ -1,16 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: maestro
app: {{ .Values.app_name }}
strategy:
rollingUpdate:
@@ -20,22 +20,17 @@ spec:
template:
metadata:
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: application
operator: In
values:
- general
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
@@ -48,13 +43,10 @@ spec:
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
requests:
cpu: {{ .Values.resources.requests.cpu }}
memory: {{ .Values.resources.requests.memory }}
limits:
cpu: {{ .Values.resources.limits.cpu }}
memory: {{ .Values.resources.limits.memory }}
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
@@ -100,6 +92,16 @@ spec:
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -108,15 +110,15 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
@@ -10,8 +10,12 @@ metadata:
generation: 1
labels:
app: maestro
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
@@ -21,7 +25,7 @@ spec:
paths:
- backend:
service:
name: maestro
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
+36
View File
@@ -0,0 +1,36 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
@@ -1,17 +1,17 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-maestro
name: prd-{{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-maestro
name: prd-{{ .Values.app_name }}
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
@@ -1,18 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
type: ClusterIP
ports:
- name: maestro
- name: {{ .Values.app_name }}
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: maestro
app: {{ .Values.app_name }}
+23
View File
@@ -0,0 +1,23 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- general
@@ -9,6 +9,7 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
@@ -42,9 +43,25 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "product-redis-prd.z4xvqj.0001.use1.cache.amazonaws.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: applications
operator: In
values:
- general
+53
View File
@@ -0,0 +1,53 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
- name: maestro.restricted_ip
value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30"
+43
View File
@@ -0,0 +1,43 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
# - name: private-maestro
# chart: ../helm-chart
# values:
# - ../helm-chart/values.yaml
# set:
# - name: app_name
# value: maestro-private
# - name: maestro.env
# value: stg
# - name: maestro.duc_url
# value: duc.stg.dadosfera.ai
# - name: hostname
# value: private-maestro.stg.dadosfera.ai
# - name: maestro.pi_factory_url
# value: pi-factory.dadosfera.ai
# - name: maestro.in_factory_url
# value: in-factory.stg.dadosfera.ai
# - name: maestro.tr_factory_url
# value: in-factory.dadosfera.ai
# - name: maestro.open_customer_id
# value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
# - name: maestro.open_group_id
# value: e3f98a2f-7748-4981-8505-7695c8ca8218
# # Customer id
# - name: maestro.dedicated_proxy
# value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
# - name: replicaCount
# value: 1
# - name: affinity
# value: null
# - name: resources
# value: null
# - name: maestro.restricted_ip
# value: "57.151.113.140/30"
+1401 -426
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -14,6 +14,8 @@ declare global {
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
}
}
}
-22
View File
@@ -1,22 +0,0 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
-22
View File
@@ -1,22 +0,0 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
-23
View File
@@ -1,23 +0,0 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
-27
View File
@@ -1,27 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: maestro
name: maestro
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
+1
View File
@@ -4,6 +4,7 @@
"compilerOptions": {
"assets": [
"**/*.proto",
"assets/**/*",
{
"include": "i18n/**/*",
"watchAssets": true
+1077 -9
View File
File diff suppressed because it is too large Load Diff
+12 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.37.0-alpha.20",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -45,22 +45,30 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.50.0",
"handlebars": "^4.7.8",
"helmet": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
@@ -70,7 +78,10 @@
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
+11 -1
View File
@@ -28,7 +28,11 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
providers: [
@@ -62,9 +66,15 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule
],
})
export class AppModule {}
+130
View File
@@ -0,0 +1,130 @@
<!DOCTYPE html>
<html lang="pt-br">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Dadosfera Relatório de PII</title>
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
<style>
@page {
size: A4 landscape; /* Alterado para paisagem (landscape) */
margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
}
body {
font-family: 'Quicksand', sans-serif;
color: #5c5c5c;
margin: 0;
padding: 10px;
font-size: 12px; /* Reduzindo o tamanho da fonte */
}
.container {
margin: 0;
width: 100%;
}
.header {
display: flex;
align-items: center;
margin-bottom: 20px;
}
.logo {
max-width: 150px; /* Reduzida para economizar espaço */
height: auto;
}
h1 {
color: #0d003b;
font-weight: 700;
margin-left: 20px;
font-size: 24px; /* Tamanho ajustado */
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 15px;
table-layout: fixed; /* Importante: define larguras fixas */
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border: 1px solid #d0d0d0;
}
th {
background-color: #1700a2;
color: white;
font-weight: bold;
text-align: left;
padding: 8px 10px;
border: 1px solid #3a26b8;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
td {
padding: 6px 10px;
border: 1px solid #d0d0d0;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
/* Definindo larguras específicas para cada coluna */
th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
tr:nth-child(even) {
background-color: #f9f9f9;
}
tr:nth-child(odd) {
background-color: white;
}
.info-section {
margin-top: 20px;
color: #5c5c5c;
}
.timestamp {
font-style: italic;
text-align: right;
margin-top: 15px;
font-size: 0.9em;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
<h1>Relatório de PII</h1>
</div>
<div class="info-section">
<p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
</div>
<table>
<thead>
<tr>
<th>Database</th>
<th>Schema</th>
<th>Tabela</th>
<th>Coluna</th>
<th>Tipo de Dado</th>
<th>Regras PII</th>
</tr>
</thead>
<tbody>
{{#each dados}}
<tr>
<td>{{database_name}}</td>
<td>{{table_schema}}</td>
<td>{{table_name}}</td>
<td>{{column_name}}</td>
<td>{{data_type}}</td>
<td>{{pii_rules}}</td>
</tr>
{{/each}}
</tbody>
</table>
<p class="timestamp">Gerado em: {{dataGeracao}}</p>
</div>
</body>
</html>
+20 -12
View File
@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
+40 -4
View File
@@ -4,6 +4,7 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -17,6 +18,7 @@ import {
import { RequestUser } from '../decorators/user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
@Injectable()
export class AuthenticationGuard
@@ -31,6 +33,7 @@ export class AuthenticationGuard
dadosferaLogger: DadosferaLogger,
private reflector: Reflector,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService
) {
this.pems = new Map();
this.logger = dadosferaLogger.logger;
@@ -48,20 +51,40 @@ export class AuthenticationGuard
});
}
canActivate(ctx: ExecutionContext): boolean {
async canActivate(ctx: ExecutionContext): Promise<boolean> {
const authFunctions = this.reflector.getAllAndMerge<
AuthenticationFunction[]
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
const mustBeAuthenticated = authFunctions.length > 0;
const request = ctx.switchToHttp().getRequest();
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!mustBeAuthenticated) {
// no need to be authenticated
return true;
}
const request = ctx.switchToHttp().getRequest();
const apiKey = request.get('X-api-key');
if (apiKey) {
const {
api_key
} = await this.apiKeyService.get(apiKey);
request.user = {
user_id: api_key.user_id,
username: api_key.username,
permissions: api_key.permissions,
customer_id: api_key.customer_id,
customer_name: api_key.customer_name,
customer_tier: api_key.customer_tier,
customer_modules: api_key.customer_modules,
access_token: apiKey,
};
return true;
}
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!accessToken) {
// couldn't load valid token
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
@@ -113,6 +136,18 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
@@ -121,6 +156,7 @@ export class AuthenticationGuard
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
// TODO: for backwards compatibility. remove in the future
+29 -10
View File
@@ -340,16 +340,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -362,6 +352,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -609,6 +618,16 @@ export interface DadosferaModule {
key: string;
permissionSeqId: number;
}
export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
{
name: 'Intelligence Module',
@@ -25,6 +25,14 @@ export function RequireSomePermission(
);
}
export function RequireModule(
key: string
) {
return createAuthenticatedDecorator((_, user: RequestUser) =>
user.customer_modules.some(module => module === key),
);
}
export function AuthenticateCondition(func: AuthenticationFunction) {
return createAuthenticatedDecorator(func);
}
+9 -1
View File
@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -52,6 +53,7 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
UserTest(token);
// UserTest(token);
});
+1
View File
@@ -11,6 +11,7 @@ export interface RequestUser {
customer_name: string;
customer_tier: string;
access_token: string;
customer_modules: string[];
}
export const User: (options?: { required?: boolean }) => ParameterDecorator =
+10
View File
@@ -3,11 +3,14 @@ import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import helmet from 'helmet';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { json, urlencoded } from 'express';
import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -22,9 +25,16 @@ async function bootstrap() {
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
}
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
+68
View File
@@ -0,0 +1,68 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@Controller('api-key')
@Authenticated()
@ApiTags('ApiKey')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
export class ApiKeyController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
method: 'create'
});
const result = await this.apiKeyService.create(createApiKeyDto, user);
this.logger.info('POST /api-key success', {
id: result.id,
method: 'create'
});
return result;
}
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
});
const result = await this.apiKeyService.findAll(user);
this.logger.info('GET /api-key success', {
count: result.length,
method: 'findAll'
});
return result;
}
@Delete(':id')
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
method: 'remove'
});
await this.apiKeyService.remove(id, user);
this.logger.info('DELETE /api-key/:id success', {
id,
method: 'remove'
});
}
}
+18
View File
@@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { ApiKeyController } from './api-key.controller';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([ducClient.providerOptions])
],
controllers: [ApiKeyController],
providers: [ApiKeyService, DadosferaLogger],
exports: [ApiKeyService]
})
export class ApiKeyModule {}
+50
View File
@@ -0,0 +1,50 @@
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc,
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
}
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
return lastValueFrom(this.apiKeyService.CreateApiKey({
permissions: createApiKeyDto.permissions
}, metadata));
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata)
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
return data.api_keys;
}
async remove(id: string, user: RequestUser) {
const metadata = PackTheMetadata(user);
await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
}
async get(key: string) {
const metadata = PackTheMetadata({});
return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
}
}
+39
View File
@@ -0,0 +1,39 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsArray, IsNumber } from 'class-validator';
export class PermissionDto {
@ApiProperty({ type: Number })
id: number;
@ApiProperty({ type: String })
name: string;
}
export class ApiKeyBaseResponseDto {
@ApiProperty({ type: String, format: 'uuid' })
id: string;
@ApiProperty({ type: String })
key_mask: string;
@ApiProperty({ type: [PermissionDto] })
permissions: PermissionDto[];
@ApiProperty({ type: String, format: 'date-time' })
created_at: string;
@ApiProperty({ type: String })
created_by: string;
}
export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
@ApiProperty({ type: String })
key: string;
}
export class CreateApiKeyDto {
@ApiProperty({ type: [Number], description: 'Array of permission IDs' })
@IsArray()
@IsNumber({}, { each: true })
permissions: number[];
}
+33
View File
@@ -0,0 +1,33 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
}
}
+15
View File
@@ -0,0 +1,15 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
+38
View File
@@ -0,0 +1,38 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -0,0 +1,3 @@
export class CreateAssignDto {
publicKey: string;
}
+219 -32
View File
@@ -12,6 +12,7 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -26,7 +27,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthVerifyTotpMfaRequest
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -45,15 +46,21 @@ import {
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request } from 'express';
import { Request, Response } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -63,7 +70,6 @@ export class AuthController {
logger: DadosferaLogger;
redirectUrl: string;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -88,13 +94,65 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ language, custom_host: frontHost });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('refresh-access-token')
@@ -103,18 +161,26 @@ export class AuthController {
async refreshAccessToken(
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('host') host: string
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
customer_name,
language,
host
custom_host: frontHost,
});
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
}
@ApiInternalOnlyEndpoint()
@@ -185,16 +251,23 @@ export class AuthController {
@ApiInternalOnlyEndpoint()
@Post('confirm-reset-password')
@HttpCode(HttpStatus.OK)
async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
async confirmResetPassword(
@Body() body: AuthConfirmResetPasswordRequest,
@Headers('origin') origin: string,
) {
this.logger.info('/auth - confirm-reset-password');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ custom_host: frontHost });
const { username, code, newPassword } = body;
return this.authClient.confirmResetPassword({
username,
code,
newPassword,
});
return this.authClient.confirmResetPassword(
{
username,
code,
newPassword,
},
metadata,
);
}
@ApiInternalOnlyEndpoint()
@@ -360,18 +433,18 @@ export class AuthController {
async blockUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest
@Body() body: BulkEditRequest,
) {
this.logger.info('blockUsers - Starting request');
try {
const metadata = PackTheMetadata(user);
this.logger.debug('Calling blockUsers service', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language')
}
language: metadata.get('language'),
},
});
const result = await this.authClient.blockUsers(body.users, metadata);
@@ -380,7 +453,7 @@ export class AuthController {
} catch (error) {
this.logger.error('blockUsers - Error', {
error: error.message,
stack: error.stack
stack: error.stack,
});
throw error;
}
@@ -393,7 +466,7 @@ export class AuthController {
async unblockUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest
@Body() body: BulkEditRequest,
) {
this.logger.info('unblockUsers');
const metadata = PackTheMetadata(user);
@@ -408,11 +481,125 @@ export class AuthController {
async resetUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest
@Body() body: BulkEditRequest,
) {
this.logger.info('resetUsers');
const metadata = PackTheMetadata(user);
return this.authClient.resetUsers(body.users, metadata);
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
}
+83 -28
View File
@@ -1,10 +1,10 @@
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -17,16 +17,22 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -39,6 +45,10 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async getPublicKeys() {
@@ -53,25 +63,59 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
}
async refreshAccessToken(
{ refreshToken }: AuthRefreshAccessTokenRequest,
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
);
}
@@ -113,19 +157,21 @@ export class AuthClientService implements OnModuleInit {
);
}
async confirmResetPassword({
username,
code,
newPassword,
}: AuthConfirmResetPasswordRequest) {
async confirmResetPassword(
{ username, code, newPassword }: AuthConfirmResetPasswordRequest,
metadata: Metadata,
) {
this.logger.info('confirmResetPassword');
return lastValueFrom(
this.authService.AuthConfirmResetPassword({
username,
code,
newPassword,
}),
this.authService.AuthConfirmResetPassword(
{
username,
code,
newPassword,
},
metadata,
),
);
}
@@ -170,15 +216,18 @@ export class AuthClientService implements OnModuleInit {
);
}
async blockUsers(users: string[],metadata: Metadata): Promise<BulkEditResponse> {
async blockUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('blockUsers - Service starting');
try {
this.logger.debug('Calling BlockUser gRPC method', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language')
}
language: metadata.get('language'),
},
});
const response = await lastValueFrom<BulkEditResponse>(
@@ -196,26 +245,32 @@ export class AuthClientService implements OnModuleInit {
}
}
async unblockUsers(users: string[],metadata: Metadata): Promise<BulkEditResponse> {
async unblockUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('unblockUsers');
return await lastValueFrom(
this.authService.UnblockUser({ users }, metadata),
);
}
async resetUsers(users: string[], metadata: Metadata): Promise<BulkEditResponse> {
async resetUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('resetUsers');
try {
this.logger.debug('Calling ResetUser gRPC method', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language')
}
language: metadata.get('language'),
},
});
const response = await lastValueFrom<BulkEditResponse>(
this.authService.ResetUser({ users }, metadata)
this.authService.ResetUser({ users }, metadata),
);
this.logger.info('resetUsers - Success', { response });
@@ -223,7 +278,7 @@ export class AuthClientService implements OnModuleInit {
} catch (error) {
this.logger.error('resetUsers - Error', {
error: error.message,
stack: error.stack
stack: error.stack,
});
throw error;
}
+3 -1
View File
@@ -87,6 +87,8 @@ export class AuthCustomer {
@ApiProperty()
themeEnabled: boolean;
@ApiProperty()
enforceMfa: boolean;
}
export class AuthSignInReq implements AuthSignInRequest {
@@ -120,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
customerName: string;
userId: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
@@ -26,6 +26,7 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
+119 -25
View File
@@ -15,6 +15,7 @@ import {
UseFilters,
HttpException,
HttpStatus,
Res,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -25,14 +26,14 @@ import {
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from '../../decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { lastValueFrom } from 'rxjs';
import {
BatchRemoveRlsRulesRequest,
GetDatasetCatalogTaskRes,
@@ -59,6 +60,8 @@ import {
GetRlsRulesRequest,
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
import { TypeParser } from 'src/utils/FileParser/parser-types';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -114,6 +117,52 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog/download - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
@@ -344,7 +393,7 @@ export class CatalogController {
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username } = user;
const { customer_name, customer_id, user_id, username, customer_modules } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -357,6 +406,7 @@ export class CatalogController {
user_id,
username,
language,
is_mask: customer_modules.some(mod => mod === 'pii')
});
const preview = await this.catalogService.getDatasetPreview(id, metadata);
@@ -748,25 +798,23 @@ export class CatalogController {
return JSON.parse(dashboards);
}
@Post('register-dataset')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async registerDatasetWithMetadataRequest(
@Body() body: RegisterDatasetWithMetatadaRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
try {
const metadata = PackTheMetadata({
customer_id,
customer_name,
@@ -774,18 +822,20 @@ export class CatalogController {
username,
});
this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
this.logger.log(
`Request from user ${user_id} for customer ${customer_name}`,
logMetadata,
);
// Create table metadata
const tableMetadataBody = {
table_metadata: body.table_metadata,
info: {
customer: customer_name,
},
logMetadata: logMetadata
logMetadata: logMetadata,
};
const table_metadata_id = await this.catalogService.createTableMetadata(
tableMetadataBody,
);
@@ -798,8 +848,12 @@ export class CatalogController {
customer: customer_name,
},
};
this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
this.logger.info(
`Creating column metadata for table ${table_metadata_id}`,
logMetadata,
);
const column_metadata_ids =
await this.catalogService.createColumnMetadata(columnMetadataBody);
// Create data preview
const dataPreviewBody = {
@@ -808,15 +862,24 @@ export class CatalogController {
customer: customer_name,
},
};
this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
this.logger.debug(
`Creating data preview for table ${table_metadata_id}`,
logMetadata,
);
const data_preview_id = await this.catalogService.createDataPreview(
dataPreviewBody,
);
// Catalog dataset item
this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
this.logger.info(
`Cataloging dataset item for table ${table_metadata_id}`,
logMetadata,
);
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
this.logger.info(
`Dataset registration completed successfully for table ${table_metadata_id}`,
logMetadata,
);
return {
message: 'Dataset registered successfully',
table_metadata_id: table_metadata_id,
@@ -824,7 +887,10 @@ export class CatalogController {
data_preview_id: data_preview_id,
};
} catch (error) {
this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
this.logger.error(
`Failed to register dataset. The following error occurred: ${error.response.data}`,
logMetadata,
);
throw new HttpException(
{
@@ -837,4 +903,32 @@ export class CatalogController {
);
}
}
}
@Get('pii-reporter')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
// use res.end to send buffer
return res.end(file);
} catch (error) {
console.error(error)
this.logger.error(error.message);
}
}
}
+6 -1
View File
@@ -3,12 +3,15 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -18,6 +21,8 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+70 -2
View File
@@ -25,6 +25,7 @@ import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
IUpdateDataRequest,
TriggerCatalogReq,
@@ -33,7 +34,10 @@ import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -80,6 +84,42 @@ class CatalogService implements OnModuleInit {
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
try {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
@@ -195,6 +235,34 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -271,7 +339,7 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
@@ -539,4 +607,4 @@ class CatalogService implements OnModuleInit {
}
}
export { CatalogService };
export { CatalogService };
+8
View File
@@ -320,3 +320,11 @@ export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
+8
View File
@@ -0,0 +1,8 @@
export class PiiDto {
database_name: string;
table_schema: string;
table_name: string;
column_name: string;
data_type: string;
pii_rules: string;
}
@@ -0,0 +1,89 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
+30
View File
@@ -0,0 +1,30 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
+275
View File
@@ -0,0 +1,275 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ConnectionTestService {
@@ -28,7 +28,7 @@ import {
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
+34 -6
View File
@@ -7,22 +7,26 @@ import {
HttpStatus,
Inject,
Param,
Post,
Put,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiTags('Customers')
@Controller('customers')
@@ -38,6 +42,15 @@ export class CustomersController {
this.logger = dadosferaLogger.logger;
}
@Post(':id/mfa')
@Authenticated()
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@RequireModule(DADOSFERA_MODULES_KEYS.DANGER_ZONE)
async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) {
this.logger.info('enableMfaEnforce', { id });
return await this.customersService.enableEnforceMfa(id, data.enabled);
}
@Get(':id/links')
@Authenticated()
@ApiOkResponse({ type: CustomerLinksResponse })
@@ -97,15 +110,30 @@ export class CustomersController {
@RequireAllPermissions(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
)
@RequireModule(DADOSFERA_MODULES_KEYS.LOG_DASHBOARD)
async getCustomerMixPanelLogsDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getLogsDashboardUrl');
): Promise<{ url: string }> {
this.logger.info('getLogsDashboardUrl');
const metadata = PackTheMetadata(user);
const result = await this.customersService.getLogsDashboardUrl(metadata);
const result = await this.customersService.getLogsDashboardUrl(metadata);
return result;
}
@Get('access-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
)
@RequireModule(DADOSFERA_MODULES_KEYS.ACCESS_DASHBOARD)
async getAccessDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getAccessDashboard');
const metadata = PackTheMetadata(user);
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
return result;
}
}
@@ -5,6 +5,7 @@ import {
HttpException,
HttpStatus,
InternalServerErrorException,
ForbiddenException,
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
@@ -33,6 +34,7 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs';
// This function will accept any string, which may result in a bug.
@Injectable()
export class CustomersService implements OnModuleInit {
private customerService: CustomersProtoService;
private logger: DadosferaLogger;
private pipelineReadService: ReadService.PipelineV2ReadService;
@@ -57,6 +59,14 @@ export class CustomersService implements OnModuleInit {
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
@@ -139,6 +149,7 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
@@ -156,6 +167,7 @@ export class CustomersService implements OnModuleInit {
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
@@ -173,4 +185,42 @@ export class CustomersService implements OnModuleInit {
return res;
}
async getAccessDashboardUrl(customerName: string, metadata: Metadata) {
/*
* TODO(Refactor): dar um jeito de exibir o dash da sbm diferente dos outros customer
* pois o signicado de department para sbm significa as instituições do usuários
*/
if (customerName !== 'sbmoffshorecom') {
throw new ForbiddenException();
}
logger.info('CustomersService - getAccessDashboardUrl');
const res = await this.getDashboardUrl('105', metadata);
logger.info('Done');
return res;
}
private async getDashboardUrl(dashboardId: string, metadata: Metadata) {
return await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: dashboardId,
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata
)
);
}
async enableEnforceMfa(id: string, enabled: boolean) {
return await lastValueFrom(
this.customerService.CustomerUpdateEnforceMfa({
customerId: id,
enforceMfa: enabled
})
)
}
}
@@ -0,0 +1,6 @@
import { ApiProperty } from "@nestjs/swagger";
export class EnforceMfa {
@ApiProperty()
enabled: boolean
}
@@ -0,0 +1,47 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -0,0 +1,10 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -0,0 +1,246 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -0,0 +1,185 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+4 -76
View File
@@ -3,14 +3,13 @@ import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
private mixpanelService: MixpanelService
) {}
@Post(':id')
async trackEvent(
@@ -20,82 +19,11 @@ export class MixpanelController {
@Req() request
) {
delete body.info;
const mixpanel = init(this.mixpanelToken);
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
await this.mixpanelService.track(id, user, request, body)
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+6 -1
View File
@@ -1,6 +1,8 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`${process.env.ENV}/root/mixpanel_token`,
`prd/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
+118
View File
@@ -0,0 +1,118 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -8,7 +8,7 @@ import {
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@@ -45,11 +45,11 @@ export class OpenDataController {
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
if (!corslist.includes(origin)) {
this.logger.info('block request by cors list: '+ origin);
throw new ForbiddenException();
}
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
@@ -32,7 +32,7 @@ import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
@@ -0,0 +1,10 @@
export class CreateShareMetadataDto {
assetId: string;
proposedId: string;
type: string;
}
export class UpdateShareMetadataDto {
shareId: string;
type: string;
}
@@ -0,0 +1,31 @@
import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Controller('share')
@Authenticated()
export class ShareMetadataController {
constructor(private readonly shareMetadataService: ShareMetadataService) {}
@Put('/')
findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.create(data, metadata);
}
@Patch('/')
updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.updateShare(data, metadata);
}
@Get('/:id')
get(@Param('id') id: string, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.get(id, metadata);
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { ShareMetadataController } from './share-metadata.controller';
import { DucClient } from '../duc/client.config';
import { ClientsModule } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [ShareMetadataController],
providers: [ShareMetadataService, DadosferaLogger],
exports: [ShareMetadataService]
})
export class ShareMetadataModule {}
@@ -0,0 +1,62 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Metadata } from '@grpc/grpc-js';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Injectable()
export class ShareMetadataService implements OnModuleInit {
ducService: ShareMetadataProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<ShareMetadataProtoService>(
ProtoServices.ShareMetadataProtoService,
);
}
create(createShareDto: CreateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.FindOrCreateShareMetadata(
{
assetId: createShareDto.assetId,
proposedId: createShareDto.proposedId,
type: createShareDto.type
},
metadata,
),
);
}
updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.ChangeShareMetadataType(
updateShare,
metadata,
),
);
}
get(id: string, metadata: Metadata) {
return lastValueFrom(
this.ducService.GetShareMetadata(
{
id,
},
metadata,
),
);
}
}
+11
View File
@@ -174,3 +174,14 @@ export class GetAllDepartmentsRes {
@ApiProperty()
departments: string[];
}
export interface UserReporter {
name: string;
email: string;
mfaStatus: string;
status: string;
lastLogin: string;
createdAt: string;
updatedAt: string;
}
@@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
+24 -1
View File
@@ -12,6 +12,7 @@ import {
Post,
Put,
Query,
Res,
UseFilters,
} from '@nestjs/common';
import {
@@ -24,12 +25,13 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -52,6 +54,7 @@ import {
UpdateUserRes,
} from './dtos/entities';
import { UsersService } from './users.service';
import { Response } from 'express';
@ApiInternalOnlyController()
@ApiTags('Users')
@@ -80,6 +83,26 @@ export class UsersController {
return await this.userService.findAllUsersByCustomerId(user.customer_id);
}
@Get('/download')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async downloadUsersInCsv(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Res() res: Response
) {
this.logger.info('downloadUsersInCsv');
this.userService.setLanguage(language);
const {
file,
filename
} = await this.userService.downloadUsersInCsv(user.customer_id);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@Get('hierarchies')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: GetAllHierarchiesRes })
+11
View File
@@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
error: (...args) => args,
+30
View File
@@ -21,6 +21,7 @@ import {
IUserByCustomer,
SetUserRolesReq,
UpdateUserReq,
UserReporter,
} from './dtos/entities';
import { RolesService } from '../roles/roles.service';
import { HIERARCHIES } from './hierarchies';
@@ -29,6 +30,7 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces
import { EnrichErrorCode } from 'src/utils/ErrorBuilder';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@Injectable()
export class UsersService implements OnModuleInit {
@@ -78,6 +80,34 @@ export class UsersService implements OnModuleInit {
};
}
async downloadUsersInCsv(customerId: string) {
const { users } = await lastValueFrom(
this.usersClientService.UserFindAllByCustomerId({ customerId }),
);
const formatUsers: UserReporter[] = users.map(user => ({
createdAt: user.createdAt,
email: user.email,
lastLogin: user.lastLogin,
mfaStatus: user.mfaStatus,
name: user.name,
status: user.status,
updatedAt: user.updatedAt
}))
const parser = ParserBuilder.build<UserReporter>('csv');
const file = await parser.parse(formatUsers);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_users_${timestamp}.csv`;
return {
file,
filename
}
}
async findOneById(id: string): Promise<{ user: IUserByCustomer }> {
const { user } = await lastValueFrom(
this.usersClientService.UserFindOneById({ id }),
+23
View File
@@ -0,0 +1,23 @@
import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';
@Injectable()
export class CacheService<T> {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
) {}
async get(key: string): Promise<T | null> {
return await this.cacheManager.get<T>(key);
}
async set(key: string, value: T): Promise<void> {
await this.cacheManager.set(key, value);
}
async delete(key: string) {
await this.cacheManager.del(key);
}
}
+22
View File
@@ -0,0 +1,22 @@
import { CacheModule, Module } from '@nestjs/common';
import { CacheService } from './cache.service';
import { redisStore } from 'cache-manager-ioredis-yet';
@Module({
imports: [
CacheModule.registerAsync({
useFactory: async () => ({
store: await redisStore({
ttl: 5 * 1000 * 60, // 5 minute
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT && Number(process.env.REDIS_PORT),
db: process.env.REDIS_DATABASE && Number(process.env.REDIS_DATABASE),
keyPrefix: 'maestro:sso',
}),
}),
}),
],
providers: [CacheService],
exports: [CacheService],
})
export class ServicesModule {}
+14
View File
@@ -0,0 +1,14 @@
import { stringify } from 'csv'
import { Parser } from './parser.interface';
export class CsvParser<T> implements Parser<T>{
async parse(data: T[]) {
return new Promise<string>((resolve, reject) => {
stringify(data, { header: true }, function (err, output) {
if (err) reject(err);
resolve(output)
});
})
}
}
+17
View File
@@ -0,0 +1,17 @@
import { readFileSync } from 'fs';
import Handlebars from 'handlebars';
import { Parser } from './parser.interface';
export class HtmlParser<T> implements Parser<T> {
constructor(private readonly templatePath: string) {}
async parse(data: T[]) {
const templateSource = readFileSync(this.templatePath, 'utf8');
const template = Handlebars.compile(templateSource);
return template({
dados: data,
dataGeracao: new Date().toLocaleString('pt-BR'),
});
}
}
+1
View File
@@ -0,0 +1 @@
export type TypeParser = "html" | "csv" | "pdf"
+25
View File
@@ -0,0 +1,25 @@
import { TypeParser } from './parser-types';
import { HtmlParser } from './html-parser';
import { CsvParser } from './csv-parser';
import { PDFParser } from './pdf-parser';
import { Parser } from './parser.interface';
export class ParserBuilder {
static build<T>(type: TypeParser): Parser<T> {
const templatePath = '/app/dist/assets/template/pii-reporter.html';
if (type === 'html') {
const htmlParser = new HtmlParser<T>(templatePath);
return htmlParser;
} else if (type === 'csv') {
const csvParser = new CsvParser<T>();
return csvParser;
} else if (type === 'pdf') {
const htmlParser = new HtmlParser<T>(templatePath);
const pdfParser = new PDFParser(htmlParser);
return pdfParser;
}
throw new Error('Unsupported Type: ' + type);
}
}
+3
View File
@@ -0,0 +1,3 @@
export interface Parser<T> {
parse(data: T[]): Promise<string | Buffer>
}
+57
View File
@@ -0,0 +1,57 @@
import puppeteer from 'puppeteer';
import { Parser } from './parser.interface';
import { HtmlParser } from './html-parser';
export class PDFParser<T> implements Parser<T> {
constructor(private htmlParser: HtmlParser<T>) {}
async parse(data: T[]) {
try {
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
],
});
const page = await browser.newPage();
page.setDefaultNavigationTimeout(0);
const html = await this.htmlParser.parse(data);
await page.setContent(html, {
waitUntil: 'networkidle0',
timeout: 90000,
});
// Configurações adicionais para garantir um PDF válido
const pdf = await page.pdf({
format: 'A4',
printBackground: true,
margin: {
top: '20mm',
right: '15mm',
bottom: '20mm',
left: '15mm',
},
preferCSSPageSize: true,
displayHeaderFooter: false,
headerTemplate: '',
footerTemplate: '',
omitBackground: false,
pageRanges: '',
tagged: true,
outline: false,
timeout: 90000,
});
await browser.close();
return Buffer.from(pdf);
} catch (error) {
console.error('Error generating PDF:', error);
throw error;
}
}
}
+7 -1
View File
@@ -24,15 +24,21 @@ export async function getOauthSecrets() {
const path = process.env.SM_OAUTH_PATH;
const secretsManagerClient = new SecretsManagerClient({});
for (const key in secrets) {
console.log(`Fetching secret for ${key} from path ${path}/${key}`);
const getSecretComand = new GetSecretValueCommand({
SecretId: `${path}/${key}`,
});
const res = await secretsManagerClient
.send(getSecretComand)
.catch(() => null);
.catch((err) => {
console.error(`Error fetching secret for ${key}:`, err);
return null;
});
if (res) {
secrets[key] = JSON.parse(res.SecretString);
console.log(JSON.parse(res.SecretString).client_id)
} else {
console.log("No secret found for", key);
secrets[key] = {
client_id: 'id',
client_secret: 'id',
@@ -14,6 +14,7 @@ interface IMetadata {
host?: string;
origin?: string;
custom_host?: string;
is_mask?: boolean;
}
export function PackTheMetadata(info: IMetadata): Metadata {
+5
View File
@@ -75,6 +75,10 @@ const CATALOG = {
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
};
const IDENTITY_PROVIDER = {
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',
INVALID_HEADER: 'IDENTITY_PROVIDER.INVALID_HEADER',
}
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
@@ -91,6 +95,7 @@ const ErrorCodes = {
TRANSFORMATION,
CONNECTION,
CATALOG,
IDENTITY_PROVIDER
};
export default ErrorCodes;