Compare commits

...
Author SHA1 Message Date
marcos-silva-rodrigues 301b6e98ec UPDATE: test subdomain 2025-08-14 17:42:42 -03:00
marcos-silva-rodrigues ba53934068 CI: add cookie secret 2025-08-14 17:23:40 -03:00
marcos-silva-rodrigues 216349f303 CHORE: update nginx ingress annotations 2025-08-14 16:03:28 -03:00
marcos-silva-rodrigues a654baef13 FIX: set cookies if exists tokens 2025-08-14 15:37:17 -03:00
marcos-silva-rodrigues 04d761ca14 Merge branch 'beta' into feature/auth-session 2025-08-14 15:26:15 -03:00
Marcos Rodrigues Silva a772804dcd Merge pull request #357 from dadosfera/chore/oracle
CI: fix multi cloud deploy and maestro env
2025-08-07 18:04:10 -03:00
marcos-silva-rodrigues 8a399fbbcf CI: fix multi cloud deploy and maestro env 2025-08-07 18:03:22 -03:00
Marcos Rodrigues Silva 94049d6e8e Merge pull request #356 from dadosfera/chore/oracle
CI: oracle
2025-08-07 12:27:08 -03:00
marcos-silva-rodrigues c28cc58ac7 CI: oracle 2025-08-07 12:24:44 -03:00
Marcos Rodrigues Silva 56a655bb90 Merge pull request #355 from dadosfera/chore/oracle
CI: fix env home
2025-08-07 12:16:02 -03:00
marcos-silva-rodrigues b3d4a6b028 CI: fix env home 2025-08-07 12:13:19 -03:00
Marcos Rodrigues Silva c7bd1f761a Merge pull request #354 from dadosfera/chore/oracle
CI: remove k8s-setup need
2025-08-07 12:01:43 -03:00
marcos-silva-rodrigues 09c045455e CI: remove k8s-setup need 2025-08-07 11:59:37 -03:00
Marcos Rodrigues Silva 52d651126e Merge pull request #353 from dadosfera/chore/oracle
CI: revert k8s-setup workflow for the step in job helmfile-deploy
2025-08-07 11:57:38 -03:00
marcos-silva-rodrigues 2f11b1e78f CI: revert k8s-setup workflow for the step in job helmfile-deploy 2025-08-07 11:56:54 -03:00
Marcos Rodrigues Silva 885e9b71bb Merge pull request #352 from dadosfera/fix/tracker-sso
Fix/tracker sso
2025-08-07 11:45:44 -03:00
marcos-silva-rodrigues 193031dfdd UPDATE: increase redis ttl 2025-08-07 10:41:48 -03:00
marcos-silva-rodrigues f626a9bb5e UPDATE: add more logger 2025-08-07 10:32:22 -03:00
Marcos Rodrigues Silva ac26ad9f44 Merge pull request #351 from dadosfera/chore/oracle
FIX: correct name
2025-08-01 18:06:27 -03:00
marcos-silva-rodrigues fbe00af6dc FIX: correct name 2025-08-01 18:05:19 -03:00
Marcos Rodrigues Silva bfa9929b69 Merge pull request #350 from dadosfera/chore/oracle
Chore/oracle
2025-08-01 18:01:38 -03:00
marcos-silva-rodrigues bada7000f6 CHORE: change ci to oracle cloud 2025-08-01 18:01:10 -03:00
Marcos Rodrigues Silva 2c908d8d95 Merge pull request #349 from dadosfera/beta
Beta
2025-07-31 15:24:28 -03:00
Marcos Rodrigues Silva da35633d42 Merge pull request #348 from dadosfera/hotfix/module-assigned
Hotfix/module assigned
2025-07-31 12:04:02 -03:00
marcos-silva-rodrigues cc7e06013f FIX: change module permission 2025-07-31 11:59:19 -03:00
Marcos Rodrigues Silva 5ec36a05b5 Merge pull request #344 from dadosfera/beta
Beta
2025-07-30 11:56:22 -03:00
Marcos Rodrigues Silva f61e46c7c1 Merge pull request #347 from dadosfera/release/07-29
FEAT: add permission embed access
2025-07-30 11:15:28 -03:00
marcos-silva-rodrigues b9013b1493 FEAT: add permission embed access 2025-07-30 11:12:08 -03:00
Marcos Rodrigues Silva 5267fa6d2f Merge pull request #346 from dadosfera/release/07-29
FIX: comment test
2025-07-29 18:05:26 -03:00
marcos-silva-rodrigues 7f40cbdeed FIX: comment test 2025-07-29 18:03:20 -03:00
Marcos Rodrigues Silva b590c9402d Merge pull request #345 from dadosfera/release/07-29
Release/07 29
2025-07-29 15:19:52 -03:00
marcos-silva-rodrigues 9546feda11 FIX: merge 2025-07-29 15:12:25 -03:00
Marcos Rodrigues Silva 3587f82976 Merge pull request #343 from dadosfera/bugfix/public-link
Bugfix/public link
2025-07-29 12:31:34 -03:00
marcos-silva-rodrigues 94a3962590 FIX: add needs steps 2025-07-29 12:26:21 -03:00
marcos-silva-rodrigues 3b2d79a451 CHORE: remove needs step 2025-07-29 12:06:33 -03:00
marcos-silva-rodrigues da56a645af CHORE: update affinity to azure 2025-07-29 11:57:10 -03:00
Marcos Rodrigues Silva af4e55af1d Merge pull request #342 from dadosfera/bugfix/public-link
CHORE: remove sudo
2025-07-29 11:35:09 -03:00
marcos-silva-rodrigues d6a08a8f82 CHORE: remove sudo 2025-07-29 11:32:02 -03:00
Marcos Rodrigues Silva c73eb8869f Merge pull request #341 from dadosfera/bugfix/public-link
CHORE: install kubectl
2025-07-29 11:15:24 -03:00
marcos-silva-rodrigues 929fc97ac6 CHORE: install kubectl 2025-07-29 11:12:47 -03:00
Marcos Rodrigues Silva 7594a75bea Merge pull request #340 from dadosfera/bugfix/public-link
CHORE: update ci to wait extract_environment job
2025-07-29 11:02:14 -03:00
marcos-silva-rodrigues ec36a056b2 CHORE: update ci to wait extract_environment job 2025-07-29 10:59:35 -03:00
Marcos Rodrigues Silva 8d30f6ef15 Merge pull request #339 from dadosfera/bugfix/public-link
CHORE: multi cloud deployment
2025-07-29 10:57:26 -03:00
marcos-silva-rodrigues 08d1e3164f CHORE: multi cloud deployment 2025-07-29 10:53:41 -03:00
Marcos Rodrigues Silva a5f3ef20fe Merge pull request #338 from dadosfera/bugfix/public-link
FIX: add user by bearer token if it exists
2025-07-29 09:51:17 -03:00
marcos-silva-rodrigues 467445fe05 FIX: add user by bearer token if it exists 2025-07-29 09:47:56 -03:00
Marcos Rodrigues Silva d0448f844a Merge pull request #337 from dadosfera/bugfix/google-oauth
UPDATE: more logs
2025-07-21 16:06:22 -03:00
marcos-silva-rodrigues b2299a5f2f UPDATE: more logs 2025-07-21 15:51:56 -03:00
Marcos Rodrigues Silva 9b42754c69 Merge pull request #336 from dadosfera/bugfix/google-oauth
Bugfix/google oauth
2025-07-21 15:06:49 -03:00
marcos-silva-rodrigues aee33334ac UPDATE: add log to debug 2025-07-21 15:01:41 -03:00
Marcos Rodrigues Silva cc530fb12d Merge pull request #335 from dadosfera/release/sso
Release/sso
2025-07-18 17:12:51 -03:00
marcos-silva-rodrigues 19877f1b29 FIX: merge with main 2025-07-18 16:46:19 -03:00
marcos-silva-rodrigues e9880bcf68 FIX: add logger in header 2025-07-18 15:56:23 -03:00
marcos-silva-rodrigues 4c7111dcb6 FIX: using referer headaer 2025-07-18 15:56:10 -03:00
marcos-silva-rodrigues e81cfdcce1 FIX: using only header origin 2025-07-18 15:55:56 -03:00
marcos-silva-rodrigues 3eba3f414f FIX: send redirect uri 2025-07-18 15:55:45 -03:00
marcos-silva-rodrigues fbeb8dfd91 FIX: redis env 2025-07-18 15:55:29 -03:00
marcos-silva-rodrigues dda3e8baaf FIX: add logger 2025-07-18 15:54:27 -03:00
marcos-silva-rodrigues c1e4f0b18a FIX: remove express session 2025-07-18 15:54:24 -03:00
Marcos Rodrigues Silva b0d9f3a468 Merge pull request #334 from dadosfera/chore/stg-env
FIX: revert env
2025-07-18 10:27:01 -03:00
marcos-silva-rodrigues 5b117acaa9 FIX: revert env 2025-07-18 10:23:10 -03:00
Marcos Rodrigues Silva a5cb607650 Merge pull request #333 from dadosfera/feature/sso-openid
FIX: change env to stg
2025-07-17 16:12:51 -03:00
marcos-silva-rodrigues fe626155c5 FIX: change env to stg 2025-07-17 16:10:42 -03:00
Marcos Rodrigues Silva debbe1c1b1 Merge pull request #332 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 18:50:04 -03:00
marcos-silva-rodrigues f937a6efe2 FIX: add logger in header 2025-07-16 18:49:27 -03:00
marcos-silva-rodrigues 30cb3c87eb FIX: using referer headaer 2025-07-16 18:45:47 -03:00
Marcos Rodrigues Silva fa46294154 Merge pull request #331 from dadosfera/feature/sso-openid
FIX: using only header origin
2025-07-16 18:37:53 -03:00
marcos-silva-rodrigues 72554fb27f FIX: using only header origin 2025-07-16 18:36:36 -03:00
Marcos Rodrigues Silva 4e6985b572 Merge pull request #330 from dadosfera/feature/sso-openid
FIX: send redirect uri
2025-07-16 18:16:27 -03:00
marcos-silva-rodrigues 4477647613 FIX: send redirect uri 2025-07-16 18:15:47 -03:00
Marcos Rodrigues Silva 2b3066abc9 Merge pull request #329 from dadosfera/feature/sso-openid
FIX: redis env
2025-07-16 17:57:26 -03:00
marcos-silva-rodrigues da1bcb38da FIX: redis env 2025-07-16 17:56:43 -03:00
Marcos Rodrigues Silva 4a7524422d Merge pull request #328 from dadosfera/feature/sso-openid
FIX: add logger
2025-07-16 17:41:40 -03:00
marcos-silva-rodrigues 275af03195 FIX: add logger 2025-07-16 17:39:15 -03:00
Marcos Rodrigues Silva 0c75cf4619 Merge pull request #327 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 12:33:44 -03:00
marcos-silva-rodrigues 80de277529 FIX: merge problems 2025-07-16 12:27:33 -03:00
marcos-silva-rodrigues 98f669db66 FIX: remove express session 2025-07-16 11:48:15 -03:00
marcos-silva-rodrigues 2c8dab5927 FIX: merge 2025-07-16 11:46:05 -03:00
marcos-silva-rodrigues cb66d4f6c0 FEAT: dynamic callback and issuer urls 2025-07-16 11:35:47 -03:00
marcos-silva-rodrigues 42a39ee891 FEAT: send user id to refresh token endpoint 2025-07-15 16:21:44 -03:00
marcos-silva-rodrigues c9bfe68ea5 FEAT: list identity providers link for sso 2025-07-14 15:16:19 -03:00
marcos-silva-rodrigues 8cb7144ff4 FIX: send accessToken in refresh token endpoint 2025-07-14 15:14:25 -03:00
Marcos Rodrigues Silva 68cc58e2fe Merge pull request #326 from dadosfera/release/11-07
Release/11 07
2025-07-14 10:54:14 -03:00
marcos-silva-rodrigues cbe5556ac9 FIX: auth callback 2025-07-14 10:51:36 -03:00
marcos-silva-rodrigues eb80967608 FIX: tests 2025-07-11 11:36:23 -03:00
marcos-silva-rodrigues 66fea0722d FEAT: enabled network check in app.dadosfera 2025-07-11 11:12:35 -03:00
marcos-silva-rodrigues c917061b97 Merge branch 'feature/export-users-and-assets' into release/11-07 2025-07-11 11:04:31 -03:00
Rafael Santana da39a07f25 Merge pull request #325 from dadosfera/feat/oracle-migration-and-affinity-standardization
FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
2025-07-09 13:02:48 -03:00
Rafael f2f65285de resolve: merge conflicts with beta branch
- Resolve ingress.yaml timeout annotation conflicts by keeping both configurations
- Resolve stg.yaml conflicts by merging affinity/resources config with restricted_ip
- Fix chart references in unimed-maestro and private-maestro sections
- Update all chart paths from ../maestro to ../helm-chart for consistency

This merge brings in the latest beta changes while preserving our Oracle migration
and affinity standardization configuration.
2025-07-09 13:00:52 -03:00
Rafael cac8f93330 CI: update validate-k8s.yml to use beta branch for staging
- Change trigger branches from main/stg to main/beta
- Update environment extraction to map beta → stg environment
- Align with standard branch naming convention (beta for staging)
- Consistent with deploy-manually.yml workflow configuration
2025-07-09 12:58:11 -03:00
Rafael bb03b18f4a FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
Oracle Migration Changes:
- Create deploy/helm-chart/ directory structure (moved from maestro/)
- Create deploy/helmfiles/ directory structure (moved from helmfiles/)
- Update chart references in helmfiles from ../maestro to ../helm-chart
- Update deploy-manually.yml from Azure AKS to Oracle OKE
- Update validate-k8s.yml from Azure AKS to Oracle OKE
- Replace Azure CLI with OCI CLI installation and configuration
- Replace Azure authentication with OCI authentication using secrets
- Replace az aks get-credentials with oci ce cluster create-kubeconfig
- Update helmfile paths from helmfiles/ to deploy/helmfiles/
- Remove DockerHub integration (push_to_dockerhub input and related steps)
- Change runner from [self-hosted, prd-azure] to [self-hosted, prd-oracle]
- Add environment field for proper Oracle deployment environment handling
- Add HOME environment variable for OCI CLI

Affinity/Resources Standardization:
- Add affinity configuration with name=general node selector in values.yaml
- Update deployment template to use conditional affinity and resources blocks
- Configure staging to disable both affinity and resources (null values)
- Enable production to use affinity targeting name=general nodes
- Enable production to use standard resource limits (100m-2000m CPU, 1500Mi-2Gi memory)

This brings maestro into full compliance with Oracle migration requirements and
implements the standardized affinity/resources pattern consistent with in-factory and duc.
2025-07-09 12:55:49 -03:00
Marcos Rodrigues Silva ab280ee152 Merge pull request #324 from dadosfera/chore/update-unimed-ips
CHORE: Update unimed ips
2025-07-08 09:22:26 -03:00
marcos-silva-rodrigues 16c8137160 CHORE: Update unimed ips 2025-07-07 17:26:26 -03:00
Marcos Rodrigues Silva b8b903c8cd Merge pull request #323 from dadosfera/feature/export-users-and-assets
Feature/export users and assets
2025-07-05 11:27:10 -03:00
marcos-silva-rodrigues c98a5ff896 FEAT: endpoint to dowload assets list 2025-07-05 11:21:05 -03:00
marcos-silva-rodrigues d68e436a8b FEAT: export users in csv 2025-07-05 09:38:16 -03:00
marcos-silva-rodrigues 8efa6790d6 FEAT: sign out endpoint 2025-07-04 18:03:08 -03:00
marcos-silva-rodrigues a381f50e88 FEAT: set cookie after user login 2025-07-03 15:52:55 -03:00
marcos-silva-rodrigues d3040cd5a8 FEAT: update and delete identity provider 2025-07-03 10:59:43 -03:00
marcos-silva-rodrigues f726a9883f FEAT: create user from identity provider 2025-06-27 13:21:59 -03:00
Marcos Rodrigues Silva c18a7baa37 Merge pull request #322 from dadosfera/chore/add-dadosfera-vpn-unimed-chart
FIX: add dadosfera vpn ip in unimed chart
2025-06-25 11:12:33 -03:00
marcos-silva-rodrigues d638d518f8 FIX: change unimed chart name 2025-06-25 11:06:41 -03:00
marcos-silva-rodrigues bdd371ffe8 FIX: add dadosfera vpn ip in unimed chart 2025-06-25 10:58:18 -03:00
marcos-silva-rodrigues 57e6e49c7c FEAT: init url callback 2025-06-25 10:28:26 -03:00
marcos-silva-rodrigues a5c8de496d FEAT: map permission to identity provider 2025-06-24 16:20:03 -03:00
Marcos Rodrigues Silva d478f32760 Merge pull request #321 from dadosfera/hotfix/nginx-timeout
FIX: add nginx proxy timeout 5 min
2025-06-24 11:02:29 -03:00
marcos-silva-rodrigues 5d8e908339 FIX: add nginx proxy timeout 5 min 2025-06-24 10:26:50 -03:00
Marcos Rodrigues Silva 606bae016b Merge pull request #320 from dadosfera/hotfix/maesto-unimed-hostname
FIX: correct helm value
2025-06-23 18:43:58 -03:00
marcos-silva-rodrigues 6910a2f04c FIX: correct helm value 2025-06-23 18:38:20 -03:00
Marcos Rodrigues Silva 8bee1788c1 Merge pull request #319 from dadosfera/hotfix/maesto-unimed-hostname
FIX: maestro custom hostname to unimed
2025-06-23 18:04:43 -03:00
marcos-silva-rodrigues b10459cb15 FIX: maestro custom hostname to unimed 2025-06-23 17:51:14 -03:00
Marcos Rodrigues Silva 3b48dd1613 Merge pull request #317 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 17:07:55 -03:00
Marcos Rodrigues Silva e5cef90a64 Merge pull request #318 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 16:04:59 -03:00
marcos-silva-rodrigues bac74c9577 FIX: default value and throw correct Error Class 2025-06-23 15:36:26 -03:00
marcos-silva-rodrigues 19d748ae09 CHORE: add restricted ip and unimed chart 2025-06-23 14:47:39 -03:00
marcos-silva-rodrigues 4db865371f Merge branch 'bugfix/update-pupperter-timeout' into release/2025-06 2025-06-23 14:12:25 -03:00
marcos-silva-rodrigues 75e8b3bf4b FIX: correct dto name 2025-06-17 14:58:35 -03:00
Marcos Rodrigues Silva 194cdc66b7 Merge pull request #316 from dadosfera/feature/public-and-assign-embed
Feature/public and assign embed
2025-06-17 08:57:53 -03:00
marcos-silva-rodrigues 53059c0a0d FIX: import 2025-06-16 18:09:25 -03:00
marcos-silva-rodrigues ed4ffc8a86 FIX: merge with beta 2025-06-16 18:01:53 -03:00
Marcos Rodrigues Silva 9967c172da Merge pull request #315 from dadosfera/feature/dedicated-maestro
FEAT: block user from login when maestro is dedicated
2025-06-16 10:37:47 -03:00
marcos-silva-rodrigues 0aa9c065a5 FEAT: block user from login when maestro is dedicated 2025-06-16 10:34:11 -03:00
Marcos Rodrigues Silva d0c2abd38d Merge pull request #314 from dadosfera/bugfix/update-pupperter-timeout
FIX: update timeout
2025-06-13 15:10:08 -03:00
marcos-silva-rodrigues cb98099a91 FIX: update timeout 2025-06-13 15:07:50 -03:00
Marcos Rodrigues Silva f446e0d5b1 Merge pull request #313 from dadosfera/feature/dedicated-maestro
FIX: send env DEDICATED_PROXY
2025-06-12 14:47:54 -03:00
marcos-silva-rodrigues 17de31f1a6 FIX: send env DEDICATED_PROXY 2025-06-12 14:45:13 -03:00
Marcos Rodrigues Silva d81a466089 Merge pull request #312 from dadosfera/feature/dedicated-maestro
FIX: open data ingress route
2025-06-12 14:08:13 -03:00
marcos-silva-rodrigues 9e597fadba FIX: open data ingress route 2025-06-12 14:04:45 -03:00
Marcos Rodrigues Silva 1e6aa0f4c2 Merge pull request #311 from dadosfera/feature/dedicated-maestro
FIX: chart private maestro
2025-06-12 13:59:11 -03:00
marcos-silva-rodrigues 41842f7c6a FIX: chart private maestro 2025-06-12 13:56:57 -03:00
Marcos Rodrigues Silva 0d5564990d Merge pull request #309 from dadosfera/feature/dedicated-maestro
Feature/dedicated maestro
2025-06-12 12:31:26 -03:00
marcos-silva-rodrigues 7f9755493a FIX: experimental hostname 2025-06-12 12:29:51 -03:00
marcos-silva-rodrigues fea587ad36 FEAT: control login by dedicated customer id injected 2025-06-12 11:51:56 -03:00
Marcos Rodrigues Silva b2b7537fee Merge pull request #308 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 15:25:27 -03:00
Marcos Rodrigues Silva 57f99fe7af Merge pull request #307 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 14:25:23 -03:00
marcos-silva-rodrigues 01263017e7 FIX: comment block list 2025-06-10 14:24:26 -03:00
Marcos Rodrigues Silva e66f9a7244 Merge pull request #306 from dadosfera/feature/api-key
FEAT: add api key endpoint
2025-06-06 15:55:53 -03:00
marcos-silva-rodrigues 54cb3f8e7a FEAT: add api key endpoint 2025-06-06 15:53:12 -03:00
Marcos Rodrigues Silva 4c861cf5e4 Merge pull request #305 from dadosfera/fix/maestro-stg-url
Fix/maestro stg url
2025-05-30 10:15:10 -03:00
marcos-silva-rodrigues a89f57e690 FIX: return pi_factory_url and tr_factory_url to stg url 2025-05-30 10:13:37 -03:00
marcos-silva-rodrigues d5141e5b6a REFACTOR: rename openid to identity provider 2025-05-26 13:37:25 -03:00
Rafael Santana 8aded98a11 Merge pull request #304 from dadosfera/beta
Beta
2025-05-26 10:51:13 -03:00
Rafael Santana 0b5a0d0d7e Merge pull request #303 from dadosfera/adjusting-nestjs-payload
Adjusting nestjs payload
2025-05-26 10:40:43 -03:00
Rafael 5a36762544 UPDATE: route /catalog/register-dataset will have an limit of 10mb 2025-05-26 10:24:48 -03:00
marcos-silva-rodrigues a4b7573ce7 FIX: sync with main 2025-05-23 14:07:05 -03:00
marcos-silva-rodrigues 6fc8c0b4a8 FEAT: add mixpanel service to track share access 2025-05-20 16:50:04 -03:00
marcos-silva-rodrigues ced2a97547 FIX: add logger and return info 2025-05-19 16:36:54 -03:00
Marcos Rodrigues Silva f579171024 Merge pull request #302 from dadosfera/beta
Default value to header origin
2025-05-15 11:18:17 -03:00
Marcos Rodrigues Silva 885e5f4568 Merge pull request #301 from dadosfera/fix/remove-default-host
Fix/remove default host
2025-05-15 11:00:45 -03:00
marcos-silva-rodrigues 81d8cd8a07 FIX: default value to header origin 2025-05-15 11:00:07 -03:00
Marcos Rodrigues Silva f153f41357 Merge pull request #300 from dadosfera/beta
Beta
2025-05-14 18:30:54 -03:00
marcos-silva-rodrigues 2b9e3ff3bc Merge branch 'main' into beta 2025-05-13 14:32:34 -03:00
marcos-silva-rodrigues d15cc4642a FEAT: assign embed and public key changes 2025-05-13 09:21:44 -03:00
marcos-silva-rodrigues 3c4f45678a CHORE: update protospack 2025-05-12 12:25:45 -03:00
marcos-silva-rodrigues b312849fff FEAT: catalog share endpoint for embeds 2025-05-12 12:15:16 -03:00
marcos-silva-rodrigues 10580dfa03 FEAT: share endpoint 2025-05-12 12:13:39 -03:00
marcos-silva-rodrigues 62d0e8b041 FEAT: assign pubic key endpoint 2025-05-12 12:12:58 -03:00
Marcos Rodrigues Silva 17cdaac380 Merge pull request #299 from dadosfera/fix/stg-envs
FIX: stg to prd for sbm environment
2025-05-09 11:07:22 -03:00
marcos-silva-rodrigues e4eb388e18 FIX: stg to prd for sbm environment 2025-05-09 11:06:23 -03:00
marcos-silva-rodrigues d94c5dee4e FEAT: init public routes to get data asset by public embed 2025-05-06 14:44:09 -03:00
Marcos Rodrigues Silva 4bea0816b6 Merge pull request #298 from dadosfera/feature/pii-rules
FIX: install chromiun in base_image
2025-04-29 16:22:22 -03:00
marcos-silva-rodrigues fb491328d6 FIX: install chromiun in base_image 2025-04-29 16:20:37 -03:00
Marcos Rodrigues Silva 4e043d93ce Merge pull request #297 from dadosfera/feature/pii-rules
REFACTOR: move template to assets folder
2025-04-29 15:41:16 -03:00
marcos-silva-rodrigues 0b525b835f REFACTOR: move template to assets folder 2025-04-29 15:40:37 -03:00
Marcos Rodrigues Silva 3c8c39ec26 Merge pull request #296 from dadosfera/feature/pii-rules
REFACTOR: create builder for parser reporter and fix pdf
2025-04-29 15:20:46 -03:00
marcos-silva-rodrigues 36997403e1 REFACTOR: create builder for parser reporterand fix pdf 2025-04-29 15:18:46 -03:00
Marcos Rodrigues Silva 1175591521 Merge pull request #295 from dadosfera/feature/pii-rules
Feature/pii rules
2025-04-29 14:04:06 -03:00
marcos-silva-rodrigues 04accb6689 FIX: correct stg urls 2025-04-29 14:02:24 -03:00
marcos-silva-rodrigues 58e3c7be94 FEAT: pii 2025-04-29 14:00:23 -03:00
marcos-silva-rodrigues 7a6db36554 FEAT: generate pii reporter 2025-04-27 13:00:49 -03:00
Marcos Rodrigues Silva 80ebd98f6f Merge pull request #294 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-23 17:46:18 -03:00
Marcos Rodrigues Silva 9e0495fc26 Merge pull request #293 from dadosfera/feat/enforce-mfa
Feat/enforce mfa
2025-04-22 14:50:39 -03:00
marcos-silva-rodrigues f2437a8b3c FIX: add decorator to require danger zone module 2025-04-22 14:46:39 -03:00
marcos-silva-rodrigues 9e87562e23 FEAT: update protospack 2025-04-22 09:43:34 -03:00
marcos-silva-rodrigues 8f5cb13cc3 FEAT: enforce mfa 2025-04-21 18:53:34 -03:00
Marcos Rodrigues Silva e0a0ec9911 Merge pull request #292 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-17 18:12:28 -03:00
marcos-silva-rodrigues 419c063c6c FEAT: add access dashboard 2025-04-17 18:04:18 -03:00
marcos-silva-rodrigues 2aed66d371 FEAT: add decorator to block endpoint if not found module 2025-04-17 15:20:27 -03:00
Marcos Rodrigues Silva 1f45436c13 Merge pull request #291 from dadosfera/fix/network-policy
FIX: send origin header
2025-04-16 17:13:55 -03:00
marcos-silva-rodrigues df8520afd3 FIX: send origin header 2025-04-15 14:24:41 -03:00
Marcos Rodrigues Silva 95187e6b80 Merge pull request #290 from dadosfera/debug/unimed-sbm
UPDATE: change in-factory, pi-factory, tr-factory to prod
2025-04-14 14:15:45 -03:00
marcos-silva-rodrigues c8a3ace052 UPDATE: change in-factory, pi-factory, tr-factory to prod 2025-04-14 14:14:46 -03:00
Marcos Rodrigues Silva c059ac45bf Merge pull request #289 from dadosfera/fix/network-policy
FIX: send origin removing http prefix
2025-04-14 08:46:56 -03:00
marcos-silva-rodrigues f4b8addda4 FIX: send origin removing http prefix 2025-04-14 08:45:59 -03:00
Marcos Rodrigues Silva aa3a1cc300 Merge pull request #288 from dadosfera/fix/network-policy
FIX: remove host and send custom host
2025-04-14 08:32:19 -03:00
marcos-silva-rodrigues 0a5f138829 FIX: remove host and send custom host 2025-04-14 08:30:38 -03:00
Marcos Rodrigues Silva 9f427f5873 Merge pull request #287 from dadosfera/fix/network-policy
UPDATE: send host
2025-04-11 18:03:11 -03:00
marcos-silva-rodrigues bacbf42a04 UPDATE: send host 2025-04-11 18:00:51 -03:00
Marcos Rodrigues Silva 56a9d52466 Merge pull request #286 from dadosfera/fix/network-policy
UPDATE: send origin header
2025-04-11 17:37:56 -03:00
marcos-silva-rodrigues 2c964253ec UPDATE: send header in metadata 2025-04-11 17:33:34 -03:00
marcos-silva-rodrigues e886bea05c UPDATE: send origin header 2025-04-11 17:18:45 -03:00
rafael-moraes-ddf 66affed338 Merge pull request #285 from dadosfera/feature/logs_dashboard
Feature/logs dashboard
2025-04-11 16:22:05 -03:00
Rafael Moraes a8ae07e6c4 FEAT: url fix 2025-04-11 16:06:55 -03:00
Rafael Moraes 0df286a5f7 FEAT: att docsfera 2025-04-11 15:23:41 -03:00
Marcos Rodrigues Silva 519a8feb14 Merge pull request #284 from dadosfera/fix/network-policy
UPDATE: send host
2025-04-11 15:08:01 -03:00
marcos-silva-rodrigues 16cb00f46f UPDATE: send host 2025-04-11 15:06:02 -03:00
Marcos Rodrigues Silva 3c59602666 Merge pull request #283 from dadosfera/fix/network-policy
FIX: remove host metadata
2025-04-10 11:55:32 -03:00
marcos-silva-rodrigues 3e0dbc7401 FIX: remove host metadata 2025-04-10 11:53:53 -03:00
Marcos Rodrigues Silva a24d29b56b Merge pull request #282 from dadosfera/fix/network-policy
FIX: send host metadata
2025-04-10 11:47:09 -03:00
marcos-silva-rodrigues deb32b5c0f FIX: send host metadata 2025-04-10 11:44:52 -03:00
Marcos Rodrigues Silva d4a6d2fb4a Merge pull request #281 from dadosfera/fix/network-policy
Fix/network policy
2025-04-10 11:34:00 -03:00
Marcos Rodrigues Silva 01ea0e9561 Merge branch 'beta' into fix/network-policy 2025-04-10 11:33:43 -03:00
marcos-silva-rodrigues 9676b51f01 FIX: comment host metadata 2025-04-10 11:27:07 -03:00
Marcos Rodrigues Silva b0d2f86eaf Merge pull request #280 from dadosfera/feat/network-policy
FEAT: networks get endpoint
2025-04-09 15:34:18 -03:00
marcos-silva-rodrigues f356efb42e Merge branch 'beta' into feat/network-policy 2025-04-09 15:32:10 -03:00
marcos-silva-rodrigues 559d98e884 FIX: merge with beta 2025-04-09 15:28:23 -03:00
Marcos Rodrigues Silva d8623b88df Merge pull request #277 from dadosfera/feat/network-policy
Feat/network policy
2025-04-09 11:36:32 -03:00
Marcos Rodrigues Silva 7044d6606e Merge pull request #278 from dadosfera/beta
Beta
2025-04-08 18:04:41 -03:00
Marcos Rodrigues Silva 92aab57d37 Merge pull request #279 from dadosfera/fix/reset-block-users
FIX: send all metadata
2025-04-08 14:35:10 -03:00
marcos-silva-rodrigues f661055ae8 FIX: send all metadata 2025-04-08 13:49:10 -03:00
Rafael Moraes d310ec2b93 FEAT: logs_dasboard 2025-04-07 11:55:06 -03:00
marcos-silva-rodrigues 301ad369c0 FEAT: networks get endpoint 2025-04-04 18:25:48 -03:00
marcos-silva-rodrigues 4c94a51a9d REFACTOR: openid module 2025-04-03 07:46:16 -03:00
Marcos Rodrigues Silva a2c1d79247 Merge pull request #276 from dadosfera/release/march
FIX: correct env
2025-04-02 17:06:25 -03:00
marcos-silva-rodrigues e5d9ac2fbf FEAT: network policy endpoint 2025-04-02 16:48:10 -03:00
Marcos Rodrigues Silva 67a15b36b3 Merge pull request #275 from dadosfera/release/march
FIX: correct env
2025-04-02 14:07:52 -03:00
marcos-silva-rodrigues f396cdb2d3 FIX: correct env 2025-04-02 14:01:54 -03:00
marcos-silva-rodrigues 9cc538cdcd FEAT: add host in metadata 2025-04-01 17:57:05 -03:00
Guilherme Maioli 01da6becff Merge pull request #274 from dadosfera/UPDATE/helmfile
UPDATE: return variable to stg.
2025-03-31 14:08:00 -03:00
Guilherme Maioli 003c9b176c UPDATE: return variable to stg. 2025-03-31 14:04:34 -03:00
Marcos Rodrigues Silva 0a5315fb95 Merge pull request #273 from dadosfera/feature/block_and_reset_users
FEAT: block, unblock and reset users
2025-03-28 17:00:49 -03:00
marcos-silva-rodrigues 5a4d578d13 FEAT: block, unblock and reset users 2025-03-28 12:30:55 -03:00
marcos-silva-rodrigues f1345472b5 UPDATE: add routes to login with idp 2025-03-25 09:18:27 -03:00
Marcos Rodrigues Silva 60214f0e57 Merge pull request #272 from dadosfera/feature/block_and_reset_users
Feature/block and reset users
2025-03-24 16:27:35 -03:00
marcos-silva-rodrigues 3a2ce72910 UPDATE: protospack 2025-03-24 14:46:24 -03:00
marcos-silva-rodrigues ea9c0502b7 UPDATE: new route to add identity provider 2025-03-24 07:58:26 -03:00
marcos-silva-rodrigues 074d3ce201 BUILD: remove unused packages 2025-03-21 18:09:09 -03:00
marcos-silva-rodrigues 05c80e2497 FEAT: reset, block and unblock all users 2025-03-21 17:04:58 -03:00
marcos-silva-rodrigues 8a363dd815 FIX: Merge main 2025-03-18 17:59:04 -03:00
Marcos Rodrigues Silva 1e5fe59d9c Merge pull request #271 from dadosfera/feature/new-mixpanel-properties
Add new mixpanel properties
2025-03-14 15:20:42 -03:00
Jonathan Witkosky 1a2b9617fe Add new mixpanel properties 2025-03-14 13:57:31 -03:00
Guilherme Maioli 1f92510a43 Merge pull request #270 from dadosfera/bugfixes/13-03
Bugfixes/13 03
2025-03-13 17:59:32 -03:00
marcos-silva-rodrigues f7fceb1ea2 Merge branch 'fix/white-label' into bugfixes/13-03 2025-03-13 17:41:12 -03:00
Rafael Moraes 2aec60a354 FEATURE: Reset and block users 2025-03-13 14:44:54 -03:00
Marcos Rodrigues Silva 9bc42a4af0 Merge pull request #269 from dadosfera/fix/private-route
FIX: cors list
2025-02-26 17:15:03 -03:00
marcos-silva-rodrigues 5c98cc1748 FIX: cors list 2025-02-26 17:13:57 -03:00
Marcos Rodrigues Silva 9ae14ab04a Merge pull request #268 from dadosfera/fix/private-route
FIX: correct ip
2025-02-26 16:00:20 -03:00
marcos-silva-rodrigues 5d014f4669 FIX: correct ip 2025-02-26 15:57:07 -03:00
Marcos Rodrigues Silva 5e8dea7255 Merge pull request #267 from dadosfera/fix/private-route
FIX: add ip range and domain list to route /open-data
2025-02-26 08:49:40 -03:00
marcos-silva-rodrigues 38fc9f521b FIX: add ip range and domain list to route /open-data 2025-02-25 17:10:39 -03:00
Marcos Rodrigues Silva 80d7ac6ff9 Merge pull request #264 from dadosfera/fix/white-label
Fix/white label
2025-02-24 09:35:37 -03:00
marcos-silva-rodrigues 9bacd7a0f5 UPDATE: merge with beta branch 2025-02-24 09:28:03 -03:00
Aldemir Humberto Soares Neto c89ac40039 Merge pull request #266 from dadosfera/fixing-build-ecr
FEAT: fixing package-lock
2025-02-22 07:42:46 -03:00
aldemirneto 6d6cf590ee FEAT: fixing package-lock 2025-02-22 07:39:41 -03:00
Aldemir Humberto Soares Neto f063a8f30e Merge pull request #265 from dadosfera/UNIVR-PII-FACILITIES
Univr pii facilities
2025-02-22 07:29:14 -03:00
aldemirneto 03536a2180 FEAT: ensuring compatibility with ^ in the version 2025-02-22 07:28:30 -03:00
aldemirneto 0efd5cc257 FEAT: Changing protospack version
PII enabling
2025-02-22 07:23:34 -03:00
marcos-silva-rodrigues 9439f13b7b UPdATE: new displayName prop 2025-02-21 17:31:49 -03:00
Rafael Santana d936806406 Merge pull request #262 from dadosfera/beta
[PRD] Enable Dataset Registration via API for External Sources
2025-02-20 14:03:56 -03:00
Rafael Santana 9d7ccd4201 Merge pull request #263 from dadosfera/feat/catalog-external-datasets-using-api-call
UPDATE: migrating maestro to backend nodepool
2025-02-20 13:47:44 -03:00
Rafael c8a4c1aae6 UPDATE: migrating maestro to backend nodepool 2025-02-20 13:46:30 -03:00
Rafael Santana 406db7ccff Merge pull request #261 from dadosfera/feat/catalog-external-datasets-using-api-call
UPDATE: improve logging for table metadata
2025-02-20 08:52:15 -03:00
Rafael f48a35c7ce UPDATE: improve logging for table metadata 2025-02-20 08:40:48 -03:00
Rafael Santana 8d25df74b7 Merge pull request #260 from dadosfera/feat/catalog-external-datasets-using-api-call
UPDATE: updating protospack for add table_schema for /catalog/registe…
2025-02-19 12:25:10 -03:00
Rafael 831a03402e UPDATE: updating protospack for add table_schema for /catalog/register-dataset rout 2025-02-19 12:19:59 -03:00
Marcos Rodrigues Silva a631338599 Merge pull request #259 from dadosfera/feat/update-customer-dash
UPDATE: customer dash
2025-02-17 19:02:52 -03:00
Marcos Rodrigues Silva d943721b4a Merge pull request #258 from dadosfera/feat/update-customer-dash
Feat/update customer dash
2025-02-17 18:47:11 -03:00
Rafael Santana 1bbf978cc4 Merge pull request #257 from dadosfera/feat/catalog-external-datasets-using-api-call
FEAT: Enable dataset registration via API for external sources
2025-02-14 16:57:58 -03:00
Rafael 856f913ce6 Merge branch 'feat/catalog-external-datasets-using-api-call' of github.com:dadosfera/maestro into feat/catalog-external-datasets-using-api-call 2025-02-14 15:07:13 -03:00
Rafael dc617b6925 REFACTOR: Improve logging with additional metadata
- Enhanced logs to include more detailed request body information.
- Standardized log calls to include extra metadata, such as service name, environment, customer name, user ID, HTTP method, and request path.
- Improved error logging to provide better debugging insights.

Example log format:
EXTRA[{"service":{"name":"maestro","environment":"stg"},"customer_name":"dadosfera","user_id":"047824d8-0021-709a-bf15-39b8dd069aaf","method":"POST","path":"/catalog/register-dataset"}]

Related tickets or issues: [TECN-9441]
2025-02-14 15:04:22 -03:00
Rafael Santana 3f80062672 Merge branch 'beta' into feat/catalog-external-datasets-using-api-call 2025-02-14 10:14:01 -03:00
Rafael 07eb423814 UPDATE: update version of protospack 2025-02-14 10:12:20 -03:00
Rafael 721232448e FEAT: Enable dataset registration via API for external sources
As part of our hybrid approach, we need to allow dataset cataloging from outside our cloud environment.
Instead of relying on our GRPC APIs, we leveraged an existing REST API to handle this process.

- Updated Maestro to call Nimbus directly for dataset registration.
- Modified protospack to generate the necessary TypeScript interfaces.
- Added the `/catalog/register-dataset` endpoint to the API.

This change enables seamless dataset registration from external sources while maintaining compatibility with our existing infrastructure.

Related tickets or issues: [TECN-9441]
2025-02-14 10:08:45 -03:00
Marcos Rodrigues Silva 0b0d261aae Merge pull request #256 from dadosfera/fix/white-label
UPDATE: remove user agent
2025-02-13 13:56:37 -03:00
marcos-silva-rodrigues 87a8d741d7 UPDATE: remove user agent 2025-02-13 13:55:37 -03:00
marcos-silva-rodrigues d03dd6129a UPDATE: customer dash 2025-02-13 12:20:04 -03:00
Marcos Rodrigues Silva 96c20995a3 Merge pull request #255 from dadosfera/beta
Beta
2025-02-03 18:22:05 -03:00
Marcos Rodrigues Silva 82470553db Merge pull request #254 from dadosfera/fix/white-label
FIX: remove support for svg
2025-02-03 12:29:19 -03:00
marcos-silva-rodrigues 2408505889 FIX: remove support for svg 2025-02-03 12:25:56 -03:00
Marcos Rodrigues Silva 7d85693a1f Merge pull request #253 from dadosfera/fix/white-label
Fix/white label
2025-02-03 09:53:38 -03:00
marcos-silva-rodrigues d86a011bfd Reapply "Merge pull request #250 from dadosfera/beta"
This reverts commit d3f7664116.
2025-02-01 16:49:18 -03:00
Marcos Rodrigues Silva 4d54ac744b Merge pull request #252 from dadosfera/feature/new-monitoring-dash
FEAT: update monitoring dash
2025-01-31 18:32:41 -03:00
marcos-silva-rodrigues 132614b551 FEAT: update monitoring dash 2025-01-31 18:24:08 -03:00
Rafael Santana 17c541548a Merge pull request #251 from dadosfera/rollback-main
Revert "Merge pull request #250 from dadosfera/beta"
2025-01-31 18:20:20 -03:00
Rafael d3f7664116 Revert "Merge pull request #250 from dadosfera/beta"
This reverts commit 2ea40004a6, reversing
changes made to 873f7ea314.
2025-01-31 18:13:33 -03:00
Marcos Rodrigues Silva 2ea40004a6 Merge pull request #250 from dadosfera/beta
Beta
2025-01-31 17:24:43 -03:00
Marcos Rodrigues Silva 82ec2d9493 Merge pull request #248 from dadosfera/feature/new-monitoring-dash
FEAT: change dashboard id
2025-01-31 14:40:02 -03:00
Marcos Rodrigues Silva 53bef174b3 Merge pull request #249 from dadosfera/feature/white-label-app
FIX: return null when not found theme
2025-01-31 13:40:07 -03:00
marcos-silva-rodrigues bcfef8359e FIX: return null when not found theme 2025-01-31 13:39:15 -03:00
Marcos Rodrigues Silva 5f18411aed Merge pull request #247 from dadosfera/feature/white-label-app
Feature/white label app
2025-01-30 17:57:09 -03:00
marcos-silva-rodrigues 1102a6e8e5 FEAT: remove personal data in logger and only endpoint for wordpress 2025-01-30 17:06:09 -03:00
marcos-silva-rodrigues 867aec092b FEAT: add svg 2025-01-30 16:06:13 -03:00
marcos-silva-rodrigues ea9e727fcf FEAT: change dashboard id 2025-01-30 15:09:27 -03:00
Marcos Rodrigues Silva 099476f618 Merge pull request #246 from dadosfera/feature/white-label-app
Feature/white label app
2025-01-27 15:07:22 -03:00
marcos-silva-rodrigues db78494b1a FIX: test log headers 2025-01-27 14:39:12 -03:00
marcos-silva-rodrigues e7ad6e5a7b CHORE: update env 2025-01-27 14:12:38 -03:00
Marcos Rodrigues Silva 3d7d051df2 Merge pull request #245 from dadosfera/feature/white-label-app
chore: update stg values
2025-01-27 11:50:43 -03:00
marcos-silva-rodrigues 8e827dc934 chore: update stg values 2025-01-27 11:47:23 -03:00
Marcos Rodrigues Silva 12b764710a Merge pull request #244 from dadosfera/feature/white-label-app
Feature/white label app
2025-01-27 09:49:29 -03:00
marcos-silva-rodrigues b5c95874bb chore: updated package 2025-01-27 09:14:42 -03:00
marcos-silva-rodrigues b896b1a9b8 REFACTOR: rename props 2025-01-26 16:44:31 -03:00
marcos-silva-rodrigues cf99acc682 FEAT: send images in stream 2025-01-26 16:37:37 -03:00
Marcos Rodrigues Silva 7fb20964dc Merge pull request #243 from dadosfera/feature/white-label-app
Feature/white label app
2025-01-23 08:39:26 -03:00
Rafael Santana 8751117946 Merge pull request #242 from dadosfera/decrease-replica-count-maestro
UPDATE: add toleration for spot instances
2025-01-22 11:24:35 -03:00
Rafael 10c4ed179b UPDATE: add toleration for spot instances 2025-01-22 11:24:01 -03:00
marcos-silva-rodrigues 8e13541b24 FEAT: update protospack 2025-01-21 17:28:13 -03:00
marcos-silva-rodrigues 4f6a9d4b66 FEAT: new property in customer obj 2025-01-21 17:00:34 -03:00
Rafael Santana 56411496c2 Merge pull request #241 from dadosfera/decrease-replica-count-maestro
Decrease replica count maestro
2025-01-20 18:50:14 -03:00
Rafael e55a1cb657 CI: removed deploy-k8s workflow 2025-01-20 18:47:44 -03:00
Rafael 3850d4b8ae UPDATE: blank commit to force deployment 2025-01-20 18:47:13 -03:00
Rafael Santana 1626d85d31 Merge pull request #240 from dadosfera/decrease-replica-count-maestro
UPDATE: decrease the number of replicas for stg environment
2025-01-20 18:41:54 -03:00
Rafael 2a6677e4d9 UPDATE: decrease the number of replicas for stg environment 2025-01-20 18:41:01 -03:00
Marcos Rodrigues Silva 74e62c001d Merge pull request #239 from dadosfera/feature/white-label-app
FIX: env fixed and add logger for the process.env
2025-01-17 08:40:08 -03:00
marcos-silva-rodrigues 8237160c6d FIX: env fixed and add logger for the process.env 2025-01-17 08:38:03 -03:00
Marcos Rodrigues Silva 41bf9d4856 Merge pull request #238 from dadosfera/feature/white-label-app
FIX: change env for stg
2025-01-16 18:47:21 -03:00
marcos-silva-rodrigues 44dcf1b281 FIX: change env for stg 2025-01-16 18:43:15 -03:00
Marcos Rodrigues Silva 104787bce8 Merge pull request #237 from dadosfera/feature/white-label-app
Feature/white label app
2025-01-15 11:04:20 -03:00
marcos-silva-rodrigues 38a330e578 FIX: change stg env for prd env 2025-01-15 10:55:08 -03:00
marcos-silva-rodrigues 7d0b8755c9 FEAT: image upload in theme 2025-01-13 17:35:56 -03:00
Marcos Rodrigues Silva 7aadf4d9d1 Merge pull request #236 from dadosfera/feature/white-label-app
FIX: add header content type and change status code to 200
2025-01-09 08:40:59 -03:00
marcos-silva-rodrigues af8fa72377 FIX: add header content type and change status code to 200 2025-01-09 08:40:02 -03:00
Marcos Rodrigues Silva c51f262474 Merge pull request #235 from dadosfera/feature/white-label-app
FIX:  param to language and message in json
2025-01-09 08:15:46 -03:00
marcos-silva-rodrigues 54d19a78ab FIX: param to language and message in json 2025-01-09 08:13:06 -03:00
Marcos Rodrigues Silva 66f33a2246 Merge pull request #234 from dadosfera/feature/white-label-app
FIX: remove language param
2025-01-08 18:15:41 -03:00
marcos-silva-rodrigues 55d9441d10 FIX: remove language param 2025-01-08 18:12:04 -03:00
Marcos Rodrigues Silva 8b7342652e Merge pull request #233 from dadosfera/feature/white-label-app
FEAT: add language by query
2025-01-08 15:34:00 -03:00
marcos-silva-rodrigues 118c2653a5 FEAT: add language by query 2025-01-08 15:26:24 -03:00
Marcos Rodrigues Silva f7d0128d99 Merge pull request #232 from dadosfera/feature/white-label-app
FIX: remove origin decorator
2025-01-06 16:23:46 -03:00
marcos-silva-rodrigues 12cc555491 FIX: remove origin decorator 2025-01-06 16:22:03 -03:00
Marcos Rodrigues Silva cf5e36db00 Merge pull request #231 from dadosfera/feature/white-label-app
Feature/white label app
2025-01-06 15:38:40 -03:00
marcos-silva-rodrigues 7c0b87ffc3 FEAT: decorators to apply an origin in endpoint router 2025-01-06 15:25:28 -03:00
marcos-silva-rodrigues beb086ad3c FIX: fixed customer and role id 2025-01-06 14:29:22 -03:00
marcos-silva-rodrigues 8c12291096 FIX: try catch in customer controller 2025-01-06 14:15:29 -03:00
Marcos Rodrigues Silva 56a063b644 Merge pull request #230 from dadosfera/feature/white-label-app
Feature/white label app
2024-12-26 16:01:47 -03:00
marcos-silva-rodrigues 3a6054c1c6 FIX: merge conflicts 2024-12-26 15:33:53 -03:00
marcos-silva-rodrigues 3a696ecb0e CHORE: update envs 2024-12-26 15:31:22 -03:00
Rafael Santana 1166073b1c Merge pull request #229 from dadosfera/stg.dadosfera.ai
UPDATE: updating stg endpoints
2024-12-23 12:20:45 -03:00
Rafael 3026cd0748 UPDATE: updating stg endpoints 2024-12-23 12:18:42 -03:00
Marcos Rodrigues Silva b48d3ae667 Merge pull request #228 from dadosfera/feature/white-label-app
Feature/white label app
2024-12-18 17:05:34 -03:00
marcos-silva-rodrigues 5868f21f22 FEAT: merge 2024-12-18 16:25:56 -03:00
Rafael Santana 873f7ea314 Merge pull request #225 from dadosfera/helm-chart
UPDATE: create helm chart for maestro
2024-12-18 16:07:39 -03:00
marcos-silva-rodrigues 2b01b340e4 FIX: add env 2024-12-18 14:19:17 -03:00
marcos-silva-rodrigues 5888088486 FIX: remove unused suite test 2024-12-18 14:19:00 -03:00
marcos-silva-rodrigues 9e2de28032 FIX: map request body to the form wordpress 2024-12-18 13:58:08 -03:00
marcos-silva-rodrigues db2ea09d25 FEAT: create new route to open data iniciative 2024-12-12 13:54:18 -03:00
Rafael Santana 8c6b514b7b Merge pull request #227 from dadosfera/helm-chart
Helm chart
2024-12-09 14:07:01 -03:00
Rafael ad03d663bb CI: updating CI to use beta branch 2024-12-09 14:00:56 -03:00
marcos-silva-rodrigues 48d4cbc41a CHORE: stable version lib protospack 2024-12-05 10:29:06 -03:00
marcos-silva-rodrigues ae81816aa7 FEAT: create get and post route to theme 2024-12-04 15:29:51 -03:00
marcos-silva-rodrigues ccdff58056 FEAT: update protospack lib 2024-12-04 15:28:57 -03:00
Rafael c10f85950a UPDATE: Commented out the aws beanstalk deployment 2024-10-12 13:28:06 -03:00
Rafael 7d69220461 UPDATE: create helm chart for maestro 2024-10-12 13:19:57 -03:00
Gabriel Amorim e316bd1a7b Merge pull request #224 from dadosfera/feat/dev-docker
FEAT: dev docker
2024-08-16 20:31:13 -03:00
Gabriel Rosa fd47a746af FIX: microservices urls 2024-08-16 22:19:11 +00:00
Gabriel Rosa 5038c7845f FEAT: dev docker 2024-08-16 21:59:23 +00:00
Gabriel Amorim 9876eab521 Merge pull request #223 from dadosfera/feat/rls
Feat/rls
2024-08-16 17:16:18 -03:00
Gabriel Rosa 99a52c3ff2 FEAT: new protospack version 2024-08-16 17:05:38 -03:00
Gabriel Rosa 0119cee3c6 FIX: removed unused imports 2024-08-14 11:29:14 -03:00
Gabriel Rosa c7e850cc79 FEAT: rls rules routes 2024-08-14 11:13:19 -03:00
Gabriel Rosa 5da47e5438 FEAT: create RLS on pi-factory 2024-08-07 18:16:20 -03:00
Gabriel Rosa b0f9f5c77e [skip ci] DOC: improving documentation in portuguese 2024-08-05 17:57:06 -03:00
Gabriel Amorim 35f35ddeb3 Merge pull request #222 from dadosfera/SOLENG-4628-feat/restrict-asset-options
[SOLENG-4628] feat/restrict asset options
2024-07-10 12:03:32 -03:00
Gabriel Rosa 90155a8811 FIX: cahnged self-hosted actions runner to run test 2024-07-10 10:43:40 -03:00
Gabriel Rosa ae8baca694 Merge branch 'main' into SOLENG-4628-feat/restrict-asset-options 2024-07-08 09:39:53 -03:00
Gabriel Amorim 82b8751f65 Merge pull request #221 from dadosfera/fix/docker-compose
FIX: use docker compose instead of docker-compose
2024-06-28 16:58:18 -03:00
Gabriel Rosa 78ee64fc45 FIX: login aws on test 2024-06-28 16:55:31 -03:00
Gabriel Rosa 2f30837fed FIX: use docker compose instead of docker-compose 2024-06-28 16:47:45 -03:00
Gabriel Amorim f46b1b62db Merge pull request #220 from dadosfera/feat/new-customer-monitoring
FEAT: new customer monitoring dashboard
2024-06-28 16:37:38 -03:00
Gabriel Rosa 8b6cd8a88e FEAT: customer monitoring dashboard 2024-06-21 09:21:27 -03:00
Gabriel Rosa 0c333b476a FEAT: new customer monitoring 2024-06-20 10:50:07 -03:00
Gabriel Rosa 1aa9371245 FIX: Preserve slash escape 2024-05-08 18:09:56 -03:00
Gabriel Rosa 14fcf4a0a1 CI: format the node_exporter url with format() 2024-05-08 18:01:44 -03:00
Gabriel Rosa 24f824bcd1 FIX: update protospack version 2024-05-08 17:17:55 -03:00
Gabriel Rosa fcdd0bf571 Merge branch 'main' into SOLENG-4628-feat/restrict-asset-options 2024-05-08 15:37:45 -03:00
Gabriel Amorim c416a2cb97 Merge pull request #219 from dadosfera/feat/pipeline-monitoring-dashboard
Feat: pipeline monitoring dashboard
2024-05-06 16:52:26 -03:00
Gabriel Rosa d03021fc9d new protospack release 2024-05-06 15:28:26 -03:00
Gabriel Rosa 574cf48726 FIX: npm audit fix 2024-05-02 12:11:46 -03:00
Gabriel Rosa 7cef404acf FEAT: get monitoring dashboard url 2024-05-02 12:11:35 -03:00
Gabriel Rosa 3c32c3c7d4 CI: Update actions/checkout to v4 2024-04-22 14:57:01 -03:00
Gabriel Amorim d44531b963 Merge pull request #218 from dadosfera/feat/choose-deploy-dockerhub
[ TECN-6603 ] CI: update action versions and choose wether to upload to dockerhub
2024-01-10 10:05:47 -03:00
Gabriel Rosa 36c80b480e CI: update action versions and choose wether to upload to dockerhub 2024-01-04 14:58:47 -03:00
Rafael Santana eeef97679c Merge pull request #217 from dadosfera/ci/dockerhub
UPDATE: updating maestro to deploy image to dockerhub
2023-11-26 21:18:32 -03:00
Rafael Santana c789dae3f6 UPDATE: updating maestro to deploy image to dockerhub 2023-11-26 20:50:10 -03:00
Gabriel Amorim 4b20388d45 DOCS: update some routes swagger 2023-09-20 09:39:43 -03:00
Gabriel Amorim 33fd2bd35f FIX: update old protospack library 2023-09-20 09:39:27 -03:00
Gabriel Amorim 654c837d3f FEAT: new node version 2023-09-18 11:34:23 -03:00
Gabriel Amorim 416b58080d CI: fixed conventional-changelog-eslint version 2023-09-18 11:33:09 -03:00
Allan Sene 216defb76f Updated file 2023-06-11 22:43:43 -03:00
Gabriel Amorim 0970c748e0 Merge pull request #216 from dadosfera/feat/generate-token
Feat: generate token
2023-05-17 10:03:00 -03:00
Gabriel Amorim aabd741cad FEAT: new Customer Get Token Route 2023-05-16 09:05:05 -03:00
Gabriel Amorim 950c6414b6 FIX: update to new nestjs version 2023-05-12 08:28:33 -03:00
Gabriel Amorim d1149a4d52 Merge pull request #215 from dadosfera/alpha
FIX: permissions.enum.ts
2023-05-08 16:02:33 -03:00
Gabriel Amorim 4c6f4bfd84 FIX: renamed reference to intelligence permission 2023-05-08 15:56:52 -03:00
Gabriel Amorim 9caf3dc67f FIX: renamed reference to intelligence permission 2023-05-08 15:48:46 -03:00
Beatriz Antunes 635c4da48c Update permissions.enum.ts 2023-05-08 15:44:05 -03:00
Gabriel Amorim 44bb205483 Merge pull request #214 from dadosfera/fix/get-links-permission
FIX: remove permission check to get customer links
2023-05-05 13:52:38 -03:00
Gabriel Amorim edb4622684 FIX: remove permission to get customer links 2023-05-05 12:13:55 -03:00
Victor Radael abb46d78da Merge pull request #212 from dadosfera/feature/customer-links
SOLENG-1481 - Customer links endpoints
2023-04-17 17:01:41 -03:00
Victor Radael 04fdc0368a Merge pull request #213 from dadosfera/update/hubspot-tables
UPDATE: HubspotTables
2023-04-15 14:49:47 -03:00
Victor Radael 50d76622c5 UPDATE: HubspotTables 2023-04-15 14:45:18 -03:00
Victor Radael 238f56cb76 Merge branch 'SOLENG-1736' into feature/customer-links 2023-04-14 14:44:57 -03:00
Victor Radael e721d6f5f5 Merge branch 'main' into feature/customer-links 2023-04-14 13:30:57 -03:00
Victor Radael 44df13e820 UPDATE: Att protospack-v2 version 2023-04-14 13:13:47 -03:00
Victor Radael 1fd0f79dd5 Att hubspot connection 2023-04-13 15:41:44 -03:00
Victor Radael 1b92371737 Att hubspot connection 2023-04-13 14:55:33 -03:00
Colombo bf5b3484a1 FEAT: add controller method interfaces 2023-04-11 20:39:47 -03:00
Colombo eb1e248d4b FEAT: add customer (links) endpoint 2023-04-11 17:04:40 -03:00
Colombo 0827414051 FEAT: add new dto property 2023-04-10 19:36:36 -03:00
Colombo 081986607b CHORE: change protospack package to beta 2023-04-06 17:07:23 -03:00
Gabriel Amorim 189080199a Merge pull request #211 from dadosfera/feat/trigger-catalog-task
Feat/trigger catalog task
2023-03-17 12:14:15 -03:00
Gabriel Amorim 55281e3965 FIX: docs 2023-03-16 17:58:11 -03:00
Gabriel Amorim db8a6175e8 new protospack version 2023-03-16 17:45:15 -03:00
Gabriel Amorim 7b0d4dd5bc FIX: set route params type 2023-03-16 17:45:03 -03:00
Gabriel Amorim 6fd277ef3a docs 2023-03-14 16:02:22 -03:00
Gabriel Amorim 7b25bfdb69 FIX: add table_schema to request 2023-03-14 16:00:57 -03:00
Gabriel Amorim 2abc19c589 FEAT: trigger and get dataset cataloging task 2023-03-14 13:39:21 -03:00
Gabriel Amorim faf6b1df75 Merge pull request #210 from dadosfera/fix/delete-connector
Fix/delete connector
2023-03-09 18:14:14 -03:00
Gabriel Amorim 422be4a422 swaggers 2023-03-06 12:11:13 -03:00
Gabriel Amorim da136e0ef0 FIX: better swagger generation strategy (Only PRD server on external docs) 2023-03-06 12:10:12 -03:00
Gabriel Amorim 737cc9b8ea FIX: removed wrong JSON.parse on connector delete response 2023-03-06 12:09:29 -03:00
Gabriel Amorim 23f6b179fd Merge pull request #209 from dadosfera/feat/google-sheets-link
Feat/google sheets link
2023-02-27 16:03:52 -03:00
Gabriel Amorim 03e451c21d Merge branch 'main' into feat/google-sheets-link 2023-02-27 11:26:03 -03:00
Gabriel Amorim 2646e3e4f6 FEAT: return google sheets entire link 2023-02-24 15:58:06 -03:00
135 changed files with 14662 additions and 11707 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ module.exports = {
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'plugin:prettier/recommended',
'prettier',
],
root: true,
env: {
+54 -108
View File
@@ -3,6 +3,7 @@ on:
push:
branches:
- main
- beta
workflow_dispatch:
inputs:
environment:
@@ -11,7 +12,6 @@ on:
type: choice
options:
- stg
- stg2
- prd
jobs:
@@ -29,6 +29,8 @@ jobs:
echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "environment=prd" >> $GITHUB_OUTPUT
elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
echo "environment=stg" >> $GITHUB_OUTPUT
fi
id: extract_environment
@@ -39,7 +41,7 @@ jobs:
new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- if: github.event_name != 'workflow_dispatch'
name: Semantic Release
@@ -47,45 +49,43 @@ jobs:
id: semantic
with:
extra_plugins: |
conventional-changelog-eslint
conventional-changelog-eslint@4.0.0
branches: |
[
'main'
'main',
{
name: 'alpha',
prerelease: true
},
{
name: 'beta',
prerelease: true
}
]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
deploy-info:
if: ${{ github.event_name == 'workflow_dispatch'}}
build_ecr_image:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
needs: [extract_environment, semantic_release]
runs-on: ubuntu-latest
runs-on:
[self-hosted, "prd"]
steps:
- name: Create summary
- name: Printing stats
env:
EVENT: ${{ github.event_name }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ENV: ${{ needs.extract_environment.outputs.environment }}
run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY
run: echo ${GITHUB_REF#refs/heads/}
deploy:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Update Pip
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
run: |
python3 -m pip install awscli --upgrade
@@ -95,14 +95,14 @@ jobs:
python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region
uses: aws-actions/configure-aws-credentials@v1-node16
uses: aws-actions/configure-aws-credentials@v4
id: aws
with:
aws-region: us-east-1
- name: Login to AWS ECR
id: login_ecr
uses: aws-actions/amazon-ecr-login@v1
uses: aws-actions/amazon-ecr-login@v2
- name: Build, Tag, and Push Image to AWS ECR
env:
@@ -110,97 +110,43 @@ jobs:
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker-compose -f build.docker-compose.yml build
docker-compose -f build.docker-compose.yml push
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
zip deploy.zip docker-compose.yml -r .ebextensions
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
# IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
# ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
# NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
# run: |
# sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
# sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
# sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
# sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
# zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
AWS_REGION: us-east-1
APP_NAME: ${{ github.event.repository.name }}
run: |
eb use $APP_NAME-$ENV
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
eb deploy
# - name: Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
# AWS_REGION: us-east-1
# APP_NAME: ${{ github.event.repository.name }}
# run: |
# eb use $APP_NAME-$ENV
# echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
# eb deploy
- name: Remove Docker's Trash
if: always()
run: |
docker system prune --volumes -a -f
docker system df
api_docs:
needs: [extract_environment, semantic_release, deploy]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Extract Docs BlockId and PageId
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
shell: bash
run: |
if [ $ENV == "stg2" ]; then
echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "stg" ]; then
echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "prd" ]; then
echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
fi
id: extract_docs_info
- name: Generate API docs
env:
DOCS_URL: ${{ secrets.DOCS_URL }}
DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
EVENT: ${{ github.event_name }}
RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
run: |
if [ ${EVENT} != "workflow_dispatch" ]; then
sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
fi
sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
- name: Generate External API docs
env:
DOCS_URL: ${{ secrets.DOCS_URL }}
DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}}
DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}}
EVENT: ${{ github.event_name }}
RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
run: |
if [ ${EVENT} != "workflow_dispatch" ]; then
sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json
fi
sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL
k8s-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: ${{ (needs.extract_environment.outputs.environment == 'prd' && 'azure' ) || 'oracle' }}
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
+137
View File
@@ -0,0 +1,137 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+6 -11
View File
@@ -6,23 +6,18 @@ on:
jobs:
test:
runs-on: self-hosted
runs-on: [self-hosted, prd]
env:
APP_NAME: ${{ github.event.repository.name }}
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Build
env:
APP_NAME: ${{ github.event.repository.name }}
run: |
docker build -t $APP_NAME --target test .
run: docker build -t ${APP_NAME}_teste --target test .
- name: Run Test
env:
ENV: test
APP_NAME: ${{ github.event.repository.name }}
run: |
docker run --rm --entrypoint="npm" $APP_NAME run test
run: docker run ${APP_NAME}_teste
- name: Remove Docker's Trash
if: always()
+102
View File
@@ -0,0 +1,102 @@
name: Validate K8S Modifications
on:
pull_request:
branches:
- main
- beta
jobs:
extract_environment:
runs-on: ubuntu-22.04
outputs:
environment: ${{ steps.extract_environment.outputs.environment }}
steps:
- name: Extract Environment
run: |
if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then
echo "environment=prd" >> $GITHUB_OUTPUT
elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then
echo "environment=stg" >> $GITHUB_OUTPUT
fi
id: extract_environment
helmfile-check:
env:
HOME: /home/runner
needs: [extract_environment]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+1 -1
View File
@@ -1 +1 @@
18.3.0
18.17
+1
View File
@@ -12,6 +12,7 @@
"start:debug"
],
"runtimeExecutable": "npm",
"runtimeVersion": "18.17",
"skipFiles": [
"<node_internals>/**"
],
+61 -16
View File
@@ -1,21 +1,66 @@
# install all dependencies
FROM node:18.3.0-alpine3.15 as packages
FROM node:18.17-alpine AS base_image
FROM base_image AS build_base
WORKDIR /app
COPY package.json package-lock.json ./
RUN apk add --no-cache aws-cli \
&& aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
&& npm install
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
aws-cli \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY package*.json ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# run aws cli without mounting secret, because CI already has AWS credentials
FROM build_base AS ci_image
RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
RUN npm ci
COPY . .
# unit test specific build
FROM node:18.3.0-alpine3.15 as test
WORKDIR /app
COPY --from=packages /app/node_modules ./node_modules
COPY . ./
FROM ci_image AS test
ENV DUC_URL=0.0.0.0:50051
ENTRYPOINT ["npm", "run", "test"]
FROM node:18.3.0-alpine3.15
WORKDIR /app
COPY . /app/
COPY --from=packages /app/node_modules ./node_modules
# dev build
FROM build_base AS dev
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
# flag --build-from-source is required to force-build sqlite3
RUN npm ci
COPY . .
ENTRYPOINT npm run start:dev
FROM ci_image AS prod_build
RUN npm run build
EXPOSE 3333
ENTRYPOINT npm run start
FROM base_image
WORKDIR /app
COPY --from=prod_build /app/dist ./dist
COPY --from=prod_build /app/node_modules ./node_modules
COPY --from=prod_build /app/package*.json ./
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT npm run start:prod
+10
View File
@@ -0,0 +1,10 @@
LICENSE
Copyright (C) 2023 Dadosfera
All rights reserved.
This software and related documentation are provided under a license agreement containing restrictions on use and disclosure and are protected by intellectual property laws. Except as expressly permitted in your license agreement or allowed by law, you may not use, copy, reproduce, translate, broadcast, modify, license, transmit, distribute, exhibit, perform, publish, or display any part, in any form, or by any means.
Reverse engineering, disassembly, or decompilation of this software, unless required by law for interoperability, is prohibited.
If you have any questions about this, please contact Dadosfera.ai at legal@dadosfera.ai
+88 -84
View File
@@ -2,52 +2,57 @@
<image src="./assets/maestro.svg" style="width:10rem">
</p>
# Maestro
Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
## 🚀 Starting
```mermaid
graph TD;
Frontend<-->Maestro;
Maestro<-->duc;
Maestro<-->pi-factory;
Maestro<-->in-factory;
```
These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
É uma API REST, desenvolvida em NodeJs utilizando o Framework [NestJs](https://docs.nestjs.com/).
### 📋 Requirements
## 🚀 Iniciando
- [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
- Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine
- Request access to the dev, stg and prd VPNs
Estas instruções permitirão que você obtenha uma cópia funcional do projeto em sua máquina local para desenvolvimento e testes.
### 🔧 Installation<a id="installation"></a>
### 📋 Requisitos
- Clone the repository
- [NodeJS v18.17 / NPM v9.6.7](https://nodejs.org/pt-br/download/)
- SSH
> Dica: Utilize [NVM](https://github.com/nvm-sh/nvm) para gerenciar facilmente as versões do node
```
git clone git@github.com:dadosfera/maestro.git
```
- Solicite acesso à conta de desenvolvimento do AWS Console para **todos os serviços necessários**
- Crie sua Access Key no menu "Security credentials"
- Defina a Access Key em sua máquina de desenvolvimento local
- Solicite acesso às VPNs de stg e prd
or
### 🔧 Instalação<a id="installation"></a>
- HTTPS
```
git clone https://github.com/dadosfera/maestro.git
```
- Clone o repositório
- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
```sh
git clone git@github.com:dadosfera/maestro.git
```
- Selecione a versão correta do node (opcional, apenas se estiver usando o [NVM](https://github.com/nvm-sh/nvm)):
```sh
nvm use
```
- Install the project dependencies:
- Instale as dependências do projeto:
```sh
npm i
```
- Setup the following enviroment variables:
- Configure as seguintes variáveis de ambiente:
```
ENV=
@@ -58,42 +63,41 @@ These instructions will allow you to get a working copy of the project on your l
SM_OAUTH_PATH=
```
- Start the server:
- Inicie o servidor:
```sh
# dev mode
npm run start:dev
# or in debug mode
npm run start:debug
```
The service should start successfully.
> Se preferir, utilize o debbuger do VSCode apertando F5
## 📄 Documentation
O serviço deve iniciar com sucesso.
NestJs makes it easy to document each route using decorators on all requests and responses properties. It automatically generates a swagger for given information and provides a route to access it http://localhost:3333/api.
For more info check the [official documentation](https://docs.nestjs.com/openapi/introduction).
## 📄 Documentação
### - Multiple documentations
O NestJs facilita a documentação de cada rota usando decoradores em todas as propriedades de solicitações e respostas. Ele gera automaticamente um swagger para as informações fornecidas e fornece uma rota para acessá-lo em http://localhost:3333/api. Para mais informações, consulte a [documentação oficial](https://docs.nestjs.com/openapi/introduction).
We are currently generating **2 different** documentations: Internal and External.
### - Documentações múltiplas
All routes that have the decorator `@ApiInternalOnly()` will not be visible on the **External** API swagger.
Atualmente, estamos gerando **2 documentações diferentes**: Interna e Externa.
- When you start the application with `npm run start` it will serve and generate the swagger JSON of the **External** API
- When you start the application with `npm run start:internal` it will serve and generate the swagger JSON of the **Internal** API
- When you run `npm run docs` it will generate the swagger JSON of both **Internal** and **External** API, and save them to `docsfera.json` and `docsfera.external.json` respectively;
Todas as rotas que têm o decorador `@ApiInternalOnly()` não serão visíveis no swagger da API **Externa**.
It is important to run `npm run docs` before every deploy so we can always have the most updated docs published.
- Quando você inicia a aplicação com `npm run start:dev`, ela servirá e gerará o JSON do swagger da API **Interna**
## Authentication decorators
- Quando você executa `npm run docs`, ele gerará o JSON do swagger de ambas as APIs **Interna** e **Externa** e os salvará em `docsfera.json` e `docsfera.external.json`, respectivamente;
Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
> Link para documentação interna: https://dadosfera.github.io/docsfera
É importante executar `npm run docs` antes de cada implantação para que sempre tenhamos as documentações mais atualizadas publicadas.
## Decoradores de autenticação
O Maestro possui utilitários para facilitar a autenticação do usuário em todos os controladores e rotas. Os seguintes decoradores estão disponíveis:
### `@Authenticated`
If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
Se o usuário precisar estar autenticado para fazer uma solicitação, podemos usar o decorador `@Authenticated` no controlador ou rota, conforme necessário.
```ts
import { Authenticated } from '../../authentication/authentication.decorator';
@@ -105,7 +109,7 @@ class FooController {
@Get('bar')
async getBar() {
this.logger.info('user is authenticated!');
this.logger.info('usuário está autenticado!');
return { authenticated: true };
}
@@ -122,14 +126,14 @@ class FooController {
@Get('bar')
@Authenticated()
async getBar() {
this.logger.info('user is authenticated!');
this.logger.info('usuário está autenticado!');
return { authenticated: true };
}
@Post('bar')
async postBar() {
this.logger.info('user is NOT authenticated!');
this.logger.info('usuário NÃO está autenticado!');
return { authenticated: false };
}
@@ -138,7 +142,7 @@ class FooController {
### `@RequireAllPermissions`
This decorator requires that **all permissions** listed are granted to the requesting user.
Este decorador exige que **todas as permissões** listadas sejam concedidas ao usuário solicitante.
```ts
import { RequireAllPermissions } from '../../authentication/authentication.decorator';
@@ -158,7 +162,7 @@ class FooController {
### `@RequireSomePermission`
In the following case, the user is required to have **at least one** listed permission.
No caso seguinte, é necessário que o usuário tenha **pelo menos uma** das permissões listadas.
```ts
import { RequireSomePermission } from '../../authentication/authentication.decorator';
@@ -178,18 +182,18 @@ class FooController {
### `@AuthenticateCondition`
If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
Se for necessária uma verificação de autenticação mais complicada, podemos usar o decorador `@AuthenticateCondition` para defini-la. A função personalizada deve retornar `true` para autenticar a solicitação.
In the following example:
No exemplo a seguir:
- all routes on the `FooController` controller can only be requested from localhost
- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
- todas as rotas no controlador `FooController` só podem ser solicitadas a partir do localhost
- `POST /foo/bar` só pode ser solicitado a partir do localhost **e** por usuários do cliente com id `111...eef`
```ts
import { AuthenticateCondition } from '../../authentication/authentication.decorator';
@Controller('foo')
// allow requests only from localhost
// permitir solicitações apenas do localhost
@AuthenticateCondition((request: Request) => request.ip === '::ffff:127.0.0.1')
class FooController {
/* ... */
@@ -200,7 +204,7 @@ class FooController {
}
@Post('bar')
// allow requests only from a specific customer
// permitir solicitações apenas de um cliente específico
@AuthenticateCondition(
(request: Request, user: RequestUser) =>
user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
@@ -211,11 +215,11 @@ class FooController {
}
```
### Note on authentication decorators
### Nota sobre decoradores de autenticação
- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
- O método antigo de autenticação colocava os dados do usuário no campo `request.body.info`, o que impõe certos problemas em relação ao corpo da solicitação, pois esses dados devem vir do frontend sem qualquer modificação pelo Maestro. Agora, esse uso está ⚠️ **DESCONTINUADO** ⚠️. Estamos trabalhando para migrar para o decorador de parâmetro `@User`. O método antigo está funcionando enquanto a migração está em andamento.
- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (_and wouldn't like to..._) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
- Os decoradores de autenticação podem ser usados juntos e todos eles **devem** passar para que a solicitação seja autenticada, mas, no caso geral, você não precisa (_e não gostaria de..._) usar todos eles juntos, pois você pode codificar toda a lógica de autenticação no decorador `@AuthenticateCondition`.
```ts
import {
@@ -251,17 +255,17 @@ class FooController {
}
```
- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
- Se `@RequireAllPermissions` e `@RequireSomePermission` forem usados com **apenas uma única permissão**, eles apresentam o **exatamente mesmo comportamento**.
```ts
// same behavior
// mesmo comportamento
@RequireAllPermissions(Permissions.BAR.MANAGE)
@RequireSomePermission(Permissions.BAR.MANAGE)
```
## `@User` parameter decorator
## Decorador de parâmetro `@User`
The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
Os dados do usuário solicitante podem ser obtidos usando o decorador de parâmetro @User, como no exemplo a seguir:
```ts
import { User, RequestUser } from '../../authentication/user.decorator';
@@ -279,7 +283,7 @@ class FooController {
}
```
If the user is required to be logged in, set `required` to `true`, as you would want in the `change-password` operation:
Se o usuário precisar estar logado, defina `required` como `true`, como por exemplo:
```ts
import { User, RequestUser } from '../../authentication/user.decorator';
@@ -305,46 +309,46 @@ class UserController {
}
```
## 📦 Development
## 📦 Desenvolvimento
### ⌨️ Coding Style
### ⌨️ Estilo de Codificação
By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
Por padrão, usamos [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) com configurações padrão.
**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
**Recomendamos usar o Visual Studio Code e instalar as extensões recomendadas para facilitar o processo de desenvolvimento.**
### Commits pattern
### Padrão de commits
Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
Nosso pipeline de fluxo de trabalho segue as especificações de commits convencionais ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) para liberar versões conforme necessário.
Format: `<type>[optional scope]: <description>`
Formato: `<type>[optional scope]: <description>`
Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
Exemplo: `FIX: ensure Range headers adhere more closely to RFC 2616`
### Branching naming convention
### Convenção de nomenclatura de branchs
- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
- **Feature**: Quaisquer alterações de código para um novo módulo ou caso de uso devem ser feitas em uma branch de feature. Esta branch é criada com base na branch `main`. Quando todas as alterações estiverem concluídas, será necessário um Pull Request/Merge Request para colocar todas essas alterações de volta na branch `main`. Exemplos: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
**It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (\_) could be used to separate the ID and description.**
**Recomenda-se usar todas as letras em minúsculas e hífen (-) para separar palavras, a menos que seja um nome ou ID de item específico. O sublinhado (\_) pode ser usado para separar o ID e a descrição.**
- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Bug Fix**: Se as alterações de código feitas na branch de feature foram rejeitadas após um lançamento, sprint ou demo, quaisquer correções necessárias após isso devem ser feitas na branch de correção de bug. Exemplos: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
- **Hot Fix**: Se houver necessidade de corrigir um bloqueador, fazer um patch temporário, aplicar uma mudança crítica de framework ou configuração que deva ser tratada imediatamente, ela deve ser criada como um Hotfix. Exemplos: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
- **Experimental**: Uma branch para experimentar. Qualquer nova feature ou ideia que não faça parte de um lançamento ou sprint. Exemplo: `experimental/dark-theme-support`.
### Making a Pull Request
### Fazendo um Pull Request
1. Commit your changes
2. Open the Pull Request on GitHub
3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
1. Comite suas alterações
2. Abra o Pull Request no GitHub
3. Envie o link do Pull Request no grupo de chat do Google da Microsfera para revisão e possível aprovação
## 🛠️ Built with
## 🛠️ Construído com
Some technologies used in this project:
Algumas tecnologias usadas neste projeto:
- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
- [NestJS](https://docs.nestjs.com) - Framework para construir aplicações NodeJS eficientes e escaláveis no lado do servidor
## ⚙️ Back-end Architecture
## ⚙️ Arquitetura de Back-end
The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
A arquitetura pode ser encontrada neste [link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
+4
View File
@@ -0,0 +1,4 @@
services:
maestro:
build: .
image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+1 -2
View File
@@ -1,5 +1,4 @@
version: "3.8"
services:
maestro:
build: .
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd:${IMAGE_TAG}
+24
View File
@@ -0,0 +1,24 @@
apiVersion: v2
name: maestro
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
+124
View File
@@ -0,0 +1,124 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.app_name }}
namespace: applications
labels:
app: {{ .Values.app_name }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: {{ .Values.app_name }}
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: {{ .Values.app_name }}
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
operator: "Equal"
value: "spot"
effect: "NoSchedule"
containers:
- name: maestro
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
- name: AWS_REGION
value: "us-east-1"
- name: BASE_HOST
value: "maestro_prd"
- name: BUCKET_CUSTOMER_CSV_ASSETS
value: {{ .Values.maestro.bucket_customer_csv_assets }}
- name: CONNECTORS_INDEX
value: "connectors"
- name: DUC_URL
value: {{ .Values.maestro.duc_url }}
- name: ELASTIC_APM_ENVIRONMENT
value: {{ .Values.maestro.env }}
- name: ELASTIC_APM_SERVER_URL
value: "https://apm-server.dadosfera.ai"
- name: ELASTIC_APM_SERVICE_NAME
value: {{ .Values.maestro.apm_service }}
- name: ENV
value: {{ .Values.maestro.env }}
- name: INFACTORY_URL
value: {{ .Values.maestro.in_factory_url }}
- name: LOGGER_GELF_HOST
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
value: {{ .Values.maestro.npm_token }}
- name: PB_TOKEN_PATH
value: {{ .Values.maestro.pb_token_path }}
- name: PIFACTORY_URL
value: {{ .Values.maestro.pi_factory_url }}
- name: SM_OAUTH_PATH
value: {{ .Values.maestro.sm_oauth_path }}
- name: TRFACTORY_URL
value: {{ .Values.maestro.tr_factory_url }}
- name: UPLOAD_FILE_AGENT_CONNECTION
value: {{ .Values.maestro.upload_file_agent_connection }}
- name: OPEN_CUSTOMER_ID
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
name: prd-duc
key: jwt_token
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
@@ -0,0 +1,32 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
pathType: Prefix
+36
View File
@@ -0,0 +1,36 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
+40
View File
@@ -0,0 +1,40 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-{{ .Values.app_name }}
namespace: applications
labels:
app: {{ .Values.app_name }}
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-{{ .Values.app_name }}
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
remoteRef:
key: prd/microservices/aws_credentials/maestro
version: "AWSCURRENT"
property: AWS_ACCESS_KEY_ID
- secretKey: AWS_SECRET_ACCESS_KEY
remoteRef:
key: prd/microservices/aws_credentials/maestro
version: "AWSCURRENT"
property: AWS_SECRET_ACCESS_KEY
- secretKey: AWS_DEFAULT_REGION
remoteRef:
key: prd/microservices/aws_credentials/maestro
version: "AWSCURRENT"
property: AWS_DEFAULT_REGION
- secretKey: jwt_token
remoteRef:
key: {{ .Values.maestro.jwt_token_secret_id }}
version: "AWSCURRENT"
property: token
+18
View File
@@ -0,0 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.app_name }}
namespace: applications
labels:
app: {{ .Values.app_name }}
spec:
type: ClusterIP
ports:
- name: {{ .Values.app_name }}
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: {{ .Values.app_name }}
+23
View File
@@ -0,0 +1,23 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- general
+67
View File
@@ -0,0 +1,67 @@
# Default values for metabase.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 3
hostname: maestro.dadosfera.ai
image:
repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
type: ClusterIP
port: 3333
targetPort: 3333
ingress:
enabled: false
port: 3333
resources:
requests:
cpu: 100m
memory: 1500Mi
limits:
cpu: 2000m
memory: 2Gi
maestro:
aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
duc_url: duc.dadosfera.ai
in_factory_url: in-factory.dadosfera.ai
bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153"
env: prd
apm_service: maestro
jwt_token_secret_id: prd/root/jwt_token
logger_gelf_port: "1026"
npm_token: npm_Xp17h3daMkORcZ55NY95Ez4gRfdzsQ3UvINP
pb_token_path: prd/root/productboard_token
pi_factory_url: pi-factory.dadosfera.ai
sm_oauth_path: prd/root/oauth_applications
tr_factory_url: in-factory.dadosfera.ai
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "product-redis-prd.z4xvqj.0001.use1.cache.amazonaws.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: applications
operator: In
values:
- general
+53
View File
@@ -0,0 +1,53 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
- name: maestro.restricted_ip
value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30"
+43
View File
@@ -0,0 +1,43 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
# - name: private-maestro
# chart: ../helm-chart
# values:
# - ../helm-chart/values.yaml
# set:
# - name: app_name
# value: maestro-private
# - name: maestro.env
# value: stg
# - name: maestro.duc_url
# value: duc.stg.dadosfera.ai
# - name: hostname
# value: private-maestro.stg.dadosfera.ai
# - name: maestro.pi_factory_url
# value: pi-factory.dadosfera.ai
# - name: maestro.in_factory_url
# value: in-factory.stg.dadosfera.ai
# - name: maestro.tr_factory_url
# value: in-factory.dadosfera.ai
# - name: maestro.open_customer_id
# value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
# - name: maestro.open_group_id
# value: e3f98a2f-7748-4981-8505-7695c8ca8218
# # Customer id
# - name: maestro.dedicated_proxy
# value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
# - name: replicaCount
# value: 1
# - name: affinity
# value: null
# - name: resources
# value: null
# - name: maestro.restricted_ip
# value: "57.151.113.140/30"
+28
View File
@@ -0,0 +1,28 @@
services:
maestro:
image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
container_name: maestro
restart: always
ports:
- 3333:3333
env_file:
- .env
node_exporter:
image: ${NODE_EXPORTER_URL}
container_name: node_exporter
restart: always
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
- /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
- '--collector.systemd'
- '--collector.processes'
network_mode: host
pid: host
-1
View File
@@ -1,4 +1,3 @@
version: "3.8"
services:
maestro:
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+517 -14
View File
@@ -133,7 +133,32 @@
},
"get": {
"operationId": "ConnectionController_getAllConnections",
"parameters": [],
"parameters": [
{
"name": "search",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
@@ -241,7 +266,16 @@
},
"get": {
"operationId": "ConnectionController_getConnectionDetails",
"parameters": [],
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
@@ -449,7 +483,16 @@
"/inputs/{id}": {
"get": {
"operationId": "InputsController_findOne",
"parameters": [],
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
@@ -472,6 +515,45 @@
]
}
},
"/pipelinesV2/monitoring-dashboard": {
"get": {
"operationId": "PipelinesController_getMonitoringDashboard",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2": {
"post": {
"operationId": "PipelinesController_create",
@@ -646,6 +728,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -678,6 +768,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -710,6 +808,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -749,6 +855,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -786,6 +900,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -823,6 +945,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -985,6 +1115,14 @@
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "shared",
"required": true,
@@ -1025,6 +1163,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1125,6 +1271,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -1167,6 +1321,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -1209,6 +1371,14 @@
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -1402,6 +1572,108 @@
]
}
},
"/catalog/dataset-catalog-task": {
"post": {
"operationId": "CatalogController_triggerCatalog",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TriggerCatalogReq"
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TriggerCatalogRes"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/dataset-catalog-task/{session}": {
"get": {
"operationId": "CatalogController_getCatalogTask",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "session",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GetDatasetCatalogTaskRes"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/oauth/hubspot": {
"get": {
"operationId": "OauthController_oauthHubspot",
@@ -1528,6 +1800,118 @@
]
}
},
"/customers/{id}/links": {
"get": {
"operationId": "CustomersController_getCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinksResponse"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
}
]
},
"put": {
"operationId": "CustomersController_setCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinkRequest"
}
}
}
},
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/customers/token": {
"get": {
"operationId": "CustomersController_getCustomerToken",
"parameters": [
{
"name": "exp",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/health": {
"get": {
"operationId": "HealthController_check",
@@ -1544,24 +1928,16 @@
}
},
"info": {
"title": "Maestro - fix/upload-files",
"title": "Maestro - feat/new-customer-monitoring",
"description": "This is the Maestro API",
"version": "1.0.0",
"contact": {}
},
"tags": [],
"servers": [
{
"url": "https://maestro.stg.dadosfera.ai",
"description": "Ambiente STG"
},
{
"url": "https://maestro-2.stg.dadosfera.ai",
"description": "Ambiente STG2"
},
{
"url": "https://maestro.dadosfera.ai",
"description": "Ambiente PRD"
"description": "Dadosfera API"
}
],
"components": {
@@ -1612,6 +1988,12 @@
},
"scheduleLimit": {
"type": "string"
},
"links": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
@@ -1619,7 +2001,8 @@
"id",
"name",
"tier",
"scheduleLimit"
"scheduleLimit",
"links"
]
},
"AuthUser": {
@@ -1826,6 +2209,9 @@
"ConnectionToCatalogDto": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
@@ -2850,6 +3236,123 @@
"required": [
"comment"
]
},
"TriggerCatalogReq": {
"type": "object",
"properties": {
"table_name": {
"type": "string",
"description": "Name of the table on Snowflake",
"example": "TB__4DXSD4_TEST"
},
"table_schema": {
"type": "string",
"description": "Schema where the asset is located. If not set defaults to \"PUBLIC\""
},
"pipeline_id": {
"type": "string",
"description": "Id of the pipeline that generated the asset."
}
},
"required": [
"table_name"
]
},
"TriggerCatalogRes": {
"type": "object",
"properties": {
"session": {
"type": "string"
}
},
"required": [
"session"
]
},
"GetDatasetCatalogTaskRes": {
"type": "object",
"properties": {
"created_at": {
"type": "string"
},
"customer_name": {
"type": "string"
},
"session_id": {
"type": "string"
},
"status": {
"type": "string"
},
"table_name": {
"type": "string"
},
"updated_at": {
"type": "string"
},
"updated_by": {
"type": "string"
}
},
"required": [
"created_at",
"customer_name",
"session_id",
"status",
"table_name",
"updated_at",
"updated_by"
]
},
"CustomerLink": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
}
}
}
+2480 -189
View File
File diff suppressed because it is too large Load Diff
+4
View File
@@ -12,6 +12,10 @@ declare global {
INTERNAL_SWAGGER: 'true' | 'false';
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
}
}
}
+1
View File
@@ -4,6 +4,7 @@
"compilerOptions": {
"assets": [
"**/*.proto",
"assets/**/*",
{
"include": "i18n/**/*",
"watchAssets": true
+6201 -11005
View File
File diff suppressed because it is too large Load Diff
+54 -43
View File
@@ -6,10 +6,10 @@
"private": true,
"license": "UNLICENSED",
"engines": {
"node": "18.3.0"
"node": "18.17"
},
"scripts": {
"preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
"prebuild": "rimraf dist",
"build": "nest build",
@@ -20,80 +20,91 @@
"start:prod": "node dist/main",
"docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"test": "jest --detectOpenHandles --forceExit",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.1",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
"@nestjs/config": "^1.2.1",
"@nestjs/core": "^8.4.7",
"@nestjs/mapped-types": "*",
"@nestjs/microservices": "^8.4.7",
"@nestjs/passport": "^8.2.2",
"@nestjs/platform-express": "^8.4.7",
"@nestjs/schedule": "^1.1.0",
"@nestjs/swagger": "^5.2.1",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
"@nestjs/common": "^9.4.3",
"@nestjs/config": "^2.3.4",
"@nestjs/core": "^9.4.3",
"@nestjs/mapped-types": "^1.2.2",
"@nestjs/microservices": "^9.4.3",
"@nestjs/passport": "^9.0.3",
"@nestjs/platform-express": "^9.4.3",
"@nestjs/schematics": "^9.2.0",
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.13.2",
"cron-parser": "^4.4.0",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.36.0",
"helmet": "^5.1.0",
"jsonwebtoken": "^9.0.0",
"elastic-apm-node": "^3.50.0",
"handlebars": "^4.7.8",
"helmet": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"protospack": "2.5.2",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.4.0"
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@nestjs/cli": "^8.2.8",
"@nestjs/schematics": "^8.0.11",
"@nestjs/testing": "^8.4.7",
"@types/express": "^4.17.13",
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.8",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
"@types/multer": "^1.4.7",
"@types/node": "^16.11.41",
"@types/multer": "^1.4.12",
"@types/node": "^16.18.52",
"@types/passport-facebook": "^2.1.11",
"@types/passport-google-oauth20": "^2.0.11",
"@types/passport-oauth2": "^1.4.11",
"@types/passport-oauth2": "^1.4.12",
"@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^5.29.0",
"@typescript-eslint/parser": "^5.29.0",
"eslint": "^8.18.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.0.0",
"@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.62.0",
"eslint": "^8.49.0",
"eslint-config-prettier": "^8.10.0",
"eslint-plugin-prettier": "^4.2.1",
"jest": "^27.5.1",
"nock": "^13.2.7",
"prettier": "^2.7.1",
"nock": "^13.3.3",
"prettier": "^2.8.8",
"source-map-support": "^0.5.20",
"supertest": "^6.2.3",
"supertest": "^6.3.3",
"ts-jest": "^27.1.5",
"ts-loader": "^9.3.1",
"ts-node": "^10.8.1",
"tsconfig-paths": "^3.14.1",
"typescript": "^4.6.3"
"ts-loader": "^9.4.4",
"ts-node": "^10.9.1",
"tsconfig-paths": "^3.14.2",
"typescript": "^4.9.5"
}
}
+18 -1
View File
@@ -25,9 +25,16 @@ import { ConfigModule } from '@nestjs/config';
import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
import { ProductboardModule } from './modules/productboard/productboard.module';
import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
controllers: [],
providers: [
DadosferaLogger,
{
@@ -56,6 +63,16 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
OauthModule,
ProductboardModule,
MixpanelModule,
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
],
+130
View File
@@ -0,0 +1,130 @@
<!DOCTYPE html>
<html lang="pt-br">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Dadosfera Relatório de PII</title>
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
<style>
@page {
size: A4 landscape; /* Alterado para paisagem (landscape) */
margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
}
body {
font-family: 'Quicksand', sans-serif;
color: #5c5c5c;
margin: 0;
padding: 10px;
font-size: 12px; /* Reduzindo o tamanho da fonte */
}
.container {
margin: 0;
width: 100%;
}
.header {
display: flex;
align-items: center;
margin-bottom: 20px;
}
.logo {
max-width: 150px; /* Reduzida para economizar espaço */
height: auto;
}
h1 {
color: #0d003b;
font-weight: 700;
margin-left: 20px;
font-size: 24px; /* Tamanho ajustado */
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 15px;
table-layout: fixed; /* Importante: define larguras fixas */
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border: 1px solid #d0d0d0;
}
th {
background-color: #1700a2;
color: white;
font-weight: bold;
text-align: left;
padding: 8px 10px;
border: 1px solid #3a26b8;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
td {
padding: 6px 10px;
border: 1px solid #d0d0d0;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
/* Definindo larguras específicas para cada coluna */
th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
tr:nth-child(even) {
background-color: #f9f9f9;
}
tr:nth-child(odd) {
background-color: white;
}
.info-section {
margin-top: 20px;
color: #5c5c5c;
}
.timestamp {
font-style: italic;
text-align: right;
margin-top: 15px;
font-size: 0.9em;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
<h1>Relatório de PII</h1>
</div>
<div class="info-section">
<p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
</div>
<table>
<thead>
<tr>
<th>Database</th>
<th>Schema</th>
<th>Tabela</th>
<th>Coluna</th>
<th>Tipo de Dado</th>
<th>Regras PII</th>
</tr>
</thead>
<tbody>
{{#each dados}}
<tr>
<td>{{database_name}}</td>
<td>{{table_schema}}</td>
<td>{{table_name}}</td>
<td>{{column_name}}</td>
<td>{{data_type}}</td>
<td>{{pii_rules}}</td>
</tr>
{{/each}}
</tbody>
</table>
<p class="timestamp">Gerado em: {{dataGeracao}}</p>
</div>
</body>
</html>
+23 -15
View File
@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -44,7 +45,7 @@ class NoClassAuthController {
@Get('has-all-permissions')
@RequireAllPermissions(
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
)
async hasAllPermissions(@Body() body) {
return { body };
@@ -53,7 +54,7 @@ class NoClassAuthController {
@Get('has-some-permission')
@RequireSomePermission(
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
)
async hasSomePermission(@Body() body) {
return { body };
@@ -62,7 +63,7 @@ class NoClassAuthController {
@Controller('class-auth-condition')
@AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
@RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
@RequireSomePermission(PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE)
class ClassAuthConditionController {
@Get('body')
async getBody(@Body() body) {
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
+40 -4
View File
@@ -4,6 +4,7 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -17,6 +18,7 @@ import {
import { RequestUser } from '../decorators/user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
@Injectable()
export class AuthenticationGuard
@@ -31,6 +33,7 @@ export class AuthenticationGuard
dadosferaLogger: DadosferaLogger,
private reflector: Reflector,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService
) {
this.pems = new Map();
this.logger = dadosferaLogger.logger;
@@ -48,20 +51,40 @@ export class AuthenticationGuard
});
}
canActivate(ctx: ExecutionContext): boolean {
async canActivate(ctx: ExecutionContext): Promise<boolean> {
const authFunctions = this.reflector.getAllAndMerge<
AuthenticationFunction[]
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
const mustBeAuthenticated = authFunctions.length > 0;
const request = ctx.switchToHttp().getRequest();
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!mustBeAuthenticated) {
// no need to be authenticated
return true;
}
const request = ctx.switchToHttp().getRequest();
const apiKey = request.get('X-api-key');
if (apiKey) {
const {
api_key
} = await this.apiKeyService.get(apiKey);
request.user = {
user_id: api_key.user_id,
username: api_key.username,
permissions: api_key.permissions,
customer_id: api_key.customer_id,
customer_name: api_key.customer_name,
customer_tier: api_key.customer_tier,
customer_modules: api_key.customer_modules,
access_token: apiKey,
};
return true;
}
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!accessToken) {
// couldn't load valid token
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
@@ -113,6 +136,18 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
@@ -121,6 +156,7 @@ export class AuthenticationGuard
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
// TODO: for backwards compatibility. remove in the future
+72 -15
View File
@@ -55,9 +55,18 @@ export const PERMISSIONS_GROUPS = {
'es-es': '',
},
},
GENERATE_TOKEN: {
seqid: 46,
claim: 'customer:generate-token',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Gerar Token de Acesso',
'en-us': 'Generate Acess Token',
'es-es': 'Gerar Token de Acceso',
},
},
},
},
PIPELINE: {
title: {
'pt-br': 'Coletar | Pipelines',
@@ -107,7 +116,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
CONNECTION: {
title: {
'pt-br': 'Coletar | Fontes de dados',
@@ -147,7 +155,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
NETWORK_CONFIG: {
title: {
'pt-br': 'Coletar | Redes',
@@ -177,7 +184,6 @@ export const PERMISSIONS_GROUPS = {
// },
},
},
OUTPUT: {
title: {
'pt-br': 'Output',
@@ -227,7 +233,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
TRANSFORMATIONS: {
title: {
'pt-br': 'Micro-transformações',
@@ -277,7 +282,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
CATALOG: {
title: {
'pt-br': 'Explorar | Catálogo',
@@ -336,19 +340,37 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Executar a catalogação de um ativo',
'en-us': 'Trigger an asset cataloging',
'es-es': 'Realizar el listado de un activo',
},
},
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -398,7 +420,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
SNOWFLAKE: {
title: {
'pt-br': 'Explorar | Consolidar',
@@ -418,7 +439,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
ZENDESK: {
title: {
'pt-br': 'Zendesk',
@@ -438,8 +458,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
ANALYZE: {
DATAVIZ: {
title: {
'pt-br': 'Analisar | Visualização',
'en-us': 'Analyze | Visualization',
@@ -456,6 +475,15 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Acceso Metabase',
},
},
},
},
INTELLIGENCE: {
title: {
'pt-br': 'Analisar | Inteligência',
'en-us': 'Analyze | Intelligence',
'es-es': 'Analizar | Inteligencia',
},
permissions: {
INTELLIGENCE: {
seqid: 31,
claim: 'intelligence:open',
@@ -468,7 +496,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
MODULES: {
title: {
'pt-br': 'Módulos da Dadosfera',
@@ -565,6 +592,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
CUSTOMER: {
title: {
'pt-br': 'Organização',
'en-us': 'Organization',
'es-es': 'Organización',
},
permissions: {
MONITORING_DASHBOARD: {
seqid: 47,
claim: 'customer:monitoring-dashboard',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Ver o dashboard de monitoramento',
'en-us': 'View the monitoring dashboard',
'es-es': 'Ver el dashboard de monitoreo',
},
},
},
},
};
export interface DadosferaModule {
name: string;
@@ -572,12 +618,23 @@ export interface DadosferaModule {
key: string;
permissionSeqId: number;
}
export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
{
name: 'Intelligence Module',
description: 'Orchest Module',
key: 'intelligence',
permissionSeqId: PERMISSIONS_GROUPS.ANALYZE.permissions.INTELLIGENCE.seqid,
permissionSeqId:
PERMISSIONS_GROUPS.INTELLIGENCE.permissions.INTELLIGENCE.seqid,
},
{
name: 'Proccessing Module',
@@ -25,6 +25,14 @@ export function RequireSomePermission(
);
}
export function RequireModule(
key: string
) {
return createAuthenticatedDecorator((_, user: RequestUser) =>
user.customer_modules.some(module => module === key),
);
}
export function AuthenticateCondition(func: AuthenticationFunction) {
return createAuthenticatedDecorator(func);
}
+10 -2
View File
@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -46,12 +47,13 @@ describe('user.decorator', () => {
const fakeUserPayload = {
user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
username: 'super.admin',
permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
permissions: [PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE].map(
({ seqid }) => seqid,
),
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
UserTest(token);
// UserTest(token);
});
+1
View File
@@ -11,6 +11,7 @@ export interface RequestUser {
customer_name: string;
customer_tier: string;
access_token: string;
customer_modules: string[];
}
export const User: (options?: { required?: boolean }) => ParameterDecorator =
+17 -6
View File
@@ -3,11 +3,14 @@ import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import helmet from 'helmet';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { json, urlencoded } from 'express';
import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -22,9 +25,16 @@ async function bootstrap() {
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
}
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
@@ -42,7 +52,7 @@ function configureSwagger(app: INestApplication) {
}
const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
const config = new DocumentBuilder()
const swaggerConfig = new DocumentBuilder()
.setTitle(swaggerTitle)
.setDescription('Documentation for Maestro gateway')
.addSecurity('access-token', {
@@ -50,11 +60,12 @@ function configureSwagger(app: INestApplication) {
name: 'Authorization',
in: 'header',
})
.setDescription('This is the Maestro API')
.addServer('https://maestro.stg.dadosfera.ai', 'Ambiente STG')
.addServer('https://maestro-2.stg.dadosfera.ai', 'Ambiente STG2')
.addServer('https://maestro.dadosfera.ai', 'Ambiente PRD')
.build();
.setDescription('This is the Maestro API');
if (process.env.INTERNAL_SWAGGER !== 'true')
swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
const config = swaggerConfig.build();
const document = SwaggerModule.createDocument(app, config);
+68
View File
@@ -0,0 +1,68 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@Controller('api-key')
@Authenticated()
@ApiTags('ApiKey')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
export class ApiKeyController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
method: 'create'
});
const result = await this.apiKeyService.create(createApiKeyDto, user);
this.logger.info('POST /api-key success', {
id: result.id,
method: 'create'
});
return result;
}
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
});
const result = await this.apiKeyService.findAll(user);
this.logger.info('GET /api-key success', {
count: result.length,
method: 'findAll'
});
return result;
}
@Delete(':id')
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
method: 'remove'
});
await this.apiKeyService.remove(id, user);
this.logger.info('DELETE /api-key/:id success', {
id,
method: 'remove'
});
}
}
+18
View File
@@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { ApiKeyController } from './api-key.controller';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([ducClient.providerOptions])
],
controllers: [ApiKeyController],
providers: [ApiKeyService, DadosferaLogger],
exports: [ApiKeyService]
})
export class ApiKeyModule {}
+50
View File
@@ -0,0 +1,50 @@
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc,
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
}
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
return lastValueFrom(this.apiKeyService.CreateApiKey({
permissions: createApiKeyDto.permissions
}, metadata));
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata)
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
return data.api_keys;
}
async remove(id: string, user: RequestUser) {
const metadata = PackTheMetadata(user);
await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
}
async get(key: string) {
const metadata = PackTheMetadata({});
return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
}
}
+39
View File
@@ -0,0 +1,39 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsArray, IsNumber } from 'class-validator';
export class PermissionDto {
@ApiProperty({ type: Number })
id: number;
@ApiProperty({ type: String })
name: string;
}
export class ApiKeyBaseResponseDto {
@ApiProperty({ type: String, format: 'uuid' })
id: string;
@ApiProperty({ type: String })
key_mask: string;
@ApiProperty({ type: [PermissionDto] })
permissions: PermissionDto[];
@ApiProperty({ type: String, format: 'date-time' })
created_at: string;
@ApiProperty({ type: String })
created_by: string;
}
export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
@ApiProperty({ type: String })
key: string;
}
export class CreateApiKeyDto {
@ApiProperty({ type: [Number], description: 'Array of permission IDs' })
@IsArray()
@IsNumber({}, { each: true })
permissions: number[];
}
+33
View File
@@ -0,0 +1,33 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
}
}
+15
View File
@@ -0,0 +1,15 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
+38
View File
@@ -0,0 +1,38 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -0,0 +1,3 @@
export class CreateAssignDto {
publicKey: string;
}
+276 -18
View File
@@ -12,6 +12,7 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -26,7 +27,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthVerifyTotpMfaRequest
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -43,15 +44,23 @@ import {
AuthRefreshAccessTokenRes,
AuthSignInReq,
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request } from 'express';
import { Request, Response } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -85,10 +94,65 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
return this.authClient.signIn({ username, password, totp }, metadata);
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('refresh-access-token')
@@ -97,16 +161,26 @@ export class AuthController {
async refreshAccessToken(
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
customer_name,
language,
custom_host: frontHost,
});
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
}
@ApiInternalOnlyEndpoint()
@@ -177,16 +251,23 @@ export class AuthController {
@ApiInternalOnlyEndpoint()
@Post('confirm-reset-password')
@HttpCode(HttpStatus.OK)
async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
async confirmResetPassword(
@Body() body: AuthConfirmResetPasswordRequest,
@Headers('origin') origin: string,
) {
this.logger.info('/auth - confirm-reset-password');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ custom_host: frontHost });
const { username, code, newPassword } = body;
return this.authClient.confirmResetPassword({
username,
code,
newPassword,
});
return this.authClient.confirmResetPassword(
{
username,
code,
newPassword,
},
metadata,
);
}
@ApiInternalOnlyEndpoint()
@@ -344,4 +425,181 @@ export class AuthController {
return { token, email, url, language };
}
@ApiInternalOnlyEndpoint()
@Post('users/block')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async blockUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest,
) {
this.logger.info('blockUsers - Starting request');
try {
const metadata = PackTheMetadata(user);
this.logger.debug('Calling blockUsers service', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language'),
},
});
const result = await this.authClient.blockUsers(body.users, metadata);
this.logger.info('blockUsers - Success', { result });
return result;
} catch (error) {
this.logger.error('blockUsers - Error', {
error: error.message,
stack: error.stack,
});
throw error;
}
}
@ApiInternalOnlyEndpoint()
@Post('users/unblock')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async unblockUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest,
) {
this.logger.info('unblockUsers');
const metadata = PackTheMetadata(user);
return this.authClient.unblockUsers(body.users, metadata);
}
@ApiInternalOnlyEndpoint()
@Post('users/reset')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async resetUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest,
) {
this.logger.info('resetUsers');
const metadata = PackTheMetadata(user);
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
}
+135 -18
View File
@@ -1,10 +1,10 @@
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -17,15 +17,22 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -38,6 +45,10 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async getPublicKeys() {
@@ -52,25 +63,59 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
}
async refreshAccessToken(
{ refreshToken }: AuthRefreshAccessTokenRequest,
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
);
}
@@ -112,19 +157,21 @@ export class AuthClientService implements OnModuleInit {
);
}
async confirmResetPassword({
username,
code,
newPassword,
}: AuthConfirmResetPasswordRequest) {
async confirmResetPassword(
{ username, code, newPassword }: AuthConfirmResetPasswordRequest,
metadata: Metadata,
) {
this.logger.info('confirmResetPassword');
return lastValueFrom(
this.authService.AuthConfirmResetPassword({
username,
code,
newPassword,
}),
this.authService.AuthConfirmResetPassword(
{
username,
code,
newPassword,
},
metadata,
),
);
}
@@ -164,6 +211,76 @@ export class AuthClientService implements OnModuleInit {
async oauthSignIn(data: { username: string; token: string }) {
const { username, token } = data;
return lastValueFrom(this.authService.AuthOauthSignIn({ token, username }));
return lastValueFrom(
this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
);
}
async blockUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('blockUsers - Service starting');
try {
this.logger.debug('Calling BlockUser gRPC method', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language'),
},
});
const response = await lastValueFrom<BulkEditResponse>(
this.authService.BlockUser({ users }, metadata),
);
this.logger.info('blockUsers - Service success', { response });
return response;
} catch (error) {
this.logger.error('blockUsers - Service error', {
error: error.message,
stack: error.stack,
});
throw error;
}
}
async unblockUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('unblockUsers');
return await lastValueFrom(
this.authService.UnblockUser({ users }, metadata),
);
}
async resetUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('resetUsers');
try {
this.logger.debug('Calling ResetUser gRPC method', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language'),
},
});
const response = await lastValueFrom<BulkEditResponse>(
this.authService.ResetUser({ users }, metadata),
);
this.logger.info('resetUsers - Success', { response });
return response;
} catch (error) {
this.logger.error('resetUsers - Error', {
error: error.message,
stack: error.stack,
});
throw error;
}
}
}
+26 -1
View File
@@ -1,3 +1,4 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import {
AuthSignInRequest,
AuthSignInResponse,
@@ -66,14 +67,28 @@ export class AuthUser {
export class AuthCustomer {
@ApiProperty()
modules: string[];
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
displayName: string;
@ApiProperty()
tier: string;
@ApiProperty()
scheduleLimit: string;
@ApiProperty()
links: Link[];
@ApiProperty()
themeEnabled: boolean;
@ApiProperty()
enforceMfa: boolean;
}
export class AuthSignInReq implements AuthSignInRequest {
@@ -107,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
customerName: string;
userId: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
@@ -115,3 +130,13 @@ export class AuthRefreshAccessTokenRes {
@ApiProperty()
accessToken: string;
}
export interface BulkEditRequest {
users: string[];
}
export interface BulkEditResponse {
message: string;
successfulUsers: string[];
failedUsers: string[];
}
@@ -26,6 +26,7 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
+6 -1
View File
@@ -5,8 +5,11 @@ import {
} from '@nestjs/microservices';
import { credentials } from '@grpc/grpc-js';
import { Catalog } from '@dadosfera/protospack-v2';
import { PlatformInterfaces } from '@dadosfera/protospack-v2';
const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
export class CatalogClientConfiguration {
public name = 'CatalogClientConfiguration';
@@ -17,11 +20,13 @@ export class CatalogClientConfiguration {
package: [
Catalog.ProtoPackages.ReadPackage,
Catalog.ProtoPackages.WritePackage,
PlatformInterfaces.ProtoPackages.WritePackage
],
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: [
Catalog.ProtoPaths.ReadFilePath,
Catalog.ProtoPaths.WriteFilePath,
PlatformInterfaces.ProtoPaths.WriteFilePath
],
loader: {
keepCase: true,
+364 -12
View File
@@ -13,18 +13,30 @@ import {
Put,
Query,
UseFilters,
HttpException,
HttpStatus,
Res,
} from '@nestjs/common';
import { ApiHeaders, ApiTags } from '@nestjs/swagger';
import {
ApiCreatedResponse,
ApiHeaders,
ApiOkResponse,
ApiTags,
} from '@nestjs/swagger';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from '../../decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
GetDatasetCatalogTaskRes,
ICatalogAllRequest,
ICatalogAllResponse,
IColumnsMetadataResponse,
@@ -35,11 +47,21 @@ import {
IOneDataAsset,
IPreviewResponse,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
import { TypeParser } from 'src/utils/FileParser/parser-types';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -95,6 +117,52 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog/download - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
@@ -180,7 +248,7 @@ export class CatalogController {
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
@Query('shared') shared?: 'true',
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -196,7 +264,7 @@ export class CatalogController {
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id: undefined,
user_id,
customer_id,
customer_name,
});
@@ -237,7 +305,7 @@ export class CatalogController {
async getDataAssetRls(
@User() user: RequestUser,
@Headers() headers,
@Param('id') id,
@Param('id') id: string,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -292,7 +360,7 @@ export class CatalogController {
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id,
@Param('id') id: string,
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -323,9 +391,9 @@ export class CatalogController {
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id,
@Param('id') id: string,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username } = user;
const { customer_name, customer_id, user_id, username, customer_modules } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -338,6 +406,7 @@ export class CatalogController {
user_id,
username,
language,
is_mask: customer_modules.some(mod => mod === 'pii')
});
const preview = await this.catalogService.getDatasetPreview(id, metadata);
@@ -353,7 +422,7 @@ export class CatalogController {
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id,
@Param('id') id: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -383,7 +452,7 @@ export class CatalogController {
async updateDataAsset(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') data_asset_id,
@Param('id') data_asset_id: string,
@Body() body: IUpdateDataRequest,
): Promise<IOneDataAsset> {
const { customer_id, customer_name, user_id, username } = user;
@@ -579,4 +648,287 @@ export class CatalogController {
return response;
}
}
@Post('dataset-catalog-task')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
)
@ApiCreatedResponse({ type: TriggerCatalogRes })
async triggerCatalog(
@User() user: RequestUser,
@Body() body: TriggerCatalogReq,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const session = await this.catalogService.triggerCatalog(body, metadata);
return { session };
}
@Get('dataset-catalog-task/:session')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
)
@ApiOkResponse({ type: GetDatasetCatalogTaskRes })
async getCatalogTask(
@User() user: RequestUser,
@Param('session') session: string,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getDatasetCatalogTask(
session,
metadata,
);
return response;
}
@Post('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.addRlsRule(body, metadata);
return response;
}
@Get('rls-rule/:id')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getOneRlsRule(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const idInt = parseInt(id);
const response = await this.catalogService.getOneRlsRule(idInt, metadata);
return response;
}
@Get('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getRlsRules(
@User() user: RequestUser,
@Query() query: GetRlsRulesRequest,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getRlsRules(query, metadata);
return response;
}
@Delete('rls-rule/:id')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async removeRlsRule(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const idInt = parseInt(id);
const response = await this.catalogService.removeRlsRule(idInt, metadata);
return response;
}
@Delete('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async batchRemoveRlsRules(
@Query() query: BatchRemoveRlsRulesRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
await this.catalogService.batchRemoveRlsRule(query, metadata);
return { message: 'OK' };
}
@Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getNimbusDashboards(
@User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const dashboards = await this.catalogService.getNimbusDashboards(
body,
metadata,
);
return JSON.parse(dashboards);
}
@Post('register-dataset')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async registerDatasetWithMetadataRequest(
@Body() body: RegisterDatasetWithMetatadaRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
try {
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.log(
`Request from user ${user_id} for customer ${customer_name}`,
logMetadata,
);
// Create table metadata
const tableMetadataBody = {
table_metadata: body.table_metadata,
info: {
customer: customer_name,
},
logMetadata: logMetadata,
};
const table_metadata_id = await this.catalogService.createTableMetadata(
tableMetadataBody,
);
this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
// Create column metadata
const columnMetadataBody = {
column_metadata: body.column_metadata,
info: {
customer: customer_name,
},
};
this.logger.info(
`Creating column metadata for table ${table_metadata_id}`,
logMetadata,
);
const column_metadata_ids =
await this.catalogService.createColumnMetadata(columnMetadataBody);
// Create data preview
const dataPreviewBody = {
data_preview: body.data_preview,
info: {
customer: customer_name,
},
};
this.logger.debug(
`Creating data preview for table ${table_metadata_id}`,
logMetadata,
);
const data_preview_id = await this.catalogService.createDataPreview(
dataPreviewBody,
);
// Catalog dataset item
this.logger.info(
`Cataloging dataset item for table ${table_metadata_id}`,
logMetadata,
);
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
this.logger.info(
`Dataset registration completed successfully for table ${table_metadata_id}`,
logMetadata,
);
return {
message: 'Dataset registered successfully',
table_metadata_id: table_metadata_id,
column_metadata_ids: column_metadata_ids,
data_preview_id: data_preview_id,
};
} catch (error) {
this.logger.error(
`Failed to register dataset. The following error occurred: ${error.response.data}`,
logMetadata,
);
throw new HttpException(
{
message: 'Ocorreu um erro ao registrar o dataset',
error: error.message,
code: 'REGISTRATION_FAILED',
details: error.message,
},
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
@Get('pii-reporter')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
// use res.end to send buffer
return res.end(file);
} catch (error) {
console.error(error)
this.logger.error(error.message);
}
}
}
+6 -1
View File
@@ -3,12 +3,15 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -18,6 +21,8 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+275 -3
View File
@@ -6,6 +6,12 @@ import {
Messages,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
Messages as PlatformInterfaceMessages,
WriteService as PlatformInterfaceWriteService,
ProtoServices as PlatformInterfacesProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
import {
BadRequestException,
HttpException,
HttpStatus,
Inject,
@@ -18,11 +24,25 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { IUpdateDataRequest } from './dtos';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
catalogWriteService: WriteService.CatalogWriteServices;
platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
logger: any;
constructor(
@Inject(DadosferaLogger)
@@ -44,9 +64,14 @@ class CatalogService implements OnModuleInit {
this.grpcClient.getService<WriteService.CatalogWriteServices>(
ProtoServices.CatalogWriteServices,
);
this.platformWriteService =
this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
);
}
_getNimbusUrl(body) {
this.logger.debug(`Body: ${JSON.stringify(body)}`);
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
@@ -59,6 +84,42 @@ class CatalogService implements OnModuleInit {
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
try {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
@@ -174,6 +235,34 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -250,7 +339,7 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
@@ -333,6 +422,189 @@ class CatalogService implements OnModuleInit {
} as typeof data_asset;
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
);
return session;
}
async getDatasetCatalogTask(session: string, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
);
return res;
}
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.AddRlsRule(data, metadata),
);
return res;
}
async removeRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
);
return res;
}
async batchRemoveRlsRule(
query: BatchRemoveRlsRulesRequest,
metadata: Metadata,
) {
const { id_rls, nimbus_dashboard_id } = query;
if (id_rls && nimbus_dashboard_id) {
throw new BadRequestException(
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
);
}
if (id_rls) {
await lastValueFrom(
this.catalogWriteService.RemoveRlsRulesByRlsId({ id_rls }, metadata),
);
} else if (nimbus_dashboard_id) {
await lastValueFrom(
this.catalogWriteService.RemoveRlsRulesByDashboardId(
{ nimbus_dashboard_id: parseInt(nimbus_dashboard_id) },
metadata,
),
);
}
return 'OK';
}
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetRlsRules(data, metadata),
);
return res.rls_rules;
}
async getOneRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetOneRlsRule({ id }, metadata),
);
return res.rls_rule;
}
async getNimbusDashboards(
data: GetNimbusDashboardsRequest,
metadata: Metadata,
) {
const res = await lastValueFrom(
this.catalogReadService.GetNimbusDashboards(data, metadata),
);
return res.dashboards;
}
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
async createColumnMetadata(body: any): Promise<number[]> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{...body.logMetadata},
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{...body.logMetadata},
);
throw new Error(error.response?.data?.message || error.message);
}
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
const customer_name = customer_name_raw?.[0]?.toString();
if (!customer_name) {
throw new BadRequestException('Customer name not found in metadata');
}
const res = await lastValueFrom(
this.platformWriteService.CatalogDataAssets(
{
data_assets: [
{
data_asset_id: table_metadata_id.toString(),
customer_name: customer_name,
data_asset_type: 'dataset',
},
],
},
metadata,
),
);
return res;
}
}
export { CatalogService };
export { CatalogService };
+57
View File
@@ -271,3 +271,60 @@ export class IColumnsMetadataResponse {
@ApiProperty({ type: [IColumnMetadata] })
columns_metadata: IColumnMetadata[];
}
export class TriggerCatalogReq {
@ApiProperty({
description: 'Name of the table on Snowflake',
example: 'TB__4DXSD4_TEST',
})
table_name: string;
@ApiPropertyOptional({
description:
'Schema where the asset is located. If not set defaults to "PUBLIC"',
})
table_schema?: string;
@ApiPropertyOptional({
description: 'Id of the pipeline that generated the asset.',
})
pipeline_id: string;
}
export class TriggerCatalogRes {
@ApiProperty()
session: string;
}
export class GetDatasetCatalogTaskRes {
@ApiProperty()
created_at: string;
@ApiProperty()
customer_name: string;
@ApiProperty()
session_id: string;
@ApiProperty()
status: string;
@ApiProperty()
table_name: string;
@ApiProperty()
updated_at: string;
@ApiProperty()
updated_by: string;
}
export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
nimbus_dashboard_id?: string;
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
+8
View File
@@ -0,0 +1,8 @@
export class PiiDto {
database_name: string;
table_schema: string;
table_name: string;
column_name: string;
data_type: string;
pii_rules: string;
}
@@ -0,0 +1,89 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
+30
View File
@@ -0,0 +1,30 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
+275
View File
@@ -0,0 +1,275 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -6,7 +6,9 @@ import {
} from '@nestjs/microservices';
import { ConnectionTest } from '@dadosfera/protospack-v2';
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
export class ConnectionTestClientConfiguration {
private config: GrpcOptions = {
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ConnectionTestService {
+3 -1
View File
@@ -6,7 +6,9 @@ import {
} from '@nestjs/microservices';
import { ConnectionManager } from '@dadosfera/protospack-v2';
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
export class ConnectionClientConfiguration {
public name = 'ConnectionClientConfiguration';
@@ -24,10 +24,11 @@ import {
ConnectionDetailsRes,
ConnectionRes,
ConnectionsRes,
GetAllConnectionsReq,
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -138,7 +139,7 @@ export class ConnectionController {
async getAllConnections(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Query() queries,
@Query() queries: GetAllConnectionsReq,
): Promise<ConnectionsRes> {
this.logger.info('/connections - Get All Connection');
@@ -164,7 +165,7 @@ export class ConnectionController {
async getConnectionDetails(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
@Query() queries,
): Promise<ConnectionDetailsRes> {
this.logger.info('/connections - Get Connection Details');
+14 -2
View File
@@ -9,6 +9,7 @@ import {
ConnectionToCatalog,
} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
import {
GetAllConnectionRequest,
GetAllConnectionResponse,
GetConnectionDetailsResponse,
GetConnectionResponse,
@@ -21,7 +22,7 @@ export type ConnectionCredentialsType =
| 'oauth'
| 'api_key'
| 'service_account'
| 'headers_authF';
| 'headers_auth';
export interface ConnectionApiConnection {
config_id: string;
plugin: string;
@@ -94,7 +95,6 @@ export class ConnectionDto implements Connection {
export class ConnectionToCatalogDto implements ConnectionToCatalog {
// ---Automatically generated information - will not be sent by the frontend--- //
id: string;
customer_id: string;
updated_at: string;
created_at: string;
@@ -104,6 +104,9 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
customer_name: string;
// ---Information sent by the frontend--- //
@ApiPropertyOptional()
id: string;
@ApiProperty()
name: string;
@@ -147,6 +150,15 @@ export class UpdateConnectionDto extends PickType(ConnectionDto, [
'properties',
]) {}
export class GetAllConnectionsReq implements GetAllConnectionRequest {
@ApiPropertyOptional()
search?: string;
@ApiPropertyOptional()
page?: string;
@ApiPropertyOptional()
size?: string;
}
export class ConnectionsRes implements GetAllConnectionResponse {
@ApiProperty({ type: [ConnectionToCatalogDto] })
connections: ConnectionToCatalogDto[];
+3 -1
View File
@@ -6,7 +6,9 @@ import {
} from '@nestjs/microservices';
import { ConnectorManager } from '@dadosfera/protospack-v2';
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
export class ConnectorClientConfiguration {
public name = 'ConnectorClientConfiguration';
@@ -273,16 +273,13 @@ export class ConnectorController {
@Param('plugin') plugin: string,
@Query('version') version: string,
) {
this.logger.info('/upload - Upload Connector Route');
this.logger.info('DELETE /:plugin - Delete Connector Route');
const response: any = await this.connectorClientService.deleteConnector(
const response = await this.connectorClientService.deleteConnector(
plugin,
version,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
return { message: response.message, connector: response.connector };
}
}
@@ -0,0 +1,139 @@
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiTags('Customers')
@Controller('customers')
@UseFilters(GrpcToHttpExceptionFilter)
export class CustomersController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private customersService: CustomersService,
) {
this.logger = dadosferaLogger.logger;
}
@Post(':id/mfa')
@Authenticated()
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@RequireModule(DADOSFERA_MODULES_KEYS.DANGER_ZONE)
async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) {
this.logger.info('enableMfaEnforce', { id });
return await this.customersService.enableEnforceMfa(id, data.enabled);
}
@Get(':id/links')
@Authenticated()
@ApiOkResponse({ type: CustomerLinksResponse })
async getCustomerLinks(@Param('id') id: string) {
this.logger.info('getCustomerLinks', { id });
const links = await this.customersService.getLinks(id);
return { links };
}
@Put(':id/links')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse()
@HttpCode(HttpStatus.OK)
async setCustomerLinks(
@Body() body: CustomerLinkRequest,
@Param('id') id: string,
) {
const { links } = body;
this.logger.info('setCustomerLinks', { id, links });
await this.customersService.setLinks(id, links);
}
@Get('token')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
@ApiProduces('text/plain')
async getCustomerToken(
@Query('exp') exp: StringValue,
@User() user: RequestUser,
): Promise<string> {
this.logger.info('getCustomerToken', { exp, user });
const data = {
customerId: user.customer_id,
userId: user.user_id,
customerName: user.customer_name,
};
return this.customersService.generateToken(exp, data);
}
@Get('monitoring-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
)
async getCustomerMonitoringDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getCustomerMonitoringDashboard');
const metadata = PackTheMetadata(user);
return this.customersService.getMonitoringDashboardUrl(metadata);
}
@Get('logs-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
)
@RequireModule(DADOSFERA_MODULES_KEYS.LOG_DASHBOARD)
async getCustomerMixPanelLogsDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getLogsDashboardUrl');
const metadata = PackTheMetadata(user);
const result = await this.customersService.getLogsDashboardUrl(metadata);
return result;
}
@Get('access-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
)
@RequireModule(DADOSFERA_MODULES_KEYS.ACCESS_DASHBOARD)
async getAccessDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getAccessDashboard');
const metadata = PackTheMetadata(user);
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
return result;
}
}
+23
View File
@@ -0,0 +1,23 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import { CustomersController } from './customers.controller';
import { CustomersService } from './customers.service';
import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
const ducClient = new DucClient();
const piFactoryClient = new PipelinesClientConfiguration();
@Module({
imports: [
ClientsModule.register([
ducClient.providerOptions,
piFactoryClient.providerOptions,
]),
],
controllers: [CustomersController],
providers: [CustomersService, DadosferaLogger],
exports: [CustomersService],
})
export class CustomersModule {}
+226
View File
@@ -0,0 +1,226 @@
import {
OnModuleInit,
Inject,
Injectable,
HttpException,
HttpStatus,
InternalServerErrorException,
ForbiddenException,
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import ErrorCodes from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import {
GetSecretValueCommand,
SecretsManagerClient,
} from '@aws-sdk/client-secrets-manager';
import getEnv from 'src/utils/getEnv';
import { logger } from 'elastic-apm-node';
import {
ReadService,
ProtoServices as PipelineProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { Metadata } from '@grpc/grpc-js';
import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
// This function will accept any string, which may result in a bug.
@Injectable()
export class CustomersService implements OnModuleInit {
private customerService: CustomersProtoService;
private logger: DadosferaLogger;
private pipelineReadService: ReadService.PipelineV2ReadService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
@Inject(PipelinesClientConfiguration.name)
private readonly pipelinesGrpcClient: ClientGrpc,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.customerService = this.grpcClient.getService<CustomersProtoService>(
ProtoServices.CustomersProtoService,
);
this.pipelineReadService =
this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
PipelineProtoServices.PipelineV2ReadService,
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId }),
);
return result.customer?.links || [];
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
throw err;
}
}
async setLinks(customerId: string, links: Link[]) {
if (!customerId || !links) {
throw new HttpException(null, HttpStatus.BAD_REQUEST);
}
try {
return await firstValueFrom(
this.customerService.CustomerUpdate({
id: customerId,
links,
} as CustomerUpdateRequest),
);
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
async generateToken(
expiresIn = '30m',
data: { customerId: string; userId: string; customerName: string },
) {
const { customerId, userId: generatedById, customerName } = data;
const secretsManagerClient = new SecretsManagerClient({});
const getSecretComand = new GetSecretValueCommand({
SecretId: `${getEnv()}/${customerName}/jwt-signing-key`,
});
let private_key_pem;
await secretsManagerClient
.send(getSecretComand)
.then((res) => {
const secret = JSON.parse(res.SecretString);
private_key_pem = secret.private_key_pem;
})
.catch((err) => {
logger.error(err.stack);
throw new InternalServerErrorException(
'Error finding keys for customer',
);
});
let jwt_token;
try {
jwt_token = jwt.sign(
{
customerId,
generatedById,
},
private_key_pem,
{
algorithm: 'RS256',
expiresIn,
issuer: 'maestro',
},
);
} catch (err) {
logger.error(err.stack);
throw new InternalServerErrorException('Error generating token');
}
// const verify = jwt.verify(jwt_token, public_key_pem, {
// algorithms: ['RS256'],
// issuer: 'maestro',
// });
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '98',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
logger.info('Done');
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '103',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
logger.info('Done');
return res;
}
async getAccessDashboardUrl(customerName: string, metadata: Metadata) {
/*
* TODO(Refactor): dar um jeito de exibir o dash da sbm diferente dos outros customer
* pois o signicado de department para sbm significa as instituições do usuários
*/
if (customerName !== 'sbmoffshorecom') {
throw new ForbiddenException();
}
logger.info('CustomersService - getAccessDashboardUrl');
const res = await this.getDashboardUrl('105', metadata);
logger.info('Done');
return res;
}
private async getDashboardUrl(dashboardId: string, metadata: Metadata) {
return await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: dashboardId,
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata
)
);
}
async enableEnforceMfa(id: string, enabled: boolean) {
return await lastValueFrom(
this.customerService.CustomerUpdateEnforceMfa({
customerId: id,
enforceMfa: enabled
})
)
}
}
+23
View File
@@ -0,0 +1,23 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CustomerLink implements Link {
@ApiProperty()
href: string;
@ApiProperty()
name: string;
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
export class CustomerLinksResponse {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
@@ -0,0 +1,6 @@
import { ApiProperty } from "@nestjs/swagger";
export class EnforceMfa {
@ApiProperty()
enabled: boolean
}
+5 -1
View File
@@ -9,7 +9,9 @@ import {
ProtoPaths,
} from '@dadosfera/protospack-v2/dist/lib/Duc';
const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.DUC_URL.startsWith('duc:') ||
process.env.DUC_URL.includes('0.0.0.0');
export class DucClient {
public name = 'DucClient';
@@ -27,6 +29,8 @@ export class DucClient {
objects: true,
arrays: true,
},
maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem
maxReceiveMessageLength: 15 * 1024 * 1024,
},
};
@@ -0,0 +1,47 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -0,0 +1,10 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -0,0 +1,246 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -0,0 +1,185 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
interface Values {
jdbc_user: string;
+3 -1
View File
@@ -6,7 +6,9 @@ import {
type GrpcOptions,
} from '@nestjs/microservices';
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
export class InputsGrpcClient {
public readonly name = 'InputsGrpcClient';
private config: GrpcOptions = {
+1 -2
View File
@@ -123,8 +123,7 @@ export class InputsController {
}
@Get('/:id')
async findOne(@Body() body, @Param() params) {
const { id } = params;
async findOne(@Body() body, @Param('id') id: string) {
this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
-2
View File
@@ -5,7 +5,6 @@ import {
Inject,
Injectable,
} from '@nestjs/common';
import { Timeout } from '@nestjs/schedule';
import CronParser, { CronExpression } from 'cron-parser';
import { ClientGrpc } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
@@ -222,7 +221,6 @@ export class InputsService {
return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
}
@Timeout(60000 * 10) // Timeout set for 10 minutes
async testConnection(data) {
try {
const testConnectionInputResponse =
+11 -75
View File
@@ -1,93 +1,29 @@
import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
private mixpanelService: MixpanelService
) {}
@Post(':id')
async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
delete body.info;
const mixpanel = init(this.mixpanelToken);
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
...body,
});
await this.mixpanelService.track(id, user, request, body)
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+6 -1
View File
@@ -1,6 +1,8 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`${process.env.ENV}/root/mixpanel_token`,
`prd/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
+118
View File
@@ -0,0 +1,118 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -6,7 +6,9 @@ import {
} from '@nestjs/microservices';
import { NetworkConfig } from '@dadosfera/protospack-v2';
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
export class NetworkConfigGrpcClient {
public name = 'NetworkConfigGrpcClient';
@@ -0,0 +1,6 @@
import { ApiProperty } from "@nestjs/swagger";
export class NetworkPoliciesDTO {
@ApiProperty()
policies: string []
}
@@ -0,0 +1,64 @@
import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Post } from '@nestjs/common';
import { ApiOkResponse } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/decorators/authentication.decorator';
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
constructor(private networkPolicyService: NetworkPolicyService) {}
@Get()
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async getNetworks(
@User() user: RequestUser,
) {
return await this.networkPolicyService.getByCustomer(
user.customer_id
);
}
@Post()
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse()
@HttpCode(HttpStatus.CREATED)
async applyNetworkPolicies(
@User() user: RequestUser,
@Body() data: NetworkPoliciesDTO,
) {
const metadata = PackTheMetadata(user);
return await this.networkPolicyService.apply(
data.policies,
user.customer_id,
metadata,
);
}
@Delete()
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse()
@HttpCode(HttpStatus.OK)
async removeNetworkPolicies(
@User() user: RequestUser,
@Body() data: NetworkPoliciesDTO,
) {
const metadata = PackTheMetadata(user);
return await this.networkPolicyService.delete(
data.policies,
user.customer_id,
metadata,
);
}
}
@@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { NetworkPolicyController } from './network-policy.controller';
import { NetworkPolicyService } from './network-policy.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([
ducClient.providerOptions
]),
],
controllers: [NetworkPolicyController],
providers: [NetworkPolicyService, DadosferaLogger]
})
export class NetworkPolicyModule {}
@@ -0,0 +1,74 @@
import { Inject, Injectable } from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
@Injectable()
export class NetworkPolicyService {
private customerService: CustomersProtoService;
private logger: DadosferaLogger;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.customerService = this.grpcClient.getService<CustomersProtoService>(
ProtoServices.CustomersProtoService,
);
}
async getByCustomer(id: string): Promise<NetworkPoliciesDTO> {
const {
customer
} = await lastValueFrom(this.customerService.CustomerFindOneById({
id
}));
return {
policies: customer.networkPolicies
}
}
async apply(
networkPolicies: string[],
customerId: string,
metadata: Metadata,
) {
return await lastValueFrom(
this.customerService.CustomerCreateNetworkPolicy(
{
customerId,
networkPolicies,
},
metadata,
),
);
}
async delete(
networkPolicies: string[],
customerId: string,
metadata: Metadata,
) {
return await lastValueFrom(
this.customerService.CustomerRemoveNetworkPolicy(
{
customerId,
networkPolicies,
},
metadata,
),
);
}
}
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -17,7 +17,7 @@ export class HubspotStrategy extends PassportStrategy(Strategy) {
clientSecret: oauthSecrets.hubspot.client_secret,
callbackURL: oauthSecrets.hubspot.redirect_uri,
scope:
'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
'tickets automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
passReqToCallback: true,
},
(accessToken, refreshToken, tokenInfo, profile, done) => {
@@ -0,0 +1,48 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateUserOpenDataDTO {
@ApiProperty()
firstName: string;
@ApiProperty()
lastName: string;
@ApiProperty()
email: string;
@ApiProperty()
organization: string;
@ApiProperty()
enquiryType: string;
}
type FormField = {
id: string;
type: string;
title: string;
value: string;
raw_value: string;
required: string;
};
type MetaData = {
title: string;
value: string;
};
export type WordpressForm = {
form: {
id: string;
name: string;
};
fields: {
[key: string]: FormField;
};
meta: {
date: MetaData;
time: MetaData;
page_url: MetaData;
user_agent: MetaData;
remote_ip: MetaData;
credit: MetaData;
};
};
@@ -0,0 +1,101 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Body, Controller, ForbiddenException, Header, Headers, HttpCode, HttpException, Inject, Param, Post, Query, Req, Res, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { LanguageEnum } from 'src/utils/languages.enum';
import { UsersService } from '../users/users.service';
import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@Controller('open-data')
@ApiInternalOnlyController()
@ApiTags('OpenData')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(GrpcToHttpExceptionFilter)
export class OpenDataController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private openDataService: OpenDataService,
) {
this.logger = dadosferaLogger.logger;
}
@Post("/sharing-ocean-data")
@HttpCode(200)
@Header('content-type', 'application/json')
@ApiOkResponse()
async createUser(
@Body()
body: WordpressForm,
@Query('language')
language: string,
@Req()
request: Request,
@Headers('origin')
origin: string
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
const roles = [process.env.OPEN_GROUP_ID];
const metadata = PackTheMetadata({
language: language || 'en-us'
});
const data = {}
try {
Object.keys(body.fields)
.filter(key => body.fields[key].required === "1")
.forEach(key => {
const field = body.fields[key]
data[field.id] = field.value
});
} catch (e) {
this.logger.error('user data ' + e.message);
}
const user: CreateUserOpenDataDTO = {
email: data["email"],
enquiryType: data["enquiry_type"],
firstName: data["first_name"],
lastName: data["last_name"],
organization: data["organization"]
}
this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`);
try {
const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata);
this.logger.info('user created with id: '+ id);
return {
success: true,
status: 'success',
message: 'user created with succesfull'
}
} catch (e) {
this.logger.error('failed with exception: ' + e.message);
return {
success: false,
status: 'failed',
message: e.message
};
}
}
}
+22
View File
@@ -0,0 +1,22 @@
import { Module } from '@nestjs/common';
import { OpenDataController } from './open-data.controller';
import { UsersService } from '../users/users.service';
import { ClientsModule } from '@nestjs/microservices'
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
import { OpenDataService } from './open-data.service';
const client = new DucClient();
@Module({
controllers: [OpenDataController],
imports: [
ClientsModule.register([client.providerOptions]),
RolesModule,
// PermissionsModule,
],
providers: [DadosferaLogger, UsersService, OpenDataService]
})
export class OpenDataModule {}
@@ -0,0 +1,51 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { lastValueFrom } from 'rxjs';
import { CreateUserOpenDataDTO } from './dto/wordpres-form';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
export class OpenDataService implements OnModuleInit {
logger: DadosferaLogger;
private usersClientService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
private dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.usersClientService = this.grpcClient.getService(
ProtoServices.UsersProtoService,
);
}
async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) {
const body = {
email: data.email,
name: data.firstName + " " + data.lastName,
department: data.organization,
jobTitle: data.enquiryType,
customerId: customerId,
roleIds: roleIds
}
try {
const { user } = await lastValueFrom(
this.usersClientService.SimpleUserCreate(body, metadata),
);
return user.id;
} catch(err) {
return err;
}
}
}
+6 -2
View File
@@ -1,6 +1,9 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
@@ -11,9 +14,10 @@ import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
+4 -2
View File
@@ -3,10 +3,12 @@ import {
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from 'protospack';
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js';
const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
export class PipelinesClientConfiguration {
public name = 'PipelinesClientConfiguration';

Some files were not shown because too many files have changed in this diff Show More