Compare commits

...
Author SHA1 Message Date
marcos-silva-rodrigues 301b6e98ec UPDATE: test subdomain 2025-08-14 17:42:42 -03:00
marcos-silva-rodrigues ba53934068 CI: add cookie secret 2025-08-14 17:23:40 -03:00
marcos-silva-rodrigues 216349f303 CHORE: update nginx ingress annotations 2025-08-14 16:03:28 -03:00
marcos-silva-rodrigues a654baef13 FIX: set cookies if exists tokens 2025-08-14 15:37:17 -03:00
marcos-silva-rodrigues 04d761ca14 Merge branch 'beta' into feature/auth-session 2025-08-14 15:26:15 -03:00
Marcos Rodrigues Silva a772804dcd Merge pull request #357 from dadosfera/chore/oracle
CI: fix multi cloud deploy and maestro env
2025-08-07 18:04:10 -03:00
marcos-silva-rodrigues 8a399fbbcf CI: fix multi cloud deploy and maestro env 2025-08-07 18:03:22 -03:00
Marcos Rodrigues Silva 94049d6e8e Merge pull request #356 from dadosfera/chore/oracle
CI: oracle
2025-08-07 12:27:08 -03:00
marcos-silva-rodrigues c28cc58ac7 CI: oracle 2025-08-07 12:24:44 -03:00
Marcos Rodrigues Silva 56a655bb90 Merge pull request #355 from dadosfera/chore/oracle
CI: fix env home
2025-08-07 12:16:02 -03:00
marcos-silva-rodrigues b3d4a6b028 CI: fix env home 2025-08-07 12:13:19 -03:00
Marcos Rodrigues Silva c7bd1f761a Merge pull request #354 from dadosfera/chore/oracle
CI: remove k8s-setup need
2025-08-07 12:01:43 -03:00
marcos-silva-rodrigues 09c045455e CI: remove k8s-setup need 2025-08-07 11:59:37 -03:00
Marcos Rodrigues Silva 52d651126e Merge pull request #353 from dadosfera/chore/oracle
CI: revert k8s-setup workflow for the step in job helmfile-deploy
2025-08-07 11:57:38 -03:00
marcos-silva-rodrigues 2f11b1e78f CI: revert k8s-setup workflow for the step in job helmfile-deploy 2025-08-07 11:56:54 -03:00
Marcos Rodrigues Silva 885e9b71bb Merge pull request #352 from dadosfera/fix/tracker-sso
Fix/tracker sso
2025-08-07 11:45:44 -03:00
marcos-silva-rodrigues 193031dfdd UPDATE: increase redis ttl 2025-08-07 10:41:48 -03:00
marcos-silva-rodrigues f626a9bb5e UPDATE: add more logger 2025-08-07 10:32:22 -03:00
Marcos Rodrigues Silva ac26ad9f44 Merge pull request #351 from dadosfera/chore/oracle
FIX: correct name
2025-08-01 18:06:27 -03:00
marcos-silva-rodrigues fbe00af6dc FIX: correct name 2025-08-01 18:05:19 -03:00
Marcos Rodrigues Silva bfa9929b69 Merge pull request #350 from dadosfera/chore/oracle
Chore/oracle
2025-08-01 18:01:38 -03:00
marcos-silva-rodrigues bada7000f6 CHORE: change ci to oracle cloud 2025-08-01 18:01:10 -03:00
Marcos Rodrigues Silva 2c908d8d95 Merge pull request #349 from dadosfera/beta
Beta
2025-07-31 15:24:28 -03:00
Marcos Rodrigues Silva da35633d42 Merge pull request #348 from dadosfera/hotfix/module-assigned
Hotfix/module assigned
2025-07-31 12:04:02 -03:00
marcos-silva-rodrigues cc7e06013f FIX: change module permission 2025-07-31 11:59:19 -03:00
Marcos Rodrigues Silva 5ec36a05b5 Merge pull request #344 from dadosfera/beta
Beta
2025-07-30 11:56:22 -03:00
Marcos Rodrigues Silva f61e46c7c1 Merge pull request #347 from dadosfera/release/07-29
FEAT: add permission embed access
2025-07-30 11:15:28 -03:00
marcos-silva-rodrigues b9013b1493 FEAT: add permission embed access 2025-07-30 11:12:08 -03:00
Marcos Rodrigues Silva 5267fa6d2f Merge pull request #346 from dadosfera/release/07-29
FIX: comment test
2025-07-29 18:05:26 -03:00
marcos-silva-rodrigues 7f40cbdeed FIX: comment test 2025-07-29 18:03:20 -03:00
Marcos Rodrigues Silva b590c9402d Merge pull request #345 from dadosfera/release/07-29
Release/07 29
2025-07-29 15:19:52 -03:00
marcos-silva-rodrigues 9546feda11 FIX: merge 2025-07-29 15:12:25 -03:00
Marcos Rodrigues Silva 3587f82976 Merge pull request #343 from dadosfera/bugfix/public-link
Bugfix/public link
2025-07-29 12:31:34 -03:00
marcos-silva-rodrigues 94a3962590 FIX: add needs steps 2025-07-29 12:26:21 -03:00
marcos-silva-rodrigues 3b2d79a451 CHORE: remove needs step 2025-07-29 12:06:33 -03:00
marcos-silva-rodrigues da56a645af CHORE: update affinity to azure 2025-07-29 11:57:10 -03:00
Marcos Rodrigues Silva af4e55af1d Merge pull request #342 from dadosfera/bugfix/public-link
CHORE: remove sudo
2025-07-29 11:35:09 -03:00
marcos-silva-rodrigues d6a08a8f82 CHORE: remove sudo 2025-07-29 11:32:02 -03:00
Marcos Rodrigues Silva c73eb8869f Merge pull request #341 from dadosfera/bugfix/public-link
CHORE: install kubectl
2025-07-29 11:15:24 -03:00
marcos-silva-rodrigues 929fc97ac6 CHORE: install kubectl 2025-07-29 11:12:47 -03:00
Marcos Rodrigues Silva 7594a75bea Merge pull request #340 from dadosfera/bugfix/public-link
CHORE: update ci to wait extract_environment job
2025-07-29 11:02:14 -03:00
marcos-silva-rodrigues ec36a056b2 CHORE: update ci to wait extract_environment job 2025-07-29 10:59:35 -03:00
Marcos Rodrigues Silva 8d30f6ef15 Merge pull request #339 from dadosfera/bugfix/public-link
CHORE: multi cloud deployment
2025-07-29 10:57:26 -03:00
marcos-silva-rodrigues 08d1e3164f CHORE: multi cloud deployment 2025-07-29 10:53:41 -03:00
Marcos Rodrigues Silva a5f3ef20fe Merge pull request #338 from dadosfera/bugfix/public-link
FIX: add user by bearer token if it exists
2025-07-29 09:51:17 -03:00
marcos-silva-rodrigues 467445fe05 FIX: add user by bearer token if it exists 2025-07-29 09:47:56 -03:00
Marcos Rodrigues Silva d0448f844a Merge pull request #337 from dadosfera/bugfix/google-oauth
UPDATE: more logs
2025-07-21 16:06:22 -03:00
marcos-silva-rodrigues b2299a5f2f UPDATE: more logs 2025-07-21 15:51:56 -03:00
Marcos Rodrigues Silva 9b42754c69 Merge pull request #336 from dadosfera/bugfix/google-oauth
Bugfix/google oauth
2025-07-21 15:06:49 -03:00
marcos-silva-rodrigues aee33334ac UPDATE: add log to debug 2025-07-21 15:01:41 -03:00
Marcos Rodrigues Silva cc530fb12d Merge pull request #335 from dadosfera/release/sso
Release/sso
2025-07-18 17:12:51 -03:00
marcos-silva-rodrigues 19877f1b29 FIX: merge with main 2025-07-18 16:46:19 -03:00
marcos-silva-rodrigues e9880bcf68 FIX: add logger in header 2025-07-18 15:56:23 -03:00
marcos-silva-rodrigues 4c7111dcb6 FIX: using referer headaer 2025-07-18 15:56:10 -03:00
marcos-silva-rodrigues e81cfdcce1 FIX: using only header origin 2025-07-18 15:55:56 -03:00
marcos-silva-rodrigues 3eba3f414f FIX: send redirect uri 2025-07-18 15:55:45 -03:00
marcos-silva-rodrigues fbeb8dfd91 FIX: redis env 2025-07-18 15:55:29 -03:00
marcos-silva-rodrigues dda3e8baaf FIX: add logger 2025-07-18 15:54:27 -03:00
marcos-silva-rodrigues c1e4f0b18a FIX: remove express session 2025-07-18 15:54:24 -03:00
Marcos Rodrigues Silva b0d9f3a468 Merge pull request #334 from dadosfera/chore/stg-env
FIX: revert env
2025-07-18 10:27:01 -03:00
marcos-silva-rodrigues 5b117acaa9 FIX: revert env 2025-07-18 10:23:10 -03:00
Marcos Rodrigues Silva a5cb607650 Merge pull request #333 from dadosfera/feature/sso-openid
FIX: change env to stg
2025-07-17 16:12:51 -03:00
marcos-silva-rodrigues fe626155c5 FIX: change env to stg 2025-07-17 16:10:42 -03:00
Marcos Rodrigues Silva debbe1c1b1 Merge pull request #332 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 18:50:04 -03:00
marcos-silva-rodrigues f937a6efe2 FIX: add logger in header 2025-07-16 18:49:27 -03:00
marcos-silva-rodrigues 30cb3c87eb FIX: using referer headaer 2025-07-16 18:45:47 -03:00
Marcos Rodrigues Silva fa46294154 Merge pull request #331 from dadosfera/feature/sso-openid
FIX: using only header origin
2025-07-16 18:37:53 -03:00
marcos-silva-rodrigues 72554fb27f FIX: using only header origin 2025-07-16 18:36:36 -03:00
Marcos Rodrigues Silva 4e6985b572 Merge pull request #330 from dadosfera/feature/sso-openid
FIX: send redirect uri
2025-07-16 18:16:27 -03:00
marcos-silva-rodrigues 4477647613 FIX: send redirect uri 2025-07-16 18:15:47 -03:00
Marcos Rodrigues Silva 2b3066abc9 Merge pull request #329 from dadosfera/feature/sso-openid
FIX: redis env
2025-07-16 17:57:26 -03:00
marcos-silva-rodrigues da1bcb38da FIX: redis env 2025-07-16 17:56:43 -03:00
Marcos Rodrigues Silva 4a7524422d Merge pull request #328 from dadosfera/feature/sso-openid
FIX: add logger
2025-07-16 17:41:40 -03:00
marcos-silva-rodrigues 275af03195 FIX: add logger 2025-07-16 17:39:15 -03:00
Marcos Rodrigues Silva 0c75cf4619 Merge pull request #327 from dadosfera/feature/sso-openid
Feature/sso openid
2025-07-16 12:33:44 -03:00
marcos-silva-rodrigues 80de277529 FIX: merge problems 2025-07-16 12:27:33 -03:00
marcos-silva-rodrigues 98f669db66 FIX: remove express session 2025-07-16 11:48:15 -03:00
marcos-silva-rodrigues 2c8dab5927 FIX: merge 2025-07-16 11:46:05 -03:00
marcos-silva-rodrigues cb66d4f6c0 FEAT: dynamic callback and issuer urls 2025-07-16 11:35:47 -03:00
marcos-silva-rodrigues 42a39ee891 FEAT: send user id to refresh token endpoint 2025-07-15 16:21:44 -03:00
marcos-silva-rodrigues c9bfe68ea5 FEAT: list identity providers link for sso 2025-07-14 15:16:19 -03:00
marcos-silva-rodrigues 8cb7144ff4 FIX: send accessToken in refresh token endpoint 2025-07-14 15:14:25 -03:00
Marcos Rodrigues Silva 68cc58e2fe Merge pull request #326 from dadosfera/release/11-07
Release/11 07
2025-07-14 10:54:14 -03:00
marcos-silva-rodrigues cbe5556ac9 FIX: auth callback 2025-07-14 10:51:36 -03:00
marcos-silva-rodrigues eb80967608 FIX: tests 2025-07-11 11:36:23 -03:00
marcos-silva-rodrigues 66fea0722d FEAT: enabled network check in app.dadosfera 2025-07-11 11:12:35 -03:00
marcos-silva-rodrigues c917061b97 Merge branch 'feature/export-users-and-assets' into release/11-07 2025-07-11 11:04:31 -03:00
Rafael Santana da39a07f25 Merge pull request #325 from dadosfera/feat/oracle-migration-and-affinity-standardization
FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
2025-07-09 13:02:48 -03:00
Rafael f2f65285de resolve: merge conflicts with beta branch
- Resolve ingress.yaml timeout annotation conflicts by keeping both configurations
- Resolve stg.yaml conflicts by merging affinity/resources config with restricted_ip
- Fix chart references in unimed-maestro and private-maestro sections
- Update all chart paths from ../maestro to ../helm-chart for consistency

This merge brings in the latest beta changes while preserving our Oracle migration
and affinity standardization configuration.
2025-07-09 13:00:52 -03:00
Rafael cac8f93330 CI: update validate-k8s.yml to use beta branch for staging
- Change trigger branches from main/stg to main/beta
- Update environment extraction to map beta → stg environment
- Align with standard branch naming convention (beta for staging)
- Consistent with deploy-manually.yml workflow configuration
2025-07-09 12:58:11 -03:00
Rafael bb03b18f4a FEAT: migrate maestro to Oracle OKE deployment structure and standardize affinity/resources
Oracle Migration Changes:
- Create deploy/helm-chart/ directory structure (moved from maestro/)
- Create deploy/helmfiles/ directory structure (moved from helmfiles/)
- Update chart references in helmfiles from ../maestro to ../helm-chart
- Update deploy-manually.yml from Azure AKS to Oracle OKE
- Update validate-k8s.yml from Azure AKS to Oracle OKE
- Replace Azure CLI with OCI CLI installation and configuration
- Replace Azure authentication with OCI authentication using secrets
- Replace az aks get-credentials with oci ce cluster create-kubeconfig
- Update helmfile paths from helmfiles/ to deploy/helmfiles/
- Remove DockerHub integration (push_to_dockerhub input and related steps)
- Change runner from [self-hosted, prd-azure] to [self-hosted, prd-oracle]
- Add environment field for proper Oracle deployment environment handling
- Add HOME environment variable for OCI CLI

Affinity/Resources Standardization:
- Add affinity configuration with name=general node selector in values.yaml
- Update deployment template to use conditional affinity and resources blocks
- Configure staging to disable both affinity and resources (null values)
- Enable production to use affinity targeting name=general nodes
- Enable production to use standard resource limits (100m-2000m CPU, 1500Mi-2Gi memory)

This brings maestro into full compliance with Oracle migration requirements and
implements the standardized affinity/resources pattern consistent with in-factory and duc.
2025-07-09 12:55:49 -03:00
Marcos Rodrigues Silva ab280ee152 Merge pull request #324 from dadosfera/chore/update-unimed-ips
CHORE: Update unimed ips
2025-07-08 09:22:26 -03:00
marcos-silva-rodrigues 16c8137160 CHORE: Update unimed ips 2025-07-07 17:26:26 -03:00
Marcos Rodrigues Silva b8b903c8cd Merge pull request #323 from dadosfera/feature/export-users-and-assets
Feature/export users and assets
2025-07-05 11:27:10 -03:00
marcos-silva-rodrigues c98a5ff896 FEAT: endpoint to dowload assets list 2025-07-05 11:21:05 -03:00
marcos-silva-rodrigues d68e436a8b FEAT: export users in csv 2025-07-05 09:38:16 -03:00
marcos-silva-rodrigues 8efa6790d6 FEAT: sign out endpoint 2025-07-04 18:03:08 -03:00
marcos-silva-rodrigues a381f50e88 FEAT: set cookie after user login 2025-07-03 15:52:55 -03:00
marcos-silva-rodrigues d3040cd5a8 FEAT: update and delete identity provider 2025-07-03 10:59:43 -03:00
marcos-silva-rodrigues f726a9883f FEAT: create user from identity provider 2025-06-27 13:21:59 -03:00
Marcos Rodrigues Silva c18a7baa37 Merge pull request #322 from dadosfera/chore/add-dadosfera-vpn-unimed-chart
FIX: add dadosfera vpn ip in unimed chart
2025-06-25 11:12:33 -03:00
marcos-silva-rodrigues d638d518f8 FIX: change unimed chart name 2025-06-25 11:06:41 -03:00
marcos-silva-rodrigues bdd371ffe8 FIX: add dadosfera vpn ip in unimed chart 2025-06-25 10:58:18 -03:00
marcos-silva-rodrigues 57e6e49c7c FEAT: init url callback 2025-06-25 10:28:26 -03:00
marcos-silva-rodrigues a5c8de496d FEAT: map permission to identity provider 2025-06-24 16:20:03 -03:00
Marcos Rodrigues Silva d478f32760 Merge pull request #321 from dadosfera/hotfix/nginx-timeout
FIX: add nginx proxy timeout 5 min
2025-06-24 11:02:29 -03:00
marcos-silva-rodrigues 5d8e908339 FIX: add nginx proxy timeout 5 min 2025-06-24 10:26:50 -03:00
Marcos Rodrigues Silva 606bae016b Merge pull request #320 from dadosfera/hotfix/maesto-unimed-hostname
FIX: correct helm value
2025-06-23 18:43:58 -03:00
marcos-silva-rodrigues 6910a2f04c FIX: correct helm value 2025-06-23 18:38:20 -03:00
Marcos Rodrigues Silva 8bee1788c1 Merge pull request #319 from dadosfera/hotfix/maesto-unimed-hostname
FIX: maestro custom hostname to unimed
2025-06-23 18:04:43 -03:00
marcos-silva-rodrigues b10459cb15 FIX: maestro custom hostname to unimed 2025-06-23 17:51:14 -03:00
Marcos Rodrigues Silva 3b48dd1613 Merge pull request #317 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 17:07:55 -03:00
Marcos Rodrigues Silva e5cef90a64 Merge pull request #318 from dadosfera/release/2025-06
Release/2025 06
2025-06-23 16:04:59 -03:00
marcos-silva-rodrigues bac74c9577 FIX: default value and throw correct Error Class 2025-06-23 15:36:26 -03:00
marcos-silva-rodrigues 19d748ae09 CHORE: add restricted ip and unimed chart 2025-06-23 14:47:39 -03:00
marcos-silva-rodrigues 4db865371f Merge branch 'bugfix/update-pupperter-timeout' into release/2025-06 2025-06-23 14:12:25 -03:00
marcos-silva-rodrigues 75e8b3bf4b FIX: correct dto name 2025-06-17 14:58:35 -03:00
Marcos Rodrigues Silva 194cdc66b7 Merge pull request #316 from dadosfera/feature/public-and-assign-embed
Feature/public and assign embed
2025-06-17 08:57:53 -03:00
marcos-silva-rodrigues 53059c0a0d FIX: import 2025-06-16 18:09:25 -03:00
marcos-silva-rodrigues ed4ffc8a86 FIX: merge with beta 2025-06-16 18:01:53 -03:00
Marcos Rodrigues Silva 9967c172da Merge pull request #315 from dadosfera/feature/dedicated-maestro
FEAT: block user from login when maestro is dedicated
2025-06-16 10:37:47 -03:00
marcos-silva-rodrigues 0aa9c065a5 FEAT: block user from login when maestro is dedicated 2025-06-16 10:34:11 -03:00
Marcos Rodrigues Silva d0c2abd38d Merge pull request #314 from dadosfera/bugfix/update-pupperter-timeout
FIX: update timeout
2025-06-13 15:10:08 -03:00
marcos-silva-rodrigues cb98099a91 FIX: update timeout 2025-06-13 15:07:50 -03:00
Marcos Rodrigues Silva f446e0d5b1 Merge pull request #313 from dadosfera/feature/dedicated-maestro
FIX: send env DEDICATED_PROXY
2025-06-12 14:47:54 -03:00
marcos-silva-rodrigues 17de31f1a6 FIX: send env DEDICATED_PROXY 2025-06-12 14:45:13 -03:00
Marcos Rodrigues Silva d81a466089 Merge pull request #312 from dadosfera/feature/dedicated-maestro
FIX: open data ingress route
2025-06-12 14:08:13 -03:00
marcos-silva-rodrigues 9e597fadba FIX: open data ingress route 2025-06-12 14:04:45 -03:00
Marcos Rodrigues Silva 1e6aa0f4c2 Merge pull request #311 from dadosfera/feature/dedicated-maestro
FIX: chart private maestro
2025-06-12 13:59:11 -03:00
marcos-silva-rodrigues 41842f7c6a FIX: chart private maestro 2025-06-12 13:56:57 -03:00
Marcos Rodrigues Silva 0d5564990d Merge pull request #309 from dadosfera/feature/dedicated-maestro
Feature/dedicated maestro
2025-06-12 12:31:26 -03:00
marcos-silva-rodrigues 7f9755493a FIX: experimental hostname 2025-06-12 12:29:51 -03:00
marcos-silva-rodrigues fea587ad36 FEAT: control login by dedicated customer id injected 2025-06-12 11:51:56 -03:00
Marcos Rodrigues Silva b2b7537fee Merge pull request #308 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 15:25:27 -03:00
Marcos Rodrigues Silva 57f99fe7af Merge pull request #307 from dadosfera/fix/wordpress-block-list
FIX: comment block list
2025-06-10 14:25:23 -03:00
marcos-silva-rodrigues 01263017e7 FIX: comment block list 2025-06-10 14:24:26 -03:00
Marcos Rodrigues Silva e66f9a7244 Merge pull request #306 from dadosfera/feature/api-key
FEAT: add api key endpoint
2025-06-06 15:55:53 -03:00
Marcos Rodrigues Silva 4c861cf5e4 Merge pull request #305 from dadosfera/fix/maestro-stg-url
Fix/maestro stg url
2025-05-30 10:15:10 -03:00
marcos-silva-rodrigues a89f57e690 FIX: return pi_factory_url and tr_factory_url to stg url 2025-05-30 10:13:37 -03:00
marcos-silva-rodrigues d5141e5b6a REFACTOR: rename openid to identity provider 2025-05-26 13:37:25 -03:00
marcos-silva-rodrigues a4b7573ce7 FIX: sync with main 2025-05-23 14:07:05 -03:00
marcos-silva-rodrigues 6fc8c0b4a8 FEAT: add mixpanel service to track share access 2025-05-20 16:50:04 -03:00
marcos-silva-rodrigues ced2a97547 FIX: add logger and return info 2025-05-19 16:36:54 -03:00
marcos-silva-rodrigues d15cc4642a FEAT: assign embed and public key changes 2025-05-13 09:21:44 -03:00
marcos-silva-rodrigues 3c4f45678a CHORE: update protospack 2025-05-12 12:25:45 -03:00
marcos-silva-rodrigues b312849fff FEAT: catalog share endpoint for embeds 2025-05-12 12:15:16 -03:00
marcos-silva-rodrigues 10580dfa03 FEAT: share endpoint 2025-05-12 12:13:39 -03:00
marcos-silva-rodrigues 62d0e8b041 FEAT: assign pubic key endpoint 2025-05-12 12:12:58 -03:00
marcos-silva-rodrigues d94c5dee4e FEAT: init public routes to get data asset by public embed 2025-05-06 14:44:09 -03:00
marcos-silva-rodrigues 4c94a51a9d REFACTOR: openid module 2025-04-03 07:46:16 -03:00
marcos-silva-rodrigues f1345472b5 UPDATE: add routes to login with idp 2025-03-25 09:18:27 -03:00
marcos-silva-rodrigues ea9c0502b7 UPDATE: new route to add identity provider 2025-03-24 07:58:26 -03:00
75 changed files with 3670 additions and 887 deletions
+7 -72
View File
@@ -13,11 +13,6 @@ on:
options:
- stg
- prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs:
extract_environment:
@@ -118,23 +113,6 @@ jobs:
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2
with:
username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker compose -f build.docker-compose.dockerhub.yml build
docker compose -f build.docker-compose.dockerhub.yml push
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -164,54 +142,11 @@ jobs:
docker system prune --volumes -a -f
docker system df
helmfile-deploy:
k8s-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: ${{ (needs.extract_environment.outputs.environment == 'prd' && 'azure' ) || 'oracle' }}
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
+137
View File
@@ -0,0 +1,137 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+45 -22
View File
@@ -4,7 +4,7 @@ on:
pull_request:
branches:
- main
- stg
- beta
jobs:
extract_environment:
@@ -21,17 +21,13 @@ jobs:
fi
id: extract_environment
helmfile-deploy:
helmfile-check:
env:
HOME: /home/runner
needs: [extract_environment]
runs-on: [self-hosted, "prd-azure"]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
steps:
- name: Summary
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
- name: Checkout code
uses: actions/checkout@v3
@@ -40,13 +36,28 @@ jobs:
with:
version: 'v3.9.0'
- name: Install Azure ClI
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
@@ -57,23 +68,35 @@ jobs:
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f helmfiles/${ENV}.yaml diff
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
@@ -1,16 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: maestro
app: {{ .Values.app_name }}
strategy:
rollingUpdate:
@@ -20,22 +20,17 @@ spec:
template:
metadata:
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: application
operator: In
values:
- general
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
@@ -48,13 +43,10 @@ spec:
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
requests:
cpu: {{ .Values.resources.requests.cpu }}
memory: {{ .Values.resources.requests.memory }}
limits:
cpu: {{ .Values.resources.limits.cpu }}
memory: {{ .Values.resources.limits.memory }}
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
@@ -100,6 +92,16 @@ spec:
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -108,15 +110,15 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-maestro
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
@@ -10,8 +10,12 @@ metadata:
generation: 1
labels:
app: maestro
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
@@ -21,7 +25,7 @@ spec:
paths:
- backend:
service:
name: maestro
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
+36
View File
@@ -0,0 +1,36 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
@@ -1,17 +1,17 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-maestro
name: prd-{{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-maestro
name: prd-{{ .Values.app_name }}
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
@@ -1,18 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: maestro
name: {{ .Values.app_name }}
namespace: applications
labels:
app: maestro
app: {{ .Values.app_name }}
spec:
type: ClusterIP
ports:
- name: maestro
- name: {{ .Values.app_name }}
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: maestro
app: {{ .Values.app_name }}
+23
View File
@@ -0,0 +1,23 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- general
@@ -9,6 +9,7 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
@@ -42,9 +43,25 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "product-redis-prd.z4xvqj.0001.use1.cache.amazonaws.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: applications
operator: In
values:
- general
+53
View File
@@ -0,0 +1,53 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
- name: maestro.restricted_ip
value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30"
+43
View File
@@ -0,0 +1,43 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
# - name: private-maestro
# chart: ../helm-chart
# values:
# - ../helm-chart/values.yaml
# set:
# - name: app_name
# value: maestro-private
# - name: maestro.env
# value: stg
# - name: maestro.duc_url
# value: duc.stg.dadosfera.ai
# - name: hostname
# value: private-maestro.stg.dadosfera.ai
# - name: maestro.pi_factory_url
# value: pi-factory.dadosfera.ai
# - name: maestro.in_factory_url
# value: in-factory.stg.dadosfera.ai
# - name: maestro.tr_factory_url
# value: in-factory.dadosfera.ai
# - name: maestro.open_customer_id
# value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
# - name: maestro.open_group_id
# value: e3f98a2f-7748-4981-8505-7695c8ca8218
# # Customer id
# - name: maestro.dedicated_proxy
# value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
# - name: replicaCount
# value: 1
# - name: affinity
# value: null
# - name: resources
# value: null
# - name: maestro.restricted_ip
# value: "57.151.113.140/30"
+1108 -511
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -14,6 +14,8 @@ declare global {
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
}
}
}
-22
View File
@@ -1,22 +0,0 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
-22
View File
@@ -1,22 +0,0 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
-23
View File
@@ -1,23 +0,0 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
-27
View File
@@ -1,27 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: maestro
name: maestro
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
+331 -4
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -27,8 +27,11 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -39,6 +42,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -46,13 +50,17 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.6.3"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
@@ -1400,9 +1408,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.1.tgz",
"integrity": "sha512-4+yNZMlhvEkHlxJcaeta+jtJ9owskZcg0yjKdZzBfT8PCclctgertLHmrG0+kDb3Y/na/rpPPHYULRp7AO4zDQ==",
"version": "3.38.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz",
"integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -1600,6 +1608,12 @@
"dev": true,
"license": "BSD-3-Clause"
},
"node_modules/@ioredis/commands": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
"license": "MIT"
},
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
@@ -2969,6 +2983,65 @@
"node": ">=18"
}
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz",
"integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz",
"integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz",
"integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@sinonjs/commons": {
"version": "1.8.6",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz",
@@ -3654,6 +3727,13 @@
"@types/node": "*"
}
},
"node_modules/@types/cache-manager": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.6.tgz",
"integrity": "sha512-8qL93MF05/xrzFm/LSPtzNEOE1eQF3VwGHAcQEylgp5hDSTe41jtFwbSYAPfyYcVa28y1vYSjIt0c1fLLUiC/Q==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/connect": {
"version": "3.4.38",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
@@ -3664,6 +3744,16 @@
"@types/node": "*"
}
},
"node_modules/@types/cookie-parser": {
"version": "1.4.9",
"resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.9.tgz",
"integrity": "sha512-tGZiZ2Gtc4m3wIdLkZ8mkj1T6CEHb35+VApbL2T14Dew8HA7c+04dmKqsKRNC+8RJPm16JEK0tFSwdZqubfc4g==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"@types/express": "*"
}
},
"node_modules/@types/cookiejar": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
@@ -3723,6 +3813,16 @@
"@types/send": "*"
}
},
"node_modules/@types/express-session": {
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.18.1.tgz",
"integrity": "sha512-S6TkD/lljxDlQ2u/4A70luD8/ZxZcrU5pQwI1rVXCiaVIywoFgbA+PIUNDjPhQpPdK0dGleLtYc/y7XWBfclBg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/express": "*"
}
},
"node_modules/@types/graceful-fs": {
"version": "4.1.9",
"resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz",
@@ -5237,6 +5337,41 @@
"node": ">= 0.8"
}
},
"node_modules/cache-manager": {
"version": "5.7.6",
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
"lru-cache": "^10.2.2",
"promise-coalesce": "^1.1.2"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/cache-manager-ioredis-yet": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cache-manager-ioredis-yet/-/cache-manager-ioredis-yet-1.2.2.tgz",
"integrity": "sha512-o03N/tQxfFONZ1XLGgIxOFHuQQpjpRdnSAL1THG1YWZIVp1JMUfjU3ElSAjFN1LjbJXa55IpC8waG+VEoLUCUw==",
"deprecated": "With cache-manager v6 we now are using Keyv",
"license": "MIT",
"dependencies": {
"cache-manager": "^5.2.3",
"ioredis": "^5.3.2"
},
"engines": {
"node": ">= 16.17.0"
}
},
"node_modules/cache-manager/node_modules/lru-cache": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
"node_modules/call-bind": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz",
@@ -5514,6 +5649,15 @@
"node": ">=0.8"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -5740,6 +5884,28 @@
"node": ">= 0.6"
}
},
"node_modules/cookie-parser": {
"version": "1.4.7",
"resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz",
"integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==",
"license": "MIT",
"dependencies": {
"cookie": "0.7.2",
"cookie-signature": "1.0.6"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/cookie-parser/node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
@@ -6027,6 +6193,15 @@
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -6802,6 +6977,12 @@
"node": ">= 0.6"
}
},
"node_modules/eventemitter3": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
"integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==",
"license": "MIT"
},
"node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -7470,6 +7651,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -8045,6 +8235,30 @@
"node": ">= 0.10"
}
},
"node_modules/ioredis": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ip-address": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
@@ -9044,6 +9258,15 @@
"url": "https://github.com/chalk/supports-color?sponsor=1"
}
},
"node_modules/jose": {
"version": "4.15.9",
"resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz",
"integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -9366,12 +9589,30 @@
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
"license": "MIT"
},
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==",
"license": "MIT"
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
"license": "MIT"
},
"node_modules/lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
"license": "MIT"
},
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
"license": "MIT"
},
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -10023,6 +10264,15 @@
"integrity": "sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==",
"license": "MIT"
},
"node_modules/object-hash": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz",
"integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==",
"license": "MIT",
"engines": {
"node": ">= 6"
}
},
"node_modules/object-identity-map": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/object-identity-map/-/object-identity-map-1.0.2.tgz",
@@ -10068,6 +10318,15 @@
"node": ">= 0.4"
}
},
"node_modules/oidc-token-hash": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.1.0.tgz",
"integrity": "sha512-y0W+X7Ppo7oZX6eovsRkuzcSM40Bicg2JEJkDJ4irIt1wsYAP5MLSNv+QAogO8xivMffw/9OvV3um1pxXgt1uA==",
"license": "MIT",
"engines": {
"node": "^10.13.0 || >=12.0.0"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -10113,6 +10372,21 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/openid-client": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.7.1.tgz",
"integrity": "sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==",
"license": "MIT",
"dependencies": {
"jose": "^4.15.9",
"lru-cache": "^6.0.0",
"object-hash": "^2.2.0",
"oidc-token-hash": "^5.0.3"
},
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/optional-js": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/optional-js/-/optional-js-2.3.0.tgz",
@@ -10815,6 +11089,15 @@
"node": ">=0.4.0"
}
},
"node_modules/promise-coalesce": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/promise-coalesce/-/promise-coalesce-1.1.2.tgz",
"integrity": "sha512-zLaJ9b8hnC564fnJH6NFSOGZYYdzrAJn2JUUIwzoQb32fG2QAakpDNM+CZo1km6keXkRXRM+hml1BFAPVnPkxg==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=16"
}
},
"node_modules/prompts": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
@@ -11194,6 +11477,44 @@
"node": ">= 0.10"
}
},
"node_modules/redis": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz",
"integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==",
"license": "MIT",
"workspaces": [
"./packages/*"
],
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.6.1",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.7",
"@redis/search": "1.2.0",
"@redis/time-series": "1.1.0"
}
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"license": "MIT",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/reflect-metadata": {
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
@@ -11881,6 +12202,12 @@
"integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==",
"license": "MIT"
},
"node_modules/standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
"license": "MIT"
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+9 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -45,8 +45,11 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -57,6 +60,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -64,6 +68,7 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
@@ -73,7 +78,10 @@
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
+10 -2
View File
@@ -28,7 +28,10 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
@@ -63,10 +66,15 @@ import { ApiKeyModule } from './modules/api-key/api-key.module';
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule,
ApiKeyModule
],
})
export class AppModule {}
+20 -12
View File
@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
@@ -4,6 +4,7 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -135,6 +136,18 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
+22 -11
View File
@@ -340,16 +340,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -362,6 +352,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -614,7 +623,9 @@ export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii'
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+9 -1
View File
@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -52,6 +53,7 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
UserTest(token);
// UserTest(token);
});
+4
View File
@@ -9,6 +9,8 @@ import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -23,9 +25,11 @@ async function bootstrap() {
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
+1 -1
View File
@@ -3,10 +3,10 @@ import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PackTheMetadata } from '../../utils/ PackTheMetadata';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ApiKeyService implements OnModuleInit {
+33
View File
@@ -0,0 +1,33 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
}
}
+15
View File
@@ -0,0 +1,15 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
+38
View File
@@ -0,0 +1,38 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -0,0 +1,3 @@
export class CreateAssignDto {
publicKey: string;
}
+196 -14
View File
@@ -12,6 +12,7 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -26,7 +27,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthVerifyTotpMfaRequest
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -45,14 +46,21 @@ import {
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request } from 'express';
import { Request, Response } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -86,13 +94,65 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Headers('origin') origin = '',
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ language, custom_host: frontHost });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('refresh-access-token')
@@ -102,17 +162,25 @@ export class AuthController {
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
language,
customer_name,
custom_host: frontHost,
});
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
}
@ApiInternalOnlyEndpoint()
@@ -420,4 +488,118 @@ export class AuthController {
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
}
+51 -7
View File
@@ -1,10 +1,10 @@
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -17,16 +17,22 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -39,6 +45,10 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async getPublicKeys() {
@@ -53,25 +63,59 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
}
async refreshAccessToken(
{ refreshToken }: AuthRefreshAccessTokenRequest,
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
);
}
+1 -1
View File
@@ -122,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
customerName: string;
userId: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
@@ -26,6 +26,7 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
+48 -2
View File
@@ -32,7 +32,7 @@ import {
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
@@ -117,6 +117,52 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog/download - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
@@ -885,4 +931,4 @@ export class CatalogController {
this.logger.error(error.message);
}
}
}
}
+6 -1
View File
@@ -3,12 +3,15 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -18,6 +21,8 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+31 -8
View File
@@ -25,6 +25,7 @@ import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
IUpdateDataRequest,
TriggerCatalogReq,
@@ -35,12 +36,6 @@ import {
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@@ -240,6 +235,34 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -316,7 +339,7 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
@@ -584,4 +607,4 @@ class CatalogService implements OnModuleInit {
}
}
export { CatalogService };
export { CatalogService };
+8
View File
@@ -320,3 +320,11 @@ export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
@@ -0,0 +1,89 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
+30
View File
@@ -0,0 +1,30 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
+275
View File
@@ -0,0 +1,275 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ConnectionTestService {
@@ -28,7 +28,7 @@ import {
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -25,7 +25,7 @@ import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiTags('Customers')
@@ -59,6 +59,14 @@ export class CustomersService implements OnModuleInit {
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
@@ -141,6 +149,7 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
@@ -158,6 +167,7 @@ export class CustomersService implements OnModuleInit {
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
@@ -0,0 +1,47 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -0,0 +1,10 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -0,0 +1,246 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -0,0 +1,185 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+4 -76
View File
@@ -3,14 +3,13 @@ import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
private mixpanelService: MixpanelService
) {}
@Post(':id')
async trackEvent(
@@ -20,82 +19,11 @@ export class MixpanelController {
@Req() request
) {
delete body.info;
const mixpanel = init(this.mixpanelToken);
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
await this.mixpanelService.track(id, user, request, body)
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+6 -1
View File
@@ -1,6 +1,8 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`${process.env.ENV}/root/mixpanel_token`,
`prd/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
+118
View File
@@ -0,0 +1,118 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -8,7 +8,7 @@ import {
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@@ -45,11 +45,11 @@ export class OpenDataController {
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
if (!corslist.includes(origin)) {
this.logger.info('block request by cors list: '+ origin);
throw new ForbiddenException();
}
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
@@ -32,7 +32,7 @@ import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
@@ -0,0 +1,10 @@
export class CreateShareMetadataDto {
assetId: string;
proposedId: string;
type: string;
}
export class UpdateShareMetadataDto {
shareId: string;
type: string;
}
@@ -0,0 +1,31 @@
import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Controller('share')
@Authenticated()
export class ShareMetadataController {
constructor(private readonly shareMetadataService: ShareMetadataService) {}
@Put('/')
findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.create(data, metadata);
}
@Patch('/')
updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.updateShare(data, metadata);
}
@Get('/:id')
get(@Param('id') id: string, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.get(id, metadata);
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { ShareMetadataController } from './share-metadata.controller';
import { DucClient } from '../duc/client.config';
import { ClientsModule } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [ShareMetadataController],
providers: [ShareMetadataService, DadosferaLogger],
exports: [ShareMetadataService]
})
export class ShareMetadataModule {}
@@ -0,0 +1,62 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Metadata } from '@grpc/grpc-js';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Injectable()
export class ShareMetadataService implements OnModuleInit {
ducService: ShareMetadataProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<ShareMetadataProtoService>(
ProtoServices.ShareMetadataProtoService,
);
}
create(createShareDto: CreateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.FindOrCreateShareMetadata(
{
assetId: createShareDto.assetId,
proposedId: createShareDto.proposedId,
type: createShareDto.type
},
metadata,
),
);
}
updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.ChangeShareMetadataType(
updateShare,
metadata,
),
);
}
get(id: string, metadata: Metadata) {
return lastValueFrom(
this.ducService.GetShareMetadata(
{
id,
},
metadata,
),
);
}
}
+11
View File
@@ -174,3 +174,14 @@ export class GetAllDepartmentsRes {
@ApiProperty()
departments: string[];
}
export interface UserReporter {
name: string;
email: string;
mfaStatus: string;
status: string;
lastLogin: string;
createdAt: string;
updatedAt: string;
}
@@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
+24 -1
View File
@@ -12,6 +12,7 @@ import {
Post,
Put,
Query,
Res,
UseFilters,
} from '@nestjs/common';
import {
@@ -24,12 +25,13 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -52,6 +54,7 @@ import {
UpdateUserRes,
} from './dtos/entities';
import { UsersService } from './users.service';
import { Response } from 'express';
@ApiInternalOnlyController()
@ApiTags('Users')
@@ -80,6 +83,26 @@ export class UsersController {
return await this.userService.findAllUsersByCustomerId(user.customer_id);
}
@Get('/download')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async downloadUsersInCsv(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Res() res: Response
) {
this.logger.info('downloadUsersInCsv');
this.userService.setLanguage(language);
const {
file,
filename
} = await this.userService.downloadUsersInCsv(user.customer_id);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@Get('hierarchies')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: GetAllHierarchiesRes })
+11
View File
@@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
error: (...args) => args,
+30
View File
@@ -21,6 +21,7 @@ import {
IUserByCustomer,
SetUserRolesReq,
UpdateUserReq,
UserReporter,
} from './dtos/entities';
import { RolesService } from '../roles/roles.service';
import { HIERARCHIES } from './hierarchies';
@@ -29,6 +30,7 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces
import { EnrichErrorCode } from 'src/utils/ErrorBuilder';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@Injectable()
export class UsersService implements OnModuleInit {
@@ -78,6 +80,34 @@ export class UsersService implements OnModuleInit {
};
}
async downloadUsersInCsv(customerId: string) {
const { users } = await lastValueFrom(
this.usersClientService.UserFindAllByCustomerId({ customerId }),
);
const formatUsers: UserReporter[] = users.map(user => ({
createdAt: user.createdAt,
email: user.email,
lastLogin: user.lastLogin,
mfaStatus: user.mfaStatus,
name: user.name,
status: user.status,
updatedAt: user.updatedAt
}))
const parser = ParserBuilder.build<UserReporter>('csv');
const file = await parser.parse(formatUsers);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_users_${timestamp}.csv`;
return {
file,
filename
}
}
async findOneById(id: string): Promise<{ user: IUserByCustomer }> {
const { user } = await lastValueFrom(
this.usersClientService.UserFindOneById({ id }),
+23
View File
@@ -0,0 +1,23 @@
import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';
@Injectable()
export class CacheService<T> {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
) {}
async get(key: string): Promise<T | null> {
return await this.cacheManager.get<T>(key);
}
async set(key: string, value: T): Promise<void> {
await this.cacheManager.set(key, value);
}
async delete(key: string) {
await this.cacheManager.del(key);
}
}
+22
View File
@@ -0,0 +1,22 @@
import { CacheModule, Module } from '@nestjs/common';
import { CacheService } from './cache.service';
import { redisStore } from 'cache-manager-ioredis-yet';
@Module({
imports: [
CacheModule.registerAsync({
useFactory: async () => ({
store: await redisStore({
ttl: 5 * 1000 * 60, // 5 minute
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT && Number(process.env.REDIS_PORT),
db: process.env.REDIS_DATABASE && Number(process.env.REDIS_DATABASE),
keyPrefix: 'maestro:sso',
}),
}),
}),
],
providers: [CacheService],
exports: [CacheService],
})
export class ServicesModule {}
+2 -2
View File
@@ -22,7 +22,7 @@ export class PDFParser<T> implements Parser<T> {
const html = await this.htmlParser.parse(data);
await page.setContent(html, {
waitUntil: 'networkidle0',
timeout: 30000,
timeout: 90000,
});
// Configurações adicionais para garantir um PDF válido
@@ -43,7 +43,7 @@ export class PDFParser<T> implements Parser<T> {
pageRanges: '',
tagged: true,
outline: false,
timeout: 30000,
timeout: 90000,
});
await browser.close();
+7 -1
View File
@@ -24,15 +24,21 @@ export async function getOauthSecrets() {
const path = process.env.SM_OAUTH_PATH;
const secretsManagerClient = new SecretsManagerClient({});
for (const key in secrets) {
console.log(`Fetching secret for ${key} from path ${path}/${key}`);
const getSecretComand = new GetSecretValueCommand({
SecretId: `${path}/${key}`,
});
const res = await secretsManagerClient
.send(getSecretComand)
.catch(() => null);
.catch((err) => {
console.error(`Error fetching secret for ${key}:`, err);
return null;
});
if (res) {
secrets[key] = JSON.parse(res.SecretString);
console.log(JSON.parse(res.SecretString).client_id)
} else {
console.log("No secret found for", key);
secrets[key] = {
client_id: 'id',
client_secret: 'id',
+5
View File
@@ -75,6 +75,10 @@ const CATALOG = {
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
};
const IDENTITY_PROVIDER = {
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',
INVALID_HEADER: 'IDENTITY_PROVIDER.INVALID_HEADER',
}
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
@@ -91,6 +95,7 @@ const ErrorCodes = {
TRANSFORMATION,
CONNECTION,
CATALOG,
IDENTITY_PROVIDER
};
export default ErrorCodes;