Compare commits

...
Author SHA1 Message Date
Claude e0bfe03898 DOCS: Map maestro-pifactory integration points for migration
Add comprehensive documentation mapping all integration points between
maestro and pi-factory services to facilitate removing the dependency.

Includes:
- 3 gRPC client configurations using PIFACTORY_URL
- 36+ gRPC method calls across Catalog, PipelineV2, and legacy Pipeline services
- 44 REST endpoints that proxy to pi-factory
- Configuration files and environment variables
- Proto package dependencies (protospack and protospack-v2)
- Migration strategy options and recommendations
- Comparison with in-factory migration scope
2025-12-17 18:21:38 +00:00
Claude 177ec6368c DOCS: Map maestro-infactory integration points for migration
Add comprehensive documentation mapping all integration points between
maestro and in-factory services to facilitate removing the dependency.

Includes:
- All 6 gRPC client configurations using INFACTORY_URL
- 27+ gRPC method calls across 6 services
- 25 REST endpoints that proxy to in-factory
- Configuration files and environment variables
- Proto package dependencies
- Migration strategy options and recommendations
2025-12-17 17:09:50 +00:00
Marcos Rodrigues Silva e7f410831f Merge pull request #395 from dadosfera/beta
Beta
2025-12-01 17:47:24 -03:00
Marcos Rodrigues Silva cd2c53b5c5 Merge pull request #398 from dadosfera/feat/user-properties
FIX: user response
2025-12-01 17:36:56 -03:00
Marcos Rodrigues adf1b3b97e FIX: user response 2025-12-01 17:30:37 -03:00
Marcos Rodrigues Silva d84b5e184b Merge pull request #397 from dadosfera/feat/user-properties
FEAT: include more user info in /me
2025-12-01 12:36:16 -03:00
Marcos Rodrigues 6d608a0457 FEAT: include more user info in /me 2025-12-01 12:26:33 -03:00
Marcos Rodrigues Silva b8be2c7803 Merge pull request #396 from dadosfera/fix/api-key-catalog-roles
FIX: api key catalog roles
2025-12-01 09:34:21 -03:00
Marcos Rodrigues 24fce721e3 FIX: api key catalog roles 2025-12-01 09:32:26 -03:00
Marcos Rodrigues Silva 23a9a27db1 Merge pull request #394 from dadosfera/fix/private-front-cors
FIX: 404 in  get public key
2025-11-28 16:44:54 -03:00
Marcos Rodrigues 0f5ed50af9 FIX: 404 in get public key 2025-11-28 16:37:38 -03:00
Marcos Rodrigues Silva 5f8f6a64ab Merge pull request #393 from dadosfera/fix/private-front-cors
FIX: include private frontend url in cors origins list
2025-11-28 10:53:45 -03:00
Marcos Rodrigues 8800ac2736 FIX: include private frontend url in cors origins list 2025-11-28 10:50:31 -03:00
Marcos Rodrigues Silva bdb82c2ce4 Merge pull request #392 from dadosfera/fix/authenticated-track-events
Fix/authenticated track events
2025-11-27 09:45:46 -03:00
Marcos Rodrigues 6d9ecc3568 FIX: mixpanel events 2025-11-27 09:43:43 -03:00
Marcos Rodrigues Silva 7764447adc Merge pull request #391 from dadosfera/fix/private-ingress-annotations
FIX: ci to oracle github worker
2025-11-25 11:37:29 -03:00
Marcos Rodrigues 576fdecf89 FIX: ci to oracle github worker 2025-11-25 11:27:04 -03:00
Marcos Rodrigues Silva 3de1e90fa8 Merge pull request #390 from dadosfera/fix/private-ingress-annotations
Fix/private ingress annotations
2025-11-25 11:12:58 -03:00
Marcos Rodrigues 5e90950660 FIX: restore whitelist annotation 2025-11-25 10:59:06 -03:00
Marcos Rodrigues 3ed26e648f FIX: ci 2025-11-25 10:48:06 -03:00
Marcos Rodrigues Silva fff3523152 Merge pull request #389 from dadosfera/fix/private-ingress-annotations
FIX: private ingress annotations
2025-11-25 10:38:52 -03:00
Marcos Rodrigues 19e4daeea4 FIX: private ingress annotations 2025-11-25 10:35:08 -03:00
Marcos Rodrigues Silva 8624d3f016 Merge pull request #388 from dadosfera/fix/reset-theme
Fix/reset theme
2025-11-21 11:01:48 -03:00
Marcos Rodrigues 4f4da5bebe FIX: reset theme 2025-11-21 10:51:31 -03:00
Marcos Rodrigues f00d2bf41d FIX: reset cstumer theme 2025-11-21 09:55:39 -03:00
Marcos Rodrigues Silva dc1d1400d8 Merge pull request #387 from dadosfera/feat/lineage
FIX: change wget to curl
2025-11-13 18:19:20 -03:00
Marcos Rodrigues d4451153a3 FIX: change wget to curl 2025-11-13 18:14:47 -03:00
Marcos Rodrigues Silva 06ba759ea3 Merge pull request #386 from dadosfera/feat/lineage
Feat/lineage
2025-11-13 17:58:33 -03:00
Marcos Rodrigues dd0d08ad6e FEAT: add permissions to lineage module 2025-11-13 17:48:55 -03:00
Marcos Rodrigues ec81082877 FIX: resolve cors from localhost 2025-11-13 17:48:38 -03:00
Marcos Rodrigues Silva 7f58dc7090 Merge pull request #385 from dadosfera/hotfix/import-files
FIX: permissions
2025-10-22 18:35:35 -03:00
Marcos Rodrigues 43aa379c06 FIX: permissions 2025-10-22 18:29:43 -03:00
Marcos Rodrigues Silva c284f8753c Merge pull request #384 from dadosfera/hotfix/import-files
FIX: pipelines v2 permissions
2025-10-22 17:48:26 -03:00
Marcos Rodrigues 3a8f2495c4 FIX: pipelines v2 permissions 2025-10-20 18:13:54 -03:00
Marcos Rodrigues Silva af248716ef Merge pull request #383 from dadosfera/beta
Beta
2025-10-16 18:15:08 -03:00
Marcos Rodrigues Silva a57ad41ad4 Merge pull request #382 from dadosfera/features/roles
Features/roles
2025-10-16 17:21:37 -03:00
Marcos Rodrigues ffddceec3b FEAT: roles 2025-10-16 17:17:01 -03:00
Marcos Rodrigues Silva b04d5bb402 Merge pull request #381 from dadosfera/feature/tableau-dash
UPDATE: default roles and user
2025-09-23 18:10:35 -03:00
Marcos Rodrigues 53df3caf3f UPDATE: default roles and user 2025-09-23 18:09:25 -03:00
Marcos Rodrigues Silva ab2e35b54f Merge pull request #380 from dadosfera/beta
Beta
2025-09-17 18:11:05 -03:00
Marcos Rodrigues Silva bfa77d8f7b Merge pull request #379 from dadosfera/feature/auth-cookie
Feature/auth cookie
2025-09-16 17:40:10 -03:00
marcos-silva-rodrigues 5002d147ad UPDATE: add all dadosfera domains 2025-09-16 17:33:19 -03:00
Marcos Rodrigues Silva cd55dc0dc4 Merge pull request #378 from dadosfera/hotfix/uptime-ip
UPDATE: uptime internal ip
2025-09-15 18:04:14 -03:00
marcos-silva-rodrigues f1d56e4c2c UPDATE: uptime internal ip 2025-09-15 10:34:36 -03:00
Marcos Rodrigues Silva bc49f79cb3 Merge pull request #373 from dadosfera/hotfix/oracle
Network policies
2025-09-09 18:08:28 -03:00
marcos-silva-rodrigues 00163ad894 UPDATE: network policies 2025-09-09 11:17:45 -03:00
Marcos Rodrigues Silva a2fbeb97cc Merge pull request #377 from dadosfera/feature/auth-cookie
UPDATE: test cors domain
2025-09-08 15:25:05 -03:00
marcos-silva-rodrigues 288a796f46 UPDATE: test cors domain 2025-09-08 15:22:26 -03:00
Marcos Rodrigues Silva b5a1e93770 Merge pull request #376 from dadosfera/feature/auth-cookie
UPDATE: cookie path
2025-09-08 10:32:32 -03:00
marcos-silva-rodrigues 1ee49ceab8 UPDATE: cookie path 2025-09-08 10:30:56 -03:00
Marcos Rodrigues Silva 6eaf9cf6d0 Merge pull request #375 from dadosfera/feature/auth-cookie
UDATE: test cookie domain dadosfera.ai
2025-09-08 09:32:21 -03:00
marcos-silva-rodrigues 075ca747df FIX: ghost commit 2025-09-08 09:26:47 -03:00
marcos-silva-rodrigues 9ead4588c1 UDATE: test cookie domain dadosfera.ai 2025-09-08 09:25:14 -03:00
Marcos Rodrigues Silva 92b64bb362 Merge pull request #374 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-05 10:25:38 -03:00
marcos-silva-rodrigues cb8798d871 FIX: enable restricted ips 2025-09-05 10:21:11 -03:00
marcos-silva-rodrigues be70f4da09 FIX: disabled network policies 2025-09-04 10:02:20 -03:00
Marcos Rodrigues Silva b436d7de7a Merge pull request #372 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-03 16:49:31 -03:00
Marcos Rodrigues Silva 6780167f2b Merge pull request #371 from dadosfera/hotfix/oracle
FIX: infinite loading when login is incorrect
2025-09-03 16:31:28 -03:00
marcos-silva-rodrigues 2a3ab4228f FIX: infinite loading when login is incorrect 2025-09-03 16:25:46 -03:00
Marcos Rodrigues Silva caeaf62a9d Merge pull request #370 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-03 12:21:17 -03:00
marcos-silva-rodrigues 8e605aa361 FIX: affinity and tls config local 2025-09-03 12:19:30 -03:00
Marcos Rodrigues Silva fa4267f54a Merge pull request #369 from dadosfera/hotfix/oracle
UPDATE: oracle vpn
2025-08-29 10:16:45 -03:00
marcos-silva-rodrigues 47d8ca2760 UPDATE: oracle vpn 2025-08-29 10:12:58 -03:00
Marcos Rodrigues Silva f3c51e5328 Merge pull request #368 from dadosfera/beta
UPDATE: fixed cloud oracle
2025-08-28 18:54:02 -03:00
marcos-silva-rodrigues dcf70fdaae UPDATE: fixed cloud oracle 2025-08-28 18:40:43 -03:00
Marcos Rodrigues Silva 70f663374d Merge pull request #366 from dadosfera/beta
MIgração para oracle
2025-08-28 18:32:59 -03:00
Marcos Rodrigues Silva 38f2317bb6 Merge pull request #367 from dadosfera/chore/oracle
UPDATE: affinity and log
2025-08-28 17:05:32 -03:00
marcos-silva-rodrigues 31e0ca6c91 UPDATE: affinity and log 2025-08-28 16:52:52 -03:00
Marcos Rodrigues Silva dc063cbf3f Merge pull request #365 from dadosfera/chore/oracle
Chore/oracle
2025-08-27 17:45:34 -03:00
marcos-silva-rodrigues a4c82ae4a8 UPDATE:migration to oracle 2025-08-27 17:38:26 -03:00
Marcos Rodrigues Silva 11a65e11d1 Merge pull request #364 from dadosfera/chore/oracle
UPDATE: redis host
2025-08-27 14:15:27 -03:00
marcos-silva-rodrigues 73d47f0ff5 UPDATE: merge 2025-08-27 14:13:17 -03:00
marcos-silva-rodrigues d36e532c8e UPDATE: redis host 2025-08-27 13:58:31 -03:00
Marcos Rodrigues Silva b2a24e3a49 Merge pull request #363 from dadosfera/feature/auth-session
UPDATE: test subdomain
2025-08-14 17:51:36 -03:00
marcos-silva-rodrigues 301b6e98ec UPDATE: test subdomain 2025-08-14 17:42:42 -03:00
Marcos Rodrigues Silva 381a401ebb Merge pull request #362 from dadosfera/feature/auth-session
CI: add cookie secret
2025-08-14 17:27:09 -03:00
marcos-silva-rodrigues ba53934068 CI: add cookie secret 2025-08-14 17:23:40 -03:00
Marcos Rodrigues Silva 1b6846532f Merge pull request #361 from dadosfera/feature/auth-session
CHORE: update nginx ingress annotations
2025-08-14 16:07:31 -03:00
marcos-silva-rodrigues 216349f303 CHORE: update nginx ingress annotations 2025-08-14 16:03:28 -03:00
Marcos Rodrigues Silva 878c9cf450 Merge pull request #360 from dadosfera/feature/auth-session
FIX: set cookies if exists tokens
2025-08-14 15:39:57 -03:00
marcos-silva-rodrigues a654baef13 FIX: set cookies if exists tokens 2025-08-14 15:37:17 -03:00
Marcos Rodrigues Silva df852449b2 Merge pull request #359 from dadosfera/feature/auth-session
Feature/auth session
2025-08-14 15:28:23 -03:00
marcos-silva-rodrigues 04d761ca14 Merge branch 'beta' into feature/auth-session 2025-08-14 15:26:15 -03:00
Marcos Rodrigues Silva 9b168612a4 Merge pull request #358 from dadosfera/beta
Beta
2025-08-08 11:23:02 -03:00
Marcos Rodrigues Silva a772804dcd Merge pull request #357 from dadosfera/chore/oracle
CI: fix multi cloud deploy and maestro env
2025-08-07 18:04:10 -03:00
marcos-silva-rodrigues 8a399fbbcf CI: fix multi cloud deploy and maestro env 2025-08-07 18:03:22 -03:00
Marcos Rodrigues Silva 94049d6e8e Merge pull request #356 from dadosfera/chore/oracle
CI: oracle
2025-08-07 12:27:08 -03:00
marcos-silva-rodrigues c28cc58ac7 CI: oracle 2025-08-07 12:24:44 -03:00
Marcos Rodrigues Silva 56a655bb90 Merge pull request #355 from dadosfera/chore/oracle
CI: fix env home
2025-08-07 12:16:02 -03:00
marcos-silva-rodrigues b3d4a6b028 CI: fix env home 2025-08-07 12:13:19 -03:00
Marcos Rodrigues Silva c7bd1f761a Merge pull request #354 from dadosfera/chore/oracle
CI: remove k8s-setup need
2025-08-07 12:01:43 -03:00
marcos-silva-rodrigues 09c045455e CI: remove k8s-setup need 2025-08-07 11:59:37 -03:00
Marcos Rodrigues Silva 52d651126e Merge pull request #353 from dadosfera/chore/oracle
CI: revert k8s-setup workflow for the step in job helmfile-deploy
2025-08-07 11:57:38 -03:00
marcos-silva-rodrigues 2f11b1e78f CI: revert k8s-setup workflow for the step in job helmfile-deploy 2025-08-07 11:56:54 -03:00
Marcos Rodrigues Silva 885e9b71bb Merge pull request #352 from dadosfera/fix/tracker-sso
Fix/tracker sso
2025-08-07 11:45:44 -03:00
marcos-silva-rodrigues 193031dfdd UPDATE: increase redis ttl 2025-08-07 10:41:48 -03:00
marcos-silva-rodrigues f626a9bb5e UPDATE: add more logger 2025-08-07 10:32:22 -03:00
Marcos Rodrigues Silva ac26ad9f44 Merge pull request #351 from dadosfera/chore/oracle
FIX: correct name
2025-08-01 18:06:27 -03:00
marcos-silva-rodrigues fbe00af6dc FIX: correct name 2025-08-01 18:05:19 -03:00
Marcos Rodrigues Silva bfa9929b69 Merge pull request #350 from dadosfera/chore/oracle
Chore/oracle
2025-08-01 18:01:38 -03:00
marcos-silva-rodrigues bada7000f6 CHORE: change ci to oracle cloud 2025-08-01 18:01:10 -03:00
Marcos Rodrigues Silva 2c908d8d95 Merge pull request #349 from dadosfera/beta
Beta
2025-07-31 15:24:28 -03:00
Marcos Rodrigues Silva da35633d42 Merge pull request #348 from dadosfera/hotfix/module-assigned
Hotfix/module assigned
2025-07-31 12:04:02 -03:00
marcos-silva-rodrigues cc7e06013f FIX: change module permission 2025-07-31 11:59:19 -03:00
Marcos Rodrigues Silva 5ec36a05b5 Merge pull request #344 from dadosfera/beta
Beta
2025-07-30 11:56:22 -03:00
Marcos Rodrigues Silva f61e46c7c1 Merge pull request #347 from dadosfera/release/07-29
FEAT: add permission embed access
2025-07-30 11:15:28 -03:00
marcos-silva-rodrigues b9013b1493 FEAT: add permission embed access 2025-07-30 11:12:08 -03:00
Marcos Rodrigues Silva 5267fa6d2f Merge pull request #346 from dadosfera/release/07-29
FIX: comment test
2025-07-29 18:05:26 -03:00
marcos-silva-rodrigues 7f40cbdeed FIX: comment test 2025-07-29 18:03:20 -03:00
Marcos Rodrigues Silva b590c9402d Merge pull request #345 from dadosfera/release/07-29
Release/07 29
2025-07-29 15:19:52 -03:00
marcos-silva-rodrigues 9546feda11 FIX: merge 2025-07-29 15:12:25 -03:00
Marcos Rodrigues Silva 3587f82976 Merge pull request #343 from dadosfera/bugfix/public-link
Bugfix/public link
2025-07-29 12:31:34 -03:00
marcos-silva-rodrigues 94a3962590 FIX: add needs steps 2025-07-29 12:26:21 -03:00
marcos-silva-rodrigues 3b2d79a451 CHORE: remove needs step 2025-07-29 12:06:33 -03:00
marcos-silva-rodrigues da56a645af CHORE: update affinity to azure 2025-07-29 11:57:10 -03:00
Marcos Rodrigues Silva af4e55af1d Merge pull request #342 from dadosfera/bugfix/public-link
CHORE: remove sudo
2025-07-29 11:35:09 -03:00
marcos-silva-rodrigues d6a08a8f82 CHORE: remove sudo 2025-07-29 11:32:02 -03:00
Marcos Rodrigues Silva c73eb8869f Merge pull request #341 from dadosfera/bugfix/public-link
CHORE: install kubectl
2025-07-29 11:15:24 -03:00
marcos-silva-rodrigues 929fc97ac6 CHORE: install kubectl 2025-07-29 11:12:47 -03:00
Marcos Rodrigues Silva 7594a75bea Merge pull request #340 from dadosfera/bugfix/public-link
CHORE: update ci to wait extract_environment job
2025-07-29 11:02:14 -03:00
marcos-silva-rodrigues ec36a056b2 CHORE: update ci to wait extract_environment job 2025-07-29 10:59:35 -03:00
Marcos Rodrigues Silva 8d30f6ef15 Merge pull request #339 from dadosfera/bugfix/public-link
CHORE: multi cloud deployment
2025-07-29 10:57:26 -03:00
marcos-silva-rodrigues 08d1e3164f CHORE: multi cloud deployment 2025-07-29 10:53:41 -03:00
Marcos Rodrigues Silva a5f3ef20fe Merge pull request #338 from dadosfera/bugfix/public-link
FIX: add user by bearer token if it exists
2025-07-29 09:51:17 -03:00
marcos-silva-rodrigues 467445fe05 FIX: add user by bearer token if it exists 2025-07-29 09:47:56 -03:00
Marcos Rodrigues Silva d0448f844a Merge pull request #337 from dadosfera/bugfix/google-oauth
UPDATE: more logs
2025-07-21 16:06:22 -03:00
Marcos Rodrigues Silva 9b42754c69 Merge pull request #336 from dadosfera/bugfix/google-oauth
Bugfix/google oauth
2025-07-21 15:06:49 -03:00
Marcos Rodrigues Silva cc530fb12d Merge pull request #335 from dadosfera/release/sso
Release/sso
2025-07-18 17:12:51 -03:00
marcos-silva-rodrigues 19877f1b29 FIX: merge with main 2025-07-18 16:46:19 -03:00
marcos-silva-rodrigues e9880bcf68 FIX: add logger in header 2025-07-18 15:56:23 -03:00
marcos-silva-rodrigues 4c7111dcb6 FIX: using referer headaer 2025-07-18 15:56:10 -03:00
marcos-silva-rodrigues e81cfdcce1 FIX: using only header origin 2025-07-18 15:55:56 -03:00
marcos-silva-rodrigues 3eba3f414f FIX: send redirect uri 2025-07-18 15:55:45 -03:00
marcos-silva-rodrigues fbeb8dfd91 FIX: redis env 2025-07-18 15:55:29 -03:00
marcos-silva-rodrigues dda3e8baaf FIX: add logger 2025-07-18 15:54:27 -03:00
marcos-silva-rodrigues c1e4f0b18a FIX: remove express session 2025-07-18 15:54:24 -03:00
Marcos Rodrigues Silva 68cc58e2fe Merge pull request #326 from dadosfera/release/11-07
Release/11 07
2025-07-14 10:54:14 -03:00
marcos-silva-rodrigues eb80967608 FIX: tests 2025-07-11 11:36:23 -03:00
marcos-silva-rodrigues 66fea0722d FEAT: enabled network check in app.dadosfera 2025-07-11 11:12:35 -03:00
marcos-silva-rodrigues c917061b97 Merge branch 'feature/export-users-and-assets' into release/11-07 2025-07-11 11:04:31 -03:00
Marcos Rodrigues Silva ab280ee152 Merge pull request #324 from dadosfera/chore/update-unimed-ips
CHORE: Update unimed ips
2025-07-08 09:22:26 -03:00
marcos-silva-rodrigues 16c8137160 CHORE: Update unimed ips 2025-07-07 17:26:26 -03:00
marcos-silva-rodrigues 8efa6790d6 FEAT: sign out endpoint 2025-07-04 18:03:08 -03:00
marcos-silva-rodrigues a381f50e88 FEAT: set cookie after user login 2025-07-03 15:52:55 -03:00
48 changed files with 2345 additions and 546 deletions
+7 -64
View File
@@ -142,68 +142,11 @@ jobs:
docker system prune --volumes -a -f
docker system df
helmfile-deploy:
k8s-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
env:
HOME: /home/runner
runs-on: [self-hosted, "prd-oracle"]
environment: ${{ needs.extract_environment.outputs.environment }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: 'oracle'
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
+137
View File
@@ -0,0 +1,137 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+482
View File
@@ -0,0 +1,482 @@
# Maestro ↔ In-Factory Migration Map
This document maps all integration points between Maestro and In-Factory that need to be addressed to remove the dependency.
## Architecture Overview
```mermaid
graph TD;
Frontend<-->Maestro;
Maestro<-->DUC;
Maestro<-->pi-factory;
Maestro<-->in-factory;
```
Maestro connects to **3 external microservices**:
- **DUC** (`DUC_URL`) - User management, authentication, permissions
- **PI-Factory** (`PIFACTORY_URL`) - Pipelines, Catalog services
- **IN-Factory** (`INFACTORY_URL`) - Connections, Connectors, Inputs, Network Config, Transformations
---
## Summary Table
| Category | Count | Impact Level |
|----------|-------|--------------|
| gRPC Client Configurations | 6 | HIGH |
| NestJS Modules | 6 | HIGH |
| REST Controllers/Endpoints | 6 | HIGH |
| Service Classes | 6 | HIGH |
| Configuration Files | 5 | MEDIUM |
| Proto Package Dependencies | 2 | HIGH |
| Environment Variables | 2 | LOW |
---
## 1. gRPC CLIENT CONFIGURATIONS (Files to Migrate)
These files configure gRPC connections to In-Factory services:
### 1.1 Connection Client (`INFACTORY_URL`)
**File:** `src/modules/connection/client.config.ts`
```typescript
// Lines 10-11, 18
process.env.INFACTORY_URL.startsWith('in-factory:')
process.env.INFACTORY_URL.includes('0.0.0.0')
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `ConnectionManager.ProtoPackages.WritePackage`
- `ConnectionManager.ProtoPackages.ReadPackage`
### 1.2 Connector Client (`INFACTORY_URL`)
**File:** `src/modules/connector/client.config.ts`
```typescript
// Lines 10-11, 18
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `ConnectorManager.ProtoPackages.WritePackage`
- `ConnectorManager.ProtoPackages.ReadPackage`
### 1.3 Inputs Client (`INFACTORY_URL`)
**File:** `src/modules/inputs/inputs-client.config.ts`
```typescript
// Lines 10-11, 17
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `Input.ProtoPackages.WritePackage`
- `Input.ProtoPackages.ReadPackage`
### 1.4 Network Config Client (`INFACTORY_URL`)
**File:** `src/modules/network-config/network-config-client.config.ts`
```typescript
// Lines 10-11, 19
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `NetworkConfig.ProtoPackages.WritePackage`
- `NetworkConfig.ProtoPackages.ReadPackage`
### 1.5 Connection Test Client (`INFACTORY_URL`)
**File:** `src/modules/connection-test/connection-test-client.config.ts`
```typescript
// Lines 10-11, 17
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `ConnectionTest.ProtoPackages.ReadPackage`
### 1.6 Transformations Client (`INFACTORY_URL`)
**File:** `src/modules/transformations/transformations-client.ts`
```typescript
// Lines 10-11, 18
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `Transformation.ProtoPackages.WritePackage`
---
## 2. SERVICE CLASSES (Business Logic to Migrate)
### 2.1 Connection Client Service
**File:** `src/modules/connection/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `CreateConnection()` | ConnectionManagerWriteServices | Write |
| `UpdateConnection()` | ConnectionManagerWriteServices | Write |
| `DeleteConnection()` | ConnectionManagerWriteServices | Write |
| `GetConnectionDetails()` | ConnectionManagerReadServices | Read |
| `GetAllConnections()` | ConnectionManagerReadServices | Read |
| `GetConnectorAvailableConnectionsByCustomer()` | ConnectionManagerReadServices | Read |
| `ValidatePlatformConnections()` | ConnectionManagerReadServices | Read |
**Key Methods:**
- `createConnection()`
- `updateConnection()`
- `deleteConnection()`
- `getConnections()`
- `getConnectionDetails()`
- `validatePlatformConnections()`
### 2.2 Connector Client Service
**File:** `src/modules/connector/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `RegisterConnector()` | ConnectorManagerWriteServices | Write |
| `UploadFile()` | ConnectorManagerWriteServices | Write |
| `RegisterMultipleConnectorsWithoutImage()` | ConnectorManagerWriteServices | Write |
| `UpdateConnectorByID()` | ConnectorManagerWriteServices | Write |
| `DeleteConnectorById()` | ConnectorManagerWriteServices | Write |
| `GetConnectors()` | ConnectorManagerReadServices | Read |
### 2.3 Inputs Service
**File:** `src/modules/inputs/inputs.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `CreateInput()` | InputWriteService | Write |
| `UpdateInput()` | InputWriteService | Write |
| `DeleteInput()` | InputWriteService | Write |
| `GetAvailableEntities()` | InputReadService | Read |
### 2.4 Network Config Service
**File:** `src/modules/network-config/network-config.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `NetworkConfigCreate()` | NetworkConfigWriteService | Write |
| `NetworkConfigUpdate()` | NetworkConfigWriteService | Write |
| `NetworkConfigDelete()` | NetworkConfigWriteService | Write |
| `NetworkConfigFindAll()` | NetworkConfigReadService | Read |
| `NetworkConfigFindOneById()` | NetworkConfigReadService | Read |
### 2.5 Connection Test Service
**File:** `src/modules/connection-test/connection-test.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `TestConnection()` | ConnectionTestReadService | Read |
### 2.6 Transformations Service
**File:** `src/modules/transformations/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `CreateTransformation()` | TransformationWriteService | Write |
| `UpdateTransformation()` | TransformationWriteService | Write |
| `DeleteTransformation()` | TransformationWriteService | Write |
| Various read operations | TransformationReadService | Read |
---
## 3. NESTJS MODULES (Module Registration)
These modules register the gRPC clients and export services:
| Module | File | Imports | Exports |
|--------|------|---------|---------|
| ConnectionModule | `src/modules/connection/connection.module.ts` | ConnectionClientConfiguration | ConnectionClientService |
| ConnectorModule | `src/modules/connector/connector.module.ts` | ConnectorClientConfiguration | ConnectorClientService |
| InputsModule | `src/modules/inputs/inputs.module.ts` | InputsGrpcClient | InputsService |
| NetworkConfigModule | `src/modules/network-config/network-config.module.ts` | NetworkConfigGrpcClient | NetworkConfigService |
| ConnectionTestModule | `src/modules/connection-test/connection-test.module.ts` | ConnectionTestClientConfiguration | ConnectionTestService |
| TransformationsModule | `src/modules/transformations/transformations.module.ts` | TransformationsClientConfiguration | TransformationsService |
**App Module Registration:** `src/app.module.ts` (Lines 15-21, 50-60)
---
## 4. REST CONTROLLERS (API Endpoints to Migrate)
### 4.1 Connection Controller
**File:** `src/modules/connection/connection.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/connections` | Create a new connection |
| PUT | `/connections/:id` | Update an existing connection |
| DELETE | `/connections/:id` | Delete a connection |
| GET | `/connections` | Get all connections |
| GET | `/connections/:id` | Get connection details |
| GET | `/connections/available/:connector_id` | Get available connections by connector |
### 4.2 Connector Controller
**File:** `src/modules/connector/connector.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/connectors` | Register a new connector |
| POST | `/connectors/uploads` | Upload multiple connector files |
| GET | `/connectors` | List all connectors |
| PUT | `/connectors/:id` | Update a connector |
| DELETE | `/connectors/:id` | Delete a connector |
### 4.3 Inputs Controller
**File:** `src/modules/inputs/inputs.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/inputs/available-entities/:plugin` | Get available entities for a plugin |
| POST | `/inputs` | Create an input |
| PATCH | `/inputs` | Update an input |
| DELETE | `/inputs/:id` | Delete an input |
### 4.4 Network Config Controller
**File:** `src/modules/network-config/network-config.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/network-configs` | Get all network configurations |
| GET | `/network-configs/:id` | Get network config by ID |
| POST | `/network-configs` | Create network configuration |
| PUT | `/network-configs/:id` | Update network configuration |
| DELETE | `/network-configs/:id` | Delete network configuration |
### 4.5 Connection Test Controller
**File:** `src/modules/connection-test/connection-test.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/connection-test` | Test a connection |
### 4.6 Transformations Controller
**File:** `src/modules/transformations/transformations.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/transformations` | Create transformation |
| GET | `/transformations` | Get transformations |
| PUT | `/transformations/:id` | Update transformation |
| DELETE | `/transformations/:id` | Delete transformation |
---
## 5. CONFIGURATION FILES
### 5.1 Helm Chart Values (Production)
**File:** `deploy/helm-chart/values.yaml`
```yaml
# Lines 32, 42
maestro:
in_factory_url: in-factory.dadosfera.ai
tr_factory_url: in-factory.dadosfera.ai # Transformation factory also uses in-factory
```
### 5.2 Helm Chart Values (Staging)
**File:** `deploy/helm-chart/values-stg.yaml`
```yaml
# Lines 5-6
maestro:
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
```
### 5.3 Helmfiles (Production)
**File:** `deploy/helmfiles/prd.yaml`
```yaml
# Lines 15-18, 39-42 (for both maestro and maestro-unimed releases)
maestro.in_factory_url: in-factory.dadosfera.ai
maestro.tr_factory_url: in-factory.dadosfera.ai
```
### 5.4 Deployment Template
**File:** `deploy/helm-chart/templates/deployment.yaml`
```yaml
# Lines 71-72
- name: INFACTORY_URL
value: {{ .Values.maestro.in_factory_url }}
```
### 5.5 Environment Type Definition
**File:** `environment.d.ts`
```typescript
// Line 9
INFACTORY_URL: string;
```
---
## 6. PROTO PACKAGE DEPENDENCIES
### 6.1 Package.json
**File:** `package.json`
```json
{
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.14"
}
```
### 6.2 Proto Imports from `@dadosfera/protospack-v2` (In-Factory related)
| Import Path | Used In |
|-------------|---------|
| `ConnectionManager` | connection/client.config.ts, connection/client.service.ts |
| `ConnectionManager/interfaces/messages` | connection/client.service.ts, connection/dtos/connection.ts |
| `ConnectionManager/interfaces/entities` | connection/dtos/connection.ts |
| `ConnectorManager` | connector/client.config.ts, connector/client.service.ts |
| `Input` | inputs/inputs-client.config.ts, inputs/inputs.service.ts |
| `Input/interfaces/messages` | inputs/inputs.service.ts |
| `Input/interfaces/entities` | inputs/inputs.controller.ts, inputs/inputs.service.ts |
| `NetworkConfig` | network-config/network-config-client.config.ts, network-config/network-config.service.ts |
| `NetworkConfig/interfaces/entities` | network-config/dto/network-config.ts |
| `ConnectionTest` | connection-test/connection-test-client.config.ts, connection-test/connection-test.service.ts |
| `Transformation` | transformations/transformations-client.ts, transformations/client.service.ts |
---
## 7. MIGRATION STRATEGY OPTIONS
### Option A: Move In-Factory functionality INTO Maestro
**Pros:**
- Single service to maintain
- No network latency for these operations
- Simpler deployment
**Cons:**
- Increases Maestro's responsibility/complexity
- Requires database access from Maestro
- May require significant refactoring
**Files to create/migrate:**
1. Database models for connections, connectors, inputs, network-configs, transformations
2. Repository layer for database operations
3. Convert gRPC services to internal services
4. Remove all gRPC client configurations
### Option B: Create REST API wrapper in In-Factory
**Pros:**
- Minimal changes to Maestro
- Can migrate incrementally
**Cons:**
- Still maintains dependency
- Additional REST→gRPC translation layer
### Option C: Direct database access from Maestro
**Pros:**
- Removes runtime dependency
- Better performance
**Cons:**
- Shared database coupling
- Complex migration
---
## 8. FILES TO MODIFY/DELETE (Summary)
### High Priority - Core Integration Files
```
src/modules/connection/client.config.ts → DELETE or REPLACE
src/modules/connection/client.service.ts → REPLACE with local implementation
src/modules/connector/client.config.ts → DELETE or REPLACE
src/modules/connector/client.service.ts → REPLACE with local implementation
src/modules/inputs/inputs-client.config.ts → DELETE or REPLACE
src/modules/inputs/inputs.service.ts → REPLACE with local implementation
src/modules/network-config/network-config-client.config.ts → DELETE or REPLACE
src/modules/network-config/network-config.service.ts → REPLACE with local implementation
src/modules/connection-test/connection-test-client.config.ts → DELETE or REPLACE
src/modules/connection-test/connection-test.service.ts → REPLACE with local implementation
src/modules/transformations/transformations-client.ts → DELETE or REPLACE
src/modules/transformations/client.service.ts → REPLACE with local implementation
```
### Medium Priority - Module Registration
```
src/modules/connection/connection.module.ts → UPDATE imports
src/modules/connector/connector.module.ts → UPDATE imports
src/modules/inputs/inputs.module.ts → UPDATE imports
src/modules/network-config/network-config.module.ts → UPDATE imports
src/modules/connection-test/connection-test.module.ts → UPDATE imports
src/modules/transformations/transformations.module.ts → UPDATE imports
src/app.module.ts → UPDATE if module structure changes
```
### Low Priority - Configuration
```
deploy/helm-chart/values.yaml → REMOVE in_factory_url, tr_factory_url
deploy/helm-chart/values-stg.yaml → REMOVE in_factory_url, tr_factory_url
deploy/helmfiles/prd.yaml → REMOVE in_factory_url references
deploy/helm-chart/templates/deployment.yaml → REMOVE INFACTORY_URL env var
environment.d.ts → REMOVE INFACTORY_URL type
README.md → UPDATE architecture diagram
```
---
## 9. DEPENDENCY COUNT BY MODULE
| Module | Files | gRPC Calls | REST Endpoints |
|--------|-------|------------|----------------|
| Connection | 4 | 7 | 6 |
| Connector | 3 | 6 | 5 |
| Inputs | 3 | 4 | 4 |
| Network Config | 3 | 5 | 5 |
| Connection Test | 3 | 1 | 1 |
| Transformations | 4 | 4+ | 4 |
| **TOTAL** | **20** | **27+** | **25** |
---
## 10. DATA MODELS (Proto Messages Used)
### Connection Manager
- `CreateConnectionRequest` / `CreateConnectionResponse`
- `UpdateConnectionRequest` / `UpdateConnectionResponse`
- `DeleteConnectionRequest` / `DeleteConnectionResponse`
- `GetConnectionDetailsRequest` / `GetConnectionDetailsResponse`
- `GetAllConnectionsRequest` / `GetAllConnectionsResponse`
- `Connection` (entity)
- `ConnectionCredential` (entity)
### Connector Manager
- `RegisterConnectorRequest` / `RegisterConnectorResponse`
- `UploadFileRequest` / `UploadFileResponse`
- `GetConnectorsRequest` / `GetConnectorsResponse`
- `Connector` (entity)
### Input
- `CreateInputRequest` / `CreateInputResponse`
- `UpdateInputRequest` / `UpdateInputResponse`
- `DeleteInputRequest` / `DeleteInputResponse`
- `GetAvailableEntitiesRequest` / `GetAvailableEntitiesResponse`
- `Info` (entity)
### Network Config
- `NetworkConfigCreateRequest` / `NetworkConfigCreateResponse`
- `NetworkConfigUpdateRequest` / `NetworkConfigUpdateResponse`
- `NetworkConfigDeleteRequest` / `NetworkConfigDeleteResponse`
- `NetworkConfigFindAllRequest` / `NetworkConfigFindAllResponse`
- `NetworkConfig` (entity)
### Connection Test
- `TestConnectionRequest` / `TestConnectionResponse`
### Transformation
- `CreateTransformationRequest` / `CreateTransformationResponse`
- `UpdateTransformationRequest` / `UpdateTransformationResponse`
- `DeleteTransformationRequest` / `DeleteTransformationResponse`
---
## NEXT STEPS
1. **Decide on migration strategy** (Option A, B, or C)
2. **Prioritize modules** - Recommend starting with Connection Test (smallest), then Inputs, Network Config, Transformations, Connection, Connector (largest)
3. **Create database schema** if moving to Option A
4. **Implement local services** one module at a time
5. **Update tests** for each migrated module
6. **Update deployment configs** to remove INFACTORY_URL
7. **Coordinate with In-Factory team** for data migration
+320
View File
@@ -0,0 +1,320 @@
# In-Factory Side - Migration Requirements
This document outlines what needs to be addressed in the **In-Factory** service to remove its coupling with Maestro.
> **Note:** This analysis is based on the Maestro codebase. For a complete analysis, the In-Factory repository should also be reviewed.
---
## Current Architecture (In-Factory → Maestro)
Based on the Maestro codebase analysis, **In-Factory** exposes the following gRPC services that Maestro consumes:
```mermaid
graph LR;
Maestro -->|gRPC| InFactory;
subgraph InFactory Services
CM[ConnectionManager]
ConM[ConnectorManager]
IN[Input]
NC[NetworkConfig]
CT[ConnectionTest]
TR[Transformation]
end
Maestro --> CM;
Maestro --> ConM;
Maestro --> IN;
Maestro --> NC;
Maestro --> CT;
Maestro --> TR;
```
---
## gRPC Services Exposed by In-Factory
### 1. ConnectionManager Service
**Package:** `ConnectionManager` from `@dadosfera/protospack-v2`
#### Write Services (`ConnectionManagerWriteServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `CreateConnection` | Create a new data connection | Maestro POST /connections |
| `UpdateConnection` | Update an existing connection | Maestro PUT /connections/:id |
| `DeleteConnection` | Delete a connection | Maestro DELETE /connections/:id |
#### Read Services (`ConnectionManagerReadServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `GetConnectionDetails` | Get details of a single connection | Maestro GET /connections/:id |
| `GetAllConnections` | List all connections for a customer | Maestro GET /connections |
| `GetConnectorAvailableConnectionsByCustomer` | Get available connections by connector | Maestro GET /connections/available/:connector_id |
| `ValidatePlatformConnections` | Validate connections against platform | Maestro internal |
---
### 2. ConnectorManager Service
**Package:** `ConnectorManager` from `@dadosfera/protospack-v2`
#### Write Services (`ConnectorManagerWriteServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `RegisterConnector` | Register a new connector | Maestro POST /connectors |
| `UploadFile` | Upload connector files | Maestro POST /connectors/uploads |
| `RegisterMultipleConnectorsWithoutImage` | Bulk register connectors | Maestro internal |
| `UpdateConnectorByID` | Update a connector | Maestro PUT /connectors/:id |
| `DeleteConnectorById` | Delete a connector | Maestro DELETE /connectors/:id |
#### Read Services (`ConnectorManagerReadServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `GetConnectors` | List all connectors | Maestro GET /connectors |
---
### 3. Input Service
**Package:** `Input` from `@dadosfera/protospack-v2`
#### Write Services (`InputWriteService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `CreateInput` | Create a new input configuration | Maestro POST /inputs |
| `UpdateInput` | Update input configuration | Maestro PATCH /inputs |
| `DeleteInput` | Delete an input | Maestro DELETE /inputs/:id |
#### Read Services (`InputReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `GetAvailableEntities` | Get available entities for a plugin | Maestro GET /inputs/available-entities/:plugin |
---
### 4. NetworkConfig Service
**Package:** `NetworkConfig` from `@dadosfera/protospack-v2`
#### Write Services (`NetworkConfigWriteService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `NetworkConfigCreate` | Create network configuration | Maestro POST /network-configs |
| `NetworkConfigUpdate` | Update network configuration | Maestro PUT /network-configs/:id |
| `NetworkConfigDelete` | Delete network configuration | Maestro DELETE /network-configs/:id |
#### Read Services (`NetworkConfigReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `NetworkConfigFindAll` | List all network configs | Maestro GET /network-configs |
| `NetworkConfigFindOneById` | Get network config by ID | Maestro GET /network-configs/:id |
---
### 5. ConnectionTest Service
**Package:** `ConnectionTest` from `@dadosfera/protospack-v2`
#### Read Services (`ConnectionTestReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `TestConnection` | Test a connection's connectivity | Maestro POST /connection-test |
---
### 6. Transformation Service
**Package:** `Transformation` from `@dadosfera/protospack-v2`
#### Write Services (`TransformationWriteService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `CreateTransformation` | Create a transformation | Maestro POST /transformations |
| `UpdateTransformation` | Update a transformation | Maestro PUT /transformations/:id |
| `DeleteTransformation` | Delete a transformation | Maestro DELETE /transformations/:id |
#### Read Services (`TransformationReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| Various read operations | Get transformation details | Maestro GET /transformations |
---
## Proto Package Ownership
The proto definitions are managed in `@dadosfera/protospack-v2`:
```
@dadosfera/protospack-v2/
├── dist/lib/
│ ├── ConnectionManager/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── ConnectorManager/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── Input/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── NetworkConfig/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── ConnectionTest/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ └── Transformation/
│ ├── interfaces/messages.ts
│ └── interfaces/entities.ts
```
---
## In-Factory Migration Options
### Option 1: Expose REST API (Keep In-Factory, Add REST Layer)
**Changes Required in In-Factory:**
1. Add REST controllers for all services
2. Implement HTTP endpoints mirroring gRPC methods
3. Update deployment to expose HTTP port
4. Create OpenAPI documentation
**Pros:**
- Minimal architectural changes
- Can run both gRPC and REST in parallel during migration
- Maestro can switch to REST calls gradually
**Cons:**
- Still maintains service dependency
- Adds another communication layer
---
### Option 2: Move Logic to Maestro (Deprecate In-Factory for these features)
**Changes Required in In-Factory:**
1. Export database schema/migrations
2. Document all business logic
3. Provide data migration scripts
4. Deprecate gRPC endpoints after migration
**Changes Required in Maestro:**
1. Create database models
2. Implement repositories
3. Create service layer with same business logic
4. Run data migration
**Pros:**
- Removes runtime dependency completely
- Simplifies architecture
- One less service to maintain
**Cons:**
- Significant development effort
- Risk of business logic divergence during migration
- Database sharing concerns
---
### Option 3: Merge Services (Combine In-Factory into a larger service)
**Changes Required:**
1. Create new combined service
2. Migrate both In-Factory and relevant Maestro code
3. Update all clients
**Pros:**
- Clean architectural redesign
- Opportunity to optimize
**Cons:**
- Largest effort
- Risk of disruption
---
## Data Migration Considerations
### Entities Managed by In-Factory
Based on proto definitions, In-Factory manages:
1. **Connections**
- Connection credentials
- Connection metadata
- Customer associations
2. **Connectors**
- Connector definitions
- Connector images/files
- Plugin configurations
3. **Inputs**
- Input configurations
- Entity mappings
4. **Network Configs**
- Network configuration settings
- Security settings
5. **Transformations**
- Transformation definitions
- Transformation scripts
### Migration Steps
1. Export database schema from In-Factory
2. Create equivalent schema in target database
3. Write data migration scripts
4. Validate data integrity
5. Switch traffic
6. Decommission old service
---
## Recommended Investigation for In-Factory Team
1. **Check for Maestro dependencies in In-Factory**
- Does In-Factory call any Maestro APIs?
- Are there any shared databases?
- Any shared message queues?
2. **Document database schema**
- All tables related to connections, connectors, inputs, network-configs, transformations
- Foreign key relationships
- Indexes and constraints
3. **List all consumers**
- Besides Maestro, who else calls In-Factory?
- Are there other internal services?
- Any external integrations?
4. **Business logic documentation**
- Validation rules
- Business constraints
- Side effects (events, notifications, etc.)
---
## Timeline Considerations
| Phase | Description | Dependencies |
|-------|-------------|--------------|
| Phase 1 | Analysis & Planning | Both teams available |
| Phase 2 | Schema/API Design | Proto definitions finalized |
| Phase 3 | Implementation | Development resources |
| Phase 4 | Data Migration | Database access, downtime window |
| Phase 5 | Testing | QA resources, test environments |
| Phase 6 | Cutover | Deployment coordination |
| Phase 7 | Decommission | Monitoring, rollback plan |
---
## Questions for In-Factory Team
1. What database does In-Factory use? (PostgreSQL, MongoDB, etc.)
2. Are there any async operations? (message queues, event sourcing)
3. What is the current data volume for each entity type?
4. Are there any scheduled jobs or background processes?
5. What monitoring/alerting is in place?
6. Are there any data retention policies?
7. What is the backup/recovery strategy?
+486
View File
@@ -0,0 +1,486 @@
# Maestro ↔ PI-Factory Migration Map
This document maps all integration points between Maestro and PI-Factory that need to be addressed to remove the dependency.
## Architecture Overview
```mermaid
graph TD;
Frontend<-->Maestro;
Maestro<-->DUC;
Maestro<-->pi-factory;
Maestro<-->in-factory;
```
PI-Factory (`PIFACTORY_URL`) is responsible for:
- **Pipeline Management** - Create, read, update, delete pipelines
- **Catalog Services** - Data asset management, metadata, previews
- **Platform Interfaces** - Dataset cataloging operations
---
## Summary Table
| Category | Count | Impact Level |
|----------|-------|--------------|
| gRPC Client Configurations | 3 | HIGH |
| NestJS Modules | 3 | HIGH |
| REST Controllers/Endpoints | 3 (~50 endpoints) | HIGH |
| Service Classes | 4 | HIGH |
| Configuration Files | 4 | MEDIUM |
| Proto Package Dependencies | 2 | HIGH |
| Environment Variables | 1 | LOW |
---
## 1. gRPC CLIENT CONFIGURATIONS (Files to Migrate)
These files configure gRPC connections to PI-Factory services:
### 1.1 Catalog Client (`PIFACTORY_URL`)
**File:** `src/modules/catalog/catalog-client.ts`
```typescript
// Lines 11-12, 19
process.env.PIFACTORY_URL.startsWith('pi-factory:')
process.env.PIFACTORY_URL.includes('0.0.0.0')
url: process.env.PIFACTORY_URL
```
**Proto Services Used:**
- `Catalog.ProtoPackages.ReadPackage`
- `Catalog.ProtoPackages.WritePackage`
- `PlatformInterfaces.ProtoPackages.WritePackage`
### 1.2 Pipelines V2 Client (`PIFACTORY_URL`)
**File:** `src/modules/pipelinesV2/pipelines-client.ts`
```typescript
// Lines 13-14, 21
url: process.env.PIFACTORY_URL
```
**Proto Services Used:**
- `PipelineV2.ProtoPackages.ReadPackage`
- `PipelineV2.ProtoPackages.WritePackage`
### 1.3 Pipelines Client (Legacy) (`PIFACTORY_URL`)
**File:** `src/modules/pipelines/pipelines-client.ts`
```typescript
// Lines 10-11, 18
url: process.env.PIFACTORY_URL
```
**Proto Services Used:**
- `PipelinePackages` from `@dadosfera/protospack`
---
## 2. SERVICE CLASSES (Business Logic to Migrate)
### 2.1 Catalog Service
**File:** `src/modules/catalog/catalog.service.ts`
**gRPC Services Initialized:**
- `CatalogReadServices` (from `Catalog.ReadService`)
- `CatalogWriteServices` (from `Catalog.WriteService`)
- `PlatformInterfacesWriteServices` (from `PlatformInterfaces.WriteService`)
**gRPC Methods Called:**
| Method | Service | Description |
|--------|---------|-------------|
| `GetAllDataAssets()` | CatalogReadServices | Search/list data assets |
| `GetOneDataAsset()` | CatalogReadServices | Get single data asset by ID |
| `GetOneDataAssetByPipelineAndObject()` | CatalogReadServices | Get asset by pipeline/object |
| `GetDatasetDoc()` | CatalogReadServices | Get dataset documentation |
| `GetDatasetPreview()` | CatalogReadServices | Get data preview |
| `GetDatasetColumnsMetadata()` | CatalogReadServices | Get column metadata |
| `GetCustomerTags()` | CatalogReadServices | Get all tags for customer |
| `GetDatasetCatalogTask()` | CatalogReadServices | Get catalog task status |
| `GetRlsRules()` | CatalogReadServices | Get RLS rules |
| `GetOneRlsRule()` | CatalogReadServices | Get single RLS rule |
| `GetNimbusDashboards()` | CatalogReadServices | Get Nimbus dashboards |
| `CreateDataAsset()` | CatalogWriteServices | Create new data asset |
| `UpdateDataAsset()` | CatalogWriteServices | Update existing data asset |
| `DeleteDataAsset()` | CatalogWriteServices | Delete data asset |
| `ManagePermission()` | CatalogWriteServices | Manage asset permissions |
| `RevokePermission()` | CatalogWriteServices | Revoke asset permissions |
| `MakeAComment()` | CatalogWriteServices | Add comment to asset |
| `UpdateAComment()` | CatalogWriteServices | Update/delete comment |
| `TriggerDatasetCataloging()` | CatalogWriteServices | Trigger catalog process |
| `AddRlsRule()` | CatalogWriteServices | Add RLS rule |
| `RemoveRlsRule()` | CatalogWriteServices | Remove RLS rule |
| `RemoveRlsRulesByRlsId()` | CatalogWriteServices | Batch remove by RLS ID |
| `RemoveRlsRulesByDashboardId()` | CatalogWriteServices | Batch remove by dashboard |
| `GetPiiReporter()` | CatalogWriteServices | Get PII report data |
| `CatalogDataAssets()` | PlatformInterfacesWriteServices | Catalog datasets |
**Additional HTTP Calls to Nimbus:**
- `POST ${nimbusUrl}/api/catalog/data-docs/` - Create data docs
- `POST ${nimbusUrl}/api/catalog/table-metadata/` - Create table metadata
- `POST ${nimbusUrl}/api/catalog/column-metadata/` - Create column metadata
- `POST ${nimbusUrl}/api/catalog/data-preview/` - Create data preview
### 2.2 Pipelines V2 Service
**File:** `src/modules/pipelinesV2/pipelines.service.ts`
**gRPC Services Initialized:**
- `PipelineV2ReadService` (from `PipelineV2.ReadService`)
- `PipelineV2WriteService` (from `PipelineV2.WriteService`)
**gRPC Methods Called:**
| Method | Service | Description |
|--------|---------|-------------|
| `PipelineV2Create()` | PipelineV2WriteService | Create new pipeline |
| `PipelineV2Update()` | PipelineV2WriteService | Update pipeline |
| `PipelineV2Remove()` | PipelineV2WriteService | Delete pipeline |
| `PipelineV2UploadFile()` | PipelineV2WriteService | Initialize file upload |
| `PipelineV2CompleteUploadFile()` | PipelineV2WriteService | Complete file upload |
| `PipelineV2FindAll()` | PipelineV2ReadService | List all pipelines |
| `PipelineV2FindOne()` | PipelineV2ReadService | Get single pipeline |
| `PipelineV2FindObjects()` | PipelineV2ReadService | Get pipeline objects |
| `PipelineV2DownloadLogs()` | PipelineV2ReadService | Download pipeline logs |
| `PipelineV2GetDashboardUrl()` | PipelineV2ReadService | Get monitoring dashboard URL |
### 2.3 Pipelines Service (Legacy)
**File:** `src/modules/pipelines/pipelines.service.ts`
**Uses:** `PipelinesClientService`
**Methods:**
- `getPipelineStatus()` - Get pipeline execution status
- `runPipeline()` - Trigger pipeline execution
### 2.4 Pipelines Client Service (Legacy)
**File:** `src/modules/pipelines/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Description |
|--------|---------|-------------|
| `getPipelineStatus()` | PipelineService | Get pipeline status |
| `triggerPipeline()` | PipelineService | Trigger pipeline run |
---
## 3. NESTJS MODULES (Module Registration)
| Module | File | Client Configuration | Exports |
|--------|------|---------------------|---------|
| CatalogModule | `src/modules/catalog/catalog.module.ts` | CatalogClientConfiguration | CatalogService |
| PipelinesV2Module | `src/modules/pipelinesV2/pipelines.module.ts` | PipelinesClientConfiguration | PipelinesService |
| PipelinesModule | `src/modules/pipelines/pipelines.module.ts` | PipelinesClientConfiguration | PipelinesService, PipelinesClientService |
**App Module Registration:** `src/app.module.ts` (Lines 20, 23, 25, 53-54, 59)
---
## 4. REST CONTROLLERS (API Endpoints to Migrate)
### 4.1 Catalog Controller
**File:** `src/modules/catalog/catalog.controller.ts`
**Base Path:** `/catalog`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/catalog` | Search data assets |
| GET | `/catalog/download` | Download assets as CSV |
| GET | `/catalog/data-asset` | Get asset by pipeline/object |
| GET | `/catalog/data-asset/:id` | Get single data asset |
| GET | `/catalog/data-asset/rls/:id` | Get data asset RLS info |
| GET | `/catalog/data-asset/:id/columns-metadata` | Get column metadata |
| GET | `/catalog/data-asset/:id/preview` | Get data preview |
| GET | `/catalog/data-asset/:id/docs` | Get documentation |
| GET | `/catalog/tags` | Get all tags |
| PUT | `/catalog/data-asset/:id` | Update data asset |
| PUT | `/catalog/data-asset/:id/manage-permissions` | Manage permissions |
| PUT | `/catalog/data-asset/:id/revoke-permissions` | Revoke permissions |
| POST | `/catalog` | Create data asset |
| POST | `/catalog/data-asset/:id/docs` | Create documentation |
| POST | `/catalog/data-asset/:id/comment` | Add comment |
| POST | `/catalog/dataset-catalog-task` | Trigger catalog task |
| POST | `/catalog/rls-rule` | Add RLS rule |
| POST | `/catalog/register-dataset` | Register dataset with metadata |
| DELETE | `/catalog/data-asset/:id` | Delete data asset |
| DELETE | `/catalog/data-asset/:id/comment` | Delete comment |
| DELETE | `/catalog/rls-rule/:id` | Remove RLS rule |
| DELETE | `/catalog/rls-rule` | Batch remove RLS rules |
| GET | `/catalog/dataset-catalog-task/:session` | Get catalog task status |
| GET | `/catalog/rls-rule/:id` | Get single RLS rule |
| GET | `/catalog/rls-rule` | Get RLS rules |
| GET | `/catalog/nimbus-dashboards` | Get Nimbus dashboards |
| GET | `/catalog/pii-reporter` | Get PII report |
**Total: 27 endpoints**
### 4.2 Pipelines V2 Controller
**File:** `src/modules/pipelinesV2/pipelines.controller.ts`
**Base Path:** `/pipelinesV2`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/pipelinesV2/monitoring-dashboard` | Get monitoring dashboard URL |
| GET | `/pipelinesV2` | List all pipelines |
| GET | `/pipelinesV2/download-logs` | Download pipeline logs |
| GET | `/pipelinesV2/:id` | Get single pipeline |
| GET | `/pipelinesV2/:id/config` | Get pipeline properties |
| GET | `/pipelinesV2/:id/objects` | Get pipeline objects |
| GET | `/pipelinesV2/:id/status` | Get pipeline status (legacy) |
| POST | `/pipelinesV2` | Create pipeline |
| POST | `/pipelinesV2/init-upload` | Initialize file upload |
| POST | `/pipelinesV2/complete-upload` | Complete file upload |
| POST | `/pipelinesV2/file` | Upload file pipeline |
| POST | `/pipelinesV2/start/:id` | Start pipeline |
| PATCH | `/pipelinesV2/:id` | Update pipeline |
| PUT | `/pipelinesV2/:id` | Update pipeline (deprecated) |
| DELETE | `/pipelinesV2/:id` | Delete pipeline |
**Total: 15 endpoints**
### 4.3 Pipelines Controller (Legacy)
**File:** `src/modules/pipelines/pipelines.controller.ts`
**Base Path:** `/pipelines`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/pipelines/start/:id` | Start pipeline (deprecated) |
| GET | `/pipelines/:id/status` | Get pipeline status (deprecated) |
**Total: 2 endpoints (deprecated)**
---
## 5. CONFIGURATION FILES
### 5.1 Helm Chart Values (Production)
**File:** `deploy/helm-chart/values.yaml`
```yaml
# Line 40
maestro:
pi_factory_url: pi-factory.dadosfera.ai
```
### 5.2 Helm Chart Values (Staging)
**File:** `deploy/helm-chart/values-stg.yaml`
```yaml
# Line 4
maestro:
pi_factory_url: pi-factory.stg.dadosfera.ai
```
### 5.3 Helmfiles (Production)
**File:** `deploy/helmfiles/prd.yaml`
```yaml
# Lines 14, 38
maestro.pi_factory_url: pi-factory.dadosfera.ai
```
### 5.4 Deployment Template
**File:** `deploy/helm-chart/templates/deployment.yaml`
```yaml
# Line 85
- name: PIFACTORY_URL
value: {{ .Values.maestro.pi_factory_url }}
```
### 5.5 Environment Type Definition
**File:** `environment.d.ts`
```typescript
// Line 10
PIFACTORY_URL: string;
```
---
## 6. PROTO PACKAGE DEPENDENCIES
### 6.1 Package.json
**File:** `package.json`
```json
{
"@dadosfera/protospack": "2.5.3", // Legacy pipelines
"@dadosfera/protospack-v2": "3.38.0-beta.14" // PipelineV2, Catalog, PlatformInterfaces
}
```
### 6.2 Proto Imports from `@dadosfera/protospack-v2` (PI-Factory related)
| Import Path | Used In |
|-------------|---------|
| `Catalog` | catalog/catalog-client.ts, catalog/catalog.service.ts |
| `Catalog/interfaces/messages` | catalog/catalog.service.ts, catalog/catalog.controller.ts, catalog/dtos |
| `PlatformInterfaces` | catalog/catalog-client.ts, catalog/catalog.service.ts |
| `PipelineV2` | pipelinesV2/pipelines-client.ts, pipelinesV2/pipelines.service.ts |
| `PipelineV2/interfaces/messages` | pipelinesV2/pipelines.service.ts, pipelinesV2/pipelines.controller.ts |
### 6.3 Proto Imports from `@dadosfera/protospack` (Legacy)
| Import Path | Used In |
|-------------|---------|
| `PipelinePackages` | pipelines/pipelines-client.ts |
| `PipelineProtoFilePath` | pipelines/pipelines-client.ts |
| `PipelineServicesNames` | pipelines/client.service.ts |
| `PipelinesServiceInterface` | pipelines/client.service.ts |
---
## 7. FILES TO MODIFY/DELETE (Summary)
### High Priority - Core Integration Files
```
src/modules/catalog/catalog-client.ts → DELETE or REPLACE
src/modules/catalog/catalog.service.ts → REPLACE with local implementation
src/modules/pipelinesV2/pipelines-client.ts → DELETE or REPLACE
src/modules/pipelinesV2/pipelines.service.ts → REPLACE with local implementation
src/modules/pipelines/pipelines-client.ts → DELETE or REPLACE
src/modules/pipelines/client.service.ts → REPLACE with local implementation
src/modules/pipelines/pipelines.service.ts → REPLACE with local implementation
```
### Medium Priority - Module Registration
```
src/modules/catalog/catalog.module.ts → UPDATE imports
src/modules/pipelinesV2/pipelines.module.ts → UPDATE imports
src/modules/pipelines/pipelines.module.ts → UPDATE imports
src/app.module.ts → UPDATE if module structure changes
```
### Low Priority - Configuration
```
deploy/helm-chart/values.yaml → REMOVE pi_factory_url
deploy/helm-chart/values-stg.yaml → REMOVE pi_factory_url
deploy/helmfiles/prd.yaml → REMOVE pi_factory_url references
deploy/helm-chart/templates/deployment.yaml → REMOVE PIFACTORY_URL env var
environment.d.ts → REMOVE PIFACTORY_URL type
README.md → UPDATE architecture diagram
```
---
## 8. DEPENDENCY COUNT BY MODULE
| Module | Files | gRPC Calls | REST Endpoints |
|--------|-------|------------|----------------|
| Catalog | 3 | 24+ | 27 |
| PipelinesV2 | 3 | 10 | 15 |
| Pipelines (Legacy) | 3 | 2 | 2 |
| **TOTAL** | **9** | **36+** | **44** |
---
## 9. COMPARISON: PI-Factory vs In-Factory
| Aspect | PI-Factory | In-Factory |
|--------|------------|------------|
| Environment Variable | `PIFACTORY_URL` | `INFACTORY_URL` |
| Modules | 3 | 6 |
| gRPC Calls | 36+ | 27+ |
| REST Endpoints | 44 | 25 |
| Complexity | HIGH | MEDIUM-HIGH |
| Domain | Pipelines, Catalog | Connections, Connectors, Inputs |
---
## 10. DATA MODELS (Proto Messages Used)
### Catalog Messages
- `CreateDataAssetRequest` / `CreateDataAssetResponse`
- `GetAllDataAssetsRequest` / `GetAllDataAssetsResponse`
- `GetOneDataAssetRequest` / Response
- `UpdateDataAssetRequest` / Response
- `DeleteDataAssetRequest` / Response
- `ManagePermissionRequest` / Response
- `RevokePermissionRequest` / Response
- `MakeACommentRequest` / Response
- `UpdateACommentRequest` / Response
- `TriggerDatasetCatalogingRequest` / Response
- `GetDatasetCatalogTaskRequest` / Response
- `AddRlsRuleRequest` / Response
- `RemoveRlsRuleRequest` / Response
- `GetRlsRulesRequest` / Response
- `GetNimbusDashboardsRequest` / Response
- `PiiMetadata`
- `RegisterDatasetWithMetatadaRequest`
### PipelineV2 Messages
- `PipelineV2CreateRequest` / `PipelineV2CreateResponse`
- `PipelineV2FindAllRequest` / `PipelineV2FindAllResponse`
- `PipelineV2FindOneRequest` / `PipelineV2FindOneResponse`
- `PipelineV2UpdateRequest` / `PipelineV2UpdateResponse`
- `PipelineV2RemoveRequest` / Response
- `PipelineV2UploadFileRequest` / Response
- `PipelineV2CompleteUploadFileRequest` / Response
- `PipelineV2FindObjectsRequest` / Response
- `PipelineV2DownloadLogsRequest` / Response
- `PipelineV2GetDashboardUrlRequest` / Response
### Platform Interfaces Messages
- `CatalogDataAssetsRequest` / Response
---
## 11. MIGRATION STRATEGY OPTIONS
### Option A: Move PI-Factory functionality INTO Maestro
**Pros:**
- Single service to maintain
- No network latency for these operations
- Simpler deployment
**Cons:**
- Significantly increases Maestro's responsibility
- Requires database access from Maestro
- Large refactoring effort (44 endpoints)
### Option B: Create REST API wrapper in PI-Factory
**Pros:**
- Minimal changes to Maestro
- Can migrate incrementally
**Cons:**
- Still maintains dependency
- Additional REST→gRPC translation layer
### Option C: Direct database access from Maestro
**Pros:**
- Removes runtime dependency
- Better performance
**Cons:**
- Shared database coupling
- Complex migration
---
## 12. RECOMMENDED MIGRATION ORDER
Given the complexity, we recommend migrating in this order:
1. **Legacy Pipelines** (2 endpoints, deprecated) - Lowest risk
2. **PipelinesV2** (15 endpoints) - Core pipeline functionality
3. **Catalog** (27 endpoints) - Most complex, migrate last
### Phase 1: Legacy Pipelines (Deprecated)
- Remove `/pipelines/start/:id`
- Remove `/pipelines/:id/status`
- Update all clients to use `/pipelinesV2/*` endpoints
### Phase 2: PipelinesV2
- Migrate pipeline CRUD operations
- Migrate file upload functionality
- Migrate monitoring dashboard
### Phase 3: Catalog
- Migrate data asset CRUD
- Migrate permissions management
- Migrate RLS rules
- Migrate dataset registration
- Migrate PII reporting
---
## NEXT STEPS
1. **Decide on migration strategy** (Option A, B, or C)
2. **Deprecate legacy pipelines module** first
3. **Create database schema** if moving to Option A
4. **Implement local services** one module at a time
5. **Update tests** for each migrated module
6. **Update deployment configs** to remove PIFACTORY_URL
7. **Coordinate with PI-Factory team** for data migration
@@ -74,6 +74,8 @@ spec:
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: LOGGER_CONSOLE_EXTRA
value: "true"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
@@ -94,6 +96,8 @@ spec:
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
@@ -4,9 +4,18 @@ metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
+3
View File
@@ -9,6 +9,9 @@ metadata:
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
+16
View File
@@ -0,0 +1,16 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity: null
+3 -2
View File
@@ -45,9 +45,10 @@ maestro:
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "product-redis-prd.z4xvqj.0001.use1.cache.amazonaws.com"
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
@@ -63,4 +64,4 @@ affinity:
- key: name
operator: In
values:
- general
- product
+5 -2
View File
@@ -1,4 +1,4 @@
charts:
releases:
- name: maestro
chart: ../helm-chart
values:
@@ -49,5 +49,8 @@ charts:
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
# 10.70.0.0/16 internal network
# 137.131.167.254/32 loadbalancer
# 159.112.184.81/32 cluster ip for the uptime request ingest
- name: maestro.restricted_ip
value: "177.52.172.0/24,57.151.113.140/30"
value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+23 -51
View File
@@ -3,56 +3,28 @@ charts:
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.stg.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
# - name: private-maestro
# chart: ../helm-chart
# values:
# - ../helm-chart/values.yaml
# set:
# - name: app_name
# value: maestro-private
# - name: maestro.env
# value: stg
# - name: maestro.duc_url
# value: duc.stg.dadosfera.ai
# - name: hostname
# value: private-maestro.stg.dadosfera.ai
# - name: maestro.pi_factory_url
# value: pi-factory.dadosfera.ai
# - name: maestro.in_factory_url
# value: in-factory.stg.dadosfera.ai
# - name: maestro.tr_factory_url
# value: in-factory.dadosfera.ai
# - name: maestro.open_customer_id
# value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
# - name: maestro.open_group_id
# value: e3f98a2f-7748-4981-8505-7695c8ca8218
# # Customer id
# - name: maestro.dedicated_proxy
# value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
# - name: replicaCount
# value: 1
# - name: affinity
# value: null
# - name: resources
# value: null
# - name: maestro.restricted_ip
# value: "57.151.113.140/30"
- name: private-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
set:
- name: app_name
value: maestro-private
- name: hostname
value: private-maestro.stg.dadosfera.ai
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: affinity
value: null
- name: resources
value: null
- name: maestro.restricted_ip
value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+87 -191
View File
@@ -34,15 +34,35 @@
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AuthSignInRes"
}
}
"description": ""
}
},
"tags": [
"Auth"
]
}
},
"/auth/sign-out": {
"post": {
"operationId": "AuthController_signOut",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"204": {
"description": ""
}
},
"tags": [
"Auth"
@@ -675,6 +695,33 @@
]
}
},
"/auth/me": {
"get": {
"operationId": "AuthController_getMe",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Auth"
]
}
},
"/connections": {
"post": {
"operationId": "ConnectionController_createConnection",
@@ -3395,9 +3442,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3447,9 +3491,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3499,9 +3540,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6231,6 +6269,39 @@
]
}
},
"/customers/{id}/theme/reset": {
"post": {
"operationId": "ThemeController_resetTheme",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerThemeResponse"
}
}
}
},
"201": {
"description": ""
}
},
"tags": [
"Theme"
]
}
},
"/network-policy": {
"get": {
"operationId": "NetworkPolicyController_getNetworks",
@@ -6791,181 +6862,6 @@
"password"
]
},
"AuthCustomer": {
"type": "object",
"properties": {
"modules": {
"type": "array",
"items": {
"type": "string"
}
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"displayName": {
"type": "string"
},
"tier": {
"type": "string"
},
"scheduleLimit": {
"type": "string"
},
"links": {
"type": "array",
"items": {
"type": "string"
}
},
"themeEnabled": {
"type": "boolean"
},
"enforceMfa": {
"type": "boolean"
}
},
"required": [
"modules",
"id",
"name",
"displayName",
"tier",
"scheduleLimit",
"links",
"themeEnabled",
"enforceMfa"
]
},
"AuthUser": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"username": {
"type": "string"
},
"createdAt": {
"type": "string"
}
},
"required": [
"id",
"name",
"username",
"createdAt"
]
},
"AuthTokens": {
"type": "object",
"properties": {
"accessToken": {
"type": "string"
},
"refreshToken": {
"type": "string"
},
"termsOfUseToken": {
"type": "string"
},
"idToken": {
"type": "string",
"deprecated": true
}
},
"required": [
"accessToken",
"refreshToken",
"idToken"
]
},
"TermsOfUse": {
"type": "object",
"properties": {
"version": {
"type": "number"
},
"publicUrl": {
"type": "string"
},
"enforceDate": {
"type": "string"
},
"createdAt": {
"type": "string"
}
},
"required": [
"version",
"publicUrl",
"enforceDate",
"createdAt"
]
},
"TermsOfUseStatus": {
"type": "object",
"properties": {
"status": {
"type": "string",
"enum": [
"pending",
"required",
"ok"
]
},
"lastSigned": {
"$ref": "#/components/schemas/TermsOfUse"
},
"next": {
"$ref": "#/components/schemas/TermsOfUse"
}
},
"required": [
"status"
]
},
"AuthSignInRes": {
"type": "object",
"properties": {
"permissions": {
"type": "array",
"items": {
"type": "string"
}
},
"mfaStatus": {
"type": "string",
"enum": [
"pending",
"none",
"totp"
]
},
"customer": {
"$ref": "#/components/schemas/AuthCustomer"
},
"user": {
"$ref": "#/components/schemas/AuthUser"
},
"tokens": {
"$ref": "#/components/schemas/AuthTokens"
},
"termsOfUse": {
"$ref": "#/components/schemas/TermsOfUseStatus"
}
},
"required": [
"permissions",
"mfaStatus"
]
},
"AuthRefreshAccessTokenReq": {
"type": "object",
"properties": {
+1
View File
@@ -15,6 +15,7 @@ declare global {
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
}
}
}
+75 -34
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -31,6 +31,7 @@
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -57,6 +58,7 @@
},
"devDependencies": {
"@types/cache-manager": "^4.0.6",
"@types/cookie-parser": "^1.4.9",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
@@ -848,6 +850,7 @@
"integrity": "sha512-vMqyb7XCDMPvJFFOaT9kxtiRh42GwlZEg1/uIgtZshS5a/8OaduUfCi7kynKgc3Tw/6Uo2D+db9qBttghhmxwQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@ampproject/remapping": "^2.2.0",
"@babel/code-frame": "^7.26.2",
@@ -1406,9 +1409,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz",
"integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==",
"version": "3.38.0-beta.14",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.14.tgz",
"integrity": "sha512-BiE1fUIHuam3cJjAGIknsRNIlf1Z0JHJOV/VZ2pyAp5mIkGOSoNFBAMXJZb+VMTnd2xsp+vPxNnWVu7OJ/CvMQ==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -2258,6 +2261,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/common/-/common-9.4.3.tgz",
"integrity": "sha512-Gd6D4IaYj01o14Bwv81ukidn4w3bPHCblMUq+SmUmWLyosK+XQmInCS09SbDDZyL8jy86PngtBLTdhJ2bXSUig==",
"license": "MIT",
"peer": true,
"dependencies": {
"iterare": "1.2.1",
"tslib": "2.5.3",
@@ -2336,6 +2340,7 @@
"integrity": "sha512-Qi63+wi55Jh4sDyaj5Hhx2jOpKqT386aeo+VOKsxnd+Ql9VvkO/FjmuwBGUyzkJt29ENYc+P0Sx/k5LtstNpPQ==",
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@nuxtjs/opencollective": "0.3.2",
"fast-safe-stringify": "2.1.1",
@@ -2473,6 +2478,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",
"integrity": "sha512-FpdczWoRSC0zz2dNL9u2AQLXKXRVtq4HgHklAhbL59X0uy+mcxhlSThG7DHzDMkoSnuuHY8ojDVf7mDxk+GtCw==",
"license": "MIT",
"peer": true,
"dependencies": {
"body-parser": "1.20.2",
"cors": "2.8.5",
@@ -2836,6 +2842,7 @@
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
"license": "Apache-2.0",
"peer": true,
"engines": {
"node": ">=8.0.0"
}
@@ -2995,6 +3002,7 @@
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"peer": true,
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
@@ -3742,6 +3750,16 @@
"@types/node": "*"
}
},
"node_modules/@types/cookie-parser": {
"version": "1.4.9",
"resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.9.tgz",
"integrity": "sha512-tGZiZ2Gtc4m3wIdLkZ8mkj1T6CEHb35+VApbL2T14Dew8HA7c+04dmKqsKRNC+8RJPm16JEK0tFSwdZqubfc4g==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"@types/express": "*"
}
},
"node_modules/@types/cookiejar": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
@@ -3781,6 +3799,7 @@
"integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.33",
@@ -3861,6 +3880,7 @@
"integrity": "sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"jest-diff": "^27.0.0",
"pretty-format": "^27.0.0"
@@ -3924,7 +3944,8 @@
"version": "16.18.126",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz",
"integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/@types/oauth": {
"version": "0.9.6",
@@ -4154,6 +4175,7 @@
"integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.62.0",
"@typescript-eslint/types": "5.62.0",
@@ -4500,6 +4522,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz",
"integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -4596,6 +4619,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
"integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
@@ -5224,6 +5248,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"caniuse-lite": "^1.0.30001688",
"electron-to-chromium": "^1.5.73",
@@ -5330,6 +5355,7 @@
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"peer": true,
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
@@ -5553,13 +5579,15 @@
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz",
"integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/class-validator": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.1.tgz",
"integrity": "sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/validator": "^13.11.8",
"libphonenumber-js": "^1.10.53",
@@ -5872,6 +5900,28 @@
"node": ">= 0.6"
}
},
"node_modules/cookie-parser": {
"version": "1.4.7",
"resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz",
"integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==",
"license": "MIT",
"dependencies": {
"cookie": "0.7.2",
"cookie-signature": "1.0.6"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/cookie-parser/node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
@@ -6201,7 +6251,8 @@
"version": "0.0.1425554",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz",
"integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==",
"license": "BSD-3-Clause"
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/dezalgo": {
"version": "1.0.4",
@@ -6478,7 +6529,6 @@
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -6666,6 +6716,7 @@
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -7012,7 +7063,6 @@
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"license": "MIT",
"peer": true,
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -7059,7 +7109,6 @@
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
"integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"license": "MIT",
"peer": true,
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.5",
@@ -7084,7 +7133,6 @@
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz",
"integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.6"
}
@@ -7094,7 +7142,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -7103,22 +7150,19 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/path-to-regexp": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
"integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/qs": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
"integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"side-channel": "^1.0.6"
},
@@ -7147,8 +7191,7 @@
"url": "https://feross.org/support"
}
],
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/external-editor": {
"version": "3.1.0",
@@ -7383,7 +7426,6 @@
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
"integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"encodeurl": "~2.0.0",
@@ -7402,7 +7444,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -7411,8 +7452,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/find-up": {
"version": "5.0.0",
@@ -8206,6 +8246,7 @@
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
@@ -8547,6 +8588,7 @@
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@jest/core": "^27.5.1",
"import-local": "^3.0.2",
@@ -9836,7 +9878,6 @@
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
"integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
@@ -10594,6 +10635,7 @@
"resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz",
"integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==",
"license": "MIT",
"peer": true,
"dependencies": {
"passport-strategy": "1.x.x",
"pause": "0.0.1",
@@ -10983,6 +11025,7 @@
"integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin-prettier.js"
},
@@ -11485,7 +11528,8 @@
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
"integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==",
"license": "Apache-2.0"
"license": "Apache-2.0",
"peer": true
},
"node_modules/relative-microtime": {
"version": "2.0.0",
@@ -11672,6 +11716,7 @@
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"tslib": "^2.1.0"
}
@@ -11733,6 +11778,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@@ -11776,7 +11822,6 @@
"resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
"integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
@@ -11801,7 +11846,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -11810,15 +11854,13 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/send/node_modules/encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -11827,8 +11869,7 @@
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/serialize-javascript": {
"version": "6.0.2",
@@ -11844,7 +11885,6 @@
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
"integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"license": "MIT",
"peer": true,
"dependencies": {
"encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
@@ -12822,6 +12862,7 @@
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
@@ -13051,6 +13092,7 @@
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -13378,7 +13420,6 @@
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz",
"integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/json-schema": "^7.0.9",
"ajv": "^8.9.0",
+3 -1
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -49,6 +49,7 @@
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -78,6 +79,7 @@
},
"devDependencies": {
"@types/cache-manager": "^4.0.6",
"@types/cookie-parser": "^1.4.9",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
-8
View File
@@ -1,8 +0,0 @@
import 'express-session';
declare module 'express-session' {
interface SessionData {
state: string | undefined;
code_verifier: string | undefined;
}
}
+20 -12
View File
@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
+4 -4
View File
@@ -143,10 +143,10 @@ export class AuthenticationGuard
}
// Bloquear o customer de acesso o maestro publico
// const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
// if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
// throw new ForbiddenException();
// }
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
+20
View File
@@ -0,0 +1,20 @@
import jwt, { JwtPayload } from 'jsonwebtoken';
export function extractUserFrom(aRawJwt: string) {
const decodedToken = jwt.decode(aRawJwt, {
complete: true,
});
const payload = decodedToken.payload as JwtPayload;
return {
user_id: payload.user_id,
username: payload.username,
permissions: payload.permissions,
customer_id: payload.customer_id,
customer_name: payload.customer_name,
customer_tier: payload.customer_tier,
customer_modules: payload.customer_modules,
access_token: aRawJwt,
}
}
+78 -11
View File
@@ -116,6 +116,44 @@ export const PERMISSIONS_GROUPS = {
},
},
},
IMPORT_FILES: {
title: {
'pt-br': 'Coletar | Importar arquivos',
'en-us': 'Collect | Import files',
'es-es': 'Colecta | Importar archivos',
},
permissions: {
VIEW: {
seqid: 48,
claim: 'import-file:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Importar arquivos',
'en-us': 'Import files',
'es-es': 'Importar archivos',
},
},
},
},
AI_CHAT: {
title: {
'pt-br': 'AutodriveDDF',
'en-us': 'AutodriveDDF',
'es-es': 'AutodriveDDF',
},
permissions: {
VIEW: {
seqid: 49,
claim: 'ai-chat:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'AutodriveDDF',
'en-us': 'AutodriveDDF',
'es-es': 'AutodriveDDF',
},
},
},
},
CONNECTION: {
title: {
'pt-br': 'Coletar | Fontes de dados',
@@ -340,16 +378,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -362,6 +390,44 @@ export const PERMISSIONS_GROUPS = {
},
},
},
LINEAGE: {
title: {
'pt-br': 'Explorar | Linhagem',
'en-us': 'Explore | Lineage',
'es-es': 'Explorar | Linaje',
},
permissions: {
VIEW: {
seqid: 50,
claim: 'lineage:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar ao módulo de Linhagem',
'en-us': 'Access to Lineage module',
'es-es': 'Acceda al módulo de Linaje',
},
}
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -615,7 +681,8 @@ export const DADOSFERA_MODULES_KEYS = {
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
PUBLIC_ASSIGN_EMBED: 'public-assign-embed',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+9 -1
View File
@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -52,6 +53,7 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
UserTest(token);
// UserTest(token);
});
+25 -3
View File
@@ -9,7 +9,8 @@ import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import session from 'express-session';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -17,20 +18,41 @@ async function bootstrap() {
});
const logger = new DadosferaLogger();
const corsOrigins = [];
if (process.env.ENV === 'local') {
corsOrigins.push('http://localhost:4200');
} else {
corsOrigins.push(
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://private-frontend.stg.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
);
}
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: '*',
origin: corsOrigins,
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true,
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV !== 'local') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
app.use(
'/catalog/register-dataset',
urlencoded({ extended: true, limit: '10mb' }),
);
}
configureSwagger(app);
+8 -4
View File
@@ -1,4 +1,4 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
@@ -15,7 +15,7 @@ export class AssignController {
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
@@ -25,9 +25,13 @@ export class AssignController {
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
try {
return await this.assignService.get(metadata);
} catch (error) {
throw new NotFoundException(error.message)
}
}
}
+1 -1
View File
@@ -33,6 +33,6 @@ export class AssignService implements OnModuleInit {
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey(metadata))
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
+164 -9
View File
@@ -12,6 +12,7 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -47,13 +48,19 @@ import {
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request } from 'express';
import { Request, Response } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@@ -86,11 +93,66 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
throw error;
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
}
@Post('refresh-access-token')
@@ -100,16 +162,25 @@ export class AuthController {
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, userId } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
language,
custom_host: frontHost,
});
return this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
}
@ApiInternalOnlyEndpoint()
@@ -418,4 +489,88 @@ export class AuthController {
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
try {
// Decodifica e valida o JWT de acesso
const userDto = await this.authClient.extractUserFrom(accessToken);
return res.status(200).json(userDto);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const userDto = await this.authClient.extractUserFrom(accessToken);
return res.status(200).json(userDto);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
}
+55 -8
View File
@@ -4,7 +4,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService, UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -21,18 +21,18 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
import { BulkEditResponse, UserDTO } from './dtos/login';
import jwt from 'jsonwebtoken';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
private userService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -46,8 +46,8 @@ export class AuthClientService implements OnModuleInit {
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
this.userService = this.grpcClient.getService<UsersProtoService>(
ProtoServices.UsersProtoService,
);
}
@@ -283,4 +283,51 @@ export class AuthClientService implements OnModuleInit {
throw error;
}
}
private async getUser(id: string, metadata: Metadata) {
this.logger.info('getUser');
return await lastValueFrom(
this.userService.UserFindOneById({ id }, metadata),
);
}
public async extractUserFrom(token: string) {
const decoded: any = token && jwt.decode(token, { complete: true });
if (!decoded) throw new Error('Invalid token');
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue)
throw new Error('Public key not found');
jwt.verify(token, pemValue);
const payload = decoded.payload;
const metadata = PackTheMetadata({
customer_id: payload.customer_id
})
const {
user
} = await this.getUser(
payload.user_id,
metadata
);
const userDto: UserDTO = {
id: user.id,
name: user.username,
jobTitle: user?.jobTitle || null,
department: user?.department || null,
hierarchy: user?.hierarchy || null,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return userDto;
}
}
+13
View File
@@ -140,3 +140,16 @@ export interface BulkEditResponse {
successfulUsers: string[];
failedUsers: string[];
}
export type UserDTO = {
id: string,
name: string,
jobTitle?: string,
department?: string,
hierarchy?: string,
customer: {
id: string,
name: string,
tier: string,
}
}
+9 -4
View File
@@ -197,9 +197,11 @@ class CatalogService implements OnModuleInit {
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
const roles_ids = result.user.roles.map((role) => role.id);
if (result) {
return result.user.roles.map((role) => role.id);
}
return roles_ids;
return [];
}
async searchDataAssets(
@@ -406,11 +408,14 @@ class CatalogService implements OnModuleInit {
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const data_asset_roles = data_asset?.roles || []
for (const role_id of data_asset_roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const data_asset_users = data_asset?.users || []
for (const user_id of data_asset_users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
@@ -41,23 +41,22 @@ export class ShareController {
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@User() user: RequestUser,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, user, request);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, user);
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
@@ -67,10 +66,10 @@ export class ShareController {
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, user);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@@ -79,10 +78,10 @@ export class ShareController {
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, user);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
+3 -1
View File
@@ -9,6 +9,7 @@ import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@@ -19,7 +20,8 @@ const client = new CatalogClientConfiguration();
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
+53 -25
View File
@@ -22,6 +22,9 @@ import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -36,6 +39,7 @@ export class ShareService implements OnModuleInit {
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
@@ -47,8 +51,8 @@ export class ShareService implements OnModuleInit {
);
}
async getDatasetColumnsMetadata(id: string, user: RequestUser) {
const shareMetadata = await this.getShareMetadata(id, user);
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
@@ -63,8 +67,8 @@ export class ShareService implements OnModuleInit {
return result;
}
async getDatasetPreview(id: string, user: RequestUser) {
const shareMetadata = await this.getShareMetadata(id, user);
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
@@ -79,14 +83,14 @@ export class ShareService implements OnModuleInit {
return result;
}
async getOneDataAssetPublic(id: string, user: RequestUser, request: Request) {
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id,
userId: user?.user_id,
customerId: user?.customer_id
id
}))
try {
const shareMetadata = await this.getShareMetadata(id, user);
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
@@ -148,8 +152,8 @@ export class ShareService implements OnModuleInit {
return { data_asset: asset[0] };
}
async getDataDocs(id: string, user: RequestUser) {
const shareMetadata = await this.getShareMetadata(id, user);
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
@@ -197,21 +201,8 @@ export class ShareService implements OnModuleInit {
});
}
// private async validateShareAssign(token: string) {
// const tokenDecoded = jwt.decode(token, {
// complete: true,
// });
// jwt.verify(token, this.pemValue, {
// algorithms: ['RS256'],
// });
// const shareId = (tokenDecoded.payload as JwtPayload).sub;
// const metadata = PackTheMetadata({});
// return await this.shareMetadataService.get(shareId, metadata);
// }
private async getShareMetadata(id: string, user: RequestUser) {
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
@@ -219,6 +210,8 @@ export class ShareService implements OnModuleInit {
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
@@ -244,4 +237,39 @@ export class ShareService implements OnModuleInit {
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -153,12 +153,27 @@ export class IdentityProviderController {
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
const origin = req.headers['origin'] as string;
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang = language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl = process.env.ENV !== "prd" ? `${origin}/auth/callback` : `${origin}/${lang}/auth/callback`;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
return await this.identityProviderService.getTokenByIdp(code, state, callbackUrl);
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@@ -166,41 +181,66 @@ export class IdentityProviderController {
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
const frontDomain = req.headers['origin'] as string;
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(@Param('id') id: string, @Req() req: Request, @Language() language: LanguageEnum) {
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const frontDomain = req.headers['origin'] as string || req.headers['referer'] as string;
this.logger.info(`Front domain: ${frontDomain}`);
const host = frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
const lang = language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl = process.env.ENV !== "prd" ? `${host}/auth/callback` : `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(id, callbackUrl);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -14,16 +14,21 @@ import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { Request } from 'express';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
) {}
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
@@ -33,22 +38,26 @@ export class IdentityProviderService implements OnModuleInit {
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
@@ -57,14 +66,19 @@ export class IdentityProviderService implements OnModuleInit {
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
@@ -76,6 +90,7 @@ export class IdentityProviderService implements OnModuleInit {
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
@@ -86,16 +101,20 @@ export class IdentityProviderService implements OnModuleInit {
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
@@ -103,22 +122,22 @@ export class IdentityProviderService implements OnModuleInit {
redirect_uris: ssoSign.redirectUrls,
});
if (ssoSign === null) {
throw new BadRequestException('SSO sign-in is expired or not found');
}
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
@@ -128,12 +147,13 @@ export class IdentityProviderService implements OnModuleInit {
}),
);
} catch (error) {
console.log(error);
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
@@ -144,6 +164,7 @@ export class IdentityProviderService implements OnModuleInit {
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
@@ -156,6 +177,7 @@ export class IdentityProviderService implements OnModuleInit {
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
+25 -8
View File
@@ -1,29 +1,46 @@
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
import { extractUserFrom } from 'src/authentication/extract-user';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@ApiInternalOnlyController()
@Controller('trackEvent')
export class MixpanelController {
logger: DadosferaLogger;
constructor(
private mixpanelService: MixpanelService
) {}
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private mixpanelService: MixpanelService,
) {
this.logger = dadosferaLogger.logger;
}
@Post(':id')
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
this.logger.info(`POST Track Event: ${id}`)
delete body.info;
const anonymousUser = {
username: "anonymous",
customer_name: "anonymous"
} as RequestUser
const hasToken = request.headers['authorization'];
const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
this.logger.info(`Has user: ${typeof hasToken == "string"}`)
await this.mixpanelService.track(id, user, request, body)
this.logger.info(`Event successful`)
return { id, body, user: user.username };
}
}
+3 -20
View File
@@ -3,6 +3,7 @@ import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
@@ -13,26 +14,6 @@ import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
);
})
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -44,6 +25,7 @@ export class PipelinesController {
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
@@ -71,6 +53,7 @@ export class PipelinesController {
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
+21 -34
View File
@@ -26,6 +26,7 @@ import {
import {
AuthenticateCondition,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
@@ -52,30 +53,6 @@ import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
case 'PATCH':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
);
})
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -88,6 +65,7 @@ export class PipelinesController {
}
@Get('monitoring-dashboard')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getMonitoringDashboard(@User() user: RequestUser) {
this.logger.info('PipelinesController - getMonitoringDashboard', { user });
@@ -100,6 +78,7 @@ export class PipelinesController {
}
@Post()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiCreatedResponse({ type: IPipelineV2 })
async create(
@Language() language: LanguageEnum,
@@ -126,6 +105,7 @@ export class PipelinesController {
}
@Get()
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async findAll(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -150,6 +130,7 @@ export class PipelinesController {
}
@Get('/download-logs')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async downloadLogs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -180,6 +161,7 @@ export class PipelinesController {
}
@Get(':id/config')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW,PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineproperties(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -191,6 +173,7 @@ export class PipelinesController {
}
@Get(':id/objects')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineObjects(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -202,16 +185,14 @@ export class PipelinesController {
}
@Get(':id/status')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const response = await this.oldPipelinesService.getPipelineStatus(body);
@@ -219,6 +200,7 @@ export class PipelinesController {
}
@Get('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async findOne(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -256,10 +238,12 @@ export class PipelinesController {
});
return res;
});
return result;
}
@Patch('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async update(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@@ -299,6 +283,7 @@ export class PipelinesController {
deprecated: true,
description: 'This method is deprecated. Please use PATCH instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updateDeprecated(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@@ -314,6 +299,7 @@ export class PipelinesController {
@Delete(':id')
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
async delete(@Param('id') id: string, @User() user: RequestUser) {
this.logger.info('PipelinesController - delete', { user });
const metadata = PackTheMetadata({
@@ -327,7 +313,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/init-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
async initUploadFile(
@User() user: RequestUser,
@Body() body: IInitUploadCSVFile,
@@ -359,7 +345,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/complete-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
async completeUploadFile(
@User() user: RequestUser,
@Body() body: ICompleteUploadCSVFile,
@@ -377,7 +363,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/file')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
async uploadedFile(
@User() user: RequestUser,
@Body() body: ICreatePipelineCSVFile,
@@ -427,6 +413,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
+20
View File
@@ -124,4 +124,24 @@ export class ThemeController {
}
}
@Post('/:id/theme/reset')
@ApiOkResponse({ type: CustomerThemeResponse })
async resetTheme(@Param('id') id: string) {
this.logger.info('getCustomerTheme with id' + id);
try {
await this.themeService.resetTheme(id);
return { theme: null };
}catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
} else {
this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message);
return { theme: null };
};
}
}
}
+12
View File
@@ -44,6 +44,18 @@ export class ThemeService implements OnModuleInit {
);
}
async resetTheme(id: string) {
const { theme } = await firstValueFrom(
this.themeService.ResetCustomerTheme({
id
}),
);
return {
theme
}
}
async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) {
if (!id) {
this.logger.error('Error - saveCustomertheme - not found id:' + id);
@@ -8,6 +8,17 @@ import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
+11
View File
@@ -9,6 +9,17 @@ import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
error: (...args) => args,
+15 -5
View File
@@ -5,15 +5,25 @@ import { redisStore } from 'cache-manager-ioredis-yet';
@Module({
imports: [
CacheModule.registerAsync({
useFactory: async () => ({
store: await redisStore({
ttl: 1000 * 60, //1 minute
useFactory: async () => {
const baseRedisConfig = {
ttl: 5 * 1000 * 60, // 5 minute
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT && Number(process.env.REDIS_PORT),
db: process.env.REDIS_DATABASE && Number(process.env.REDIS_DATABASE),
keyPrefix: 'maestro:sso',
}),
}),
}
if (process.env.ENV !== 'local') {
baseRedisConfig['tls'] = {
servername: process.env.REDIS_HOST,
}
}
return {
store: await redisStore(baseRedisConfig),
}
},
}),
],
providers: [CacheService],