Compare commits

...
162 Commits
Author SHA1 Message Date
Gabriel Amorim d7062efc60 Merge pull request #183 from dadosfera/feat/rest-api
Feat/rest api
2022-12-05 15:22:05 -03:00
Gabriel Rosa 4a3dd98977 Merge branch 'main' into feat/rest-api 2022-12-05 13:58:22 -03:00
Gabriel Amorim 06f4260b61 Merge pull request #181 from dadosfera/stg2
Stg2
2022-11-29 19:16:48 -03:00
Gabriel Rosa 64717c8d92 Merge branch 'main' into stg2 2022-11-29 14:41:11 -03:00
Anderson Feitosa e6e66c77e1 Merge pull request #180 from dadosfera/feature/update-pipeline-cron
feature/update-pipeline-cron
2022-11-29 14:04:31 -03:00
Gabriel Rosa 131e82b4c5 FIX: connection details improvements 2022-11-29 09:47:19 -03:00
Gabriel Rosa 32d37ecadf Removed dev deploy environment 2022-11-29 08:42:26 -03:00
Gabriel Rosa 4d667acb86 removed unused import 2022-11-25 18:21:58 -03:00
Gabriel Rosa 9d417ff61b FIX: treating new connectors upload reponse 2022-11-25 18:21:43 -03:00
Anderson Feitosa b01c4c6c25 UPGRADE: change protospack version 2022-11-25 13:00:09 -03:00
Anderson Feitosa 449a1abb63 FEAT: update auth guard to patch method 2022-11-25 12:11:51 -03:00
Anderson Feitosa 04ca17a69d FEAT: update protospack 2022-11-25 11:37:34 -03:00
Anderson Feitosa 34a0eb40a7 DOCS: docsfera patch 2022-11-23 14:58:17 -03:00
Anderson Feitosa 2df69fba84 FEAT: add log on update 2022-11-23 13:35:02 -03:00
Anderson Feitosa dcbcc5ff36 FEAT: add patch method on pipeline controller 2022-11-23 13:02:32 -03:00
Gabriel Rosa 703b5f36b3 Merge branch 'main' into feat/rest-api 2022-11-21 13:36:13 -03:00
Gabriel Rosa 54d10cb873 FIX: try to delete pipeline even if it does not exists in opensearch 2022-11-17 09:25:05 -03:00
Gabriel Amorim 569c3cf573 Merge pull request #179 from dadosfera/feat/incremental-table
Feat/incremental table
2022-11-16 16:52:35 -03:00
Gabriel Rosa c420ef62ed new protospack version 2022-11-16 16:46:38 -03:00
Alexsander Pereira 966ba00fec Merge pull request #178 from dadosfera/stg2
FEAT: Adding stg2
2022-11-16 15:47:49 -03:00
Alexsander Pereira 0aadf7fc40 FEAT: Adding stg2 2022-11-16 15:18:18 -03:00
Gabriel Rosa 3e3268cb26 FIX: New getAvailableEntities payload 2022-11-16 10:37:21 -03:00
Anderson Feitosa 09bd623b62 Merge pull request #177 from dadosfera/catalog-permissions
FIX: create data asset permission public
2022-11-14 16:19:09 -03:00
Gabriel Rosa 4bf9a5831a FIX: create data asset permission public 2022-11-14 16:15:57 -03:00
Anderson Feitosa 264077289b Merge pull request #176 from dadosfera/catalog-permissions
FIX: renamed catalog create permission
2022-11-14 15:52:04 -03:00
Gabriel Rosa 14c686cff4 FIX: renamed catalog create permission
- Catalog controller using permission decorator on every route
2022-11-14 15:40:38 -03:00
Gabriel Amorim 0e57667419 Merge pull request #175 from dadosfera/feat/modules-permissions
Feat/modules permissions
2022-11-11 12:38:04 -03:00
Gabriel Rosa ac7dc4905e FIX: changed PROCESS permisison group title 2022-11-11 12:35:04 -03:00
Gabriel Rosa 94282718ad Changed permission group name 2022-11-11 12:27:57 -03:00
Gabriel Rosa 7c228c5c53 FIX: set new module permissions public 2022-11-11 12:26:14 -03:00
Gabriel Amorim 14e0f3d2d3 Merge pull request #174 from dadosfera/fix/event-tracker
Fix/event tracker
2022-11-11 10:18:42 -03:00
Victor Radael 85ef0d7c74 Merge remote-tracking branch 'origin' into fix/event-tracker 2022-11-11 10:16:07 -03:00
Victor Radael e00b6481d0 FIX: Tracker 2022-11-11 10:15:49 -03:00
Victor Radael 8906c73d84 Revert "FIX: Event Tracker"
This reverts commit 3bc37db891.
2022-11-11 10:13:24 -03:00
Victor Radael 3bc37db891 FIX: Event Tracker 2022-11-11 10:12:20 -03:00
Gabriel Rosa 3c877b8b8d Merge branch 'main' into feat/rest-api 2022-11-10 15:14:45 -03:00
Gabriel Amorim fe44664970 Merge pull request #173 from dadosfera/fix/modules-permissions
FIX: new modules permissions
2022-11-10 15:13:01 -03:00
Gabriel Rosa 0503732c1e new protospack version 2022-11-10 14:43:46 -03:00
Gabriel Rosa fa0855d631 FIX: new modules permissions 2022-11-10 12:57:15 -03:00
Gabriel Amorim 38aee30f2a Merge pull request #172 from dadosfera/fix/pipeline-details-bug
Fix/pipeline details bug
2022-11-10 10:32:42 -03:00
Gabriel Rosa 19f5c947bd Set release version on swagger when deploying main 2022-11-10 10:12:14 -03:00
Gabriel Rosa 44aa0e6010 improved run prod command 2022-11-10 09:47:30 -03:00
Gabriel Rosa cf4e7da1d8 Typing login route 2022-11-10 09:46:22 -03:00
Gabriel Rosa 8bd7d54aa7 new protospack version 2022-11-10 09:16:02 -03:00
Gabriel Rosa a00a66e0d0 Merge branch 'main' into fix/pipeline-details-bug 2022-11-10 08:41:36 -03:00
Victor Radael f4c5d03269 Merge pull request #171 from dadosfera/ft/mixpanel-track
FEAT: Mixpanel Tracker
2022-11-10 08:35:55 -03:00
Victor Radael 060ba7da8a FEAT: Mixpanel Tracker 2022-11-09 20:46:40 -03:00
Victor Radael 5ec2c01b20 FEAT: Mixpanel Tracker 2022-11-09 20:37:39 -03:00
Gabriel Rosa 4a7e660a49 FIX: fixed condition of getObjValueFromPath 2022-11-09 20:06:23 -03:00
Gabriel Rosa abc6258492 FIX: fixed condition to set default value on pipeline details 2022-11-09 19:30:13 -03:00
Gabriel Rosa 7ee1215da2 Deploy docs using github hosted runner 2022-11-09 18:48:50 -03:00
Gabriel Rosa cd5886337b FIX: customer modules 2022-11-09 18:40:53 -03:00
Gabriel Rosa e6b0034805 Merge branch 'main' into feat/rest-api 2022-11-09 18:29:04 -03:00
Gabriel Rosa b9c1f784d3 FIX: treatingconnector upload response 2022-11-09 18:25:57 -03:00
Gabriel Rosa 5fb8dae312 FIX: pipeline details now showing correct default values on s3 pipelines 2022-11-09 17:35:18 -03:00
Gabriel Amorim 5072a54ef6 Merge pull request #170 from dadosfera/fix/revert-action-version
FIX: return semantic release action to previous version
2022-11-09 16:05:04 -03:00
Gabriel Rosa 718f5cb12a FIX: return sematic release action to previous version 2022-11-09 15:56:28 -03:00
Gabriel Amorim 4c41dd8029 Merge pull request #169 from dadosfera/fix/revert-action-version
FIX: remove `conventional-changelog-eslint` plugin
2022-11-09 15:41:08 -03:00
Gabriel Rosa fe8a0b88b0 FIX: remove conventional-changelog-eslint plugin 2022-11-09 15:39:37 -03:00
Gabriel Rosa 3edf5c2416 Merge branch 'main' into feat/rest-api 2022-11-09 15:03:09 -03:00
Gabriel Amorim cd67ca61cd Merge pull request #168 from dadosfera/fix/customer-modules
FIX: customer modules
2022-11-09 14:59:54 -03:00
Gabriel Rosa b9439685f3 FIX: new swagger info 2022-11-09 14:29:32 -03:00
Gabriel Rosa 04edce0c56 FIX: new protospack version 2022-11-09 14:08:05 -03:00
Gabriel Rosa f3e00a15c8 Remove unused imports 2022-11-09 14:06:30 -03:00
Gabriel Rosa 0422c7a369 Set branch name on swagger version 2022-11-09 14:06:12 -03:00
Gabriel Rosa f22d9370db test new upload documentation 2022-11-09 11:27:54 -03:00
Victor Radael ef098dd74d Merge pull request #167 from dadosfera/ft/create-data-asset
FEAT: Ft/create data asset
2022-11-08 19:48:48 -03:00
Victor Radael 2541960b44 FEAT: Create Data Asset 2022-11-08 19:32:26 -03:00
Victor Radael 2dc965ed90 FIX:delete and update data asset 2022-11-08 14:18:31 -03:00
Gabriel Rosa a9ee74724f FIX: better error handling 2022-11-08 12:11:05 -03:00
Victor Radael b2f1ce669d FIX: Save 2022-11-08 11:41:34 -03:00
Victor Radael 602d81e51f FIX: Save 2022-11-08 11:39:48 -03:00
Rodrigo Zamboni bbdb023dde Merge pull request #165 from dadosfera/feat/newCustomersInfo
Feat/new customers info
2022-11-07 16:05:33 -03:00
Gabriel Rosa 9cb4cd7093 FIX: fixed protospack version 2022-11-07 13:48:28 -03:00
Gabriel Rosa efc5870d87 FIX: fixed protospack version 2022-11-07 13:48:09 -03:00
Rodrigo Zamboni c3d33f8230 Merge branch 'main' into feat/newCustomersInfo 2022-11-07 12:38:13 -03:00
rodrigo.zamboni 0654e8fa04 FIX: updaetd protospack version 2022-11-07 12:35:51 -03:00
Gabriel Amorim 28ab892208 Merge pull request #164 from dadosfera/catalog-permissions
FIX: Catalog permissions
2022-11-07 10:44:02 -03:00
Gabriel Rosa d1522eb0ea FIX: new protospack version 2022-11-04 17:57:48 -03:00
Gabriel Rosa 6be9e06b22 FIX: new protospack version 2022-11-04 14:22:07 -03:00
Gabriel Rosa 7af261d56c New permissions for transformations 2022-11-04 12:03:15 -03:00
Gabriel Rosa 8fa924fdfb Changed delete description 2022-11-04 09:27:53 -03:00
Victor Radael 4b4bdfea61 FEAT: Create Data Asset 2022-11-03 17:18:20 -03:00
Gabriel Rosa d1064c852b rename workflow job 2022-11-03 12:03:25 -03:00
Gabriel Rosa 63338f5250 fixed deploy worflow file 2022-11-03 11:59:01 -03:00
Gabriel Rosa 5e1fb41930 FIX: identation 2022-11-03 11:57:52 -03:00
Gabriel Rosa a807da2414 test deploy 2022-11-03 11:48:46 -03:00
Gabriel Rosa 85f14fcf4b FIX: generate better summary on deploy 2022-11-03 11:28:09 -03:00
Gabriel Rosa d95898ce55 FIX: generate better summary on deploy 2022-11-03 11:18:02 -03:00
Gabriel Rosa 33075b0f85 FIX: generate better summary on deploy 2022-11-03 11:10:38 -03:00
Gabriel Rosa ca75f8f6e7 FIX: generate better summary on deploy 2022-11-03 11:00:55 -03:00
Gabriel Rosa af12a39959 FEAT: Deploy summary 2022-11-03 10:53:37 -03:00
Gabriel Rosa f77476b9ae FIX: echo text 2022-11-03 10:33:55 -03:00
Gabriel Rosa fb0c659cd1 FIX: permissions for catalog 2022-11-03 10:27:50 -03:00
Gabriel Rosa aea6815de7 Merge branch 'main' into catalog-permissions 2022-11-03 10:20:10 -03:00
Gabriel Amorim 0103f100f5 Merge pull request #163 from dadosfera/new-ecr-image
FIX: new ecr image
2022-11-01 15:59:49 -03:00
Gabriel Rosa 66b6be43b7 FIX: new ecr image 2022-11-01 15:33:14 -03:00
Gabriel Rosa c029e286b5 WIP 2022-11-01 15:31:45 -03:00
Gabriel Amorim a8f94a2d53 Merge pull request #162 from dadosfera/feat/platform-delete-pipeline
Feat: platform delete pipeline
2022-10-31 16:43:44 -03:00
Gabriel Rosa aadb3fcdee Better inputs error codes 2022-10-31 16:20:11 -03:00
Gabriel Rosa a26e88d2b6 Merge branch 'main' into feat/platform-delete-pipeline 2022-10-31 15:37:13 -03:00
Victor Radael b20f67e8b8 Merge pull request #161 from dadosfera/ft/delete-data-assets
Ft/delete data assets
2022-10-27 18:44:08 -03:00
Victor Radael 4a8a8eb463 FIX: Pipeline delete 2022-10-27 18:20:56 -03:00
Victor Radael d52f8b78c8 FIX: Pipeline delete 2022-10-27 18:06:39 -03:00
Victor Radael bfbe7cce87 Merge pull request #160 from dadosfera/ft/delete-data-assets
Ft/delete data assets
2022-10-27 17:14:22 -03:00
Victor Radael 1ab8c1dd10 FIX: Merge origin main 2022-10-27 15:24:04 -03:00
Gabriel Amorim 246de83142 Merge pull request #159 from dadosfera/ref/catalog
Catalog Refactor
2022-10-26 17:40:42 -03:00
Victor Radael dfa6222538 FIX: Delete data asset 2022-10-26 14:22:29 -03:00
Gabriel Rosa fc7a9c7909 FIX: do not throw errors when deleting input or transformation on pipeline delete 2022-10-26 10:52:12 -03:00
Gabriel Rosa 932616a578 Delete connection permission public 2022-10-26 09:09:28 -03:00
Victor Radael b64619ede1 FIX: Remove trash code 2022-10-25 19:29:23 -03:00
Gabriel Rosa 4a08e5f560 Merge branch 'main' into ref/catalog 2022-10-25 17:33:04 -03:00
Gabriel Rosa 4ffd0916c3 FIX: pipeline delete working 2022-10-25 16:21:58 -03:00
Victor Radael 92041f453c FIX: Remove trash code 2022-10-25 14:00:26 -03:00
Gabriel Rosa f2f09cc9c7 FIX: oauth redirect adjusted 2022-10-24 15:24:14 -03:00
Victor Radael d486231fd6 FEAT: RLS new barnch 2022-10-21 15:44:21 -03:00
Victor Radael 526a3bddcd Merge pull request #158 from dadosfera/ft/rls-old-catalog
FEAT: Rls first change
2022-10-21 15:18:01 -03:00
Victor Radael 45108d3019 FEAT: Rls first change 2022-10-21 14:49:28 -03:00
Victor Radael ac3e26b851 FIX: RLS 2022-10-21 14:36:59 -03:00
Gabriel Rosa 7fbc919716 FIX: catalog filters 2022-10-20 11:23:30 -03:00
Gabriel Rosa 5b1eb19511 FIX: catalog sort by 2022-10-20 10:27:38 -03:00
Victor Radael b714f3cf36 FIX: Filters 2022-10-20 10:04:45 -03:00
Victor Radael 48c1e29168 FIX: Filters 2022-10-20 09:45:12 -03:00
Victor Radael 04ea1d0f04 FIX: Get a total and fix owner type 2022-10-19 17:01:22 -03:00
Victor Radael 0de5e1d5d1 PATCH: Ref/Catalog new functions 2022-10-18 14:13:24 -03:00
Victor Radael b8c41fad1b PATCH: Ref/Catalog new functions 2022-10-18 13:43:18 -03:00
Victor Radael ef9ec5ad42 PATCH: Ref/Catalog new functions 2022-10-18 13:05:47 -03:00
Victor Radael 65e71f21a0 Merge branch 'ref/catalog' of github.com:dadosfera/maestro into ref/catalog 2022-10-18 13:05:27 -03:00
Victor Radael b921393ba1 PATCH: Ref/Catalog new functions 2022-10-18 13:05:05 -03:00
Gabriel Rosa 18aa7d8674 FIX: permission language fixed 2022-10-17 17:53:47 -03:00
Victor Radael f2dd871de2 FIX: Ref Catalogo new functions models 2022-10-17 16:16:31 -03:00
Victor Radael cf3fa88196 FIX: Ref Catalogo new functions models 2022-10-17 15:00:29 -03:00
Victor Radael 131810533c FIX: Ref Catalogo new functions models 2022-10-17 14:58:11 -03:00
Victor Radael ea06f3f571 FIX: Ref Catalogo new functions models 2022-10-17 14:51:28 -03:00
Victor Radael 81921a7eb2 FIX: Ref Catalogo new functions models 2022-10-17 14:38:31 -03:00
Victor Radael e9fdd0285b FIX: Ref Catalogo new functions models 2022-10-17 13:42:13 -03:00
Victor Radael 167470adba FIX: Ref Catalogo new functions models 2022-10-17 13:35:13 -03:00
Victor Radael fbb9a59214 FIX: Ref Catalogo new functions models 2022-10-17 13:32:41 -03:00
Gabriel Amorim ef6d8594a3 Merge pull request #157 from dadosfera/fix/get-user-permissions
FIX: new function to get all user permissions
2022-10-14 11:57:05 -03:00
Gabriel Rosa ac73c0b046 FIX: new function to get all user permissions 2022-10-14 11:33:15 -03:00
Gabriel Amorim 5a30bf08b2 Merge pull request #156 from dadosfera/fix/pipeline-properties
Fix/pipeline properties
2022-10-13 16:39:41 -03:00
Gabriel Rosa 62b4eb709e Return properties object on findOneProperties 2022-10-13 16:24:32 -03:00
Gabriel Rosa b37a18d1da Merge branch 'main' into fix/pipeline-properties 2022-10-13 15:59:53 -03:00
Gabriel Rosa 379bac704f FIX: getPipelineproperties 2022-10-13 14:59:27 -03:00
Gabriel Rosa 9178530a21 FEAT: get pipeline properties 2022-10-13 14:58:41 -03:00
Gabriel Amorim 667d3f874d Merge pull request #155 from dadosfera/fix/user-permissions
FIX: permissions array on user findOneById
2022-10-13 14:03:02 -03:00
Victor Radael 475af224bf Merge remote-tracking branch 'origin' into ref/catalog 2022-10-13 11:43:16 -03:00
Gabriel Rosa 11f1df5537 FIX: permissions array on user findOneById 2022-10-13 09:23:56 -03:00
Victor Radael e3b02571c8 Merge pull request #154 from dadosfera/fix/oauth
FIX: Public oauth route
2022-10-13 09:05:29 -03:00
Gabriel Rosa 979a12aa7f Merge branch 'main' into fix/oauth 2022-10-13 09:02:17 -03:00
Gabriel Rosa 7646d7a3c5 WIP 2022-10-13 08:57:21 -03:00
Victor Radael 8f0e851a88 FIX: Public oauth route 2022-10-12 23:12:33 -03:00
Victor Radael 779418f471 clear catalog 2022-10-11 22:23:36 -03:00
Gabriel Amorim 6e6d03f302 Merge pull request #153 from dadosfera/fix/user-roles
Fix/user roles
2022-10-11 16:40:12 -03:00
Victor Radael 85f41521ed Merge remote-tracking branch 'origin' into ref/catalog 2022-10-11 15:36:08 -03:00
Victor Radael e2bb119745 refactor/catalog 2022-10-11 15:35:29 -03:00
Gabriel Rosa 3d5ce59b52 FIX: user can get own info, even with no users:admin permission 2022-10-11 11:54:39 -03:00
Gabriel Rosa 1c63271f64 FIX: send language to request that need terms of use 2022-10-11 10:26:13 -03:00
Gabriel Rosa 31a51c20aa FIX: pass metadata to signIn request 2022-10-11 08:41:59 -03:00
Gabriel Rosa 62730eb969 FIX: terms of use dynamic language 2022-10-10 19:56:57 -03:00
Victor Radael 5a2c760bab Merge pull request #152 from dadosfera/fix/dashboards
FIX: Dashboard get all permisisons
2022-10-08 18:00:24 -03:00
Victor Radael 28b4f3ac4b FIX: Dashboard get all permisisons 2022-10-08 17:56:19 -03:00
46 changed files with 7354 additions and 1870 deletions
+45 -61
View File
@@ -6,12 +6,12 @@ on:
workflow_dispatch:
inputs:
environment:
description: 'Deployment environment'
description: "Deployment environment"
required: true
type: choice
options:
- dev
- stg
- stg2
- prd
jobs:
@@ -26,9 +26,9 @@ jobs:
- name: Extract Environment
run: |
if [ ${EVENT} == "workflow_dispatch" ]; then
echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "##[set-output name=environment;]$(echo "prd")"
echo "environment=prd" >> $GITHUB_OUTPUT
fi
id: extract_environment
@@ -39,7 +39,7 @@ jobs:
new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- if: github.event_name != 'workflow_dispatch'
name: Semantic Release
@@ -56,20 +56,26 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
deploy:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
deploy-info:
if: ${{ github.event_name == 'workflow_dispatch'}}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, '${{ needs.extract_environment.outputs.environment }}']
runs-on: ubuntu-latest
steps:
- name: Printing stats
- name: Create summary
env:
EVENT: ${{ github.event_name }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ENV: ${{ needs.extract_environment.outputs.environment }}
run: echo ${GITHUB_REF#refs/heads/}
run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY
deploy:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
steps:
- name: Checkout
uses: actions/checkout@v2
@@ -90,7 +96,7 @@ jobs:
python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region
uses: aws-actions/configure-aws-credentials@v1
uses: aws-actions/configure-aws-credentials@v1-node16
id: aws
with:
aws-region: us-east-1
@@ -113,10 +119,12 @@ jobs:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk
@@ -136,71 +144,47 @@ jobs:
docker system df
api_docs:
needs: [extract_environment, semantic_release, deploy]
runs-on:
[self-hosted, '${{ needs.extract_environment.outputs.environment }}']
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Extract Docs PageId
- name: Extract Docs BlockId and PageId
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
DEV_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
shell: bash
run: |
if [ $ENV == "dev" ]; then
echo "##[set-output name=result;]$(echo $DEV_DOCS_PAGE_ID)"
if [ $ENV == "stg2" ]; then
echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "stg" ]; then
echo "##[set-output name=result;]$(echo $STG_DOCS_PAGE_ID)"
echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "prd" ]; then
echo "##[set-output name=result;]$(echo $PRD_DOCS_PAGE_ID)"
echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
fi
id: extract_docs_page_id
- name: Extract Docs BlockId
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
DEV_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
shell: bash
run: |
if [ $ENV == "dev" ]; then
echo "##[set-output name=result;]$(echo $DEV_DOCS_BLOCK_ID)"
elif [ $ENV == "stg" ]; then
echo "##[set-output name=result;]$(echo $STG_DOCS_BLOCK_ID)"
elif [ $ENV == "prd" ]; then
echo "##[set-output name=result;]$(echo $PRD_DOCS_BLOCK_ID)"
fi
id: extract_docs_block_id
id: extract_docs_info
- name: Generate API docs
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
DOCS_URL: ${{ secrets.DOCS_URL }}
DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
DOCS_PAGE_ID: ${{ steps.extract_docs_page_id.outputs.result }}
DOCS_BLOCK_ID: ${{ steps.extract_docs_block_id.outputs.result }}
continue-on-error: true
DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
EVENT: ${{ github.event_name }}
RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
run: |
SWAGGER=`cat swagger.txt`
curl --request POST \
--url $DOCS_URL \
--header 'Content-Type: application/json' \
--data @<(cat <<EOF
{
"service": "$DOCS_API_TOKEN",
"pageId": "$DOCS_PAGE_ID",
"blockId": "$DOCS_BLOCK_ID",
"schema": $SWAGGER
}
EOF
)
- name: Cleanup Docker's Leftovers
if: always()
continue-on-error: true
run: |
docker system prune
docker rmi -f $(docker images -aq)
if [ ${EVENT} != "workflow_dispatch" ]; then
sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
fi
sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
+1 -1
View File
@@ -10,7 +10,7 @@ services:
- .env
node_exporter:
image: prom/node-exporter:v1.3.1
image: ${NODE_EXPORTER_URL}
container_name: node_exporter
restart: always
volumes:
+5717
View File
File diff suppressed because it is too large Load Diff
+62 -18
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.18.10",
"@dadosfera/protospack-v2": "3.21.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -33,6 +33,7 @@
"helmet": "^5.1.0",
"jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
@@ -1726,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.18.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.10.tgz",
"integrity": "sha512-0bRUvo4p7cJJOL+B0hVTiSDaA2MGtirPkFVQW4lUfzsyY5vrsbAkMz6R3iPdPPwFSxGn5LwidHGkP3/l7hlEpA==",
"version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -3543,7 +3544,6 @@
},
"node_modules/agent-base": {
"version": "6.0.2",
"dev": true,
"license": "MIT",
"dependencies": {
"debug": "4"
@@ -8205,6 +8205,29 @@
"version": "1.2.6",
"license": "MIT"
},
"node_modules/mixpanel": {
"version": "0.17.0",
"resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz",
"integrity": "sha512-DY5WeOy/hmkPrNiiZugJpWR0iMuOwuj1a3u0bgwB2eUFRV6oIew/pIahhpawdbNjb+Bye4a8ID3gefeNPvL81g==",
"dependencies": {
"https-proxy-agent": "5.0.0"
},
"engines": {
"node": ">=10.0"
}
},
"node_modules/mixpanel/node_modules/https-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
"dependencies": {
"agent-base": "6",
"debug": "4"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/mkdirp": {
"version": "0.5.6",
"license": "MIT",
@@ -8229,8 +8252,9 @@
}
},
"node_modules/moment-timezone": {
"version": "0.5.34",
"license": "MIT",
"version": "0.5.38",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.38.tgz",
"integrity": "sha512-nMIrzGah4+oYZPflDvLZUgoVUO4fvAqHstvG3xAUnMolWncuAiLDWNnJZj6EwJGMGfb1ZcuTFE6GI3hNOVWI/Q==",
"dependencies": {
"moment": ">= 2.9.0"
},
@@ -10337,9 +10361,9 @@
}
},
"node_modules/ts-proto": {
"version": "1.126.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.126.1.tgz",
"integrity": "sha512-PKrcgd0mCQJDY7tp+th349uYhnjs2Tz7wSqm9+DBcodl3YCKPt1pFtFmApOZdQ8WlghhgKAuJDIRQJBd6jD5YA==",
"version": "1.131.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.131.0.tgz",
"integrity": "sha512-oj/kWrDz3JkkPQwumQDKfwHroH9iNz2alvrbCYzNYJlb/2MAUzFNfh9T+SZUPDfoXXK6oJoW1hGCHznj67yHNg==",
"dependencies": {
"@types/object-hash": "^1.3.0",
"dataloader": "^1.4.0",
@@ -12306,9 +12330,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.18.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.10.tgz",
"integrity": "sha512-0bRUvo4p7cJJOL+B0hVTiSDaA2MGtirPkFVQW4lUfzsyY5vrsbAkMz6R3iPdPPwFSxGn5LwidHGkP3/l7hlEpA==",
"version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -13555,7 +13579,6 @@
},
"agent-base": {
"version": "6.0.2",
"dev": true,
"requires": {
"debug": "4"
}
@@ -16719,6 +16742,25 @@
"minimist": {
"version": "1.2.6"
},
"mixpanel": {
"version": "0.17.0",
"resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz",
"integrity": "sha512-DY5WeOy/hmkPrNiiZugJpWR0iMuOwuj1a3u0bgwB2eUFRV6oIew/pIahhpawdbNjb+Bye4a8ID3gefeNPvL81g==",
"requires": {
"https-proxy-agent": "5.0.0"
},
"dependencies": {
"https-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
"requires": {
"agent-base": "6",
"debug": "4"
}
}
}
},
"mkdirp": {
"version": "0.5.6",
"requires": {
@@ -16736,7 +16778,9 @@
"integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
},
"moment-timezone": {
"version": "0.5.34",
"version": "0.5.38",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.38.tgz",
"integrity": "sha512-nMIrzGah4+oYZPflDvLZUgoVUO4fvAqHstvG3xAUnMolWncuAiLDWNnJZj6EwJGMGfb1ZcuTFE6GI3hNOVWI/Q==",
"requires": {
"moment": ">= 2.9.0"
}
@@ -18131,9 +18175,9 @@
}
},
"ts-proto": {
"version": "1.126.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.126.1.tgz",
"integrity": "sha512-PKrcgd0mCQJDY7tp+th349uYhnjs2Tz7wSqm9+DBcodl3YCKPt1pFtFmApOZdQ8WlghhgKAuJDIRQJBd6jD5YA==",
"version": "1.131.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/ts-proto/-/ts-proto-1.131.0.tgz",
"integrity": "sha512-oj/kWrDz3JkkPQwumQDKfwHroH9iNz2alvrbCYzNYJlb/2MAUzFNfh9T+SZUPDfoXXK6oJoW1hGCHznj67yHNg==",
"requires": {
"@types/object-hash": "^1.3.0",
"dataloader": "^1.4.0",
+4 -2
View File
@@ -10,13 +10,14 @@
},
"scripts": {
"preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
"prebuild": "rimraf dist",
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"start": "nest start",
"start:dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "node dist/main",
"start:prod": "npm run build && node dist/main",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"test:watch": "jest --watch",
@@ -27,7 +28,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.18.10",
"@dadosfera/protospack-v2": "3.21.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -48,6 +49,7 @@
"helmet": "^5.1.0",
"jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
+4 -2
View File
@@ -24,6 +24,7 @@ import { CatalogModule } from './modules/catalog/catalog.module';
import { ConfigModule } from '@nestjs/config';
import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
import { ProductboardModule } from './modules/productboard/productboard.module';
import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
@Module({
controllers: [],
@@ -38,14 +39,15 @@ import { ProductboardModule } from './modules/productboard/productboard.module';
ConfigModule.forRoot({
isGlobal: true,
}),
ConnectorModule,
AuthModule,
ConnectorModule,
PermissionsModule,
TermsOfUseModule,
ConnectionModule,
NetworkConfigModule,
ConnectionTestModule,
PipelinesModule,
PipelinesV2Module,
TransformationsModule,
HealthModule,
CatalogModule,
@@ -53,9 +55,9 @@ import { ProductboardModule } from './modules/productboard/productboard.module';
RolesModule,
InputsModule,
OauthModule,
PipelinesV2Module,
CatalogModule,
ProductboardModule,
MixpanelModule,
],
})
export class AppModule {}
@@ -44,7 +44,7 @@ class NoClassAuthController {
@Get('has-all-permissions')
@RequireAllPermissions(
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
)
async hasAllPermissions(@Body() body) {
return { body };
@@ -53,7 +53,7 @@ class NoClassAuthController {
@Get('has-some-permission')
@RequireSomePermission(
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
)
async hasSomePermission(@Body() body) {
return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
@Controller('class-auth-condition')
@AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
@RequireSomePermission(PERMISSIONS_GROUPS.METABASE.permissions.OPEN)
@RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
class ClassAuthConditionController {
@Get('body')
async getBody(@Body() body) {
@@ -444,13 +444,13 @@ describe('authentication.guard', () => {
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.METABASE.permissions.OPEN]);
const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
+61 -41
View File
@@ -128,11 +128,11 @@ export const PERMISSIONS_GROUPS = {
DELETE: {
seqid: 38,
claim: 'connection:delete',
usage: PermissionUsages.INTERNAL,
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Excluir Fonte',
'en-us': 'Remove Source',
'es-s': 'Eliminar Source',
'es-es': 'Eliminar Source',
},
},
UPDATE: {
@@ -290,49 +290,50 @@ export const PERMISSIONS_GROUPS = {
claim: 'GET /catalog',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Buscar data assets',
'en-us': 'Search for data assets',
'es-es': 'Buscar data assets',
'pt-br': 'Acessar lista de Ativos e visualizar atributos',
'en-us': 'Access Assets list and its attributes',
'es-es': 'Acceda a la lista de Activos y vea los atributos',
},
},
CREATE: {
seqid: 19,
claim: 'POST /catalog',
claim: 'catalog:create',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Criar atributos no catálogo',
'en-us': 'Create attributes',
'es-es': 'Crear atributos',
'pt-br': 'Criar Ativos',
'en-us': 'Create Assets',
'es-es': 'Crear Activos',
},
},
UPDATE: {
seqid: 25,
claim: 'PUT /catalog',
usage: PermissionUsages.INTERNAL,
claim: 'catalog:edit',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Editar atributos no catálogo',
'en-us': 'Edit attributes',
'es-es': 'Editar atributos',
'pt-br': 'Criar e editar atributos no catálogo',
'en-us': 'Create and edit attributes on the catalog',
'es-es': 'Crear y editar atributos en el catálogo',
},
},
DELETE: {
seqid: 1,
claim: 'DELETE /catalog',
usage: PermissionUsages.INTERNAL,
claim: 'catalog:delete',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Excluir atributos no catálogo',
'en-us': 'Remove attributes',
'es-es': 'Eliminar atributos',
'pt-br': 'Excluir Ativos do catálogo',
'en-us': 'Remove Assets from the catalog',
'es-es': 'Eliminar Activos del catálogo',
},
},
DATA_MANAGER: {
seqid: 42,
claim: 'catalog:data_manager',
usage: PermissionUsages.PUBLIC,
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Vizualizar todos os recursos do catálogo.',
'en-us': 'View all data assets.',
'es-es': 'Ver todos los activos de datos',
'pt-br':
'Gerente do catálogo. Consegue vizualizar e editar todos os ativos.',
'en-us': 'Catalog manager. Able to view and edit all assets.',
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
},
@@ -428,14 +429,14 @@ export const PERMISSIONS_GROUPS = {
},
},
METABASE: {
ANALYZE: {
title: {
'pt-br': 'Analisar | Visualização',
'en-us': 'Analyze | Visualization',
'es-es': 'Analizar | Visualización',
},
permissions: {
OPEN: {
METABASE: {
seqid: 11,
claim: 'GET /metabase',
usage: PermissionUsages.PUBLIC,
@@ -445,26 +446,26 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Acceso Metabase',
},
},
},
},
ORCHEST: {
title: {
'pt-br': 'Analisar | Inteligência',
'en-us': 'Analyze | Intelligence',
'es-es': 'Analizar | Inteligencia',
},
permissions: {
INTELLIGENCE: {
seqid: 31,
claim: 'intelligence:open',
usage: PermissionUsages.INTERNAL,
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Orchest',
'en-us': 'Access Orchest',
'es-s': 'Acceso Orchest',
'pt-br': 'Acessar Módulo de Inteligência',
'en-us': 'Access Intelligence Module',
'es-s': 'Acceder Módulo de Inteligencia',
},
},
},
},
MODULES: {
title: {
'pt-br': 'Módulos da Dadosfera',
'en-us': 'Dadosfera Modules',
'es-es': 'Módulos de la Dadosfera',
},
permissions: {
APP_BUILDER: {
seqid: 32,
claim: 'app-builder:open',
@@ -472,7 +473,7 @@ export const PERMISSIONS_GROUPS = {
name: {
'pt-br': 'Acessar App Builder',
'en-us': 'Access App Builder',
'es-es': 'Acceso App Builder',
'es-es': 'Acceder App Builder',
},
},
MACHINE_LEARNING: {
@@ -482,7 +483,7 @@ export const PERMISSIONS_GROUPS = {
name: {
'pt-br': 'Acessar Machine Learning',
'en-us': 'Access Machine Learning',
'es-es': 'Acceso Machine Learning',
'es-es': 'Acceder Machine Learning',
},
},
},
@@ -506,6 +507,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
PROCESS: {
title: {
'pt-br': 'Processar',
'en-us': 'Process',
'es-es': 'Procesar',
},
permissions: {
TRANSFORMATION: {
seqid: 43,
claim: 'process:open',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Transformação',
'en-us': 'Access Transformation Module',
'es-es': 'Acceder Módulo de transformación',
},
},
},
},
DADOSFERA: {
title: {
'pt-br': 'Usuários',
+1 -1
View File
@@ -49,7 +49,7 @@ describe('user.decorator', () => {
const fakeUserPayload = {
user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
username: 'super.admin',
permissions: [PERMISSIONS_GROUPS.METABASE.permissions.OPEN].map(
permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
({ seqid }) => seqid,
),
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
@@ -3,7 +3,6 @@ import {
Catch,
ArgumentsHost,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { Response } from 'express';
+34 -11
View File
@@ -6,6 +6,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -24,26 +26,47 @@ async function bootstrap() {
});
app.use(helmet());
configureSwagger(app);
const date = new Date();
await app.listen(3333);
}
function configureSwagger(app: INestApplication) {
let branchName = '';
try {
branchName = execSync('git rev-parse --abbrev-ref HEAD')
.toString()
.replace('\n', '')
.trim();
} catch (error) {
console.log("Can't get branch name", error);
}
const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
const config = new DocumentBuilder()
.setTitle('Maestro')
.setTitle(swaggerTitle)
.setDescription('Documentation for Maestro gateway')
.setVersion(
`${date.getDate()}.${
date.getMonth() + 1
}.${date.getFullYear()} - ${date.getHours()}:${date.getMinutes()}`,
.addBearerAuth(
{ type: 'http', scheme: 'bearer', bearerFormat: 'JWT' },
'Authorization',
)
.addBearerAuth()
.build();
const document = SwaggerModule.createDocument(app, config);
if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
await writeFileSync('swagger.txt', JSON.stringify(JSON.stringify(document)));
await app.listen(3333);
writeFileSync(
'docsfera.json',
JSON.stringify(
{
service: '${DOCS_API_TOKEN}',
pageId: '${DOCS_PAGE_ID}',
blockId: '${DOCS_BLOCK_ID}',
schema: document,
},
null,
2,
),
);
}
bootstrap();
+18 -10
View File
@@ -10,7 +10,6 @@ import {
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
AuthSignInRequest,
AuthRefreshAccessTokenRequest,
AuthChangePasswordRequest,
AuthResetPasswordRequest,
@@ -27,6 +26,8 @@ import { AuthClientService } from './auth.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { Metadata } from '@grpc/grpc-js';
import { AuthSignInReq, AuthSignInRes } from './dtos/login';
@ApiTags('Auth')
@UseFilters(new GrpcToHttpExceptionFilter())
@@ -42,6 +43,18 @@ export class AuthController {
this.logger = dadosferaLogger.logger;
}
@Post('sign-in')
@HttpCode(HttpStatus.OK)
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Headers('Dadosfera-Lang') language: string,
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const metadata = new Metadata();
metadata.add('language', language || 'pt-br');
return this.authClient.signIn({ username, password, totp }, metadata);
}
@Post('/sso/snowflake')
@RequireAllPermissions(PERMISSIONS_GROUPS.SNOWFLAKE.permissions.OPEN)
@HttpCode(HttpStatus.OK)
@@ -57,22 +70,17 @@ export class AuthController {
});
}
@Post('sign-in')
@HttpCode(HttpStatus.OK)
async signIn(@Body() { username, password, totp }: AuthSignInRequest) {
this.logger.info('/auth - SignIn');
return this.authClient.signIn({ username, password, totp });
}
@Post('refresh-access-token')
@HttpCode(HttpStatus.OK)
async refreshAccessToken(
@Body() { refreshToken }: AuthRefreshAccessTokenRequest,
@Headers('Dadosfera-Lang') language: string,
) {
this.logger.info('/auth - RefreshAccessToken');
const metadata = new Metadata();
metadata.add('language', language || 'pt-br');
return this.authClient.refreshAccessToken({ refreshToken });
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
}
@Post('change-password')
+11 -4
View File
@@ -19,6 +19,7 @@ import {
AuthConfirmResetPasswordRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
export class AuthClientService implements OnModuleInit {
@@ -51,19 +52,25 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
async signIn({ username, password, totp }: AuthSignInRequest) {
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }),
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
}
async refreshAccessToken({ refreshToken }: AuthRefreshAccessTokenRequest) {
async refreshAccessToken(
{ refreshToken }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken }),
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
);
}
+88 -1
View File
@@ -1,4 +1,8 @@
import { ApiProperty } from '@nestjs/swagger';
import {
AuthSignInRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class LoginDto {
@ApiProperty()
@@ -10,3 +14,86 @@ export class LoginDto {
@ApiProperty({ required: false })
totp: string;
}
export class AuthTokens {
@ApiProperty()
accessToken: string;
@ApiProperty()
refreshToken: string;
@ApiPropertyOptional()
termsOfUseToken: string;
@ApiProperty({ deprecated: true })
idToken: string;
}
export enum MfaStatuses {
PENDING = 'pending',
NONE = 'none',
TOTP = 'totp',
}
export enum TouStatuses {
PENDING = 'pending',
REQUIRED = 'required',
OK = 'ok',
}
export class TermsOfUse {
@ApiProperty()
version: number;
@ApiProperty()
publicUrl: string;
@ApiProperty()
enforceDate: string;
@ApiProperty()
createdAt: string;
}
export class TermsOfUseStatus {
@ApiProperty({ enum: TouStatuses })
status: string;
@ApiPropertyOptional()
lastSigned?: TermsOfUse;
@ApiPropertyOptional()
next?: TermsOfUse;
}
export class AuthUser {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
username: string;
@ApiProperty()
createdAt: string;
}
export class AuthCustomer {
@ApiProperty()
modules: string[];
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
tier: string;
}
export class AuthSignInReq implements AuthSignInRequest {
@ApiProperty()
username: string;
@ApiProperty()
password: string;
@ApiPropertyOptional()
totp: string;
}
export class AuthSignInRes implements AuthSignInResponse {
@ApiProperty()
permissions: string[];
@ApiProperty({ enum: MfaStatuses })
mfaStatus: string;
@ApiPropertyOptional()
customer?: AuthCustomer;
@ApiPropertyOptional()
user?: AuthUser;
@ApiPropertyOptional()
tokens?: AuthTokens;
@ApiPropertyOptional()
termsOfUse?: TermsOfUseStatus;
}
File diff suppressed because it is too large Load Diff
+157 -341
View File
@@ -1,8 +1,10 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { ReadService } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import { WriteService } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
WriteService,
ReadService,
ProtoServices,
Messages,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
HttpException,
HttpStatus,
@@ -15,6 +17,7 @@ import { lastValueFrom } from 'rxjs';
import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -55,6 +58,19 @@ class CatalogService implements OnModuleInit {
)}.dadosfera.ai`;
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.CreateDataAsset(data, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
}
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
@@ -81,34 +97,43 @@ class CatalogService implements OnModuleInit {
});
}
async catalogAll(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/all/`);
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return data;
return lastValueFrom(
this.catalogWriteService.MakeAComment(data, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
}
async dataAppsAll(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data_apps/`);
return data;
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.UpdateAComment(data, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
}
async dataAppsOne(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/data_apps/${id}/`,
);
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return data;
}
async getAllDashboardMetabase(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/dashboard-metabase/`,
);
return data;
return lastValueFrom(
this.catalogWriteService.DeleteDataAsset(data, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
}
async getUserRolesIds(userId: string) {
@@ -145,15 +170,23 @@ class CatalogService implements OnModuleInit {
return [customer_users, customer_roles];
}
async getAllOpenSearchDataAssets(
data: Messages.GetAllDataAssetsRequest,
metadata,
customer_id,
) {
async getAllOpenSearchDataAssets(query, metadata, customer_id) {
this.logger.info('CatalogService - getAllOpenSearchDashboard');
const response = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(data, metadata),
const { search, page, size, sort_by, order, ...filters } = query;
const { data_assets, total } = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(
{
search,
page,
size,
sort_by,
order,
filters: JSON.stringify(filters),
},
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
@@ -161,12 +194,12 @@ class CatalogService implements OnModuleInit {
);
});
const data_assets = JSON.parse(response.data_assets);
const result = JSON.parse(data_assets);
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
for (const asset of data_assets.tables) {
for (const asset of result.data_assets) {
const condition =
asset.owner && !asset.owner.includes('@') ? true : false;
const result = condition
@@ -198,30 +231,18 @@ class CatalogService implements OnModuleInit {
});
}
for (const asset of data_assets.dashboards) {
const condition =
asset.owner && !asset.owner.includes('@') ? true : false;
const result = condition
? await this.userService.findOneById(asset.owner).catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
Object.assign(asset, { ...asset, owner });
}
return { data_assets };
return { data_assets: result.data_assets, total };
}
async getOneDashboardMetabase(id, body, metadata, customer_id) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/dashboard-metabase/${id}`,
);
async getOneDataAsset({
data_asset_id,
data_asset_type,
customer_id,
metadata,
}) {
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: 'dashboard' },
{ id: data_asset_id, type: data_asset_type },
metadata,
),
);
@@ -253,9 +274,7 @@ class CatalogService implements OnModuleInit {
}
}
Object.assign(data.dashboard, {
tags: opensearchTable.tags,
description: opensearchTable.description || null,
Object.assign(opensearchTable, {
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
roles,
@@ -263,163 +282,94 @@ class CatalogService implements OnModuleInit {
owner,
});
return data;
return { data_asset: opensearchTable };
}
async getAllTableMetadata(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-metadata/`,
);
return data;
}
async getOneTableMetadata(body, params, metadata, customer_id) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-metadata/`,
{ params: params },
);
if (data?.length === 1) {
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id: data[0].id, type: 'dataset' },
metadata,
),
);
const opensearchTable = JSON.parse(data_asset);
const result = opensearchTable.owner
? await this.userService
.findOneById(opensearchTable.owner)
.catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const users = [];
const roles = [];
for (const role of customer_roles) {
for (const role_id of opensearchTable.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of opensearchTable.users) {
if (user.id === user_id) users.push(user);
}
}
return [
{
...data[0],
tags: opensearchTable.tags,
table_description: opensearchTable.description,
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
users,
roles,
owner,
},
];
}
return data;
}
async deleteOneTableMetadata(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-metadata/${id}`,
);
return data;
}
async getOneColumnMetadata(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/column-metadata/`,
{ params: params },
);
return data;
}
async deleteOneColumnMetadata(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/column-metadata/${id}/`,
);
return data;
}
async getOneDataPreview(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-preview/`, {
params: params,
});
return data;
}
async getDataStatus(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-status/`);
return data;
}
async createDataStatus(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-status/`,
body,
);
return data;
}
async getDataDescription(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/data-description/`,
);
return data;
}
async createDataDescription(body, metadata) {
const response = await lastValueFrom(
async updateOneDataAsset({ data_asset_id, customer_id, body, metadata }) {
const { data_asset } = await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{
id: `dataset-${body.table_id}`,
description: body.description,
tags: undefined,
},
{ id: data_asset_id, changes: JSON.stringify(body) },
metadata,
),
);
// {
// description: 'descrição nova';
// id: 5;
// table_id: 4;
// }
// const nimbusUrl = this._getNimbusUrl(body);
// const { data } = await axios.post(
// `${nimbusUrl}/api/catalog/data-description/`,
// body,
// );
// {"id":5,"description":"descrição nova","updated_at":"2022-09-28T14:35:15.618750Z","table_id":4}
return {
id: 0,
description: body.description,
updated_at: new Date().toISOString(),
table_id: body.table_id,
};
const opensearchTable = JSON.parse(data_asset);
const result = opensearchTable.owner
? await this.userService
.findOneById(opensearchTable.owner)
.catch(() => null)
: null;
const owner = result !== null ? result.user.username : null;
const [customer_users, customer_roles] =
await this.getUsernamesAndRolesByCustomerId(customer_id);
const users = [];
const roles = [];
for (const role of customer_roles) {
for (const role_id of opensearchTable.roles) {
if (role.id === role_id) roles.push(role);
}
}
for (const user of customer_users) {
for (const user_id of opensearchTable.users) {
if (user.id === user_id) users.push(user);
}
}
Object.assign(opensearchTable, {
p_roles: opensearchTable.roles,
p_users: opensearchTable.users,
roles,
users,
owner,
});
return { data_asset: opensearchTable };
}
async getDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-docs/`);
return data;
async getDataDocs(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc(
{ id, type: data_asset_type },
metadata,
),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
async getDatasetPreview(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id, type: data_asset_type },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getDatasetColumnsMetadata(data_asset_id: string, metadata: Metadata) {
const [data_asset_type, id] = data_asset_id.split('-');
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id, type: data_asset_type },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async createDataDocs(body) {
@@ -431,70 +381,6 @@ class CatalogService implements OnModuleInit {
return data;
}
async getDataRating(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-rating/`);
return data;
}
async createDataRating(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-rating/`,
body,
);
return data;
}
async getSummaryRating(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/summary-rating/${id}`,
);
return data;
}
async getDataComment(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-comment/`);
return data;
}
async createDataComment(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-comment/`,
body,
);
return data;
}
async getDataReview(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-review/`, {
params: params,
});
return data;
}
async createTags(data: Messages.TagTableRequest, metadata) {
this.logger.info('CatalogService - createTags');
const response = await lastValueFrom(
this.catalogWriteService.TagTable(data, metadata),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return { data_assets: response };
}
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
@@ -512,76 +398,6 @@ class CatalogService implements OnModuleInit {
return response;
}
async findAllTableTags(data, metadata) {
this.logger.info('CatalogService - findAllTableTags');
const response = await lastValueFrom(
this.catalogReadService.GetTableTags(data, metadata),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return response;
}
async deleteTags(data, metadata) {
this.logger.info('CatalogService - deleteTags');
const response = await lastValueFrom(
this.catalogWriteService.UntagTable(data, metadata),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return response;
}
async createTableTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/table-tags/`,
body,
);
return data;
}
async deleteTableTags(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-tags/${id}`,
);
return data;
}
async getTableRules(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-rules/`,
body,
);
return data;
}
async deleteTableRules(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-rules/${id}`,
);
return data;
}
}
export { CatalogService };
+217
View File
@@ -0,0 +1,217 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export class IRole {
@ApiProperty()
name: string;
@ApiProperty()
id: string;
}
export class IUser {
@ApiProperty()
username: string;
@ApiProperty()
id: string;
}
export class IComment {
@ApiProperty()
id: number;
@ApiProperty()
message: string;
@ApiProperty()
created_at: string;
@ApiProperty()
username: string;
}
export class IDataAsset {
@ApiProperty()
id: number;
@ApiPropertyOptional()
database_name?: string;
@ApiPropertyOptional()
table_name?: string;
@ApiPropertyOptional()
num_columns?: number;
@ApiPropertyOptional()
location?: string;
@ApiPropertyOptional()
created_time?: string;
@ApiPropertyOptional()
table_type?: string;
@ApiPropertyOptional()
size_in_bytes?: number;
@ApiPropertyOptional()
num_files?: number;
@ApiPropertyOptional()
data_encrypted?: string;
@ApiPropertyOptional()
num_rows?: number;
@ApiPropertyOptional()
display_name?: string;
@ApiProperty()
pipeline_id: string;
@ApiProperty()
user_id: string;
@ApiProperty()
description: string;
@ApiProperty()
data_asset_type: string;
@ApiProperty()
tags: string[];
@ApiProperty()
owner: string;
@ApiProperty({ type: [IRole] })
roles: IRole[];
@ApiProperty({ type: [IUser] })
users: IUser[];
@ApiPropertyOptional()
dashboard_id?: number;
@ApiPropertyOptional()
name?: string;
@ApiPropertyOptional()
parameters?: string[];
@ApiPropertyOptional()
embedding_params?: object;
@ApiPropertyOptional()
created_at?: string;
@ApiPropertyOptional()
updated_at?: string;
@ApiProperty({ type: [IComment] })
comments: IComment[];
}
export class IOneDataAsset {
@ApiProperty()
data_asset: IDataAsset;
}
export class IMakeAComment {
@ApiProperty()
message: string;
}
export class IDeleteComment {
@ApiProperty()
comment: IComment;
}
export class ICatalogAllResponse {
@ApiProperty({ type: [IDataAsset] })
data_assets: IDataAsset[];
@ApiProperty()
total: number;
}
export class IData {
@ApiProperty()
var: number;
@ApiProperty()
date: string;
@ApiProperty()
last: number;
@ApiProperty()
maximum: number;
@ApiProperty()
minimum: number;
@ApiProperty()
day_opening: number;
}
export class IUpdateDataRequest {
@ApiProperty()
name: string;
@ApiProperty()
description: string;
@ApiProperty()
tags: string[];
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
@ApiProperty()
description: string;
@ApiProperty()
data_asset_type: string;
@ApiProperty()
external_url: string;
@ApiProperty()
location: string;
}
export class IPreview {
@ApiProperty()
id: string;
@ApiProperty()
database_name: string;
@ApiProperty()
table_name: string;
@ApiProperty({ type: [IData] })
data: IData[];
}
export class IDocs {
@ApiProperty()
id: string;
@ApiProperty()
docs: string;
@ApiProperty()
updated_at: string;
@ApiProperty()
table_id: string;
}
export class IColumnMetadata {
@ApiProperty()
id: string;
@ApiProperty()
table_name: string;
@ApiProperty()
column_name: string;
@ApiProperty()
data_type: string;
@ApiProperty()
missing_count: string;
@ApiProperty()
min: string;
@ApiProperty()
max: string;
@ApiProperty()
count: string;
@ApiProperty()
histogram: string;
@ApiProperty()
mean: string;
@ApiProperty()
median: string;
@ApiProperty()
missing_rate: string;
@ApiProperty()
stddev: string;
@ApiProperty()
unique_count: string;
@ApiProperty()
unique_rate: string;
@ApiProperty()
analyzer_rules: string[];
@ApiProperty()
version: string;
}
export class IPreviewResponse {
@ApiProperty()
preview: IPreview;
}
export class IDocsResponse {
@ApiProperty()
docs: IDocs;
}
export class IColumnsMetadataResponse {
@ApiProperty({ type: [IColumnMetadata] })
columns_metadata: IColumnMetadata[];
}
+6 -36
View File
@@ -1,9 +1,4 @@
import {
OnModuleInit,
Inject,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { OnModuleInit, Inject } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { ConnectionManager } from '@dadosfera/protospack-v2';
import { lastValueFrom } from 'rxjs';
@@ -86,12 +81,7 @@ export class ConnectionClientService implements OnModuleInit {
},
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
);
}
async updateConnection(
@@ -112,12 +102,7 @@ export class ConnectionClientService implements OnModuleInit {
},
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
);
}
async deleteConnection({ body, metadata }): Promise<ConnectionRes> {
@@ -130,12 +115,7 @@ export class ConnectionClientService implements OnModuleInit {
},
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
);
}
async getAllConnections({ body, metadata }): Promise<ConnectionsRes> {
@@ -146,12 +126,7 @@ export class ConnectionClientService implements OnModuleInit {
{ search, filters: JSON.stringify(filters), size, page },
metadata,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
);
}
async getConnectionDetails({
@@ -162,12 +137,7 @@ export class ConnectionClientService implements OnModuleInit {
return lastValueFrom(
this.connectionServiceRead.GetConnectionDetails({ id }, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
);
}
async INTERNAL_getAllConnections({
@@ -9,6 +9,7 @@ import {
Get,
Headers,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { ConnectionClientService } from './client.service';
@@ -28,8 +29,10 @@ import {
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@UseFilters(new GrpcToHttpExceptionFilter())
@ApiTags('connections')
@ApiBearerAuth()
@Authenticated()
+24 -160
View File
@@ -83,161 +83,30 @@ export class ConnectorClientService implements OnModuleInit {
);
});
const responseParsed = JSON.parse(res.message);
const connectorErrors = responseParsed.items
.filter((i) => i.update._index === 'connector' && i.update.error)
const connectorsResponse = responseParsed.items.filter(
(i) => i.update._index === process.env.CONNECTORS_INDEX,
);
const connectorErrors = connectorsResponse
.filter((i) => i.update.error)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const connectionControlsErrors = responseParsed.items
.filter(
(i) => i.update._index === 'connection_controls' && i.update.error,
)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const connectionStepsErrors = responseParsed.items
.filter((i) => i.update._index === 'connection_steps' && i.update.error)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const configControlsErrors = responseParsed.items
.filter((i) => i.update._index === 'config_controls' && i.update.error)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const connectorCreated = responseParsed.items
.filter(
(i) => i.update._index === 'connector' && i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const connectionControlsCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_controls' &&
i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const connectionStepsCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_steps' &&
i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const configControlsCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'config_controls' &&
i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const connectorUpdated = responseParsed.items
.filter(
(i) => i.update._index === 'connector' && i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const connectionControlsUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_controls' &&
i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const connectionStepsUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_steps' &&
i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const configControlsUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'config_controls' &&
i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const connectorNoop = responseParsed.items
.filter(
(i) => i.update._index === 'connector' && i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
}));
const connectionControlsNoop = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_controls' &&
i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
}));
const connectionStepsNoop = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_steps' && i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
}));
const configControlsNoop = responseParsed.items
.filter(
(i) =>
i.update._index === 'config_controls' && i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
id: i.update?._id,
reason: i.update?.error,
status: i.error?.status,
}));
const connectorCreated = connectorsResponse
.filter((i) => i.update.result === 'created')
.map((i) => i.update._id);
const connectorUpdated = connectorsResponse
.filter((i) => i.update.result === 'updated')
.map((i) => i.update._id);
const connectorNoop = connectorsResponse
.filter((i) => i.update.result === 'noop')
.map((i) => i.update._id);
return {
errors: {
connectors: connectorErrors,
connection_controls: connectionControlsErrors,
connection_steps: connectionStepsErrors,
config_controls: configControlsErrors,
},
created: {
connectors: connectorCreated,
connection_controls: connectionControlsCreated,
connection_steps: connectionStepsCreated,
config_controls: configControlsCreated,
},
updated: {
connectors: connectorUpdated,
connection_controls: connectionControlsUpdated,
connection_steps: connectionStepsUpdated,
config_controls: configControlsUpdated,
},
noop: {
connectors: connectorNoop,
connection_controls: connectionControlsNoop,
connection_steps: connectionStepsNoop,
config_controls: configControlsNoop,
},
errors: connectorErrors,
created: connectorCreated,
updated: connectorUpdated,
noop: connectorNoop,
};
}
@@ -263,13 +132,13 @@ export class ConnectorClientService implements OnModuleInit {
});
}
async getConnectorDetails(plugin: string, language: string) {
async getConnectorDetails(connectorId: string, language: string) {
const meta = new Metadata();
meta.add('Language', language);
return lastValueFrom(
this.connectorServiceRead.GetConnectorDetails(
{
plugin,
plugin: connectorId,
},
meta,
),
@@ -316,12 +185,7 @@ export class ConnectorClientService implements OnModuleInit {
version,
plugin,
}),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
);
}
async updateConnector({ plugin, changes }) {
@@ -19,7 +19,6 @@ import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ConnectorClientService } from './client.service';
import { AddTagDto } from './dtos/add-tag';
import { CreateConnectorDto } from './dtos/create-connector';
import { DeleteConnectorDto } from './dtos/delete-connector';
import { GetAllDto } from './dtos/get-all';
import { RemoveTagDto } from './dtos/remove-tag';
import { UpdateDto } from './dtos/update';
@@ -274,7 +273,7 @@ export class ConnectorController {
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
async deleteConnector(
@Param('plugin') plugin: string,
@Body() { version }: DeleteConnectorDto,
@Query('version') version: string,
) {
this.logger.info('/upload - Upload Connector Route');
+2 -2
View File
@@ -16,12 +16,12 @@ export class DucClient {
transport: Transport.GRPC,
options: {
url: process.env.DUC_URL,
package: ProtoPackages.WritePackage,
package: [ProtoPackages.WritePackage, ProtoPackages.ReadPackage],
credentials:
process.env.LOCAL_ENV || process.env.ENV === 'local'
? undefined
: credentials.createSsl(),
protoPath: ProtoPaths.WriteFilePath,
protoPath: [ProtoPaths.WriteFilePath, ProtoPaths.ReadFilePath],
loader: {
keepCase: true,
enums: String,
+10 -156
View File
@@ -1,106 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from 'protospack/dist/lib/interfaces';
export class GoogleAnalyticsClientSecrets {
@ApiProperty()
type: string;
@ApiProperty()
project_id: string;
@ApiProperty()
private_key_id: string;
@ApiProperty()
private_key: string;
@ApiProperty()
client_email: string;
@ApiProperty()
client_id: string;
@ApiProperty()
auth_uri: string;
@ApiProperty()
token_uri: string;
@ApiProperty()
auth_provider_x509_cert_url: string;
@ApiProperty()
client_x509_cert_url: string;
}
export class OauthObject {
@ApiPropertyOptional()
get_tokens_url: string;
@ApiPropertyOptional()
get_tokens_url_params: string;
@ApiPropertyOptional()
get_tokens_set_response: Record<string, any>;
@ApiPropertyOptional()
content_type: string;
}
export class InputOptions {
@ApiPropertyOptional()
oauth: OauthObject;
@ApiPropertyOptional()
skip_select_columns: true;
@ApiPropertyOptional()
skip_select_entities: false;
@ApiPropertyOptional()
skip_transformation: true;
}
export class FileFormatParams {
@ApiProperty()
sep: string;
@ApiProperty()
header: boolean;
@ApiProperty()
encoding: string;
}
export class AuthParameters {
@ApiProperty()
aws_access_key_id: string;
@ApiProperty()
aws_secret_access_key: string;
}
export class CredentialsJdbc {
@ApiPropertyOptional()
jdbc_user: string;
@ApiPropertyOptional()
jdbc_password: string;
@ApiPropertyOptional()
database: string;
@ApiPropertyOptional()
endpoint: string;
@ApiPropertyOptional()
port: string;
@ApiPropertyOptional()
engine: string;
@ApiPropertyOptional()
schema: string;
}
export class Credentials extends CredentialsJdbc {
@ApiProperty()
connection_type: string;
@ApiPropertyOptional()
client_aws_access_key_id: string;
@ApiPropertyOptional()
client_aws_secret_access_key: string;
@ApiPropertyOptional()
client_bucket: string;
@ApiPropertyOptional()
file_to_extract: string;
@ApiPropertyOptional()
file_format_params: FileFormatParams;
@ApiPropertyOptional()
view_id: string;
@ApiPropertyOptional()
client_secrets: GoogleAnalyticsClientSecrets;
@ApiPropertyOptional()
start_date: string;
@ApiPropertyOptional()
end_date: string;
@ApiPropertyOptional()
oauth_code: string;
}
export class Column {
@ApiProperty()
name: string;
@@ -114,6 +13,14 @@ export class TableColumns {
columns: string[];
@ApiProperty()
references: Column[];
@ApiProperty()
type: string;
}
export class AvailableEntity {
@ApiProperty()
name: string;
@ApiProperty()
replication_methods: string[];
}
export class Input {
@@ -133,26 +40,6 @@ export class Input {
created_at: string;
@ApiProperty()
updated_at: string;
//deprecated attributes (for retro compatibility)
@ApiPropertyOptional({ deprecated: true })
category: string;
@ApiPropertyOptional({ deprecated: true })
credentials: Credentials;
@ApiPropertyOptional({ deprecated: true })
options: InputOptions;
@ApiPropertyOptional({ deprecated: true })
info: Info;
@ApiPropertyOptional({ deprecated: true })
cron: string;
@ApiPropertyOptional({ deprecated: true })
source_bucket: string;
@ApiPropertyOptional({ deprecated: true })
source_prefix: string;
@ApiPropertyOptional({ deprecated: true })
auth_parameters: AuthParameters;
@ApiPropertyOptional({ deprecated: true })
file_format_params: FileFormatParams;
}
export class GetAvailableEntitiesReq {
@ApiProperty()
@@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq {
}
export class GetAvailableEntitiesRes {
@ApiProperty()
entities: string[];
@ApiProperty({ type: [AvailableEntity] })
entities: AvailableEntity[];
}
export class TestConnectionGetColumnsReq {
@ApiProperty()
plugin: string;
@ApiProperty()
tables: string[];
@ApiPropertyOptional()
credentials: CredentialsJdbc;
@ApiPropertyOptional()
id: string;
@ApiPropertyOptional()
info: Info;
}
export class TestConnectionGetColumnsRes {
@ApiProperty({ type: [TableColumns] })
tables: TableColumns[];
}
export class TestConnectionReq {
@ApiProperty()
plugin: string;
@ApiProperty()
credentials: CredentialsJdbc;
@ApiPropertyOptional()
info: Info;
}
export class TestConnectionRes {
@ApiProperty()
connection_state: boolean;
@ApiProperty()
total_entities: number;
@ApiProperty()
database_tables: string[];
}
export class CreateInputReq extends OmitType(Input, [
'id',
'created_at',
+19 -38
View File
@@ -7,6 +7,7 @@ import {
Param,
Patch,
Post,
UseFilters,
} from '@nestjs/common';
import { InputsService } from './inputs.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -18,16 +19,13 @@ import {
GetAvailableEntitiesReq,
GetAvailableEntitiesRes,
Input,
TestConnectionGetColumnsReq,
TestConnectionGetColumnsRes,
TestConnectionReq,
TestConnectionRes,
} from './dtos/input.model';
import { UpdateInputRequest } from './dtos/old_interfaces';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@ApiTags('inputs')
@ApiTags('Inputs')
@Controller('inputs')
@AuthenticateCondition((req, user) => {
switch (req.method) {
@@ -38,7 +36,9 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE.seqid,
) ||
user.permissions.includes(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid)
user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
)
);
default:
@@ -49,10 +49,13 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
) ||
user.permissions.includes(PERMISSIONS_GROUPS.PIPELINE.permissions.GET.seqid)
user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions.GET.seqid,
)
);
}
})
@UseFilters(new GrpcToHttpExceptionFilter())
export class InputsController {
logger: DadosferaLogger;
constructor(
@@ -66,41 +69,19 @@ export class InputsController {
@Get('available-entities/:plugin')
@ApiOkResponse({ type: GetAvailableEntitiesRes })
async getAvailableEntities(
@Body() body,
@Param() params: GetAvailableEntitiesReq,
@User() user: RequestUser,
) {
const { info } = body;
const { plugin } = params;
return await this.inputService.getAvailableEntities({ info, plugin });
}
@Post('/test-connection')
@ApiOkResponse({ type: TestConnectionRes })
async testConnection(@Body() data: TestConnectionReq) {
this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
});
const response = await this.inputService.testConnection(data);
return response;
}
@Post('/test-connection/get-columns')
@ApiOkResponse({ type: TestConnectionGetColumnsRes })
async getColumns(@Body() data: TestConnectionGetColumnsReq) {
this.logger.info(
`/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
{
user: data.info.user_id,
customer: data.info.customer,
const availableEntities = await this.inputService.getAvailableEntities({
info: {
customer: user.customer_name,
customer_id: user.customer_id,
user_id: user.user_id,
},
);
const response = await this.inputService.getColumns(data);
return response;
plugin,
});
return availableEntities;
}
@Post()
+12 -81
View File
@@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
import { Input } from '@dadosfera/protospack-v2';
import {
GetAvailableEntitiesRequest,
InputCreateGenericRequest,
InputCreateS3Request,
InputNewCreateRequest,
TestConnectionGetColumnsRequest,
TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -72,17 +72,6 @@ export class InputsService {
objectCamelToSnake(createInputResponse);
return createInputResponse;
},
findOne: async (data: IIdRequest) => {
this.logger.info('InputClientService - FindOne');
const findOneInputResponse = await lastValueFrom(
this.inputReadService.InputFindOne(data),
).catch((e) => {
this.logger.error(e.details);
throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
});
objectCamelToSnake(findOneInputResponse);
return findOneInputResponse;
},
update: async (updateInputDTO: UpdateInputRequest) => {
this.logger.info('InputClientService - Update');
const updateInputResponse = await lastValueFrom(
@@ -91,14 +80,6 @@ export class InputsService {
return updateInputResponse;
},
remove: async (idRequest: IIdRequest) => {
this.logger.info('InputClientService - Remove');
const removeInputResponse = await lastValueFrom(
this.inputWriteService.InputRemove(idRequest),
);
return removeInputResponse;
},
testConnection: async (data: TestConnectionRequest) => {
this.logger.info('InputClientService - TestConnection');
const testConnectionResponse = await lastValueFrom(
@@ -107,32 +88,6 @@ export class InputsService {
return testConnectionResponse;
},
getColumns: async (data: TestConnectionGetColumnsRequest) => {
this.logger.info('InputClientService - TestConnection/Get-Columns');
const getColumnsResponse = await new Promise((resolve, reject) => {
this.inputReadService.GetColumns(data).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return getColumnsResponse;
},
getAvailableEntities: async (data) => {
this.logger.info('InputClientService - GetAvailableEntities');
data;
@@ -207,12 +162,9 @@ export class InputsService {
// this.validateCron(data);
const { body, info } = data;
if (body.info) delete body.info;
const inputCreateGenericRequest: any | InputCreateGenericRequest = {
const inputCreateGenericRequest: InputCreateGenericRequest = {
input: {
...body,
credentials: body.credentials && JSON.stringify(body.credentials),
},
info,
};
@@ -224,8 +176,8 @@ export class InputsService {
return { input: adjustedInput };
}
async getAvailableEntities(data): Promise<{ entities: string[] }> {
return await this.OLD_inputClient.getAvailableEntities(data);
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
}
async findAll(info: Info) {
@@ -239,17 +191,13 @@ export class InputsService {
}
async findOne(idRequest: IIdRequest) {
try {
const findOneInputResponse: any = await this.OLD_inputClient.findOne(
idRequest,
);
findOneInputResponse.input = this.adjustInputPayload(
findOneInputResponse.input,
);
return findOneInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
const findOneInputResponse: any = await lastValueFrom(
this.inputReadService.InputFindOne(idRequest),
);
findOneInputResponse.input = this.adjustInputPayload(
findOneInputResponse.input,
);
return findOneInputResponse;
}
async update(id: string, data, info: Info) {
@@ -271,13 +219,7 @@ export class InputsService {
}
async remove(idRequest: IIdRequest) {
try {
const removeInputResponse = await this.OLD_inputClient.remove(idRequest);
return removeInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
}
@Timeout(60000 * 10) // Timeout set for 10 minutes
@@ -292,17 +234,6 @@ export class InputsService {
}
}
async getColumns(data) {
try {
const testConnectionGetColumnsResponse =
await this.OLD_inputClient.getColumns(data);
return testConnectionGetColumnsResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
generateInputS3Payload(payload) {
const { credentials, plugin, cron } = payload;
if (!credentials) return payload;
@@ -0,0 +1,33 @@
import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/authentication/authentication.decorator';
import { RequestUser, User } from 'src/authentication/user.decorator';
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
) {}
@Post(':id')
async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
delete body.info;
const mixpanel = init(this.mixpanelToken);
await mixpanel.people.set(user.username, {
$name: user.username,
$email: user.username,
customer_name: user.customer_name,
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
...body,
});
return { id, body, user: user.username };
}
}
+16
View File
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { getSecreteFromSecreteManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
@Module({
controllers: [MixpanelController],
providers: [
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecreteFromSecreteManager(
`${process.env.ENV}/root/mixpanel_token`,
),
},
],
})
export class MixpanelModule {}
+3 -5
View File
@@ -5,9 +5,7 @@ import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { Authenticated } from 'src/authentication/authentication.decorator';
@ApiTags('oauth')
@Authenticated()
@Controller('oauth')
export class OauthController {
redirectUrl: string;
@@ -19,13 +17,13 @@ export class OauthController {
switch (process.env.ENV) {
case 'dev':
case 'stg':
this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/coletix/auth-callback`;
this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/collect/auth-callback`;
break;
case 'prd':
this.redirectUrl = `https://app.dadosfera.ai/coletix/auth-callback`;
this.redirectUrl = `https://app.dadosfera.ai/collect/auth-callback`;
break;
default:
this.redirectUrl = `http://localhost:4200/coletix/auth-callback`;
this.redirectUrl = `http://localhost:4200/collect/auth-callback`;
}
}
@Get('hubspot')
@@ -1,14 +1,8 @@
import { Controller, Get, Headers, Inject } from '@nestjs/common';
import { ApiHeader, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
import {
PERMISSIONS_GROUPS,
PermissionUsages,
} from 'src/authentication/permissions.enum';
import { Authenticated } from 'src/authentication/authentication.decorator';
import { PermissionUsages } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GetPublicPermissionsRes } from './dto/entities';
+3 -100
View File
@@ -1,21 +1,9 @@
import {
BadRequestException,
ConflictException,
Inject,
InternalServerErrorException,
OnModuleInit,
} from '@nestjs/common';
import { ClientGrpc, Payload } from '@nestjs/microservices';
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack';
import { lastValueFrom } from 'rxjs';
import { objectSnakeToCamel } from 'src/utils/CaseConverter';
import {
ICreatePipelineDto,
IGetPipelineLogsRequest,
IIdRequest,
IUpdatePipelineRequest,
} from './interfaces';
import { IIdRequest } from './interfaces';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
@@ -39,91 +27,6 @@ export class PipelinesClientService implements OnModuleInit {
);
}
async create(@Payload() createPipelineDto: ICreatePipelineDto) {
this.logger.info('PipelinesClientService - Create');
const createPipelineResponse = await lastValueFrom(
this.pipelineService.Create(objectSnakeToCamel(createPipelineDto)),
).catch((error: { details: string }) => {
if (error.details.includes('INVALID_REQUEST')) {
const [, message] = error.details.split('|');
throw new BadRequestException(message);
}
throw new InternalServerErrorException(error.details);
});
this.logger.info('Done');
return createPipelineResponse;
}
async findAll(data) {
this.logger.info('PipelinesClientService - FindAll');
const findAllPipelineResponse = await lastValueFrom(
this.pipelineService.FindAll(data),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return findAllPipelineResponse;
}
async findOne(data: IIdRequest) {
this.logger.info('PipelinesClientService - FindOne');
const findOnePipelineResponse = await lastValueFrom(
this.pipelineService.FindOne(data),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return findOnePipelineResponse;
}
async update(UpdatePipelineRequest: IUpdatePipelineRequest) {
this.logger.info('PipelinesClientService - Update');
const updatePipelineResponse = await lastValueFrom(
this.pipelineService.Update(UpdatePipelineRequest),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return updatePipelineResponse;
}
async remove(data: IIdRequest) {
this.logger.info('PipelinesClientService - Remove');
const removePipelineResponse = await lastValueFrom(
this.pipelineService.remove(data),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return removePipelineResponse;
}
async getPipelineLogsMessages(data: IGetPipelineLogsRequest) {
this.logger.info('PipelinesClientService - GetPipelineLogsMessages');
const logsPipelineResponse = await lastValueFrom(
this.pipelineService.getPipelineLogs(data),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return logsPipelineResponse;
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
+1 -91
View File
@@ -1,13 +1,4 @@
import {
Body,
Controller,
Delete,
Get,
Inject,
Param,
Post,
Put,
} from '@nestjs/common';
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
@@ -16,7 +7,6 @@ import {
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/authentication/user.decorator';
@ApiTags('Pipelines')
@Controller('pipelines')
@@ -86,84 +76,4 @@ export class PipelinesController {
return response;
}
@Get(':id/:details')
async getPipelineLogs(@Param() params) {
const { id, details } = params;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE LOGS ROUTE`,
{},
);
const response = await this.pipelineService.getPipelineLogsMessages(
id,
details,
);
return response;
}
@Post()
async create(@Body() createPipelineDto) {
this.logger.info(process.env.DEV_URL + `/pipelines ON CREATE ROUTE`);
const response = await this.pipelineService.create(createPipelineDto);
return response;
}
@Get()
async findAll(@User() user: RequestUser) {
this.logger.info(process.env.DEV_URL + `/pipeline ON Find All ROUTE`, {
user: user.user_id,
customer: user.customer_id,
});
const { customer_id, user_id, customer_name } = user;
const response = await this.pipelineService.findAll({
customer_name,
customer_id,
user_id,
});
return response;
}
@Get('/:id')
async findOne(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(process.env.DEV_URL + `/pipeline/${id} ON Find One ROUTE`);
const response = await this.pipelineService.findOne({ id, ...data });
return response;
}
@Put(':id')
async update(@Body() updatePipelineDto, @Param() params) {
const { id } = params;
const { info } = updatePipelineDto;
delete updatePipelineDto.info;
this.logger.info(process.env.DEV_URL + `/pipeline/${id} ON UPDATE ROUTE`);
const response = await this.pipelineService.update(
id,
updatePipelineDto,
info,
);
return response;
}
@Delete(':id')
async delete(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(process.env.DEV_URL + `/pipeline/${id} ON DELETE ROUTE`);
const response = await this.pipelineService.remove({ id, ...data });
return response;
}
}
@@ -1,6 +1,4 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { DecodeGrpcStruct } from 'protospack';
import { Info } from 'protospack/dist/lib/interfaces';
import { PipelinesClientService } from './client.service';
import { IIdRequest } from './interfaces';
import { objectCamelToSnake } from 'src/utils/CaseConverter';
@@ -9,91 +7,6 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
export class PipelinesService {
constructor(private pipelineClient: PipelinesClientService) {}
adjustPayload(payload) {
if (payload.input?.input_generic) {
payload.input = DecodeGrpcStruct(payload.input?.input_generic);
} else {
payload.input = payload.input?.input_s3 || payload.input?.input_jdbc;
}
}
async create(createPipelineDto) {
const createPipelineResponse = await this.pipelineClient.create(
createPipelineDto,
);
const pipeline = objectCamelToSnake(createPipelineResponse);
this.adjustPayload(pipeline.pipeline);
return pipeline;
}
async findOne(data: IIdRequest) {
try {
const findOnePipelineResponse = await this.pipelineClient.findOne(data);
const pipeline = objectCamelToSnake(findOnePipelineResponse);
this.adjustPayload(pipeline.pipeline);
return pipeline;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async findAll(data) {
try {
const { pipelines } = await this.pipelineClient.findAll({
info: data,
});
if (pipelines)
pipelines.forEach((pipeline) => {
this.adjustPayload(pipeline);
});
return { pipelines: pipelines || [] };
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async update(id: string, data, info: Info) {
try {
const updatePipelineResponse = await this.pipelineClient.update({
id,
info,
...data,
});
const pipeline = objectCamelToSnake(updatePipelineResponse);
this.adjustPayload(pipeline);
return pipeline;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async remove(data: IIdRequest) {
try {
const removePipelineResponse = await this.pipelineClient.remove(data);
return removePipelineResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async getPipelineLogsMessages(id: string, details: string) {
try {
const pipelineLogsResponse =
await this.pipelineClient.getPipelineLogsMessages({ id, details });
return objectCamelToSnake(pipelineLogsResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async getPipelineStatus(data: IIdRequest) {
try {
const pipelineStatusResponse =
+88 -21
View File
@@ -9,8 +9,16 @@ import {
Put,
Headers,
Query,
UseFilters,
HttpCode,
HttpStatus,
Patch,
} from '@nestjs/common';
import { ApiCreatedResponse, ApiTags } from '@nestjs/swagger';
import {
ApiCreatedResponse,
ApiNoContentResponse,
ApiTags,
} from '@nestjs/swagger';
import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
@@ -21,7 +29,10 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import { ICreatePipelineV2Req, IPipelineV2 } from './interfaces';
@ApiTags('Pipelines')
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@UseFilters(new GrpcToHttpExceptionFilter())
@ApiTags('PipelinesV2')
@Controller('pipelinesV2')
@AuthenticateCondition((req, user) => {
let action;
@@ -35,6 +46,10 @@ import { ICreatePipelineV2Req, IPipelineV2 } from './interfaces';
action = 'UPDATE';
break;
case 'PATCH':
action = 'UPDATE';
break;
default:
action = req.method;
}
@@ -61,7 +76,7 @@ export class PipelinesController {
@User() user: RequestUser,
@Body() createPipelineDto: ICreatePipelineV2Req,
) {
this.logger.info(process.env.DEV_URL + `/pipelines ON CREATE ROUTE`);
this.logger.info('PipelinesController - create', { user });
if (!language) language = 'en-us';
const { customer_id, customer_name, username, user_id } = user;
const metadata = PackTheMetadata({
@@ -86,10 +101,7 @@ export class PipelinesController {
@Headers('Dadosfera-Lang') language,
@Query() data,
) {
this.logger.info(process.env.DEV_URL + `/pipeline ON Find All ROUTE`, {
user: user.user_id,
customer: user.customer_name,
});
this.logger.info('PipelinesController - findAll', { user });
if (!language) language = 'en-us';
@@ -109,13 +121,28 @@ export class PipelinesController {
};
}
@Get(':id/config')
async getPipelineproperties(
@Headers('Dadosfera-Lang') language,
@User() user: RequestUser,
@Param('id') id,
) {
this.logger.info('PipelinesController - getPipelineproperties', { user });
const metadata = PackTheMetadata({
...user,
language: language || 'pt-br',
});
return this.pipelinesClientService.findOneProperties(id, metadata);
}
@Get('/:id')
async findOne(
@Headers('Dadosfera-Lang') language,
@User() user: RequestUser,
@Param('id') id,
): Promise<Messages.PipelineV2FindOneResponse> {
this.logger.info(process.env.DEV_URL + `/pipeline/${id} ON Find One ROUTE`);
this.logger.info('PipelinesController - findOne', { user });
if (!language) language = 'en-us';
const { customer_id, customer_name, user_id, username } = user;
@@ -131,13 +158,16 @@ export class PipelinesController {
const result = await this.pipelinesClientService
.findOne({ id }, metadata)
.then((res) => {
//{pipeline:{tables: {tables: [], input_id: ''}}}
let tables = JSON.parse(res.pipeline.config.tables);
if (tables?.tables) tables = tables.tables;
Object.assign(res.pipeline, {
transformations: res.pipeline.transformations
? JSON.parse(res.pipeline.transformations)
: [],
config: {
cron: res.pipeline.config.cron,
tables: JSON.parse(res.pipeline.config.tables),
tables,
},
properties: res.pipeline.properties
? JSON.parse(res.pipeline.properties)
@@ -149,28 +179,65 @@ export class PipelinesController {
}
@Put('/:id')
async update(@Body() updatePipelineDto, @Param('id') id) {
async update(
@Headers('Dadosfera-Lang') language,
@Body() updatePipelineDto,
@Param('id') id,
@User() user: RequestUser,
) {
this.logger.info('PipelinesController - update', { user });
const { info } = updatePipelineDto;
delete updatePipelineDto.info;
this.logger.info(process.env.DEV_URL + `/pipeline/${id} ON UPDATE ROUTE`);
const { customer_id, customer_name, user_id, username } = user;
if (!language) language = 'en-us';
const response = await this.pipelinesClientService.update({
...updatePipelineDto,
info,
id,
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const response = await this.pipelinesClientService.update(
{
...updatePipelineDto,
info,
id,
},
metadata,
);
this.logger.info('PipelinesController - update: OK', { user });
return response;
}
@Patch('/:id')
async updateByPatch(
@Headers('Dadosfera-Lang') language,
@Body() updatePipelineDto,
@Param('id') id,
@User() user: RequestUser,
) {
this.logger.info('PipelinesController - patch', { user });
const response = await this.update(language, updatePipelineDto, id, user);
this.logger.info('PipelinesController - patch: OK', { user });
return response;
}
@Delete(':id')
async delete(@Body() data, @Param() params) {
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
async delete(@Param() params, @User() user: RequestUser) {
this.logger.info('PipelinesController - delete', { user });
const { id } = params;
this.logger.info(process.env.DEV_URL + `/pipeline/${id} ON DELETE ROUTE`);
const response = await this.pipelinesClientService.remove({ id, ...data });
return response;
const metadata = PackTheMetadata({
customer_id: user.customer_id,
customer_name: user.customer_name,
user_id: user.user_id,
});
await this.pipelinesClientService.remove({ id, metadata, user });
this.logger.info('PipelinesController - delete: OK');
}
}
@@ -8,6 +8,9 @@ import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module';
const client = new PipelinesClientConfiguration();
@@ -15,6 +18,9 @@ const client = new PipelinesClientConfiguration();
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
ConnectorModule,
InputsModule,
TransformationsModule,
],
controllers: [PipelinesController],
providers: [PipelinesService, DadosferaLogger],
+109 -35
View File
@@ -1,9 +1,4 @@
import {
BadRequestException,
Inject,
InternalServerErrorException,
OnModuleInit,
} from '@nestjs/common';
import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
Messages,
@@ -17,6 +12,14 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
import { InputsService } from '../inputs/inputs.service';
import { RequestUser } from 'src/authentication/user.decorator';
import { TransformationsService } from '../transformations/transformations.service';
import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger;
@@ -27,6 +30,9 @@ export class PipelinesService implements OnModuleInit {
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly connectorService: ConnectorClientService,
private readonly inputsService: InputsService,
private readonly transformationsService: TransformationsService,
) {
this.logger = dadosferaLogger.logger;
}
@@ -90,7 +96,7 @@ export class PipelinesService implements OnModuleInit {
const [errorType, message] = error.details.split('|');
throw new BadRequestException(message);
}
throw new InternalServerErrorException(error.details);
throw new ErrorBuilder(error.details);
});
this.logger.info('Done');
@@ -125,48 +131,116 @@ export class PipelinesService implements OnModuleInit {
const findOnePipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne(data, metadata),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
);
this.logger.info('Done');
return findOnePipelineResponse;
}
async update(UpdatePipelineRequest: Messages.PipelineV2UpdateRequest) {
async update(
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
metadata,
) {
this.logger.info('PipelinesClientService - Update');
const updatePipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2Update(UpdatePipelineRequest),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.pipelineWriteService.PipelineV2Update(
UpdatePipelineRequest,
metadata,
),
);
this.logger.info('Done');
return updatePipelineResponse;
}
async remove(data: Messages.PipelineV2RemoveRequest) {
this.logger.info('PipelinesClientService - Remove');
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
const { id, metadata, user } = data;
const info = {
user_id: user.user_id,
customer_id: user.customer_id,
customer: user.customer_name,
};
let existsInProduct = true;
const { pipeline } = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
).catch((e) => {
if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false;
else throw new ErrorBuilder(e.details);
const removePipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2Remove(data),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
return { pipeline: undefined };
});
await lastValueFrom(
this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
);
if (existsInProduct) {
//{pipeline:{tables: {tables: [], input_id: ''}}}
const input = pipeline.config.tables
? JSON.parse(pipeline.config.tables)
: null;
if (input)
await this.inputsService
.remove({
id: input.input_id,
info,
})
.catch((error) =>
this.logger.error('Could not delete input', {
data: { input, error },
}),
);
const transformations: { id: string }[] = pipeline.transformations
? JSON.parse(pipeline.transformations)
: null;
if (transformations && transformations.length)
for (const transformation of transformations) {
await this.transformationsService
.remove({
id: transformation.id,
info,
})
.catch((error) =>
this.logger.error('Could not delete transformation', {
data: { transformation, error },
}),
);
}
}
}
async findOneProperties(id: string, metadata: Metadata) {
this.logger.info('PipelinesClientService - findOneProperties');
const { pipeline } = await this.findOne({ id }, metadata);
const pipelineProperties = JSON.parse(pipeline.properties);
const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
const res = await this.connectorService.getConnectorDetails(
connectorId,
metadata.get('language')[0].toString(),
);
const connector = JSON.parse(res.connector);
const config_controls = connector.config_controls || [];
for (const config_control of config_controls) {
const path: string = config_control.name;
const default_value = getObjValueFromPath({
object: { properties: pipelineProperties },
path,
});
if (default_value != undefined)
config_control.default_value = default_value;
}
return removePipelineResponse;
return {
pipeline: {
...pipeline,
config_controls,
properties: pipelineProperties,
},
};
}
}
@@ -17,6 +17,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
import { RequestUser, User } from '../../authentication/user.decorator';
import { Metadata } from '@grpc/grpc-js';
@ApiTags('TermsOfUse')
@UseFilters(new GrpcToHttpExceptionFilter())
@@ -33,10 +34,15 @@ export class TermsOfUseController {
}
@Get('token')
async getToken(@User({ required: true }) user: RequestUser) {
async getToken(
@User({ required: true }) user: RequestUser,
@Headers('Dadosfera-Lang') language: string,
) {
this.logger.info('/terms-of-use - get token');
const metadata = new Metadata();
metadata.add('language', language || 'pt-br');
return this.termsOfUseClient.getToken({ userId: user.user_id });
return this.termsOfUseClient.getToken({ userId: user.user_id }, metadata);
}
@Post('accept')
+5 -2
View File
@@ -10,6 +10,7 @@ import {
TermsOfUseAcceptRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
export class TermsOfUseClientService implements OnModuleInit {
@@ -31,8 +32,10 @@ export class TermsOfUseClientService implements OnModuleInit {
);
}
async getToken(data: TermsOfUseGetTokenRequest) {
return lastValueFrom(this.termsOfUseService.TermsOfUseGetToken(data));
async getToken(data: TermsOfUseGetTokenRequest, metadata: Metadata) {
return lastValueFrom(
this.termsOfUseService.TermsOfUseGetToken(data, metadata),
);
}
async accept(data: TermsOfUseAcceptRequest) {
@@ -11,34 +11,19 @@ import {
import { ApiTags } from '@nestjs/swagger';
import { TransformationsClientService } from './client.service';
import { IIdRequest } from './interfaces';
import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/authentication/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { TransformationsService } from './transformations.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/authentication/user.decorator';
const pipelinePermissions = PERMISSIONS_GROUPS.PIPELINE.permissions;
@ApiTags('Transformations')
@Controller('transformations')
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.TRANSFORMATIONS.permissions[action].seqid,
);
})
@Authenticated()
export class TransformationsController {
logger: DadosferaLogger;
constructor(
@@ -50,6 +35,7 @@ export class TransformationsController {
}
@Post()
@RequireSomePermission(pipelinePermissions.CREATE, pipelinePermissions.UPDATE)
async create(@Body() createTransformationDto) {
this.logger.info(`/transformation - ON CREATE ROUTE`, {
user: createTransformationDto.info.user_id,
@@ -67,6 +53,7 @@ export class TransformationsController {
}
@Get()
@RequireSomePermission(pipelinePermissions.GET)
async findAll(@Body() data: IIdRequest) {
this.logger.info(`/transformation - ON Find All ROUTE`, {
user: data.info.user_id,
@@ -82,6 +69,7 @@ export class TransformationsController {
}
@Get('/:id')
@RequireSomePermission(pipelinePermissions.GET)
async findOne(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(`/transformation/${id} - ON Find One ROUTE`, {
@@ -98,6 +86,7 @@ export class TransformationsController {
}
@Put(':id')
@RequireSomePermission(pipelinePermissions.CREATE, pipelinePermissions.UPDATE)
async update(
@User() user: RequestUser,
@Body() updateTransformationDto,
@@ -120,6 +109,7 @@ export class TransformationsController {
}
@Delete(':id')
@RequireSomePermission(pipelinePermissions.CREATE, pipelinePermissions.UPDATE)
async delete(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(`/transformation/${id} - ON DELETE ROUTE`, {
+8 -5
View File
@@ -55,7 +55,10 @@ export class UserNoRoles extends OmitType(User, ['roles']) {}
export class UserNoRolesAndCustomer extends OmitType(UserNoRoles, [
'customer',
]) {}
export class UserByCustomer extends OmitType(User, ['customer']) {}
export class IUserByCustomer extends OmitType(User, ['customer']) {
@ApiPropertyOptional()
permissions?: string[];
}
export class CreateUserReq {
@ApiPropertyOptional()
@@ -111,11 +114,11 @@ export class UpdateUserReq {
export class UpdateUserRes {
@ApiProperty()
user: UserByCustomer;
user: IUserByCustomer;
}
export class GetAllUsersByCustomerIdRes {
@ApiProperty({ type: [UserByCustomer] })
users: UserByCustomer[];
@ApiProperty({ type: [IUserByCustomer] })
users: IUserByCustomer[];
@ApiProperty()
departments: string[];
@ApiProperty()
@@ -153,7 +156,7 @@ export class SetUserRolesReq {
export class SetUserRolesRes {
@ApiProperty()
user: UserByCustomer;
user: IUserByCustomer;
}
export class SearchUserRes extends GetAllUsersByCustomerIdRes {}
+12 -8
View File
@@ -45,12 +45,13 @@ import {
UnassignRoleToUserReq,
UpdateUserReq,
UpdateUserRes,
UserByCustomer,
IUserByCustomer,
} from './dtos/entities';
import { UsersService } from './users.service';
import { Metadata } from '@grpc/grpc-js';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
// TODO GET de hierarquias e do PATCH em usuário
@ApiTags('Users')
@Controller('users')
@ApiHeader({
@@ -73,10 +74,6 @@ export class UsersController {
}
@Get()
@ApiOkResponse({
type: GetAllUsersByCustomerIdRes,
description: 'List of users of a customer',
})
async getAllUsersByCustomerId(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,
@@ -123,9 +120,8 @@ export class UsersController {
}
@Get(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({
type: UserByCustomer,
type: IUserByCustomer,
})
async findOneById(
@User() user: RequestUser,
@@ -134,6 +130,14 @@ export class UsersController {
) {
this.logger.info('findOneById', { user });
this.userService.setLanguage(language);
if (
id !== user.user_id &&
!user.permissions.includes(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid,
)
) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
return await this.userService.findOneById(id);
}
+16 -2
View File
@@ -18,6 +18,7 @@ import {
BatchCreateUserReq,
CreateUserReq,
Hierarchy,
IUserByCustomer,
SetUserRolesReq,
UpdateUserReq,
} from './dtos/entities';
@@ -77,11 +78,24 @@ export class UsersService implements OnModuleInit {
};
}
async findOneById(id: string) {
async findOneById(id: string): Promise<{ user: IUserByCustomer }> {
const { user } = await lastValueFrom(
this.usersClientService.UserFindOneById({ id }),
);
return { user: this.adjustUsersPayload([user])[0] };
const res: { user: IUserByCustomer } = {
user: this.adjustUsersPayload([user])[0],
};
const { permissions } = await lastValueFrom(
this.usersClientService.GetUserPermissions({ id }),
).catch((error) => {
this.logger.error('UsersService.findOneById - GetUserPermissions', {
error,
});
const permissions: string[] = [];
return { permissions };
});
res.user.permissions = permissions;
return res;
}
async createUser(req: CreateUserReq, meta?: Metadata) {
+46 -2
View File
@@ -214,6 +214,16 @@ export function EnrichErrorCode(code: string) {
message: 'Id encontra-se num formato inválido',
code,
};
case ErrorCodes.CONNECTION.NOT_FOUND:
case ErrorCodes.CONNECTION.PLATFORM_NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Conexão não encontrada',
message:
'Não encontramos a conexão solicitada. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.CONNECTION_TEST.ERROR:
return {
statusCode: HttpStatus.BAD_REQUEST,
@@ -248,14 +258,48 @@ export function EnrichErrorCode(code: string) {
code,
};
case ErrorCodes.PIPELINE.NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Não encontramos a pipeline solicitada',
message:
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.PIPELINE.PLATFORM_NOT_FOUND:
case ErrorCodes.PIPELINE.PLATFORM_BAD_REQUEST:
case ErrorCodes.PIPELINE.PLATFORM_UNKNOWN:
return {
statusCode: HttpStatus.BAD_GATEWAY,
error: 'Oops... Não foi possível realizar sua ação',
message:
'Houve um erro interno ao buscar por sua pipeline. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.INPUT.NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Oops... O recurso que você buscou não está disponível',
message:
'Não encontramos o recurso buscado. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.NOT_IMPLEMENTED:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Oops... O serviço que você está tentando acessar não existe!',
message: 'Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.INTERNAL:
case ErrorCodes.UNKNOWN:
default:
return {
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
error: 'Desconhecido',
error: 'Oops... Não foi possível realizar sua ação',
message:
'Erro desconhecido. Tente novamente ou entre em contato com o suporte',
'Ocorreu um erro desconhecido. Caso o erro persista, entre em contato com o suporte',
code: ErrorCodes.UNKNOWN,
details: code,
};
+10
View File
@@ -0,0 +1,10 @@
export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
const { object, path } = data;
const keys = path.split('.');
let value = JSON.parse(JSON.stringify(object));
for (const key of keys) {
if (value != undefined) value = value[key];
}
return value;
}
+21 -1
View File
@@ -51,17 +51,37 @@ const CONNECTION_TEST = {
BAD_REQUEST: 'CONNECTION_TEST.BAD_REQUEST',
UNKNOWN: 'CONNECTION_TEST.UNKNOWN',
};
const PIPELINE = {
NOT_FOUND: 'PIPELINE.NOT_FOUND',
PLATFORM_NOT_FOUND: 'PIPELINE.PLATFORM_NOT_FOUND',
PLATFORM_BAD_REQUEST: 'PIPELINE.PLATFORM_BAD_REQUEST',
PLATFORM_UNKNOWN: 'PIPELINE.PLATFORM_UNKNOWN',
};
const INPUT = {
NOT_FOUND: 'INPUT.NOT_FOUND',
};
const CONNECTION = {
NOT_FOUND: 'CONNECTION.NOT_FOUND',
PLATFORM_NOT_FOUND: 'CONNECTION.PLATFORM_NOT_FOUND',
};
const TRANSFORMATION = {
NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
};
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
INTERNAL: 'INTERNAL',
NOT_IMPLEMENTED: 'NOT_IMPLEMENTED',
AUTH,
ROLE,
TERMS_OF_USE,
USER,
CUSTOMER,
CONNECTION_TEST,
PIPELINE,
INPUT,
TRANSFORMATION,
CONNECTION,
};
export default ErrorCodes;
-1
View File
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -1,4 +1,4 @@
{
"extends": "./tsconfig.json",
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
"exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
}