Compare commits

...
156 Commits
Author SHA1 Message Date
Arthur Simas d89573bc79 FIX: creating swap file 2022-06-06 17:35:26 -03:00
Gabriel Amorim 03ed4d88a0 FIX: validate cron
Merge pull request #101 from dadosfera/fix/validate-cron
2022-05-31 16:33:05 -03:00
Gabriel Rosa 329bbf2e4d Better cron validation 2022-05-31 16:10:21 -03:00
Gabriel Rosa ae069a0e00 removed unused imports 2022-05-31 12:04:32 -03:00
Gabriel Rosa d3bcbdc43c validating cron when creating or updating input 2022-05-31 11:34:28 -03:00
Arthur Simas 4e2e6787a4 Merge branch 'feat/change-password' into main 2022-05-30 18:27:02 -03:00
Arthur Simas 50da33b766 FEAT: added verifyPasswordResetCode endpoint 2022-05-30 17:17:50 -03:00
Rodrigo Zamboni bb72cfce45 Merge pull request #100 from dadosfera/tests
Tests
2022-05-30 10:57:23 -03:00
rodrigo.zamboni 03d17758ae FIX: commented broken test 2022-05-30 10:11:04 -03:00
rodrigo.zamboni b7e76a6d3f FIX: fix busboy npm critical error 2022-05-30 09:58:07 -03:00
Arthur Simas c920eb27b9 FEAT: reset password 2022-05-27 18:57:05 -03:00
Arthur Simas c25d2427c2 FEAT: change password 2022-05-27 18:55:43 -03:00
Arthur Simas 0f13d0c590 FIX(refresh token): removed username param 2022-05-27 18:55:35 -03:00
Arthur Simas 7781298e6e Merge branch 'hotfix/refresh-token-contract' into fix/create-partial-input 2022-05-27 15:25:00 -03:00
Arthur Simas 2c00d3249a FIX(refresh token): removed username param 2022-05-27 15:12:22 -03:00
Arthur Simas cd7c7d538f CI: deploy workflow updated 2022-05-26 15:38:36 -03:00
rodrigo.zamboni e496d42776 ódio 2022-05-26 10:13:42 -03:00
Gabriel Rosa a07247d538 update protospack version 2022-05-25 15:49:45 -03:00
Arthur Simas 153ba1a98b CI: printing env 2022-05-25 12:24:50 -03:00
Gabriel Rosa 46519323f8 Ci: imporved github flow 2022-05-25 11:56:26 -03:00
Arthur Simas a51d52d68e CI: fixed versioning 2022-05-25 11:53:44 -03:00
Gabriel Rosa 555ff2eda9 CI - github flow melhorado 2022-05-25 11:52:54 -03:00
Arthur Simas 5d1a54a720 CI: . 2022-05-25 11:45:37 -03:00
Arthur Simas 8399a0e4c1 CI: . 2022-05-25 11:40:38 -03:00
Arthur Simas cf54cb0bc2 CI: not considering semver during manual deploy 2022-05-25 11:37:14 -03:00
rodrigo.zamboni 4eba099390 Merge branch 'main' of https://github.com/dadosfera/maestro into tests 2022-05-25 11:14:45 -03:00
rodrigo.zamboni 08fa269ce0 initial tests 2022-05-25 11:14:40 -03:00
Gabriel Rosa e52af6844b FIX: create partial input 2022-05-25 11:09:24 -03:00
Arthur Simas 2344e4dba3 CI: using github-flow deployment workflow 2022-05-24 18:57:41 -03:00
rodrigo.zamboni d686753f25 DOCS: Added proper readme 2022-05-23 17:19:36 -03:00
Rodrigo Zamboni c1498f8f1b Merge pull request #98 from dadosfera/beta
Beta -> main
2022-05-20 17:27:29 -03:00
Gabriel Rosa 0b3cb581c4 merge alpha -> beta 2022-05-20 15:19:43 -03:00
Gabriel Rosa 97a6e05812 FIX: returning input when creating jdbc 2022-05-20 15:19:00 -03:00
Gabriel Rosa 356d438404 merge alpha 2022-05-20 14:07:02 -03:00
Gabriel Rosa cf870cff14 FIX: formating payload 2022-05-20 14:06:13 -03:00
Gabriel Rosa 26c6a289eb merge from beta 2022-05-20 11:51:48 -03:00
Gabriel Rosa 8d27e42fcd FIX: protospack version 2022-05-20 11:51:08 -03:00
Gabriel Rosa c44e557b11 merge beta 2022-05-20 10:08:20 -03:00
Gabriel Rosa 25d06ab4c5 FIX: send input to createS3 2022-05-20 10:07:38 -03:00
Gabriel Amorim 31bb795d23 Merge pull request #97 from dadosfera/alpha
Alpha -> Beta
2022-05-20 09:12:34 -03:00
Gabriel Rosa a2e25eb737 FIX: added ssl back 2022-05-20 08:51:25 -03:00
Gabriel Rosa 6be53bebaf FIX: adjusting pipeline payload fixed 2022-05-20 08:50:08 -03:00
Rodrigo Zamboni 741f009723 Merge pull request #96 from dadosfera/alpha
Alpha
2022-05-19 15:20:06 -03:00
Rodrigo Zamboni 45477b5750 Merge branch 'beta' into alpha 2022-05-19 15:09:15 -03:00
rodrigo.zamboni 36de53dfd3 FIX: lib version 2022-05-19 14:48:22 -03:00
rodrigo.zamboni cd8fe2fbf6 CI: Deploy 2022-05-19 14:46:17 -03:00
rodrigo.zamboni c4a2bee69c CI: DEploy 2022-05-19 14:18:08 -03:00
rodrigo.zamboni e48d11153d FIX: Added S3 or JDBC input selection for pipelines 2022-05-19 14:13:06 -03:00
Gabriel Amorim c7e4de7d65 Merge pull request #95 from dadosfera/feat/mfa
FEAT: MFA finished + better error handling
2022-05-19 13:59:46 -03:00
Arthur Simas b2e7870cc5 CHORE: protospack version bump 2022-05-19 12:56:32 -03:00
Arthur Simas 1e239026a3 FEAT: MFA finished + better error handling 2022-05-19 12:02:08 -03:00
Rodrigo Zamboni bc108d43e3 Merge pull request #94 from dadosfera/beta
Beta
2022-05-18 16:54:04 -03:00
Arthur Simas 1c5b929c1b REFACTOR: nimbus URL 2022-05-18 13:57:19 -03:00
rodrigo.zamboni 3e21c308f6 UPDATE: Added S3 Input Creation 2022-05-17 15:58:18 -03:00
Gabriel Rosa f4a5f91cbb FIX: remove backmerge 2022-05-16 17:03:12 -03:00
Gabriel Rosa c8aeb7f3db FIX: remove backmerge 2022-05-16 17:02:46 -03:00
Victor Radael b9453b11cf REFACTOR: Init Catalog Class 2022-05-16 14:02:18 -03:00
Victor Radael a69752332e FIX: Nimbus URL Strategy 2022-05-16 13:42:06 -03:00
Victor Radael 09c7cc79ee FIX: Nimbus URL Strategy 2022-05-16 12:04:13 -03:00
Arthur Simas 65f6f03e55 CHORE: pinning protospack version to 1.7.4 2022-05-13 11:15:05 -03:00
Arthur Simas a54300f600 CHORE: pinning protospack version to 1.7.2 2022-05-13 11:00:25 -03:00
Arthur Simas 5ef85f79d3 Merge branch 'alpha' into beta 2022-05-13 10:49:27 -03:00
rodrigo.zamboni a8a429977f UPDATE: Implementing MFA routes on maestro 2022-05-11 18:02:01 -03:00
Gabriel Amorim 692a9de0af FEAT: add refresh token - Merge pull request #90 from dadosfera/feat/refreshToken
FEATURE: add refresh token
2022-05-11 10:37:36 -03:00
rodrigo.zamboni 7dc669ca44 FIX: fix auth trim 2022-05-10 15:53:47 -03:00
rodrigo.zamboni e897da1ef0 FIX: Fix login middleware 2022-05-10 15:32:34 -03:00
Arthur Simas b366e786ba STYLE: styling conforming to ESLint 2022-05-10 10:23:55 -03:00
Arthur Simas 24d56c7a58 FEAT: add refresh token 2022-05-09 11:47:25 -03:00
rodrigo.zamboni d7f205901b UPDATE: Changing transformation factory microservice client url to inputs factory 2022-05-05 16:27:20 -03:00
rodrigo.zamboni 7212c5249c FIX: testing timeout on testconnection input client 2022-05-04 16:44:00 -03:00
rodrigo.zamboni 827c7b1bf1 FIX: Summary-rating returning default 0 values 2022-05-04 12:00:01 -03:00
rodrigo.zamboni 8d89b747b4 FIX: Fix delete routes method 2022-05-04 00:24:44 -03:00
rodrigo.zamboni 4f18328603 FIX: update protospack lib version 2022-05-03 17:59:51 -03:00
rodrigo.zamboni a68d312447 FIX: updated protospack lib 2022-05-03 17:07:57 -03:00
rodrigo.zamboni 7f677f0376 FIX: Fix mistaken routes method 2022-05-03 16:54:03 -03:00
rodrigo.zamboni bb2c0ab23e Merge branch 'main' of https://github.com/dadosfera/maestro into main 2022-05-03 15:13:10 -03:00
rodrigo.zamboni feec48114c FIX: fixed missing nimbus catalog routes 2022-05-03 15:12:18 -03:00
Rodrigo Zamboni 7bee8d83a9 Merge pull request #89 from dadosfera/beta
Beta
2022-05-03 09:14:43 -03:00
Rodrigo Zamboni 84b2dbd178 Merge pull request #88 from dadosfera/alpha
CI: Deploy on dev env
2022-05-02 17:53:46 -03:00
rodrigo.zamboni 36706ff3dd CI: Deploy on dev env 2022-05-02 17:33:03 -03:00
Victor Radael 0164e6978a FIX: Metabase URL 2022-04-29 20:53:17 -03:00
rodrigo.zamboni 7d371926f5 FIX: fix nimbus prd url 2022-04-29 17:20:23 -03:00
rodrigo.zamboni 75e9e97f04 FIX: fix nimbus prd url 2022-04-29 17:18:21 -03:00
rodrigo.zamboni b5f58b99fd FIX: Changing nimbus prd url 2022-04-29 16:33:42 -03:00
rodrigo.zamboni 503293439f FIX: updating protospack lib version 2022-04-29 16:14:14 -03:00
rodrigo.zamboni eed57ff039 FIX: fix pipeline status sort 2022-04-29 12:42:49 -03:00
rodrigo.zamboni ef2a24a0c3 FIX: Fix runPipeline fail case return 2022-04-28 17:39:05 -03:00
Rodrigo Zamboni 8a31e11445 Merge pull request #87 from dadosfera/beta
Beta
2022-04-28 17:23:54 -03:00
Rodrigo Zamboni 2013e97e20 Merge pull request #86 from dadosfera/alpha
Alpha
2022-04-28 17:12:53 -03:00
rodrigo.zamboni dd271205ba Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-04-28 17:00:44 -03:00
rodrigo.zamboni a1f9f14897 FIX: Fixing more catalog routes params 2022-04-28 17:00:08 -03:00
Gabriel Amorim b7efa2f58d Merge pull request #85 from dadosfera/beta
Beta -> Main
2022-04-28 16:04:10 -03:00
Gabriel Amorim 395e36581d Merge pull request #84 from dadosfera/alpha
Alpha -> Beta
2022-04-28 16:03:53 -03:00
rodrigo.zamboni 97a54f9267 Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-04-28 15:56:30 -03:00
rodrigo.zamboni 00b0e00bd0 FIX: fix iframe_url and pipeline status 2022-04-28 15:55:48 -03:00
Rodrigo Zamboni 9188536161 Merge pull request #83 from dadosfera/beta
Beta
2022-04-28 14:53:16 -03:00
Rodrigo Zamboni 1ae8151184 Merge pull request #82 from dadosfera/alpha
Alpha -> Beta
2022-04-28 14:40:14 -03:00
rodrigo.zamboni 3d77980dd2 CI: deploy 2022-04-28 14:24:28 -03:00
Rodrigo Zamboni 849b979e63 Merge pull request #81 from dadosfera/feat/CatalogFixes
Feat/catalog fixes
2022-04-28 14:19:31 -03:00
rodrigo.zamboni 28580782b1 FIX: catalog routes fix 2022-04-28 14:14:03 -03:00
Rodrigo Zamboni fc784e2d6d Merge pull request #79 from dadosfera/alpha
Alpha
2022-04-28 13:22:14 -03:00
rodrigo.zamboni c997e91912 FIX: fix table-metadata find one route 2022-04-28 12:59:35 -03:00
rodrigo.zamboni 22f493d48a FIX: fixing missing id on delete routes 2022-04-28 12:12:37 -03:00
rodrigo.zamboni 035213824b FIX: fixing missing id on delete routes 2022-04-28 12:00:19 -03:00
rodrigo.zamboni 1ffa9ade51 FIX: CI 2022-04-28 10:53:17 -03:00
Rodrigo Zamboni c91aa57344 Merge pull request #78 from dadosfera/beta
Beta
2022-04-27 17:36:54 -03:00
Rodrigo Zamboni 077a95cdbd Merge pull request #77 from dadosfera/alpha
Alpha
2022-04-27 17:16:09 -03:00
Rodrigo Zamboni f39108772b Merge pull request #76 from dadosfera/NewInput
FIX: fix pipeline activate route method
2022-04-27 16:57:56 -03:00
rodrigo.zamboni cdd8beee8f FIX: fix pipeline activate route method 2022-04-27 16:56:03 -03:00
rodrigo.zamboni 14331ed229 FIX: fix pipeline activate route method 2022-04-27 16:54:48 -03:00
Victor Radael 5f3db0cb43 Merge pull request #75 from dadosfera/beta
Beta -> Main
2022-04-27 11:42:18 -03:00
Rodrigo Zamboni 28f304c47c Merge pull request #74 from dadosfera/alpha
Alpha
2022-04-26 17:41:57 -03:00
Rodrigo Zamboni 0c1b62b2aa Merge pull request #73 from dadosfera/NewInput
FIX: Fix get-columns route method
2022-04-26 17:25:11 -03:00
rodrigo.zamboni 13f7e9d812 FIX: Fix get-columns route method 2022-04-26 17:19:18 -03:00
Gabriel Amorim 841dcfef06 Merge pull request #72 from dadosfera/alpha
Alpha
2022-04-26 16:30:31 -03:00
Gabriel Amorim f41ddc1483 Merge pull request #71 from dadosfera/NewInput
UPDATE: Implemented new Input format
2022-04-26 16:19:19 -03:00
rodrigo.zamboni 0a2c7c0a38 UPDATE: Implemented new Input format 2022-04-26 16:15:54 -03:00
Rodrigo Zamboni 87f4c17ca2 Merge pull request #70 from dadosfera/beta
Beta
2022-04-26 13:50:00 -03:00
Rodrigo Zamboni a89625c1e6 Merge pull request #69 from dadosfera/alpha
Alpha
2022-04-26 11:16:06 -03:00
rodrigo.zamboni 44211d5658 FIX: CI 2022-04-26 11:01:20 -03:00
rodrigo.zamboni d4e11b1935 FIX: CI 2022-04-26 10:48:57 -03:00
rodrigo.zamboni 5c5e940cf7 FIX: CI 2022-04-26 10:40:11 -03:00
Rodrigo Zamboni 2b162bd0a7 Merge pull request #68 from dadosfera/beta
Beta
2022-04-25 17:06:59 -03:00
Victor Radael 829f7611e2 Merge pull request #67 from dadosfera/alpha
Alpha
2022-04-25 16:49:04 -03:00
Rodrigo Zamboni b986cf3281 Merge pull request #66 from dadosfera/addHelmet
FIX: added helmet package to protect headers security
2022-04-25 16:35:59 -03:00
rodrigo.zamboni 75c7602e13 FIX: added helmet package to protect headers security 2022-04-25 16:25:42 -03:00
Victor Radael 710105a4a8 Merge pull request #65 from dadosfera/addHelmet
added helmet package to protect headers security
2022-04-25 16:16:07 -03:00
rodrigo.zamboni 30b048f776 added helmet package to protect headers security 2022-04-25 16:11:50 -03:00
Victor Radael 845b95b112 Merge pull request #64 from dadosfera/beta
Beta ->  main
2022-04-21 17:15:16 -03:00
Victor Radael 6d6b024884 Merge pull request #63 from dadosfera/alpha
Alpha -> Beta
2022-04-21 16:18:51 -03:00
Rodrigo Zamboni a4282a0dc5 Merge pull request #62 from dadosfera/feat/nimbusImplementation
Feat/nimbus implementation
2022-04-21 16:05:37 -03:00
rodrigo.zamboni 4672ecb6b3 FIX: Fixed maestro after protospack lib att 2022-04-21 16:02:11 -03:00
rodrigo.zamboni 6160107d75 FIX: Downgrade protospack lib 2022-04-20 16:22:26 -03:00
Rodrigo Zamboni 589a1f9d51 Merge pull request #60 from dadosfera/feat/nimbusImplementation
Feat/nimbus implementation
2022-04-20 16:01:32 -03:00
rodrigo.zamboni cb929d9b60 FIX: FIX 2022-04-20 15:56:50 -03:00
rodrigo.zamboni 678bebef7e FIX: fix proto packages names 2022-04-20 15:52:26 -03:00
Rodrigo Zamboni b6d7d54f9c Merge pull request #58 from dadosfera/feat/nimbusImplementation
adding get all tags function
2022-04-20 15:30:40 -03:00
rodrigo.zamboni 5df8b2834c FIX: adding get all tags function 2022-04-20 15:08:51 -03:00
rodrigo.zamboni 76d9571f02 FIX: adding get all tags function 2022-04-20 14:51:04 -03:00
rodrigo.zamboni 0c51014ec9 adding get all tags function 2022-04-20 14:40:01 -03:00
Rodrigo Zamboni 7f1d8e4b2a Merge pull request #57 from dadosfera/beta
Beta
2022-04-19 18:29:16 -03:00
Victor Radael 92b78ff19d Merge pull request #56 from dadosfera/alpha
Alpha
2022-04-19 17:06:28 -03:00
Rodrigo Zamboni 1b328b9915 Merge pull request #55 from dadosfera/NimbusImplementation
UPDATE: Added Nimbus Catalog routes
2022-04-19 16:24:34 -03:00
rodrigo.zamboni e653aeea3c UPDATE: Added Nimbus Catalog routes 2022-04-19 15:56:29 -03:00
Rodrigo Zamboni b49633cbc2 Merge pull request #54 from dadosfera/beta
Beta
2022-04-18 13:58:14 -03:00
Rodrigo Zamboni 4d956e55c6 Merge pull request #53 from dadosfera/alpha
Alpha
2022-04-18 12:18:28 -03:00
Rodrigo Zamboni 735703bf3b Merge pull request #52 from dadosfera/feat/activatePipeline
Feat/activate pipeline
2022-04-18 11:54:43 -03:00
rodrigo.zamboni 93cdc3f030 UPDATE: remove unescessary console log 2022-04-18 11:50:17 -03:00
rodrigo.zamboni 12b8b5926c UPDATE: Created Trigger Pipeline route 2022-04-18 11:44:19 -03:00
rodrigo.zamboni 1643096d47 Created Trigger Pipeline route 2022-04-18 11:43:36 -03:00
Rodrigo Zamboni 8a519ac217 Merge pull request #51 from dadosfera/beta
Beta
2022-04-14 15:33:36 -03:00
Victor Radael ffa69a0cf5 Merge pull request #50 from dadosfera/alpha
Alpha
2022-04-14 15:14:12 -03:00
Rodrigo Zamboni 7722e380e3 Merge pull request #49 from dadosfera/authFix
FIX: using Unauth nest function
2022-04-14 15:01:18 -03:00
rodrigo.zamboni ecdc7b4eb6 FIX: using Unauth nest function 2022-04-14 14:56:18 -03:00
rodrigo.zamboni 7d14080ac1 FIX: Invalid auth credentials return 2022-04-14 14:48:40 -03:00
Victor Radael 7e7deba2ab FIX: CI 2022-04-12 18:10:40 -03:00
37 changed files with 11916 additions and 1515 deletions
+3
View File
@@ -0,0 +1,3 @@
container_commands:
01setup_swap:
command: "bash .ebextensions/setup_swap.sh"
+7
View File
@@ -0,0 +1,7 @@
Resources:
AWSEBAutoScalingLaunchConfiguration:
Properties:
MetadataOptions:
HttpEndpoint: enabled
HttpPutResponseHopLimit: 2
HttpTokens: required
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
SWAPFILE=/var/swapfile
SWAP_MEGABYTES=1024
if [ -f $SWAPFILE ]; then
echo "Swapfile $SWAPFILE found, assuming already setup"
exit;
fi
/bin/dd if=/dev/zero of=$SWAPFILE bs=1M count=$SWAP_MEGABYTES
/bin/chmod 600 $SWAPFILE
/sbin/mkswap $SWAPFILE
/sbin/swapon $SWAPFILE
+140
View File
@@ -0,0 +1,140 @@
name: Deploy
on:
push:
branches:
- main
workflow_dispatch:
inputs:
environment:
description: 'Deployment environment'
required: true
type: choice
options:
- dev
- stg
- prd
jobs:
extract_environment:
runs-on: ubuntu-latest
outputs:
environment: ${{ steps.extract_environment.outputs.environment }}
env:
EVENT: ${{ github.event_name }}
DEPLOY_ENV: ${{ github.event.inputs.environment }}
steps:
- name: Extract Environment
run: |
if [ ${EVENT} == "workflow_dispatch" ]; then
echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "##[set-output name=environment;]$(echo "prd")"
fi
id: extract_environment
semantic_release:
runs-on: ubuntu-latest
outputs:
#new_release_published: ${{ steps.semantic.outputs.last_release_version != steps.semantic.outputs.new_release_version }}
new_release_published: ${{ steps.semantic.outputs.new_release_published }}
#new_release_version: ${{ steps.semantic.outputs.new_release_version }}
new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
steps:
- name: Checkout
uses: actions/checkout@v2
- if: github.event_name != 'workflow_dispatch'
name: Semantic Release
uses: cycjimmy/semantic-release-action@v2
id: semantic
with:
semantic_version: 16
extra_plugins: |
conventional-changelog-eslint
branches: |
[
'main'
]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
deploy:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, '${{ needs.extract_environment.outputs.environment }}']
steps:
- name: Printing stats
env:
EVENT: ${{ github.event_name }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ENV: ${{ needs.extract_environment.outputs.environment }}
run: echo ${GITHUB_REF#refs/heads/}
- name: Checkout
uses: actions/checkout@v2
- name: Update Pip
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
run: |
python3 -m pip install awscli --upgrade
- name: Install AWS Elastic Beanstalk CLI
run: |
python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region
uses: aws-actions/configure-aws-credentials@v1
id: aws
with:
aws-region: us-east-1
- name: Login to AWS ECR
id: login_ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Build, Tag, and Push Image to AWS ECR
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
docker-compose -f build.docker-compose.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
AWS_REGION: us-east-1
APP_NAME: ${{ github.event.repository.name }}
run: |
eb use $APP_NAME-$ENV
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
eb deploy
- name: Remove Docker's Trash
continue-on-error: true
run: |
docker system prune
docker rmi -f $(docker images -aq)
-148
View File
@@ -1,148 +0,0 @@
name: Deploy
on:
push:
branches:
- alpha
- beta
- main
jobs:
extract_environment:
runs-on: ubuntu-latest
outputs:
environment: ${{ steps.extract_environment.outputs.environment }}
steps:
- name: Extract Environment
run: |
if [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "##[set-output name=environment;]$(echo "prd")"
elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
echo "##[set-output name=environment;]$(echo "dev")"
elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
echo "##[set-output name=environment;]$(echo "stg")"
fi
id: extract_environment
deploy:
runs-on: self-hosted
steps:
- name: Checkout
uses: actions/checkout@v2
# Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
- name: BKP .npmrc Step 1
run: |
cp .npmrc .npmrcbkp
- name: Semantic Release
uses: cycjimmy/semantic-release-action@v2
id: semantic
with:
semantic_version: 16
extra_plugins: |
@saithodev/semantic-release-backmerge
conventional-changelog-eslint
branches: |
[
'main',
{
name: 'alpha',
prerelease: true
},
{
name: 'beta',
prerelease: true
}
]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
# Because Semantic Realese deleted the .npmrc file, it was necessary to insert two steps to backup the file.
- name: BKP .npmrc Step 2
run: |
cp .npmrcbkp .npmrc
- name: Update Pip
if: steps.semantic.outputs.new_release_published == 'true'
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
if: steps.semantic.outputs.new_release_published == 'true'
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
if: steps.semantic.outputs.new_release_published == 'true'
run: |
python3 -m pip install awscli --upgrade
- name: Install AWS Elastic Beanstalk CLI
if: steps.semantic.outputs.new_release_published == 'true'
run: |
python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region
if: steps.semantic.outputs.new_release_published == 'true'
uses: aws-actions/configure-aws-credentials@v1
id: aws
with:
aws-region: us-east-1
- name: Extract Environment
if: steps.semantic.outputs.new_release_published == 'true'
shell: bash
run: |
if [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "##[set-output name=result;]$(echo "prd")"
elif [ ${GITHUB_REF} == "refs/heads/alpha" ]; then
echo "##[set-output name=result;]$(echo "dev")"
elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
echo "##[set-output name=result;]$(echo "stg")"
fi
id: extract_environment
- name: Login to AWS ECR
if: steps.semantic.outputs.new_release_published == 'true'
id: login_ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Build, Tag, and Push Image to AWS ECR
if: steps.semantic.outputs.new_release_published == 'true'
env:
ENV: ${{ steps.extract_environment.outputs.result }}
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
docker-compose -f build.docker-compose.yml build --build-arg NPM_TOKEN=${NPM_TOKEN}
docker-compose -f build.docker-compose.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
if: steps.semantic.outputs.new_release_published == 'true'
env:
ENV: ${{ steps.extract_environment.outputs.result }}
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk
if: steps.semantic.outputs.new_release_published == 'true'
env:
ENV: ${{ steps.extract_environment.outputs.result }}
AWS_REGION: us-east-1
APP_NAME: maestro
run: |
eb use $APP_NAME-$ENV
eb deploy
- name: Remove Docker's Trash
continue-on-error: true
run: |
docker system prune
docker rmi -f $(docker images -aq)
+1 -1
View File
@@ -29,7 +29,7 @@ lerna-debug.log*
# IDE - VSCode
.vscode/*
!.vscode/settings.json
#!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
+1
View File
@@ -0,0 +1 @@
18
-12
View File
@@ -26,18 +26,6 @@
"preset": "eslint"
}
],
[
"@saithodev/semantic-release-backmerge",
{
"branches": [
{ "from": "main", "to": "alpha" },
{ "from": "main", "to": "beta" }
],
"backmergeStrategy": "merge",
"clearWorkspace": true,
"restoreWorkspace": true
}
],
"@semantic-release/npm",
"@semantic-release/github"
]
+48 -2
View File
@@ -1,4 +1,50 @@
# Maestro
<p align="center">
<image src="./assets/maestro.svg" style="width:10rem">
<h1 align="center">Maestro</h1>
</p>
<h1 align="center">Maestro</h1>
</p>
This is the Dadosfera´s gateway repository, it´s responsable for the communication between frontend application and Dadosfera´s mirosservices.
## 💻 Requirements
Before you start, make sure you have done the following steps:
* Installed Nodejs version 16.14.2
* Installed latest NPM version
## 🚀 Installing Maestro
First of all clone the repository:
* SSH:
```
git clone git@github.com:dadosfera/maestro.git
```
* HTTPS:
```
git clone git@github.com:dadosfera/maestro.git
```
## Enviroment variables
Here is a list of enviroment variables needed in order to run the application correctly.
```
ENV=
DUC_URL=
INFACTORY_URL=
TRFACTORY_URL=
OTFACTORY_URL=
PIFACTORY_URL=
JWT_PRIVATE_KEY=
AWS_IDENTITY_POOL_ID=
```
## Running Maestro
In order to run Maestro just run the following command:
```
npm run start:dev
```
If everything is fine the Maestro will start and be ready to receive HTTP requests
+9993 -1227
View File
File diff suppressed because it is too large Load Diff
+8 -2
View File
@@ -29,11 +29,13 @@
"@nestjs/mapped-types": "*",
"@nestjs/microservices": "^8.4.3",
"@nestjs/platform-express": "^8.4.3",
"@nestjs/schedule": "^1.1.0",
"@nestjs/swagger": "^5.2.1",
"@victorradael/protospack": "^1.3.9",
"@victorradael/protospack": "2.4.2-1",
"axios": "^0.25.0",
"cron-parser": "^4.4.0",
"dotenv": "^14.2.0",
"grpc": "^1.24.11",
"helmet": "^5.0.2",
"jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5",
"reflect-metadata": "^0.1.13",
@@ -41,6 +43,9 @@
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.3.0"
},
"overrides": {
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@nestjs/cli": "^8.2.4",
"@nestjs/schematics": "^8.0.8",
@@ -57,6 +62,7 @@
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.0.0",
"jest": "^27.5.1",
"nock": "^13.2.4",
"prettier": "^2.6.1",
"source-map-support": "^0.5.20",
"supertest": "^6.1.3",
+5
View File
@@ -29,6 +29,8 @@ import { TransformationsClientConfiguration } from './clients/transformations/cl
import { AuthClient } from './clients/auth/client.config';
import { InputsClientConfiguration } from './clients/inputs/client.config';
import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
import { CatalogController } from './modules/catalog/catalog.controller';
import { CatalogService } from './modules/catalog/catalog.service';
const authClient = new AuthClient();
const inputClient = new InputsClientConfiguration();
@@ -44,6 +46,7 @@ const transformationClient = new TransformationsClientConfiguration();
PipelinesController,
AuthController,
HealthController,
CatalogController,
],
providers: [
InputsService,
@@ -57,6 +60,7 @@ const transformationClient = new TransformationsClientConfiguration();
OutputsClientService,
PipelinesClientService,
AuthClientService,
CatalogService,
],
imports: [
ConfigModule.forRoot({
@@ -97,6 +101,7 @@ export class AppModule implements NestModule {
TransformationsController,
OutputsController,
PipelinesController,
CatalogController,
);
}
}
+6 -4
View File
@@ -1,7 +1,7 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { DucProtofile } from '@victorradael/protospack';
import { DucProtoFilePath, DucPackages } from '@victorradael/protospack';
export class AuthClient {
config(): ClientOptions {
@@ -9,9 +9,11 @@ export class AuthClient {
transport: Transport.GRPC,
options: {
url: process.env.DUC_URL,
package: 'duc',
credentials: credentials.createSsl(),
protoPath: DucProtofile,
package: DucPackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: DucProtoFilePath,
loader: {
enums: String,
objects: true,
+225 -9
View File
@@ -1,27 +1,124 @@
import { OnModuleInit, Inject } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CognitoServiceInterface } from '@victorradael/protospack';
import { ILogin } from './interfaces';
import {
DucServicesNames,
AuthServiceInterface,
AuthSignInRequest,
AuthRefreshAccessTokenRequest,
AuthConfirmationRequest,
AuthResendConfirmationCodeRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthDismissTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthChangePasswordRequest,
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
} from '@victorradael/protospack';
export class AuthClientService implements OnModuleInit {
private cognitoService: CognitoServiceInterface;
private authService: AuthServiceInterface;
constructor(
@Inject('AUTH_PACKAGE') private readonly grpcClient: ClientGrpc,
) {}
onModuleInit() {
this.cognitoService = this.grpcClient.getService<CognitoServiceInterface>(
'CognitoProtoService',
this.authService = this.grpcClient.getService<AuthServiceInterface>(
DucServicesNames.AuthProtoService,
);
}
async signIn({ username, password }: ILogin): Promise<any> {
console.log('CognitoClientService', 'SignIn');
async signIn({ username, password, totp }: AuthSignInRequest): Promise<any> {
console.log('AuthClientService', 'SignIn');
return new Promise((resolve, reject) => {
this.authService.signIn({ username, password, totp }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async enableTotpMFA({
accessToken,
password,
}: AuthEnableTotpMfaRequest): Promise<any> {
console.log('AuthClientService', 'enableTotpMFA');
return new Promise((resolve, reject) => {
this.authService.enableTotpMfa({ accessToken, password }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async disableTotpMFA({
accessToken,
password,
}: AuthDisableTotpMfaRequest): Promise<any> {
console.log('AuthClientService', 'disableTotpMFA');
return new Promise((resolve, reject) => {
this.authService.disableTotpMfa({ accessToken, password }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async dismissTotpMFA({
accessToken,
}: AuthDismissTotpMfaRequest): Promise<any> {
console.log('AuthClientService', 'dismissTotpMFA');
return new Promise((resolve, reject) => {
this.authService.dismissTotpMfa({ accessToken }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async verifyTotp({
accessToken,
totp,
}: AuthVerifyTotpMfaRequest): Promise<any> {
console.log('AuthClientService', 'disableTotpMFA');
return new Promise((resolve, reject) => {
this.authService.verifyTotp({ accessToken, totp }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async confirmRegister({
username,
code,
}: AuthConfirmationRequest): Promise<any> {
console.log('AuthClientService', 'confirmRegister');
const tokens = await new Promise((resolve, reject) => {
this.cognitoService.signIn({ username, password }).subscribe({
this.authService.confirmRegister({ username, code }).subscribe({
next(x) {
resolve(x);
},
@@ -40,4 +137,123 @@ export class AuthClientService implements OnModuleInit {
});
return tokens;
}
async resendeConfirmationCode({
username,
}: AuthResendConfirmationCodeRequest): Promise<any> {
console.log('AuthClientService', 'resendConfirmationCode');
const authResendConfirmationCodeRequestReturn = await new Promise(
(resolve, reject) => {
this.authService.resendConfirmationCode({ username }).subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
},
)
.then((res) => res)
.catch((err) => {
throw new Error(err);
});
return authResendConfirmationCodeRequestReturn;
}
async refreshAccessToken({
refreshToken,
}: AuthRefreshAccessTokenRequest): Promise<any> {
console.log('AuthClientService', 'RefreshAccessToken');
return new Promise((resolve, reject) => {
this.authService.refreshAccessToken({ refreshToken }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async changePassword({
accessToken,
oldPassword,
newPassword,
}: AuthChangePasswordRequest): Promise<any> {
console.log('AuthClientService', 'ChangePassword');
return new Promise((resolve, reject) => {
this.authService
.changePassword({
accessToken,
oldPassword,
newPassword,
})
.subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async resetPassword({ username }: AuthResetPasswordRequest): Promise<any> {
console.log('AuthClientService', 'resetPassword');
return new Promise((resolve, reject) => {
this.authService.resetPassword({ username }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async verifyResetPasswordCode({
username,
code,
}: AuthVerifyResetPasswordCodeRequest): Promise<any> {
console.log('AuthClientService', 'verifyResetPasswordCode');
return new Promise((resolve, reject) => {
this.authService.verifyResetPasswordCode({ username, code }).subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
async confirmResetPassword({
username,
code,
newPassword,
}: AuthConfirmResetPasswordRequest): Promise<any> {
console.log('AuthClientService', 'confirmResetPassword');
return new Promise((resolve, reject) => {
this.authService
.confirmResetPassword({ username, code, newPassword })
.subscribe({
next: resolve,
error: (err) => reject(err.details),
complete() {
console.log('done');
},
});
});
}
}
-4
View File
@@ -1,4 +0,0 @@
export interface ILogin {
username: string;
password: string;
}
+6 -4
View File
@@ -1,7 +1,7 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { InputProtofile } from '@victorradael/protospack';
import { InputPackages, InputProtoFilePath } from '@victorradael/protospack';
export class InputsClientConfiguration {
config(): ClientOptions {
@@ -9,9 +9,11 @@ export class InputsClientConfiguration {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: 'input',
credentials: credentials.createSsl(),
protoPath: InputProtofile,
package: InputPackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: InputProtoFilePath,
loader: {
enums: String,
objects: true,
+62 -11
View File
@@ -1,16 +1,18 @@
import { OnModuleInit, Inject } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { InputService } from '@victorradael/protospack';
import {
InputCreateResponse,
InputCreateS3Request,
InputNewCreateRequest,
InputService,
TestConnectionGetColumnsRequest,
TestConnectionRequest,
} from '@victorradael/protospack';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import {
ICreateInputRequest,
IIdRequest,
ITestConnectionRequest,
UpdateInputRequest,
} from './interfaces';
import { IIdRequest, UpdateInputRequest } from './interfaces';
export class InputsClientService implements OnModuleInit {
private inputService: InputService;
@@ -23,11 +25,36 @@ export class InputsClientService implements OnModuleInit {
this.grpcClient.getService<InputService>('InputService');
}
async create(createInputDto: ICreateInputRequest) {
async create(createInputDto: InputNewCreateRequest) {
console.log('InputClientService', 'Create');
const createInputResponse = await new Promise((resolve, reject) => {
this.inputService.Create(objectSnakeToCamel(createInputDto)).subscribe({
this.inputService
.NewCreate(objectSnakeToCamel(createInputDto))
.subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new Error(err);
});
return createInputResponse;
}
async createS3Inputs(createInputDto: InputCreateS3Request) {
console.log('InputClientService', 'Create');
const createInputResponse = await new Promise((resolve, reject) => {
this.inputService.CreateS3(objectSnakeToCamel(createInputDto)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
@@ -147,10 +174,10 @@ export class InputsClientService implements OnModuleInit {
return removeInputResponse;
}
async testConnection(data: ITestConnectionRequest) {
async testConnection(data: TestConnectionRequest) {
console.log('InputClientService', 'TestConnection');
const testConnectionResponse = await new Promise((resolve, reject) => {
this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({
this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
@@ -170,4 +197,28 @@ export class InputsClientService implements OnModuleInit {
return testConnectionResponse;
}
async getColumns(data: TestConnectionGetColumnsRequest) {
console.log('InputClientService', 'TestConnection/Get-Columns');
const getColumnsResponse = await new Promise((resolve, reject) => {
this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
})
.then((res) => res)
.catch((err) => {
throw new Error(err);
});
return getColumnsResponse;
}
}
+6 -4
View File
@@ -1,4 +1,4 @@
import { OutputProtofile } from '@victorradael/protospack';
import { OutputPackages, OutputProtoFilePath } from '@victorradael/protospack';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { credentials } from '@grpc/grpc-js';
@@ -8,9 +8,11 @@ export class OutputsClientConfiguration {
transport: Transport.GRPC,
options: {
url: process.env.OTFACTORY_URL,
package: 'output',
credentials: credentials.createSsl(),
protoPath: OutputProtofile,
package: OutputPackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: OutputProtoFilePath,
loader: {
enums: String,
objects: true,
+7 -3
View File
@@ -1,6 +1,9 @@
import { Controller, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc, Payload } from '@nestjs/microservices';
import { OutputsServiceInterface } from '@victorradael/protospack';
import {
OutputServicesNames,
OutputsServiceInterface,
} from '@victorradael/protospack';
import {
objectCamelToSnake,
objectSnakeToCamel,
@@ -19,8 +22,9 @@ export class OutputsClientService implements OnModuleInit {
) {}
onModuleInit() {
this.outputService =
this.grpcClient.getService<OutputsServiceInterface>('OutputService');
this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
OutputServicesNames.OutputService,
);
}
async create(@Payload() createOutputDto: ICreateOutputRequest) {
+9 -4
View File
@@ -1,5 +1,8 @@
import { ClientOptions, Transport } from '@nestjs/microservices';
import { PipelineProtofile } from '@victorradael/protospack';
import {
PipelinePackages,
PipelineProtoFilePath,
} from '@victorradael/protospack';
import { credentials } from '@grpc/grpc-js';
export class PipelinesClientConfiguration {
@@ -8,9 +11,11 @@ export class PipelinesClientConfiguration {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: 'pipeline',
credentials: credentials.createSsl(),
protoPath: PipelineProtofile,
package: PipelinePackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: PipelineProtoFilePath,
loader: {
enums: String,
objects: true,
+64 -20
View File
@@ -1,13 +1,15 @@
import { Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc, Payload } from '@nestjs/microservices';
import { PipelinesServiceInterface } from '@victorradael/protospack';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@victorradael/protospack';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import {
ICreatePipelineDto,
IdRequest,
IGetPipelineLogsRequest,
IIdRequest,
IUpdatePipelineRequest,
@@ -21,7 +23,9 @@ export class PipelinesClientService implements OnModuleInit {
onModuleInit() {
this.pipelineService =
this.grpcClient.getService<PipelinesServiceInterface>('PipelineService');
this.grpcClient.getService<PipelinesServiceInterface>(
PipelineServicesNames.PipelineService,
);
}
async create(@Payload() createPipelineDto: ICreatePipelineDto) {
@@ -29,7 +33,7 @@ export class PipelinesClientService implements OnModuleInit {
const createPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService
.create(objectSnakeToCamel(createPipelineDto))
.Create(objectSnakeToCamel(createPipelineDto))
.subscribe({
next(x) {
resolve(x);
@@ -55,9 +59,9 @@ export class PipelinesClientService implements OnModuleInit {
console.log('PipelinesClientService', 'FindAll');
const findAllPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.findAll(objectSnakeToCamel(data)).subscribe({
this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
resolve(x.pipelines);
},
error(err) {
console.log('Observable Error');
@@ -80,7 +84,7 @@ export class PipelinesClientService implements OnModuleInit {
console.log('PipelinesClientService', 'FindOne');
const findOnePipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.findOne(objectSnakeToCamel(data)).subscribe({
this.pipelineService.FindOne(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
@@ -106,7 +110,7 @@ export class PipelinesClientService implements OnModuleInit {
const updatePipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService
.update(objectSnakeToCamel(UpdatePipelineRequest))
.Update(objectSnakeToCamel(UpdatePipelineRequest))
.subscribe({
next(x) {
resolve(x);
@@ -178,22 +182,62 @@ export class PipelinesClientService implements OnModuleInit {
return logsPipelineResponse;
}
async getPipelineStatus({ id }: IdRequest) {
async getPipelineStatus(data) {
console.log('PipelinesClientService', 'GetPipelineStatus');
const statusPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.getPipelineStatus({ id }).subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
this.pipelineService
.getPipelineStatus(objectSnakeToCamel(data))
.subscribe({
next(x) {
const statusArray = x.status.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
});
resolve({ status: statusArray });
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
})
.then((res) => res)
.catch((err) => {
throw new Error(err);
});
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
console.log('PipelinesClientService', 'RunPipeline');
const statusPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService
.triggerPipeline(objectSnakeToCamel({ id, info }))
.subscribe({
next(x) {
if (x.status == false) {
reject(
'This pipeline is not ready yet to execute, Try again later!',
);
}
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
})
.then((res) => res)
.catch((err) => {
+7 -4
View File
@@ -1,5 +1,8 @@
import { ClientOptions, Transport } from '@nestjs/microservices';
import { TransformationProtofile } from '@victorradael/protospack';
import {
TransformationPackages,
TransformationProtoFilePath,
} from '@victorradael/protospack';
import { credentials } from '@grpc/grpc-js';
export class TransformationsClientConfiguration {
@@ -7,10 +10,10 @@ export class TransformationsClientConfiguration {
return {
transport: Transport.GRPC,
options: {
url: process.env.TRFACTORY_URL,
package: 'transformation',
url: process.env.INFACTORY_URL,
package: TransformationPackages,
credentials: credentials.createSsl(),
protoPath: TransformationProtofile,
protoPath: TransformationProtoFilePath,
loader: {
enums: String,
objects: true,
@@ -28,7 +28,7 @@ export class TransformationsClientService implements OnModuleInit {
const createTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService
.create(objectSnakeToCamel(createTransformationsDto))
.Create(objectSnakeToCamel(createTransformationsDto))
.subscribe({
next(x) {
resolve(x);
@@ -62,7 +62,7 @@ export class TransformationsClientService implements OnModuleInit {
const findAllTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService.findAll(objectSnakeToCamel(data)).subscribe({
this.transformationService.FindAll(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
@@ -93,7 +93,7 @@ export class TransformationsClientService implements OnModuleInit {
const findOneTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService.findOne(objectSnakeToCamel(data)).subscribe({
this.transformationService.FindOne(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
@@ -121,7 +121,7 @@ export class TransformationsClientService implements OnModuleInit {
const updateTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService
.update(objectSnakeToCamel(updateTransformationDTO))
.Update(objectSnakeToCamel(updateTransformationDTO))
.subscribe({
next(x) {
resolve(x);
@@ -149,7 +149,7 @@ export class TransformationsClientService implements OnModuleInit {
const removeTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService.remove(objectSnakeToCamel(data)).subscribe({
this.transformationService.Remove(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
+3
View File
@@ -1,6 +1,7 @@
import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import { writeFileSync } from 'fs';
import helmet from 'helmet';
import { AppModule } from './app.module';
@@ -14,6 +15,8 @@ async function bootstrap() {
},
});
app.use(helmet());
const config = new DocumentBuilder()
.setTitle('Maestro Grpc Documentation')
.setDescription('Documentation for Maestro gateway')
+18 -9
View File
@@ -9,18 +9,19 @@ import {
} from '@nestjs/common';
import jwkToPem from 'jwk-to-pem';
import { decode, verify } from 'jsonwebtoken';
import { HttpExceptionFilter } from 'src/error/http-exception.filter';
import { HttpExceptionFilter } from '../error/http-exception.filter';
@UseFilters(new HttpExceptionFilter())
export class LoggerMiddleware implements NestMiddleware {
use = async (request: Request, response: Response, next: NextFunction) => {
console.log('Middle', new Date());
const idToken = request.get('Dadosfera-User');
const accessToken = request.get('Authorization');
const privateKey = process.env.JWT_PRIVATE_KEY;
let requiredRoute = '';
const requiredMethod = request.method.trim();
const requiredRoute = request.route.path.split('/')[1].trim();
if (request.route.path.split('/').length >= 2) {
requiredRoute = request.route.path.split('/')[1].trim();
}
verify(idToken, privateKey, (err) => {
if (err) {
@@ -30,28 +31,36 @@ export class LoggerMiddleware implements NestMiddleware {
const jwtDecoded: any = decode(idToken);
const permissions = jwtDecoded.user.permissions;
const clienId = jwtDecoded.user.customerId;
const clientId = jwtDecoded.user.customerId;
const customer = jwtDecoded.user.customer;
const userId = jwtDecoded.user.id;
const customer_tier = jwtDecoded.user.customer_tier;
await verifyToken(accessToken);
let havePermission = false;
permissions.forEach((permission) => {
const permission = permissions.find((permission) => {
permission = permission.split('/');
const method = permission[0].trim();
const route = permission[1].trim();
if (method === requiredMethod && route === requiredRoute) {
havePermission = true;
return permission;
}
});
if (permission) {
havePermission = true;
}
if (havePermission) {
request.body.info = {
customer_id: clienId,
customer_id: clientId,
user_id: userId,
customer,
customer_tier,
};
next();
} else {
throw new ForbiddenException();
+211 -9
View File
@@ -1,25 +1,227 @@
import { Body, Controller, Post } from '@nestjs/common';
import {
Body,
Controller,
Headers,
Post,
HttpCode,
HttpStatus,
} from '@nestjs/common';
import {
AuthSignInRequest,
AuthRefreshAccessTokenRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthChangePasswordRequest,
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
} from '@victorradael/protospack';
import { AuthClientService } from 'src/clients/auth/client.service';
interface ISignIn {
username: string;
password: string;
}
import ErrorBuilder from '../../utils/ErrorBuilder';
@Controller('auth')
export class AuthController {
constructor(private authClient: AuthClientService) {}
// DEPRECADO!
@Post()
async signIn(@Body() { username, password }: ISignIn) {
@HttpCode(HttpStatus.OK)
async signIn_old(@Body() body: AuthSignInRequest) {
console.log(`/auth`, 'SignIn_old');
return this.signIn(body);
}
@Post('login')
@HttpCode(HttpStatus.OK)
async signIn_login(@Body() body: AuthSignInRequest) {
console.log(`/auth`, 'SignIn_login');
return this.signIn(body);
}
@Post('sign-in')
@HttpCode(HttpStatus.OK)
async signIn(@Body() { username, password, totp }: AuthSignInRequest) {
console.log(`/auth`, 'SignIn');
const tokens = await this.authClient
.signIn({ username, password })
.then((result) => result)
.catch((err) => console.log(err));
.signIn({ username, password, totp })
.catch((err) => {
throw ErrorBuilder(err);
});
return tokens;
}
@Post('refresh-access-token')
@HttpCode(HttpStatus.OK)
async refreshAccessToken(
@Body() { refreshToken }: AuthRefreshAccessTokenRequest,
) {
console.log(`/auth`, 'RefreshAccessToken');
const accessToken = await this.authClient
.refreshAccessToken({ refreshToken })
.catch((err) => {
throw ErrorBuilder(err);
});
return accessToken;
}
@Post('enable-totp')
@HttpCode(HttpStatus.OK)
async enableTotpMFA(
@Body() body: AuthEnableTotpMfaRequest,
@Headers() headers,
) {
console.log(`/auth`, 'enable-totp');
const { password } = body;
const { authorization: accessToken } = headers;
const response = await this.authClient
.enableTotpMFA({ accessToken, password })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('disable-totp')
@HttpCode(HttpStatus.OK)
async disableTotpMFA(
@Body() body: AuthDisableTotpMfaRequest,
@Headers() headers,
) {
console.log(`/auth`, 'disable-totp');
const { password } = body;
const { authorization: accessToken } = headers;
const response = await this.authClient
.disableTotpMFA({ accessToken, password })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('dismiss-totp')
@HttpCode(HttpStatus.OK)
async dismissTotpMFA(@Headers() headers) {
console.log(`/auth`, 'disable-totp');
const { authorization: accessToken } = headers;
const response = await this.authClient
.dismissTotpMFA({ accessToken })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('verify-totp')
@HttpCode(HttpStatus.OK)
async verifyTotp(
@Body() body: AuthVerifyTotpMfaRequest,
@Headers() headers,
): Promise<any> {
console.log(`/auth`, 'enable-totp');
const { totp } = body;
const { authorization: accessToken } = headers;
const response = await this.authClient
.verifyTotp({ accessToken, totp })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('change-password')
@HttpCode(HttpStatus.OK)
async changePassword(
@Body() body: AuthChangePasswordRequest,
@Headers() headers,
): Promise<any> {
console.log(`/auth`, 'change-password');
const { oldPassword, newPassword } = body;
const { authorization: accessToken } = headers;
const response = await this.authClient
.changePassword({
accessToken,
oldPassword,
newPassword,
})
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('reset-password')
@HttpCode(HttpStatus.OK)
async resetPassword(@Body() body: AuthResetPasswordRequest): Promise<any> {
console.log(`/auth`, 'reset-password');
const { username } = body;
const response = await this.authClient
.resetPassword({ username })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('verify-reset-password-code')
@HttpCode(HttpStatus.OK)
async verifyResetPasswordCode(
@Body() body: AuthVerifyResetPasswordCodeRequest,
): Promise<any> {
console.log(`/auth`, 'verify-reset-password-code');
const { username, code } = body;
const response = await this.authClient
.verifyResetPasswordCode({ username, code })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
@Post('confirm-reset-password')
@HttpCode(HttpStatus.OK)
async confirmResetPassword(
@Body() body: AuthConfirmResetPasswordRequest,
): Promise<any> {
console.log(`/auth`, 'confirm-reset-password');
const { username, code, newPassword } = body;
const response = await this.authClient
.confirmResetPassword({ username, code, newPassword })
.catch((err) => {
throw ErrorBuilder(err);
});
return response;
}
}
+147
View File
@@ -1,5 +1,152 @@
import nock from 'nock';
import { LoggerMiddleware } from '../../middlewares/authentication';
import { NextFunction, Request, Response } from 'express';
import { ConsoleLogger, UnauthorizedException } from '@nestjs/common';
describe('PipelinesGrpcServerService', () => {
// let mockRequest: Partial<Request>;
// let mockResponse: Partial<Response>;
// const nextFunction: NextFunction = jest.fn();
// beforeEach(() => {
// mockRequest = {
// headers: {},
// body: {
// info: {},
// },
// };
// mockResponse = {
// json: jest.fn(),
// };
// });
it('should be defined', () => {
expect(2 + 2).toBe(4);
});
// it('Should be able to pass auth', async () => {
// const awsRegion = process.env.AWS_REGION;
// const awsPoolId = process.env.AWS_IDENTITY_POOL_ID;
// const cognitoRes = {
// keys: [
// {
// alg: 'RS256',
// e: 'AQAB',
// kid: 'z3VA3+i9JZY3JFDJTNWOap8RY+B7C6mFedaqCuvLvqA=',
// kty: 'RSA',
// n: 'vIGCQDkPA_eQUoaGDUsCyK_Whr76mNW0kZ1uac6ZnyY6hsjUIjvwehqv-ux3cQo4rQZQVFcoh8n9cK5jguz4GVdD972vIxoEdyv32nBFVr5e1PBunKJ2Y32GTR_Hl0XiE0hRe1v6cWuTqiC4qm1NY7tXYL3mI9L6s9ztNbhmG_V44y26PdhL4vRVrJaHOAmCs-77U-QYAC7Llkpmjh-8tG1zt9_FJ237cpBUVOuhD-7Nm32_eB9wddBGfBw0F10ko_KJoU-7683_Kv8k9coJzFUSONId-bfnLOzs8j1L6ZAHipXCR7rpmuRYzXztjp4Wmm2zPUToWLdMEy0Hq1OWmw',
// use: 'sig',
// },
// {
// alg: 'RS256',
// e: 'AQAB',
// kid: 'u+1W9pi+clp8LaPhZVrv4dXMqRkTRD02YWhbm0NvsUw=',
// kty: 'RSA',
// n: 'noGq1cRMAKJPWahqfC_zWasYovSUycaS1basMfEoh3ePLc9zRgmyfiVKYzLRosHMe1uk0Y5ekCBKnWA8Yl7I84Yt7IIIaE44oJjSEGBKT3m8i8YaXzawaNs63KPkRh8553o3KzL75bQWcI_ABKqkf-uAKSCl_XotBGkzLUl4hIOYtAtRGEfKaNMPqyTCT5Zn71pMd0isppaUiTW2T5QLsZV1IBp46aSrl_D5Q_FTsJT7feobQVoHp2zfIorCpkfTXBTBMQTEBFSDyPbc6cULl48VKxp0B0GEwR_kYCEHEVzf41LQcUWZUE0OdBychijkSc9MZJnBWUYQZyedJILWnQ',
// use: 'sig',
// },
// ],
// };
// nock(
// `https://cognito-idp.${awsRegion}.amazonaws.com/${awsPoolId}/.well-known/jwks.json`,
// )
// .persist()
// .get('')
// .reply(200, cognitoRes);
// mockRequest.headers['Dadosfera-User'] =
// 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyIjp7ImlkIjoiZTljYjI1ZGEtMTQyOC00NDJjLTg5NjItYTdkNmQxNjhkODUyIiwibmFtZSI6InJvZHJpZ28uemFtYm9uaSIsInVzZXJuYW1lIjoicm9kcmlnby56YW1ib25pQGRhZG9zZmVyYS5haSIsInJ1bGVJZCI6IjIxNzdmZjg4LThmY2ItNDRlNS1hYzYxLWNjYmU2Y2M3MGU4ZCIsImN1c3RvbWVySWQiOiIxMTI5ODBhMy0wYTEyLTQxYzktYmZmZC03OTFjZjdjYTg3YTIiLCJtZmFTdGF0dXMiOiJwZW5kaW5nIiwiY3JlYXRlZEF0IjoiMjAyMi0wNS0xM1QxNTowOTowOS4xMzhaIiwidXBkYXRlZEF0IjoiMjAyMi0wNS0xM1QxNTowOTowOS4xMzhaIiwicGVybWlzc2lvbnMiOlsiUE9TVCAvcGlwZWxpbmVzIiwiR0VUIC9waXBlbGluZXMiLCJQVVQgL3BpcGVsaW5lcyIsIkRFTEVURSAvcGlwZWxpbmVzIiwiUE9TVCAvaW5wdXRzIiwiR0VUIC9pbnB1dHMiLCJQVVQgL2lucHV0cyIsIkRFTEVURSAvaW5wdXRzIiwiUE9TVCAvb3V0cHV0cyIsIkdFVCAvb3V0cHV0cyIsIlBVVCAvb3V0cHV0cyIsIkRFTEVURSAvb3V0cHV0cyIsIlBPU1QgL3RyYW5zZm9ybWF0aW9ucyIsIkdFVCAvdHJhbnNmb3JtYXRpb25zIiwiUFVUIC90cmFuc2Zvcm1hdGlvbnMiLCJERUxFVEUgL3RyYW5zZm9ybWF0aW9ucyIsIlBPU1QgL2NhdGFsb2ciLCJHRVQgL2NhdGFsb2ciLCJQVVQgL2NhdGFsb2ciLCJERUxFVEUgL2NhdGFsb2ciLCJQT1NUIC9hdXRoIiwiR0VUIC9tZXRhYmFzZSIsIkdFVCAvc25vd2ZsYWtlIl0sImN1c3RvbWVyIjoiZGFkb3NmZXJhIn0sImlhdCI6MTY1MzY1NjE2NywiZXhwIjoxNjUzNzQyNTY3fQ.eeNEEUk_95KOxM-objS7gXz-SCVkNFYpEP688MfLkdI';
// mockRequest.headers['Authorization'] =
// 'eyJraWQiOiJ1KzFXOXBpK2NscDhMYVBoWlZydjRkWE1xUmtUUkQwMllXaGJtME52c1V3PSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiIxYjkyZmFkMC1iNWVlLTQwMzAtOGRmYy1hZWM0MGQzYzdkYmUiLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtZWFzdC0xLmFtYXpvbmF3cy5jb21cL3VzLWVhc3QtMV9OVXY3WTJTeGoiLCJjbGllbnRfaWQiOiI0N3RrY3Jxc2NuajY3bWhnYmg3dXQ1YnJzNCIsIm9yaWdpbl9qdGkiOiJhZWJjNjA5NC1hYTJmLTRiZmUtOWExZi01ZWVhYTVmYzAwNDAiLCJldmVudF9pZCI6IjBhY2NjMzJiLTllNzktNGY2NS1iMDFiLWZhZGJlYzFhZWFiNiIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYXdzLmNvZ25pdG8uc2lnbmluLnVzZXIuYWRtaW4iLCJhdXRoX3RpbWUiOjE2NTM2NTYxNjcsImV4cCI6MTY1MzY1Nzk2NywiaWF0IjoxNjUzNjU2MTY3LCJqdGkiOiIyYTgzNDhmOC0xNDhlLTQ3NjctODcxYi00M2UyOGRjNjc2NTkiLCJ1c2VybmFtZSI6InJvZHJpZ28uemFtYm9uaUBkYWRvc2ZlcmEuYWkifQ.UgIgeGEH2olNgqPly8cymNgWIZJz5n9N4yeKgTusfGsu5h-TWP_jVhPoCvFoixO2XKzFjSCvKwlKQYZyB74jLQLs-j5C5KGBdOea1HX7pUnEfEVtBINd1kcHib5YpGYq3MHG1uVx8vNJS3XviwgYg4rgNWA4du-QbhMicdRyHolYM-dWxuxtkwTRWMe1Vw6KlTctFsvUWx1GMgjp37tsLONcp3B8OsYXfiR764K_pBNs5gMhJ39gJ7NHHLWkJrtrUTpIoHWaZS_HEgvVyQiJENQvK_bPDMPm_lyF1dW-JBgot4TpAxMmvV1XGabkzycIcAOwUaPsKBlMphgunz1Ffw';
// mockRequest.method = 'GET';
// mockRequest.route = {
// path: '/inputs/',
// stack: [
// {
// method: 'get',
// },
// ],
// methods: {
// get: true,
// },
// };
// const authMiddleware = new LoggerMiddleware();
// await authMiddleware.useTest(
// mockRequest as Request,
// mockResponse as Response,
// nextFunction,
// );
// expect(nextFunction).toHaveBeenCalled();
// });
// it('Should not be able to pass auth', async () => {
// const awsRegion = process.env.AWS_REGION;
// const awsPoolId = process.env.AWS_IDENTITY_POOL_ID;
// const cognitoRes = {
// keys: [
// {
// alg: 'RS256',
// e: 'AQAB',
// kid: 'z3VA3+i9JZY3JFDJTNWOap8RY+B7C6mFedaqCuvLvqA=',
// kty: 'RSA',
// n: 'vIGCQDkPA_eQUoaGDUsCyK_Whr76mNW0kZ1uac6ZnyY6hsjUIjvwehqv-ux3cQo4rQZQVFcoh8n9cK5jguz4GVdD972vIxoEdyv32nBFVr5e1PBunKJ2Y32GTR_Hl0XiE0hRe1v6cWuTqiC4qm1NY7tXYL3mI9L6s9ztNbhmG_V44y26PdhL4vRVrJaHOAmCs-77U-QYAC7Llkpmjh-8tG1zt9_FJ237cpBUVOuhD-7Nm32_eB9wddBGfBw0F10ko_KJoU-7683_Kv8k9coJzFUSONId-bfnLOzs8j1L6ZAHipXCR7rpmuRYzXztjp4Wmm2zPUToWLdMEy0Hq1OWmw',
// use: 'sig',
// },
// {
// alg: 'RS256',
// e: 'AQAB',
// kid: 'u+1W9pi+clp8LaPhZVrv4dXMqRkTRD02YWhbm0NvsUw=',
// kty: 'RSA',
// n: 'noGq1cRMAKJPWahqfC_zWasYovSUycaS1basMfEoh3ePLc9zRgmyfiVKYzLRosHMe1uk0Y5ekCBKnWA8Yl7I84Yt7IIIaE44oJjSEGBKT3m8i8YaXzawaNs63KPkRh8553o3KzL75bQWcI_ABKqkf-uAKSCl_XotBGkzLUl4hIOYtAtRGEfKaNMPqyTCT5Zn71pMd0isppaUiTW2T5QLsZV1IBp46aSrl_D5Q_FTsJT7feobQVoHp2zfIorCpkfTXBTBMQTEBFSDyPbc6cULl48VKxp0B0GEwR_kYCEHEVzf41LQcUWZUE0OdBychijkSc9MZJnBWUYQZyedJILWnQ',
// use: 'sig',
// },
// ],
// };
// nock(
// `https://cognito-idp.${awsRegion}.amazonaws.com/${awsPoolId}/.well-known/jwks.json`,
// )
// .persist()
// .get('')
// .reply(200, cognitoRes);
// mockRequest.headers['Dadosfera-User'] =
// 'asdeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyIjp7ImlkIjoiZTljYjI1ZGEtMTQyOC00NDJjLTg5NjItYTdkNmQxNjhkODUyIiwibmFtZSI6InJvZHJpZ28uemFtYm9uaSIsInVzZXJuYW1lIjoicm9kcmlnby56YW1ib25pQGRhZG9zZmVyYS5haSIsInJ1bGVJZCI6IjIxNzdmZjg4LThmY2ItNDRlNS1hYzYxLWNjYmU2Y2M3MGU4ZCIsImN1c3RvbWVySWQiOiIxMTI5ODBhMy0wYTEyLTQxYzktYmZmZC03OTFjZjdjYTg3YTIiLCJtZmFTdGF0dXMiOiJwZW5kaW5nIiwiY3JlYXRlZEF0IjoiMjAyMi0wNS0xM1QxNTowOTowOS4xMzhaIiwidXBkYXRlZEF0IjoiMjAyMi0wNS0xM1QxNTowOTowOS4xMzhaIiwicGVybWlzc2lvbnMiOlsiUE9TVCAvcGlwZWxpbmVzIiwiR0VUIC9waXBlbGluZXMiLCJQVVQgL3BpcGVsaW5lcyIsIkRFTEVURSAvcGlwZWxpbmVzIiwiUE9TVCAvaW5wdXRzIiwiR0VUIC9pbnB1dHMiLCJQVVQgL2lucHV0cyIsIkRFTEVURSAvaW5wdXRzIiwiUE9TVCAvb3V0cHV0cyIsIkdFVCAvb3V0cHV0cyIsIlBVVCAvb3V0cHV0cyIsIkRFTEVURSAvb3V0cHV0cyIsIlBPU1QgL3RyYW5zZm9ybWF0aW9ucyIsIkdFVCAvdHJhbnNmb3JtYXRpb25zIiwiUFVUIC90cmFuc2Zvcm1hdGlvbnMiLCJERUxFVEUgL3RyYW5zZm9ybWF0aW9ucyIsIlBPU1QgL2NhdGFsb2ciLCJHRVQgL2NhdGFsb2ciLCJQVVQgL2NhdGFsb2ciLCJERUxFVEUgL2NhdGFsb2ciLCJQT1NUIC9hdXRoIiwiR0VUIC9tZXRhYmFzZSIsIkdFVCAvc25vd2ZsYWtlIl0sImN1c3RvbWVyIjoiZGFkb3NmZXJhIn0sImlhdCI6MTY1MzY1NjE2NywiZXhwIjoxNjUzNzQyNTY3fQ.eeNEEUk_95KOxM-objS7gXz-SCVkNFYpEP688MfLkdI';
// mockRequest.headers['Authorization'] =
// 'asdeyJraWQiOiJ1KzFXOXBpK2NscDhMYVBoWlZydjRkWE1xUmtUUkQwMllXaGJtME52c1V3PSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiIxYjkyZmFkMC1iNWVlLTQwMzAtOGRmYy1hZWM0MGQzYzdkYmUiLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtZWFzdC0xLmFtYXpvbmF3cy5jb21cL3VzLWVhc3QtMV9OVXY3WTJTeGoiLCJjbGllbnRfaWQiOiI0N3RrY3Jxc2NuajY3bWhnYmg3dXQ1YnJzNCIsIm9yaWdpbl9qdGkiOiJhZWJjNjA5NC1hYTJmLTRiZmUtOWExZi01ZWVhYTVmYzAwNDAiLCJldmVudF9pZCI6IjBhY2NjMzJiLTllNzktNGY2NS1iMDFiLWZhZGJlYzFhZWFiNiIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYXdzLmNvZ25pdG8uc2lnbmluLnVzZXIuYWRtaW4iLCJhdXRoX3RpbWUiOjE2NTM2NTYxNjcsImV4cCI6MTY1MzY1Nzk2NywiaWF0IjoxNjUzNjU2MTY3LCJqdGkiOiIyYTgzNDhmOC0xNDhlLTQ3NjctODcxYi00M2UyOGRjNjc2NTkiLCJ1c2VybmFtZSI6InJvZHJpZ28uemFtYm9uaUBkYWRvc2ZlcmEuYWkifQ.UgIgeGEH2olNgqPly8cymNgWIZJz5n9N4yeKgTusfGsu5h-TWP_jVhPoCvFoixO2XKzFjSCvKwlKQYZyB74jLQLs-j5C5KGBdOea1HX7pUnEfEVtBINd1kcHib5YpGYq3MHG1uVx8vNJS3XviwgYg4rgNWA4du-QbhMicdRyHolYM-dWxuxtkwTRWMe1Vw6KlTctFsvUWx1GMgjp37tsLONcp3B8OsYXfiR764K_pBNs5gMhJ39gJ7NHHLWkJrtrUTpIoHWaZS_HEgvVyQiJENQvK_bPDMPm_lyF1dW-JBgot4TpAxMmvV1XGabkzycIcAOwUaPsKBlMphgunz1Ffw';
// mockRequest.method = 'GET';
// mockRequest.route = {
// path: '/inputs/',
// stack: [
// {
// method: 'get',
// },
// ],
// methods: {
// get: true,
// },
// };
// const authMiddleware = new LoggerMiddleware();
// // const t = await authMiddleware.useTest(
// // mockRequest as Request,
// // mockResponse as Response,
// // nextFunction,
// // );
// // expect(nextFunction).toHaveBeenCalled();
// expect(async () => {
// const t = await authMiddleware.useTest(
// mockRequest as Request,
// mockResponse as Response,
// nextFunction,
// );
// console.log(t);
// }).toThrow('Unauthorized');
// expect(1).toBe(2);
// // expect(t).toThrow(UnauthorizedException);
// });
});
+302
View File
@@ -0,0 +1,302 @@
import {
Body,
Controller,
Delete,
Get,
Param,
Post,
Query,
} from '@nestjs/common';
import { CatalogService } from './catalog.service';
@Controller('catalog')
export class CatalogController {
constructor(private catalogService: CatalogService) {}
@Get('all')
async catalogAll(@Body() body) {
console.log(`/catalog`, 'ON CATALOG ALL ROUTE');
const res = await this.catalogService.catalogAll(body);
return res;
}
@Get('data_apps')
async dataAppsAll(@Body() body) {
console.log(`/catalog`, 'ON FIND ALL DATA APPS ROUTE');
const res = await this.catalogService.dataAppsAll(body);
return res;
}
@Delete('data_apps/:id')
async dataAppsOne(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON FIND ONE DATA APP ROUTE');
const { id } = params;
const res = await this.catalogService.dataAppsOne(id, body);
return res;
}
@Get('dashboard-metabase')
async getAllDashboardMetabase(@Body() body) {
console.log(`/catalog`, 'ON GET ALL DASHBOARDS METABASE ROUTE');
const res = await this.catalogService.getAllDashboardMetabase(body);
return res;
}
@Get('dashboard-metabase/:id')
async getOneDashboardMetabase(@Body() body, @Param() params) {
const { id } = params;
console.log(`/catalog`, 'ON GET ONE DASHBOARD METABASE ROUTE');
const res = await this.catalogService.getOneDashboardMetabase(id, body);
return res;
}
@Get('table-metadata')
async getAllTableMetadata(@Body() body, @Query() query) {
console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE');
if (!query) {
const res = await this.catalogService.getAllTableMetadata(body);
return res;
} else {
const res = await this.catalogService.getOneTableMetadata(body, query);
return res;
}
}
@Delete('table-metadata/:id')
async deleteOneTableMetadata(@Param() params, @Body() body) {
const { id } = params;
console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE');
const res = await this.catalogService.deleteOneTableMetadata(id, body);
return res;
}
@Get('column-metadata')
async getOneColumnMetadata(@Body() body, @Query() params) {
console.log(`/catalog`, 'ON GET ONE COLUMN METADATA ROUTE');
const res = await this.catalogService.getOneColumnMetadata(body, params);
return res;
}
@Delete('column-metadata/:id')
async deleteOneColumnMetadata(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE');
const { id } = params;
const res = await this.catalogService.deleteOneColumnMetadata(id, body);
return res;
}
@Get('data-preview')
async getOneDataPreview(@Body() body, @Query() params) {
console.log(`/catalog`, 'ON GET ONE DATAPREVIEW ROUTE');
const res = await this.catalogService.getOneDataPreview(body, params);
return res;
}
@Get('data-status')
async getDataStatus(@Body() body) {
console.log(`/catalog`, 'ON GET DATA STATUS ROUTE');
const res = await this.catalogService.getDataStatus(body);
return res;
}
@Get('data-status')
async createDataStatus(@Body() body) {
console.log(`/catalog`, 'ON CREATE DATA STATUS ROUTE');
const res = await this.catalogService.createDataStatus(body);
return res;
}
@Get('data-description')
async getDataDescription(@Body() body) {
console.log(`/catalog`, 'ON GET DATA DESCRIPTION ROUTE');
const res = await this.catalogService.getDataDescription(body);
return res;
}
@Post('data-description')
async createDataDescription(@Body() body) {
console.log(`/catalog`, 'ON CREATE DATA DESCRIPTION ROUTE');
const res = await this.catalogService.createDataDescription(body);
return res;
}
@Get('data-docs')
async getDataDocs(@Body() body) {
console.log(`/catalog`, 'ON GET DATA DOCS ROUTE');
const res = await this.catalogService.getDataDocs(body);
return res;
}
@Post('data-docs')
async createDataDocs(@Body() body) {
console.log(`/catalog`, 'ON CREATE DATA DOCS ROUTE');
const res = await this.catalogService.createDataDocs(body);
return res;
}
@Get('data-rating')
async getDataRating(@Body() body) {
console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
const res = await this.catalogService.getDataRating(body);
return res;
}
@Post('data-rating')
async createDataRating(@Body() body) {
console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
const res = await this.catalogService.createDataRating(body);
return res;
}
@Get('summary-rating/:id')
async getSummaryRating() {
console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE');
//const { id } = params;
//
//const res = await this.catalogService.getSummaryRating(id, body);
return {
avg_rating: 0,
rating_count: 0,
};
}
@Get('data-comment')
async getDataComment(@Body() body) {
console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
const res = await this.catalogService.getDataComment(body);
return res;
}
@Post('data-comment')
async createDataComment(@Body() body) {
console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
const res = await this.catalogService.createDataComment(body);
return res;
}
@Get('data-review')
async getDataReview(@Body() body, @Query() params) {
console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE');
const res = await this.catalogService.getDataReview(body, params);
return res;
}
@Post('tags')
async createTags(@Body() body) {
console.log(`/catalog`, 'ON CREATE TAG ROUTE');
const res = await this.catalogService.createTags(body);
return res;
}
@Get('tags')
async findAllTags(@Body() body) {
console.log(`/catalog`, 'ON FIND ALL TAGS ROUTE');
const res = await this.catalogService.findAllTags(body);
return res;
}
@Delete('tags/:id')
async deleteTags(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON DELETE TAG ROUTE');
const { id } = params;
const res = await this.catalogService.deleteTags(id, body);
return res;
}
@Post('table-tags')
async createTableTags(@Body() body) {
console.log(`/catalog`, 'ON CREATE TABLE TAG ROUTE');
const res = await this.catalogService.createTableTags(body);
return res;
}
@Get('table-tags')
async getAllTableTags(@Body() body) {
console.log(`/catalog`, 'ON GET ALL TABLE TAGS ROUTE');
const res = await this.catalogService.findAllTableTags(body);
return res;
}
@Delete('table-tags/:id')
async deleteTableTags(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');
const { id } = params;
const res = await this.catalogService.deleteTableTags(id, body);
return res;
}
@Get('table-rules')
async getTableRules(@Body() body) {
console.log(`/catalog`, 'ON GET TABLE RULES ROUTE');
const res = await this.catalogService.getTableRules(body);
return res;
}
@Delete('table-rules/:id')
async deleteTableRules(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE');
const { id } = params;
const res = await this.catalogService.deleteTableRules(id, body);
return res;
}
}
+263
View File
@@ -0,0 +1,263 @@
import axios from 'axios';
class CatalogService {
_getNimbusUrl(body) {
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
return `https://nimbus-${customer}.dadosfera.ai`;
}
return `https://nimbus-${customer}.${process.env.ENV}.dadosfera.ai`;
}
async catalogAll(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/all/`);
return data;
}
async dataAppsAll(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data_apps/`);
return data;
}
async dataAppsOne(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/data_apps/${id}/`,
);
return data;
}
async getAllDashboardMetabase(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/dashboard-metabase/`,
);
return data;
}
async getOneDashboardMetabase(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/dashboard-metabase/${id}`,
);
console.log(data);
// const [, path] = data.iframe_url.split('.dadosfera');
// const host = `httpss://metabase-${body.info.customer.toLowerCase()}.dadosfera.ai`;
// data.iframe_url = host + path;
return data;
}
async getAllTableMetadata(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-metadata/`,
);
return data;
}
async getOneTableMetadata(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-metadata/`,
{ params: params },
);
return data;
}
async deleteOneTableMetadata(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-metadata/${id}`,
);
return data;
}
async getOneColumnMetadata(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/column-metadata/`,
{ params: params },
);
return data;
}
async deleteOneColumnMetadata(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/column-metadata/${id}/`,
);
return data;
}
async getOneDataPreview(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-preview/`, {
params: params,
});
return data;
}
async getDataStatus(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-status/`);
return data;
}
async createDataStatus(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-status/`,
body,
);
return data;
}
async getDataDescription(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/data-description/`,
);
return data;
}
async createDataDescription(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-description/`,
body,
);
return data;
}
async getDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-docs/`);
return data;
}
async createDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-docs/`,
body,
);
return data;
}
async getDataRating(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-rating/`);
return data;
}
async createDataRating(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-rating/`,
body,
);
return data;
}
async getSummaryRating(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/summary-rating/${id}`,
);
return data;
}
async getDataComment(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-comment/`);
return data;
}
async createDataComment(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-comment/`,
body,
);
return data;
}
async getDataReview(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-review/`, {
params: params,
});
return data;
}
async createTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(`${nimbusUrl}/api/catalog/tags/`, body);
return data;
}
async findAllTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/tags/`, body);
return data;
}
async findAllTableTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-tags/`,
body,
);
return data;
}
async deleteTags(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(`${nimbusUrl}/api/catalog/tags/${id}`);
return data;
}
async createTableTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/table-tags/`,
body,
);
return data;
}
async deleteTableTags(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-tags/${id}`,
);
return data;
}
async getTableRules(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-rules/`,
body,
);
return data;
}
async deleteTableRules(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-rules/${id}`,
);
return data;
}
}
export { CatalogService };
+18 -7
View File
@@ -4,15 +4,13 @@ import {
Delete,
Get,
Param,
Patch,
Post,
Put,
} from '@nestjs/common';
import { InputsService } from './inputs.service';
import { InputsClientService } from 'src/clients/inputs/client.service';
import {
ICreateInputRequest,
UpdateInputRequest,
} from 'src/clients/inputs/interfaces';
import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
import { InputNewCreateRequest } from '@victorradael/protospack';
@Controller('inputs')
export class InputsController {
@@ -28,8 +26,21 @@ export class InputsController {
return response;
}
@Post('/test-connection/get-columns')
async getColumns(@Body() data) {
console.log(
`/test-connection/get-columns`,
'ON TEST CONNECTION GET COLUMNS ROUTE',
);
const inputService = new InputsService(this.inputsClientService);
const response = await inputService.getColumns(data);
return response;
}
@Post()
async create(@Body() createInputDto: ICreateInputRequest) {
async create(@Body() createInputDto: InputNewCreateRequest) {
console.log(`/input`, 'ON CREATE ROUTE');
const inputService = new InputsService(this.inputsClientService);
@@ -60,7 +71,7 @@ export class InputsController {
return response;
}
@Put(':id')
@Patch(':id')
async update(@Body() updateInputDto: UpdateInputRequest, @Param() params) {
const { id } = params;
const { info } = updateInputDto;
+115 -13
View File
@@ -1,20 +1,72 @@
import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { Timeout } from '@nestjs/schedule';
import CronParser, { CronExpression } from 'cron-parser';
import { InputsClientService } from 'src/clients/inputs/client.service';
import {
ICreateInputRequest,
IIdRequest,
Info,
} from 'src/clients/inputs/interfaces';
import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
@Injectable()
export class InputsService {
constructor(private inputClient: InputsClientService) {}
secondsInADay = 60 * 60 * 24;
secondsInAnHour = 60 * 60;
async create(@Body() data: ICreateInputRequest) {
adjustInputPayload(payload) {
return payload?.input_s3 || payload?.input_jdbc;
}
getDifferenceInSeconds(date1: Date, date2: Date) {
const diffInMs = Math.abs(date2.getTime() - date1.getTime());
return diffInMs / 1000;
}
validateCron(data) {
const { info, cron } = data;
const { customer_tier } = info;
if (!cron) return;
let interval: CronExpression;
try {
const createInputResponse = await this.inputClient.create(data);
return createInputResponse;
interval = CronParser.parseExpression(cron);
} catch (error) {
throw new HttpException(
'Intervalo de tempo inválido',
HttpStatus.BAD_REQUEST,
);
}
const nextDate = interval.next().toDate();
const afterNextDate = interval.next().toDate();
const secondsApart = this.getDifferenceInSeconds(nextDate, afterNextDate);
if (customer_tier === 'BASIC' && secondsApart < this.secondsInADay) {
throw new HttpException(
'Intervalo de tempo não pode ser inferior a um dia.',
HttpStatus.FORBIDDEN,
);
} else if (secondsApart < this.secondsInAnHour) {
throw new HttpException(
'Intervalo de tempo não pode ser inferior a uma hora.',
HttpStatus.FORBIDDEN,
);
}
}
async create(@Body() data) {
this.validateCron(data);
try {
if (
data.plugin == 'csv' ||
data.plugin == 'json' ||
data.plugin == 'parquet'
) {
const { info, ...input } = data;
const inputPayload = this.generateInputS3Payload(input);
const createInputResponse = await this.inputClient.createS3Inputs({
input: inputPayload,
info,
});
return createInputResponse;
} else {
const createInputResponse: any = await this.inputClient.create(data);
if (createInputResponse?.input?.input_jdbc) {
return { input: createInputResponse?.input?.input_jdbc };
}
return createInputResponse;
}
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
@@ -22,8 +74,12 @@ export class InputsService {
async findAll(body) {
try {
const findAllInputResponse = await this.inputClient.findAll(body);
const findAllInputResponse: any = await this.inputClient.findAll(body);
if (findAllInputResponse?.inputs?.length) {
findAllInputResponse.inputs = findAllInputResponse.inputs.map((input) =>
this.adjustInputPayload(input),
);
}
return findAllInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -32,7 +88,12 @@ export class InputsService {
async findOne(idRequest: IIdRequest) {
try {
const findOneInputResponse = await this.inputClient.findOne(idRequest);
const findOneInputResponse: any = await this.inputClient.findOne(
idRequest,
);
findOneInputResponse.input = this.adjustInputPayload(
findOneInputResponse.input,
);
return findOneInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -40,13 +101,17 @@ export class InputsService {
}
async update(id: string, data, info: Info) {
this.validateCron({ ...data, info });
try {
const updateInputResponse = await this.inputClient.update({
const updateInputResponse: any = await this.inputClient.update({
id,
info,
...data,
});
updateInputResponse.input = this.adjustInputPayload(
updateInputResponse?.input,
);
return updateInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -63,6 +128,7 @@ export class InputsService {
}
}
@Timeout(60000 * 10) // Timeout set for 10 minutes
async testConnection(data) {
try {
const testConnectionInputResponse = await this.inputClient.testConnection(
@@ -74,4 +140,40 @@ export class InputsService {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async getColumns(data) {
try {
const testConnectionGetColumnsResponse =
await this.inputClient.getColumns(data);
return testConnectionGetColumnsResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
generateInputS3Payload(payload) {
const { credentials, plugin } = payload;
if (!credentials) return payload;
const {
client_aws_access_key_id,
client_aws_secret_access_key,
file_format_params,
format_file_params,
client_bucket,
file_to_extract,
} = credentials;
const formatedPayload = {
plugin,
source_bucket: client_bucket,
source_prefix: file_to_extract,
auth_parameters: {
aws_access_key_id: client_aws_access_key_id,
aws_secret_access_key: client_aws_secret_access_key,
},
file_format_params: file_format_params || format_file_params,
};
return formatedPayload;
}
}
+21 -3
View File
@@ -14,16 +14,34 @@ import { PipelinesService } from './pipelines.service';
export class PipelinesController {
constructor(private pipelinesClientService: PipelinesClientService) {}
@Get(':id/status')
async getPipelineStatus(@Param() params) {
@Post('start/:id')
async activate(@Param() params, @Body() body) {
const { id } = params;
const { info } = body;
console.log(
process.env.DEV_URL + `/pipeline/start/${id}`,
'ON START PIPELINE ROUTE',
);
const pipelineService = new PipelinesService(this.pipelinesClientService);
const response = await pipelineService.runPipeline({ id, info });
return response;
}
@Get(':id/status')
async getPipelineStatus(@Body() body, @Param() params) {
const { id } = params;
body.id = id;
console.log(
process.env.DEV_URL + `/pipeline/${id}`,
'ON GET PIPELINE STATUS ROUTE',
);
const pipelineService = new PipelinesService(this.pipelinesClientService);
const response = await pipelineService.getPipelineStatus(id);
const response = await pipelineService.getPipelineStatus(body);
return response;
}
+37 -6
View File
@@ -7,13 +7,20 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
export class PipelinesService {
constructor(private pipelineClient: PipelinesClientService) {}
adjustPayload(payload) {
payload.input = payload.input?.input_s3 || payload.input?.input_jdbc;
}
async create(createPipelineDto) {
try {
const createPipelineResponse = await this.pipelineClient.create(
createPipelineDto,
);
return objectCamelToSnake(createPipelineResponse);
const pipeline = objectCamelToSnake(createPipelineResponse);
this.adjustPayload(pipeline.pipeline);
return pipeline;
} catch (err) {
console.log(err);
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
@@ -23,7 +30,11 @@ export class PipelinesService {
async findOne(data: IIdRequest) {
try {
const findOnePipelineResponse = await this.pipelineClient.findOne(data);
return objectCamelToSnake(findOnePipelineResponse);
const pipeline = objectCamelToSnake(findOnePipelineResponse);
this.adjustPayload(pipeline.pipeline);
return pipeline;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
@@ -33,7 +44,11 @@ export class PipelinesService {
try {
const findAllPipelineResponse = await this.pipelineClient.findAll(data);
return objectCamelToSnake(findAllPipelineResponse);
const pipelines = objectCamelToSnake(findAllPipelineResponse);
pipelines.forEach((pipeline) => {
this.adjustPayload(pipeline);
});
return { pipelines };
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
@@ -47,7 +62,10 @@ export class PipelinesService {
...data,
});
return objectCamelToSnake(updatePipelineResponse);
const pipeline = objectCamelToSnake(updatePipelineResponse);
this.adjustPayload(pipeline);
return pipeline;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
@@ -74,14 +92,27 @@ export class PipelinesService {
}
}
async getPipelineStatus(id: string) {
async getPipelineStatus(data: IIdRequest) {
try {
const pipelineStatusResponse =
await this.pipelineClient.getPipelineStatus({ id });
await this.pipelineClient.getPipelineStatus(data);
return objectCamelToSnake(pipelineStatusResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async runPipeline({ id, info }: IIdRequest) {
try {
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
id,
info,
});
return objectCamelToSnake(triggerPipelineResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
}
@@ -11,15 +11,20 @@ export class TransformationsService {
constructor(private transformationClient: TransformationsClientService) {}
async create(createTransformationDto: ICreateTransformationsRequest) {
const { transformations, info } = createTransformationDto;
if (!transformations || transformations.length == 0) {
throw new HttpException('Missing transformations field.', 400);
}
try {
const convertedDTO = createTransformationDto.transformations.map((tr) => {
const convertedDTO = transformations.map((tr) => {
return trSnakeToCamel(tr);
});
const createTransformationResponse =
await this.transformationClient.create({
transformations: convertedDTO,
info: createTransformationDto.info,
info: info,
});
return createTransformationResponse;
+131
View File
@@ -0,0 +1,131 @@
import { HttpStatus, HttpException } from '@nestjs/common';
import ErrorCodes from './errorCodes';
function Builder({ statusCode, message, error, code }) {
return new HttpException({ statusCode, message, error, code }, statusCode);
}
export default function ErrorBuilder(code: string) {
console.log(code);
switch (code) {
case ErrorCodes.AUTH.WRONG_CREDENTIALS:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Não autenticado',
message: 'Usuário ou senha incorretos',
code,
});
case ErrorCodes.AUTH.WRONG_PASSWORD_CONFIRMATION:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Senha incorreta',
message: 'Confirmação de senha incorreta',
code,
});
case ErrorCodes.AUTH.TOTP_NOT_ENABLED:
return Builder({
statusCode: HttpStatus.PRECONDITION_FAILED,
error: 'Não permitido',
message: 'A autenticação multifator não está habilitada',
code,
});
case ErrorCodes.AUTH.TOTP_ALREADY_ENABLED:
return Builder({
statusCode: HttpStatus.PRECONDITION_FAILED,
error: 'Não permitido',
message: 'A autenticação multifator já está habilitada',
code,
});
case ErrorCodes.AUTH.TOTP_ALREADY_DISABLED:
return Builder({
statusCode: HttpStatus.PRECONDITION_FAILED,
error: 'Não permitido',
message: 'A autenticação multifator já está desabilitada',
code,
});
case ErrorCodes.AUTH.TOTP_REQUIRED:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Não autenticado',
message: 'Informe o token de autenticação multifator',
code,
});
case ErrorCodes.AUTH.CODE_MISMATCH:
case ErrorCodes.AUTH.CODE_ALREADY_USED:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Não autenticado',
message: 'Token de autenticação multifator inválido',
code,
});
case ErrorCodes.AUTH.RESET_PASSWORD_CODE_EXPIRED:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Não permitido',
message: 'Token para recuperar senha expirado',
code,
});
case ErrorCodes.AUTH.RESET_PASSWORD_CODE_INVALID:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Não permitido',
message: 'Token para recuperar senha inválido',
code,
});
case ErrorCodes.AUTH.WEAK_NEW_PASSWORD:
return Builder({
statusCode: HttpStatus.BAD_REQUEST,
error: 'Não permitido',
message: 'Senha muito fraca. Escolha uma senha mais forte',
code,
});
case ErrorCodes.RATE_LIMIT:
return Builder({
statusCode: HttpStatus.TOO_MANY_REQUESTS,
error: 'Limite excedido',
message:
'Você tentou realizar essa operação muitas vezes. Tente novamente mais tarde',
code,
});
case ErrorCodes.AUTH.UNAUTHORIZED:
return Builder({
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Não autenticado',
message: 'É necessário estar logado para realizar essa operação',
code,
});
case ErrorCodes.AUTH.FORBIDDEN:
return Builder({
statusCode: HttpStatus.FORBIDDEN,
error: 'Não autorizado',
message:
'Você não tem permissões suficientes para realizar essa operação',
code,
});
case ErrorCodes.INTERNAL:
case ErrorCodes.UNKNOWN:
default:
return Builder({
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
error: 'Desconhecido',
message:
'Erro desconhecido. Tente novamente ou entre em contato com o suporte',
code: ErrorCodes.UNKNOWN,
});
}
}
+24
View File
@@ -0,0 +1,24 @@
export const AUTH = {
UNAUTHORIZED: 'AUTH.UNAUTHORIZED',
FORBIDDEN: 'AUTH.FORBIDDEN',
WRONG_CREDENTIALS: 'AUTH.WRONG_CREDENTIALS',
WRONG_PASSWORD_CONFIRMATION: 'AUTH.WRONG_PASSWORD_CONFIRMATION',
TOTP_NOT_ENABLED: 'AUTH.TOTP_NOT_ENABLED',
TOTP_ALREADY_ENABLED: 'AUTH.TOTP_ALREADY_ENABLED',
TOTP_ALREADY_DISABLED: 'AUTH.TOTP_ALREADY_DISABLED',
TOTP_REQUIRED: 'AUTH.TOTP_REQUIRED',
CODE_MISMATCH: 'AUTH.CODE_MISMATCH',
CODE_ALREADY_USED: 'AUTH.CODE_ALREADY_USED',
RESET_PASSWORD_CODE_EXPIRED: 'AUTH.RESET_PASSWORD_CODE_EXPIRED',
RESET_PASSWORD_CODE_INVALID: 'AUTH.RESET_PASSWORD_CODE_INVALID',
WEAK_NEW_PASSWORD: 'AUTH.WEAK_NEW_PASSWORD',
};
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
INTERNAL: 'INTERNAL',
AUTH,
};
export default ErrorCodes;
+1 -1
View File
File diff suppressed because one or more lines are too long