Compare commits

...
443 Commits
Author SHA1 Message Date
arthur simas 5122dbde85 Merge pull request #126 from dadosfera/feat/scoped-packages
FEAT: scoped packages
2022-08-02 14:43:07 -03:00
Arthur Simas dcb3094890 FIX: protospack-v2 changes 2022-08-01 17:05:00 -03:00
Arthur Simas c8e4e90100 FIX: aws codeartifact region 2022-08-01 15:52:30 -03:00
Arthur Simas 09cff9fa15 CI: cleanup ci tests 2022-08-01 15:52:24 -03:00
Arthur Simas 79b624cea6 FEAT: using scoped package @dadosfera/dadosfera-logs 2022-08-01 15:52:18 -03:00
Arthur Simas 5356334258 FEAT: using scoped package @dadosfera/protospack-v2 2022-08-01 15:52:06 -03:00
Gabriel Amorim 5f6e4e88a5 Merge pull request #125 from dadosfera/fix/users-roles-management
FIX: sending access token to role function to evaluate editing permissions
2022-08-01 15:30:28 -03:00
Gabriel Rosa 32ab834834 FIX: fix user.decorator tests failing because of missing access_token on fakeUserPayload 2022-08-01 13:46:27 -03:00
Gabriel Rosa ebbfe0abff FIX: sending access token to role function to evaluate editing permissions 2022-08-01 09:17:08 -03:00
arthur simas d76a2b4ab8 Merge pull request #123 from dadosfera/feat/user-management
FEAT: user management
2022-07-28 19:24:50 -03:00
Arthur Simas 047737eadd FIX: removing log from subscribe.complete method 2022-07-28 19:21:16 -03:00
Arthur Simas e20ae71672 FIX: resend invite endpoint renamed 2022-07-27 19:35:25 -03:00
Arthur Simas 97b6047cf1 FIX: avoiding to change user object received in adjustUsersPayload, returning modified object instead 2022-07-27 19:20:35 -03:00
Arthur Simas 9c6d2b499a FEAT: resend invite 2022-07-27 18:15:07 -03:00
Arthur Simas f43c270c33 CHORE: protospack-v2 pre-release 2022-07-27 12:51:07 -03:00
Arthur Simas b0ccb39f2c FIX: adding error message on each failed user (BatchCreateUserRes) 2022-07-27 12:46:20 -03:00
Gabriel Rosa 9f900882cb FIX: sending customerId on updateUser 2022-07-25 10:09:18 -03:00
Gabriel Rosa a24abed9ed updated protospack version 2022-07-22 08:53:27 -03:00
Gabriel Rosa b04a38bcc0 FIX: get departments and jobTitles 2022-07-22 08:45:58 -03:00
Gabriel Rosa 67706c5fc3 FIX: returning hierarchies the right way 2022-07-20 16:58:53 -03:00
Gabriel Rosa 0fa4dcb933 new protospack version 2022-07-20 15:37:01 -03:00
Gabriel Rosa 04284e1feb FIX: changing hierarchy strategy 2022-07-20 15:18:14 -03:00
Gabriel Rosa 3e7581e20c FIX: changing hierarchy strategy 2022-07-20 15:17:58 -03:00
Gabriel Rosa c61c37b268 FIX: new user columns 2022-07-20 09:50:58 -03:00
Gabriel Rosa 5ce8099c25 tests fixes 2022-07-19 16:22:24 -03:00
Gabriel Rosa 9cdb0a5d38 FIX: Update user accepting roleIds 2022-07-19 10:17:57 -03:00
Gabriel Rosa 57f47d3b2e FIX: get all hierarchies 2022-07-18 18:20:15 -03:00
Gabriel Rosa 4ab955a7a1 removed unused import 2022-07-18 18:00:44 -03:00
Gabriel Rosa 1ca3694b75 merge main 2022-07-18 18:00:18 -03:00
Gabriel Rosa a6a35c507a new Protospack 2022-07-18 17:56:50 -03:00
Gabriel Rosa cfcb8c1c42 FEAT: new user endpoints 2022-07-18 17:50:12 -03:00
Gabriel Rosa a55da8e20f FEAT: batch create user 2022-07-18 15:37:56 -03:00
Gabriel Rosa 82310c6314 FEAT: Users roles id = seqid 2022-07-18 14:04:35 -03:00
Gabriel Rosa 52f9d06ad0 FIX: get permissions route 2022-07-18 11:24:52 -03:00
Victor Radael a83f91a720 Merge pull request #122 from dadosfera/ft/connectors-manager
FEAT: Ft/connectors manager
2022-07-15 18:46:01 -03:00
Victor Radael ccef7628f9 FIX: Remove AuthController 2022-07-15 18:00:09 -03:00
Victor Radael a5e424c854 Update app.module.ts 2022-07-15 17:53:38 -03:00
Gabriel Rosa 339432df16 FIX: server urls on swagger 2022-07-15 15:24:42 -03:00
Victor Radael 7dfa10d5ee Merge branch 'ft/connectors-manager' of github.com:dadosfera/maestro into ft/connectors-manager 2022-07-15 15:10:40 -03:00
Victor Radael c87ceec77b FIX: Upload Imaga Files and add permissions 2022-07-15 15:10:11 -03:00
Gabriel Rosa 071c524a2e FIX: changed get roles to search roles 2022-07-15 14:39:30 -03:00
Gabriel Rosa 1de4eedd38 FEAT: new permission for user management 2022-07-15 12:27:49 -03:00
Gabriel Rosa 29276d6070 Merge branch 'main' into feat/user-management 2022-07-15 11:42:13 -03:00
Gabriel Rosa 344a251de1 merge main 2022-07-15 11:42:01 -03:00
Arthur Simas 3b95032554 Merge branch 'main' into ft/connectors-manager 2022-07-15 11:24:46 -03:00
arthur simas a0089b5d5d Merge pull request #121 from dadosfera/fix/terms-of-use-ip
FIX: terms of use ip:port
2022-07-15 11:15:12 -03:00
Gabriel Rosa 1b620b6fb0 FEAT: ROle management 2022-07-15 11:08:57 -03:00
Arthur Simas 1eb76d4b1c FIX: terms of use ip:port 2022-07-15 11:08:46 -03:00
Victor Radael 314bc5a562 Merge remote-tracking branch 'origin' into ft/connectors-manager 2022-07-14 17:34:17 -03:00
Victor Radael d01b0357ed FIX: Remove logs and add Authenticated 2022-07-14 17:31:12 -03:00
arthur simas a206aad8db Merge pull request #120 from dadosfera/feature/terms-of-use
FEAT: terms of use
2022-07-14 14:50:01 -03:00
Gabriel Rosa de57bfed1e FEAT: new roles routes working 2022-07-14 12:53:00 -03:00
Gabriel Rosa 14cfc46492 FEAT: Roles WIP 2022-07-14 10:23:04 -03:00
Victor Radael 9d10e58b1b FIX: SSL 2022-07-13 14:14:48 -03:00
Victor Radael 6dec2c1e56 FIX: Header Dadosfera-Lang 2022-07-13 14:11:35 -03:00
Victor Radael 23a65d18a6 Search Adapt fields 2022-07-12 17:04:25 -03:00
Gabriel Rosa d5cd562860 FEAT: get permissions working 2022-07-12 15:43:21 -03:00
Gabriel Rosa f5064420a7 FIX: missing import 2022-07-12 14:48:01 -03:00
Gabriel Rosa bb5befb2f6 FEAT: new route for updating Roles and changed hierarchy to string 2022-07-12 14:27:54 -03:00
Gabriel Rosa 398360fadb FEAT: new routes do add or remove permisisons from role 2022-07-12 12:11:03 -03:00
Victor Radael 13072978f5 FEAT: Accept-Language 2022-07-12 09:21:45 -03:00
Gabriel Rosa b9612609d9 FIX: add description on create role documentation 2022-07-11 16:58:40 -03:00
Victor Radael e7b37629a1 Insomnia File 2022-07-11 14:08:25 -03:00
Victor Radael 5b6994b247 Merge remote-tracking branch 'origin/main' into ft/connectors-manager 2022-07-11 12:26:02 -03:00
Gabriel Rosa 039f837914 FIX: tests fixing 2022-07-11 11:07:08 -03:00
Gabriel Rosa 83dcc7942c FIX: better code organization 2022-07-11 10:42:01 -03:00
Gabriel Rosa 5c079bb4f5 FIX: permissions contract 2022-07-11 10:09:06 -03:00
Arthur Simas f383cf49d5 FIX: added get token method 2022-07-11 02:13:53 -03:00
Arthur Simas db78ee3d8b FEAT: terms of use 2022-07-11 02:13:33 -03:00
Gabriel Rosa 730fab8ed1 FEAT: internationalization of permissions 2022-07-08 19:03:32 -03:00
Victor Radael 04e1e1ff2b New features to Connector's Search 2022-07-08 18:33:58 -03:00
Victor Radael d309c20cbf New features to Connector's Search 2022-07-08 18:32:06 -03:00
Gabriel Rosa 4cff593695 FEAT: Roles and permissions modules 2022-07-08 17:28:23 -03:00
Victor Radael 03d42df3fd Att Modules 2022-07-08 13:06:26 -03:00
Gabriel Rosa ea04f59e04 FIX: Fixed create user req 2022-07-08 12:28:40 -03:00
Gabriel Rosa d1961a9710 FEAT: User Management WIP 2022-07-08 12:04:56 -03:00
Gabriel Rosa 358dd6a15a merge main 2022-07-08 10:08:22 -03:00
arthur simas cc3b34d87e Merge pull request #119 from dadosfera/fix/error-handling
FIX: added exception filter to catch unhandled errors
2022-07-07 20:28:44 -03:00
Arthur Simas 88bc7fe625 FIX: added exception filter to catch unhandled errors 2022-07-07 20:24:27 -03:00
Gabriel Amorim 137f129e13 Merge pull request #118 from dadosfera/conditional-apm-import
FIX: Conditional apm import
2022-07-07 20:22:55 -03:00
Gabriel Rosa f49c5ac1c1 FIX: conditionally importing apm 2022-07-07 20:04:42 -03:00
Gabriel Rosa d288a4f357 FIX: conditionally importing apm 2022-07-07 18:18:53 -03:00
Gabriel Rosa 9f009309cb FIX: conditionally importing apm 2022-07-07 17:51:28 -03:00
Victor Radael 395db05b05 Merge main 2022-07-07 17:17:57 -03:00
Victor Radael 2d69e6db70 Merge main 2022-07-07 17:13:30 -03:00
Victor Radael 9514134db1 Merge main 2022-07-07 16:58:33 -03:00
Victor Radael 0c69cebca1 Merge main 2022-07-07 16:55:52 -03:00
Victor Radael 9791b8bdb0 Merge main 2022-07-07 16:55:04 -03:00
Victor Radael 6803ab3420 Merge main 2022-07-07 16:36:28 -03:00
Victor Radael 6aa7be98d2 Merge main 2022-07-07 16:32:17 -03:00
Gabriel Rosa f5665557c9 FIX: better parameter naming 2022-07-07 16:17:45 -03:00
arthur simas 366987399e Merge pull request #117 from dadosfera/refactor/protospack-v2
FEAT: protospack-v2 updated
2022-07-07 16:05:00 -03:00
Gabriel Rosa 4356adea65 FEAT: User module init 2022-07-07 15:57:40 -03:00
Arthur Simas 4c49dbc0ea FEAT: protospack-v2 updated 2022-07-07 15:00:59 -03:00
arthur simas 59ffb24985 Merge pull request #116 from dadosfera/refactor/auth-module
FEAT: auth module refactored
2022-07-06 17:47:54 -03:00
Arthur Simas 1cd191e970 FEAT: auth module refactored 2022-07-06 17:20:00 -03:00
arthur simas 7e92ca2c36 Merge pull request #113 from dadosfera/refactor/auth
auth unit tests (+ readme updated)
2022-07-06 14:10:13 -03:00
Arthur Simas 12af510d36 DOCS: readme improved 2022-07-06 11:55:19 -03:00
Arthur Simas 61871958aa DOCS: readme improved 2022-07-06 11:55:19 -03:00
Arthur Simas 64d84f9148 FEAT: refactor authentication and user decorators + unit tests 2022-07-06 11:55:19 -03:00
Arthur Simas 2c099ba491 CI: removing npm token on build 2022-07-06 11:55:19 -03:00
Arthur Simas 65b2578f15 FIX: removing useless tests 2022-07-06 11:55:19 -03:00
Gabriel Amorim 68e5534edb Merge pull request #115 from dadosfera/fix/recreate-input
FIX: recreating input getting id the right way
2022-07-06 11:54:42 -03:00
Gabriel Rosa 51c7bdfebf FIX: recreating input getting id the right way 2022-07-06 11:53:22 -03:00
Rodrigo Zamboni 9f6d2c7c9c Merge pull request #114 from dadosfera/feat/logs-lib
HOTFIX - FIX: delete info field before using it
2022-07-06 11:11:20 -03:00
rodrigo.zamboni 02955b99fe FIX: delete info field before using it 2022-07-06 11:06:52 -03:00
Rodrigo Zamboni 93505d3268 Merge pull request #112 from dadosfera/feat/logs-lib
Add Logs Lib and APM to prd
2022-07-06 09:55:48 -03:00
rodrigo.zamboni e5bf5b23e7 FIX: Kibana Transactions working 2022-07-05 17:00:12 -03:00
rodrigo.zamboni 0d20b4dcab UPDATE: Prepared Maestro for APM 2022-07-01 17:53:15 -03:00
rodrigo.zamboni bb5cf0494d FIX: Remove health check logs message 2022-07-01 17:02:52 -03:00
rodrigo.zamboni c058e54c90 FIX: Remove health check logs message 2022-07-01 16:51:21 -03:00
rodrigo.zamboni e82166f2c7 FIX: fix Capital letter service name 2022-07-01 16:30:41 -03:00
Victor Radael 8d23eaeea2 Code artifact and Get Connectors Tags 2022-06-30 16:55:46 -03:00
rodrigo.zamboni f8b495036d Merge branch 'passport/facebook' into feat/logs-lib 2022-06-30 14:13:23 -03:00
Gabriel Rosa b03b8f3b08 FEAT: passport facebook 2022-06-30 12:35:47 -03:00
Gabriel Amorim aaf99b8c2e FEAT: passport salesforce and improvements connector manager
Merge pull request #111 from dadosfera/feature/passport-salesforce
2022-06-30 10:28:58 -03:00
rodrigo.zamboni 426ff2141f FIX: Changed registry after adding protospack to codeartifact 2022-06-29 17:59:53 -03:00
Gabriel Rosa 8060cd769f merge connector-manager 2022-06-29 17:27:21 -03:00
rodrigo.zamboni ac021ea7e1 FIX: dockerfile 2022-06-29 17:25:53 -03:00
Gabriel Rosa 89eed11561 FIX: get salesforce access token and refresh token 2022-06-29 17:25:29 -03:00
rodrigo.zamboni abe78bff9b FIX: dockerfile 2022-06-29 16:46:28 -03:00
rodrigo.zamboni 3a26c5bebf FIX: dockerfile 2022-06-29 15:45:04 -03:00
rodrigo.zamboni 2af0b3af52 FIX: dockerfile 2022-06-29 15:39:36 -03:00
rodrigo.zamboni c28bc74b3f FIX: fix commands 2022-06-29 15:23:08 -03:00
rodrigo.zamboni 7eae88b5cd FIX: Added aws cli installation on dockerfile 2022-06-29 15:15:13 -03:00
rodrigo.zamboni 1e66065a21 FIC: add codeartifact login on dockerfile 2022-06-29 15:06:01 -03:00
rodrigo.zamboni 1e407f081e added package.lock 2022-06-29 14:49:49 -03:00
rodrigo.zamboni 31c507d667 FIX: Merged with radaels branch 2022-06-29 14:27:17 -03:00
rodrigo.zamboni a39622bfa1 UPDATE: Added log-lib on more routes and services 2022-06-29 14:12:13 -03:00
Victor Radael 0895922b1c Merge main 2022-06-29 14:12:01 -03:00
Victor Radael 1cc094d40b Merge main 2022-06-29 14:09:33 -03:00
Victor Radael d13aa2b72a Swagger 2022-06-29 13:59:46 -03:00
Victor Radael a98ec4bf32 Merge branch 'merge/auth+connectors-manager' of github.com:dadosfera/maestro into merge/auth+connectors-manager 2022-06-29 13:59:15 -03:00
Victor Radael d5b1cb51b4 Search Adapt fields 2022-06-29 13:58:32 -03:00
Arthur Simas 5d3510639e FEAT: salesforce passport 2022-06-29 12:39:54 -03:00
rodrigo.zamboni bc0d456c2a merge with main 2022-06-29 12:28:00 -03:00
rodrigo.zamboni b658eacce6 pull from main 2022-06-29 10:56:43 -03:00
rodrigo.zamboni 9c124be191 UPGRADE: Added LOgs lib 2022-06-29 10:50:03 -03:00
Gabriel Amorim dd4a1ecd46 FEAT: passport+connector manager
Merge pull request #110 from dadosfera/feat/passport+connector-manager
2022-06-28 19:09:40 -03:00
Gabriel Rosa 6eaaa746d0 FIX: merge conflict fixed 2022-06-28 18:40:52 -03:00
Gabriel Rosa df79fbdab0 Merge branch 'merge/auth+connectors-manager' of github.com:dadosfera/maestro into merge/auth+connectors-manager 2022-06-28 18:37:44 -03:00
Gabriel Rosa 0f34152b02 Merge branch 'feat/passport+connector-manager' of github.com:dadosfera/maestro into feat/passport+connector-manager 2022-06-28 18:36:49 -03:00
Gabriel Rosa 08d576432e FIX: new permissions 2022-06-28 18:36:41 -03:00
arthur simas ce0362217c Merge pull request #109 from dadosfera/feat/new-permissions
FEAT: new orchest permissions
2022-06-28 18:31:03 -03:00
Arthur Simas 3399f125e5 Merge branch 'feat/new-permissions' into feat/passport+connector-manager 2022-06-28 18:02:53 -03:00
Arthur Simas 9e0a58b4b7 FEAT: new orchest permissions 2022-06-28 18:02:01 -03:00
Gabriel Rosa 49a9c93d6f merge main 2022-06-28 15:24:21 -03:00
Gabriel Rosa ddb81d3082 REFACTOR: removed unused imports 2022-06-28 12:31:56 -03:00
Gabriel Rosa 9d9d903335 FIX: Better error handling on pipeline create 2022-06-28 12:22:00 -03:00
arthur simas 3ddbfcebeb FEAT: new orchest permission + added missing field user.createdAt on signin payload (#108) 2022-06-28 11:52:53 -03:00
Gabriel Rosa e1d0aa00f5 swagfer changed 2022-06-28 10:24:54 -03:00
Gabriel Rosa 4209b5b23e merge with connector manager 2022-06-28 09:26:45 -03:00
Arthur Simas d6607b1d9f FEAT: connector manager 2022-06-27 19:22:36 -03:00
Arthur Simas 0eef0713df FIX: logging errors only if auth is required 2022-06-27 18:32:08 -03:00
Arthur Simas 17d629b430 FIX: suppressing health check not authenticated log 2022-06-27 18:32:08 -03:00
Arthur Simas 5362a99b24 CHORE: added protospack@v2 correct version 2022-06-27 18:32:08 -03:00
Arthur Simas 1566efc490 FEAT: squash - auth guard
nestjs access controll mock

swagger implementation

middleware changes and lib

permissions file back to maestro

FEAT: sending permissions to duc on startup


FEAT: new permissions.enum format


FEAT: permissions injection (maestro -> duc)


FIX: changing method invocation order


FEAT: refactored permission enum + duc permission injector + grpcHandler


FEAT: auth guards


FEAT: seq ids updated
2022-06-27 18:32:08 -03:00
Gabriel Rosa 91aefe7d61 FEAT: passport mailchimp 2022-06-27 17:50:45 -03:00
Gabriel Rosa b584c80e24 FIX: google-oauth redirect_uri fixed 2022-06-27 16:13:58 -03:00
Gabriel Rosa 74a6eb8132 FIX: google redirect uri fixed 2022-06-27 15:51:24 -03:00
Gabriel Rosa 7012a6e2ac FIX: google strategy different callbacks for google 2022-06-27 14:13:18 -03:00
Gabriel Rosa 2f5d940b46 FIX: google strategy custom scopes 2022-06-27 14:02:51 -03:00
Gabriel Rosa 06a0ed3299 merge new auth with passport oAuth 2022-06-27 11:16:16 -03:00
Gabriel Rosa f08e9dc1a8 FEAT: mailchimp config 2022-06-27 10:03:49 -03:00
Arthur Simas a0c19b587c FIX: logging errors only if auth is required 2022-06-25 20:07:34 -03:00
Arthur Simas 3fe91ad4e8 FIX: logging errors only if auth is required 2022-06-25 20:07:16 -03:00
Gabriel Rosa 07f2d56b9b FEAT: Mailchimp strategy 2022-06-24 16:20:19 -03:00
Arthur Simas 1668d55679 MERGE: auth + connector manager 2022-06-24 14:46:37 -03:00
Gabriel Rosa 498ea61c06 google oauth working 2022-06-24 13:03:16 -03:00
Gabriel Rosa 07f123e528 New strategies 2022-06-24 11:08:38 -03:00
Gabriel Rosa e439823319 update libs 2022-06-24 10:28:05 -03:00
Arthur Simas 55fb1a8b34 FIX: suppressing health check not authenticated log 2022-06-23 23:38:24 -03:00
Arthur Simas 8ce7a692b6 CHORE: added protospack@v2 correct version 2022-06-23 22:03:04 -03:00
Arthur Simas c7997ed8aa FEAT: squash - auth guard
nestjs access controll mock

swagger implementation

middleware changes and lib

permissions file back to maestro

FEAT: sending permissions to duc on startup


FEAT: new permissions.enum format


FEAT: permissions injection (maestro -> duc)


FIX: changing method invocation order


FEAT: refactored permission enum + duc permission injector + grpcHandler


FEAT: auth guards


FEAT: seq ids updated
2022-06-23 22:02:49 -03:00
Gabriel Amorim 89b571d7f8 FIX: Use passport for oauth
Merge pull request #105 from dadosfera/feature/passport-oauth
2022-06-23 18:44:25 -03:00
Gabriel Rosa 7776d61f82 fix frontend redirect url 2022-06-23 17:02:33 -03:00
Victor Radael 0e2f935807 Add insomnia file att 2022-06-23 16:59:08 -03:00
Gabriel Rosa 9bf7cc660e tiny code organization improvements 2022-06-23 16:31:48 -03:00
Gabriel Rosa d6552678a8 get credentials from AWS secrets manager 2022-06-23 16:26:49 -03:00
Victor Radael 9e25a1ffa7 Add change body to query params 2022-06-23 16:09:45 -03:00
Victor Radael 44ceee0671 Add change body to query params 2022-06-23 16:08:17 -03:00
Gabriel Rosa a42ec164c3 Merge branch 'fix/oauth-generic' into feature/passport-oauth 2022-06-23 11:43:43 -03:00
Gabriel Rosa 6abc2595cb get oauthSecrets from secrets manager 2022-06-23 11:34:12 -03:00
Gabriel Rosa b850596d64 get oauth secrets from secrets manager 2022-06-22 17:51:30 -03:00
Gabriel Rosa da3b7f3096 hubspot oauth 2022-06-22 15:24:36 -03:00
rodrigo.zamboni 3c660738f4 installed logs lib 2022-06-22 10:23:54 -03:00
Gabriel Rosa ff2b8fae6f generic axios 2022-06-22 09:49:23 -03:00
Victor Radael 44337dc4ed Add Insomnia Folder 2022-06-21 18:05:56 -03:00
Victor Radael 80e7aa7b9c Fix and add upload file 2022-06-21 18:01:21 -03:00
Victor Radael 6f4bb9ce6d Fix and add upload file 2022-06-21 18:01:06 -03:00
Gabriel Amorim 4df23c917a FEAT: new connectors
Merge pull request #103 from dadosfera/feature/hubspot-connector
2022-06-21 11:44:08 -03:00
Victor Radael 92afbb5e55 Find All 2022-06-21 10:23:14 -03:00
Gabriel Rosa 6c3eef7181 update protospack and deploy file 2022-06-21 10:13:20 -03:00
Gabriel Rosa 072a9fee8e hubspot guard 2022-06-20 17:09:28 -03:00
Victor Radael 953c0ac41e Register Connector 2022-06-20 15:37:45 -03:00
Gabriel Rosa 8c91551cdc incomplete oauth 2022-06-20 15:07:23 -03:00
Gabriel Rosa 668c5fe6a5 WIP: Documentation of input API 2022-06-20 11:36:26 -03:00
Gabriel Rosa d057495b2e FIX: redirect_uri.redirect_uri fixed 2022-06-15 15:19:45 -03:00
Gabriel Rosa 2f1c71eefe update protospack and better adjustInputPayload 2022-06-15 11:14:22 -03:00
Gabriel Rosa 3bb014e714 get redirect_uri from get_tokens_url_params 2022-06-15 10:08:03 -03:00
Gabriel Rosa 2ff2aa049f update protospack 2022-06-15 08:52:30 -03:00
Gabriel Rosa f6c7420f59 finding inputs using generic strategy 2022-06-15 08:44:21 -03:00
Gabriel Rosa abbe88f45b removed unused statement 2022-06-14 14:02:06 -03:00
Gabriel Rosa 383741f48c get entities 2022-06-14 14:01:40 -03:00
Gabriel Rosa ff663e7166 update protospack 2022-06-14 11:35:09 -03:00
Gabriel Rosa 305daeeeab update protospack 2022-06-13 16:50:38 -03:00
Gabriel Rosa 852ddbf88b transforming to camelCase before sending to in-factory 2022-06-13 15:55:02 -03:00
Gabriel Rosa 4dfe6c3870 fallback for env vars 2022-06-13 13:12:01 -03:00
Gabriel Rosa 2710159a73 minor improvements 2022-06-13 11:17:41 -03:00
Gabriel Rosa 1ab5d49908 removing info that won't be saved on database 2022-06-13 10:49:27 -03:00
Gabriel Amorim 43372b5a0f FIX: Better error handling on input creation
Merge pull request #102 from dadosfera/fix/error-handling
2022-06-10 16:05:29 -03:00
Gabriel Rosa 0ac2dd2de7 Better error handling on input creation 2022-06-10 15:57:19 -03:00
Gabriel Rosa 0342386a76 oauth almost complete 2022-06-09 17:14:11 -03:00
Gabriel Rosa 28e55f8fd9 update node version 2022-06-08 11:12:51 -03:00
Gabriel Rosa 4e2e296442 Fixed cron on create S3 2022-06-08 10:31:59 -03:00
Arthur Simas d89573bc79 FIX: creating swap file 2022-06-06 17:35:26 -03:00
Gabriel Amorim 03ed4d88a0 FIX: validate cron
Merge pull request #101 from dadosfera/fix/validate-cron
2022-05-31 16:33:05 -03:00
Gabriel Rosa 329bbf2e4d Better cron validation 2022-05-31 16:10:21 -03:00
Gabriel Rosa ae069a0e00 removed unused imports 2022-05-31 12:04:32 -03:00
Gabriel Rosa d3bcbdc43c validating cron when creating or updating input 2022-05-31 11:34:28 -03:00
Arthur Simas 4e2e6787a4 Merge branch 'feat/change-password' into main 2022-05-30 18:27:02 -03:00
Arthur Simas 50da33b766 FEAT: added verifyPasswordResetCode endpoint 2022-05-30 17:17:50 -03:00
Rodrigo Zamboni bb72cfce45 Merge pull request #100 from dadosfera/tests
Tests
2022-05-30 10:57:23 -03:00
rodrigo.zamboni 03d17758ae FIX: commented broken test 2022-05-30 10:11:04 -03:00
rodrigo.zamboni b7e76a6d3f FIX: fix busboy npm critical error 2022-05-30 09:58:07 -03:00
Arthur Simas c920eb27b9 FEAT: reset password 2022-05-27 18:57:05 -03:00
Arthur Simas c25d2427c2 FEAT: change password 2022-05-27 18:55:43 -03:00
Arthur Simas 0f13d0c590 FIX(refresh token): removed username param 2022-05-27 18:55:35 -03:00
Arthur Simas 7781298e6e Merge branch 'hotfix/refresh-token-contract' into fix/create-partial-input 2022-05-27 15:25:00 -03:00
Arthur Simas 2c00d3249a FIX(refresh token): removed username param 2022-05-27 15:12:22 -03:00
Arthur Simas cd7c7d538f CI: deploy workflow updated 2022-05-26 15:38:36 -03:00
rodrigo.zamboni e496d42776 ódio 2022-05-26 10:13:42 -03:00
Gabriel Rosa a07247d538 update protospack version 2022-05-25 15:49:45 -03:00
Arthur Simas 153ba1a98b CI: printing env 2022-05-25 12:24:50 -03:00
Gabriel Rosa 46519323f8 Ci: imporved github flow 2022-05-25 11:56:26 -03:00
Arthur Simas a51d52d68e CI: fixed versioning 2022-05-25 11:53:44 -03:00
Gabriel Rosa 555ff2eda9 CI - github flow melhorado 2022-05-25 11:52:54 -03:00
Arthur Simas 5d1a54a720 CI: . 2022-05-25 11:45:37 -03:00
Arthur Simas 8399a0e4c1 CI: . 2022-05-25 11:40:38 -03:00
Arthur Simas cf54cb0bc2 CI: not considering semver during manual deploy 2022-05-25 11:37:14 -03:00
rodrigo.zamboni 4eba099390 Merge branch 'main' of https://github.com/dadosfera/maestro into tests 2022-05-25 11:14:45 -03:00
rodrigo.zamboni 08fa269ce0 initial tests 2022-05-25 11:14:40 -03:00
Gabriel Rosa e52af6844b FIX: create partial input 2022-05-25 11:09:24 -03:00
Arthur Simas 2344e4dba3 CI: using github-flow deployment workflow 2022-05-24 18:57:41 -03:00
rodrigo.zamboni d686753f25 DOCS: Added proper readme 2022-05-23 17:19:36 -03:00
Rodrigo Zamboni c1498f8f1b Merge pull request #98 from dadosfera/beta
Beta -> main
2022-05-20 17:27:29 -03:00
Gabriel Rosa 0b3cb581c4 merge alpha -> beta 2022-05-20 15:19:43 -03:00
Gabriel Rosa 97a6e05812 FIX: returning input when creating jdbc 2022-05-20 15:19:00 -03:00
Gabriel Rosa 356d438404 merge alpha 2022-05-20 14:07:02 -03:00
Gabriel Rosa cf870cff14 FIX: formating payload 2022-05-20 14:06:13 -03:00
Gabriel Rosa 26c6a289eb merge from beta 2022-05-20 11:51:48 -03:00
Gabriel Rosa 8d27e42fcd FIX: protospack version 2022-05-20 11:51:08 -03:00
Gabriel Rosa c44e557b11 merge beta 2022-05-20 10:08:20 -03:00
Gabriel Rosa 25d06ab4c5 FIX: send input to createS3 2022-05-20 10:07:38 -03:00
Gabriel Amorim 31bb795d23 Merge pull request #97 from dadosfera/alpha
Alpha -> Beta
2022-05-20 09:12:34 -03:00
Gabriel Rosa a2e25eb737 FIX: added ssl back 2022-05-20 08:51:25 -03:00
Gabriel Rosa 6be53bebaf FIX: adjusting pipeline payload fixed 2022-05-20 08:50:08 -03:00
Rodrigo Zamboni 741f009723 Merge pull request #96 from dadosfera/alpha
Alpha
2022-05-19 15:20:06 -03:00
Rodrigo Zamboni 45477b5750 Merge branch 'beta' into alpha 2022-05-19 15:09:15 -03:00
rodrigo.zamboni 36de53dfd3 FIX: lib version 2022-05-19 14:48:22 -03:00
rodrigo.zamboni cd8fe2fbf6 CI: Deploy 2022-05-19 14:46:17 -03:00
rodrigo.zamboni c4a2bee69c CI: DEploy 2022-05-19 14:18:08 -03:00
rodrigo.zamboni e48d11153d FIX: Added S3 or JDBC input selection for pipelines 2022-05-19 14:13:06 -03:00
Gabriel Amorim c7e4de7d65 Merge pull request #95 from dadosfera/feat/mfa
FEAT: MFA finished + better error handling
2022-05-19 13:59:46 -03:00
Arthur Simas b2e7870cc5 CHORE: protospack version bump 2022-05-19 12:56:32 -03:00
Arthur Simas 1e239026a3 FEAT: MFA finished + better error handling 2022-05-19 12:02:08 -03:00
Rodrigo Zamboni bc108d43e3 Merge pull request #94 from dadosfera/beta
Beta
2022-05-18 16:54:04 -03:00
Arthur Simas 1c5b929c1b REFACTOR: nimbus URL 2022-05-18 13:57:19 -03:00
rodrigo.zamboni 3e21c308f6 UPDATE: Added S3 Input Creation 2022-05-17 15:58:18 -03:00
Gabriel Rosa f4a5f91cbb FIX: remove backmerge 2022-05-16 17:03:12 -03:00
Gabriel Rosa c8aeb7f3db FIX: remove backmerge 2022-05-16 17:02:46 -03:00
Victor Radael b9453b11cf REFACTOR: Init Catalog Class 2022-05-16 14:02:18 -03:00
Victor Radael a69752332e FIX: Nimbus URL Strategy 2022-05-16 13:42:06 -03:00
Victor Radael 09c7cc79ee FIX: Nimbus URL Strategy 2022-05-16 12:04:13 -03:00
Arthur Simas 65f6f03e55 CHORE: pinning protospack version to 1.7.4 2022-05-13 11:15:05 -03:00
Arthur Simas a54300f600 CHORE: pinning protospack version to 1.7.2 2022-05-13 11:00:25 -03:00
Arthur Simas 5ef85f79d3 Merge branch 'alpha' into beta 2022-05-13 10:49:27 -03:00
rodrigo.zamboni a8a429977f UPDATE: Implementing MFA routes on maestro 2022-05-11 18:02:01 -03:00
Gabriel Amorim 692a9de0af FEAT: add refresh token - Merge pull request #90 from dadosfera/feat/refreshToken
FEATURE: add refresh token
2022-05-11 10:37:36 -03:00
rodrigo.zamboni 7dc669ca44 FIX: fix auth trim 2022-05-10 15:53:47 -03:00
rodrigo.zamboni e897da1ef0 FIX: Fix login middleware 2022-05-10 15:32:34 -03:00
Arthur Simas b366e786ba STYLE: styling conforming to ESLint 2022-05-10 10:23:55 -03:00
Arthur Simas 24d56c7a58 FEAT: add refresh token 2022-05-09 11:47:25 -03:00
rodrigo.zamboni d7f205901b UPDATE: Changing transformation factory microservice client url to inputs factory 2022-05-05 16:27:20 -03:00
rodrigo.zamboni 7212c5249c FIX: testing timeout on testconnection input client 2022-05-04 16:44:00 -03:00
rodrigo.zamboni 827c7b1bf1 FIX: Summary-rating returning default 0 values 2022-05-04 12:00:01 -03:00
rodrigo.zamboni 8d89b747b4 FIX: Fix delete routes method 2022-05-04 00:24:44 -03:00
rodrigo.zamboni 4f18328603 FIX: update protospack lib version 2022-05-03 17:59:51 -03:00
rodrigo.zamboni a68d312447 FIX: updated protospack lib 2022-05-03 17:07:57 -03:00
rodrigo.zamboni 7f677f0376 FIX: Fix mistaken routes method 2022-05-03 16:54:03 -03:00
rodrigo.zamboni bb2c0ab23e Merge branch 'main' of https://github.com/dadosfera/maestro into main 2022-05-03 15:13:10 -03:00
rodrigo.zamboni feec48114c FIX: fixed missing nimbus catalog routes 2022-05-03 15:12:18 -03:00
Rodrigo Zamboni 7bee8d83a9 Merge pull request #89 from dadosfera/beta
Beta
2022-05-03 09:14:43 -03:00
Rodrigo Zamboni 84b2dbd178 Merge pull request #88 from dadosfera/alpha
CI: Deploy on dev env
2022-05-02 17:53:46 -03:00
rodrigo.zamboni 36706ff3dd CI: Deploy on dev env 2022-05-02 17:33:03 -03:00
Victor Radael 0164e6978a FIX: Metabase URL 2022-04-29 20:53:17 -03:00
rodrigo.zamboni 7d371926f5 FIX: fix nimbus prd url 2022-04-29 17:20:23 -03:00
rodrigo.zamboni 75e9e97f04 FIX: fix nimbus prd url 2022-04-29 17:18:21 -03:00
rodrigo.zamboni b5f58b99fd FIX: Changing nimbus prd url 2022-04-29 16:33:42 -03:00
rodrigo.zamboni 503293439f FIX: updating protospack lib version 2022-04-29 16:14:14 -03:00
rodrigo.zamboni eed57ff039 FIX: fix pipeline status sort 2022-04-29 12:42:49 -03:00
rodrigo.zamboni ef2a24a0c3 FIX: Fix runPipeline fail case return 2022-04-28 17:39:05 -03:00
Rodrigo Zamboni 8a31e11445 Merge pull request #87 from dadosfera/beta
Beta
2022-04-28 17:23:54 -03:00
Rodrigo Zamboni 2013e97e20 Merge pull request #86 from dadosfera/alpha
Alpha
2022-04-28 17:12:53 -03:00
rodrigo.zamboni dd271205ba Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-04-28 17:00:44 -03:00
rodrigo.zamboni a1f9f14897 FIX: Fixing more catalog routes params 2022-04-28 17:00:08 -03:00
Gabriel Amorim b7efa2f58d Merge pull request #85 from dadosfera/beta
Beta -> Main
2022-04-28 16:04:10 -03:00
Gabriel Amorim 395e36581d Merge pull request #84 from dadosfera/alpha
Alpha -> Beta
2022-04-28 16:03:53 -03:00
rodrigo.zamboni 97a54f9267 Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-04-28 15:56:30 -03:00
rodrigo.zamboni 00b0e00bd0 FIX: fix iframe_url and pipeline status 2022-04-28 15:55:48 -03:00
Rodrigo Zamboni 9188536161 Merge pull request #83 from dadosfera/beta
Beta
2022-04-28 14:53:16 -03:00
Rodrigo Zamboni 1ae8151184 Merge pull request #82 from dadosfera/alpha
Alpha -> Beta
2022-04-28 14:40:14 -03:00
rodrigo.zamboni 3d77980dd2 CI: deploy 2022-04-28 14:24:28 -03:00
Rodrigo Zamboni 849b979e63 Merge pull request #81 from dadosfera/feat/CatalogFixes
Feat/catalog fixes
2022-04-28 14:19:31 -03:00
rodrigo.zamboni 28580782b1 FIX: catalog routes fix 2022-04-28 14:14:03 -03:00
Rodrigo Zamboni fc784e2d6d Merge pull request #79 from dadosfera/alpha
Alpha
2022-04-28 13:22:14 -03:00
rodrigo.zamboni c997e91912 FIX: fix table-metadata find one route 2022-04-28 12:59:35 -03:00
rodrigo.zamboni 22f493d48a FIX: fixing missing id on delete routes 2022-04-28 12:12:37 -03:00
rodrigo.zamboni 035213824b FIX: fixing missing id on delete routes 2022-04-28 12:00:19 -03:00
rodrigo.zamboni 1ffa9ade51 FIX: CI 2022-04-28 10:53:17 -03:00
Rodrigo Zamboni c91aa57344 Merge pull request #78 from dadosfera/beta
Beta
2022-04-27 17:36:54 -03:00
Rodrigo Zamboni 077a95cdbd Merge pull request #77 from dadosfera/alpha
Alpha
2022-04-27 17:16:09 -03:00
Rodrigo Zamboni f39108772b Merge pull request #76 from dadosfera/NewInput
FIX: fix pipeline activate route method
2022-04-27 16:57:56 -03:00
rodrigo.zamboni cdd8beee8f FIX: fix pipeline activate route method 2022-04-27 16:56:03 -03:00
rodrigo.zamboni 14331ed229 FIX: fix pipeline activate route method 2022-04-27 16:54:48 -03:00
Victor Radael 5f3db0cb43 Merge pull request #75 from dadosfera/beta
Beta -> Main
2022-04-27 11:42:18 -03:00
Rodrigo Zamboni 28f304c47c Merge pull request #74 from dadosfera/alpha
Alpha
2022-04-26 17:41:57 -03:00
Rodrigo Zamboni 0c1b62b2aa Merge pull request #73 from dadosfera/NewInput
FIX: Fix get-columns route method
2022-04-26 17:25:11 -03:00
rodrigo.zamboni 13f7e9d812 FIX: Fix get-columns route method 2022-04-26 17:19:18 -03:00
Gabriel Amorim 841dcfef06 Merge pull request #72 from dadosfera/alpha
Alpha
2022-04-26 16:30:31 -03:00
Gabriel Amorim f41ddc1483 Merge pull request #71 from dadosfera/NewInput
UPDATE: Implemented new Input format
2022-04-26 16:19:19 -03:00
rodrigo.zamboni 0a2c7c0a38 UPDATE: Implemented new Input format 2022-04-26 16:15:54 -03:00
Rodrigo Zamboni 87f4c17ca2 Merge pull request #70 from dadosfera/beta
Beta
2022-04-26 13:50:00 -03:00
Rodrigo Zamboni a89625c1e6 Merge pull request #69 from dadosfera/alpha
Alpha
2022-04-26 11:16:06 -03:00
rodrigo.zamboni 44211d5658 FIX: CI 2022-04-26 11:01:20 -03:00
rodrigo.zamboni d4e11b1935 FIX: CI 2022-04-26 10:48:57 -03:00
rodrigo.zamboni 5c5e940cf7 FIX: CI 2022-04-26 10:40:11 -03:00
Rodrigo Zamboni 2b162bd0a7 Merge pull request #68 from dadosfera/beta
Beta
2022-04-25 17:06:59 -03:00
Victor Radael 829f7611e2 Merge pull request #67 from dadosfera/alpha
Alpha
2022-04-25 16:49:04 -03:00
Rodrigo Zamboni b986cf3281 Merge pull request #66 from dadosfera/addHelmet
FIX: added helmet package to protect headers security
2022-04-25 16:35:59 -03:00
rodrigo.zamboni 75c7602e13 FIX: added helmet package to protect headers security 2022-04-25 16:25:42 -03:00
Victor Radael 710105a4a8 Merge pull request #65 from dadosfera/addHelmet
added helmet package to protect headers security
2022-04-25 16:16:07 -03:00
rodrigo.zamboni 30b048f776 added helmet package to protect headers security 2022-04-25 16:11:50 -03:00
Victor Radael 845b95b112 Merge pull request #64 from dadosfera/beta
Beta ->  main
2022-04-21 17:15:16 -03:00
Victor Radael 6d6b024884 Merge pull request #63 from dadosfera/alpha
Alpha -> Beta
2022-04-21 16:18:51 -03:00
Rodrigo Zamboni a4282a0dc5 Merge pull request #62 from dadosfera/feat/nimbusImplementation
Feat/nimbus implementation
2022-04-21 16:05:37 -03:00
rodrigo.zamboni 4672ecb6b3 FIX: Fixed maestro after protospack lib att 2022-04-21 16:02:11 -03:00
rodrigo.zamboni 6160107d75 FIX: Downgrade protospack lib 2022-04-20 16:22:26 -03:00
Rodrigo Zamboni 589a1f9d51 Merge pull request #60 from dadosfera/feat/nimbusImplementation
Feat/nimbus implementation
2022-04-20 16:01:32 -03:00
rodrigo.zamboni cb929d9b60 FIX: FIX 2022-04-20 15:56:50 -03:00
rodrigo.zamboni 678bebef7e FIX: fix proto packages names 2022-04-20 15:52:26 -03:00
Rodrigo Zamboni b6d7d54f9c Merge pull request #58 from dadosfera/feat/nimbusImplementation
adding get all tags function
2022-04-20 15:30:40 -03:00
rodrigo.zamboni 5df8b2834c FIX: adding get all tags function 2022-04-20 15:08:51 -03:00
rodrigo.zamboni 76d9571f02 FIX: adding get all tags function 2022-04-20 14:51:04 -03:00
rodrigo.zamboni 0c51014ec9 adding get all tags function 2022-04-20 14:40:01 -03:00
Rodrigo Zamboni 7f1d8e4b2a Merge pull request #57 from dadosfera/beta
Beta
2022-04-19 18:29:16 -03:00
Victor Radael 92b78ff19d Merge pull request #56 from dadosfera/alpha
Alpha
2022-04-19 17:06:28 -03:00
Rodrigo Zamboni 1b328b9915 Merge pull request #55 from dadosfera/NimbusImplementation
UPDATE: Added Nimbus Catalog routes
2022-04-19 16:24:34 -03:00
rodrigo.zamboni e653aeea3c UPDATE: Added Nimbus Catalog routes 2022-04-19 15:56:29 -03:00
Rodrigo Zamboni b49633cbc2 Merge pull request #54 from dadosfera/beta
Beta
2022-04-18 13:58:14 -03:00
Rodrigo Zamboni 4d956e55c6 Merge pull request #53 from dadosfera/alpha
Alpha
2022-04-18 12:18:28 -03:00
Rodrigo Zamboni 735703bf3b Merge pull request #52 from dadosfera/feat/activatePipeline
Feat/activate pipeline
2022-04-18 11:54:43 -03:00
rodrigo.zamboni 93cdc3f030 UPDATE: remove unescessary console log 2022-04-18 11:50:17 -03:00
rodrigo.zamboni 12b8b5926c UPDATE: Created Trigger Pipeline route 2022-04-18 11:44:19 -03:00
rodrigo.zamboni 1643096d47 Created Trigger Pipeline route 2022-04-18 11:43:36 -03:00
Rodrigo Zamboni 8a519ac217 Merge pull request #51 from dadosfera/beta
Beta
2022-04-14 15:33:36 -03:00
Victor Radael ffa69a0cf5 Merge pull request #50 from dadosfera/alpha
Alpha
2022-04-14 15:14:12 -03:00
Rodrigo Zamboni 7722e380e3 Merge pull request #49 from dadosfera/authFix
FIX: using Unauth nest function
2022-04-14 15:01:18 -03:00
rodrigo.zamboni ecdc7b4eb6 FIX: using Unauth nest function 2022-04-14 14:56:18 -03:00
rodrigo.zamboni 7d14080ac1 FIX: Invalid auth credentials return 2022-04-14 14:48:40 -03:00
Victor Radael 7e7deba2ab FIX: CI 2022-04-12 18:10:40 -03:00
Victor Radael e425e90eeb FIX: CI 2022-04-12 17:59:41 -03:00
Victor Radael 45f35b4e39 Merge branch 'alpha' into main 2022-04-12 17:44:29 -03:00
Victor Radael d450acd007 FIX: CI 2022-04-12 17:43:46 -03:00
Victor Radael e9c1dfc320 Merge pull request #48 from dadosfera/beta
Beta
2022-04-12 17:20:18 -03:00
Rodrigo Zamboni 67484cbef8 Merge pull request #47 from dadosfera/alpha
Alpha
2022-04-12 17:07:57 -03:00
rodrigo.zamboni 740be1d98b FIX: CI 2022-04-12 16:20:03 -03:00
rodrigo.zamboni b9989d86c1 Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-04-12 16:15:25 -03:00
rodrigo.zamboni 9b3a127f6a fixed authorization on case of missing Authorization header 2022-04-12 16:15:04 -03:00
Victor Radael 55028c4b83 Merge pull request #46 from dadosfera/alpha
Alpha
2022-04-08 13:02:06 -03:00
Victor Radael 7d5dee4b0c FIX: Deploy 2022-04-08 12:48:40 -03:00
Victor Radael 708bf2f585 FIX: Deploy 2022-04-07 18:41:39 -03:00
Rodrigo Zamboni 1d53226540 Merge pull request #45 from dadosfera/alpha
FIX: CI
2022-04-07 18:12:58 -03:00
Victor Radael 5f6ae5d512 FIX: CI 2022-04-07 17:54:14 -03:00
Rodrigo Zamboni c51092260a Merge pull request #44 from dadosfera/alpha
Alpha
2022-04-07 16:54:25 -03:00
Victor Radael 3add8b026e Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-04-07 16:44:10 -03:00
Victor Radael 42726b805b FIX: Ci 2022-04-07 16:43:59 -03:00
Victor Radael 8d8c5c000e Merge pull request #43 from dadosfera/alpha
Deploy STG
2022-04-07 16:17:27 -03:00
rodrigo.zamboni b7f608f246 FIX: TestConnection Fix 2022-04-07 11:22:57 -03:00
rodrigo.zamboni e513689f4a REFACTOR: Att Functions to work with info given by auth 2022-03-30 11:32:59 -03:00
rodrigo.zamboni 761a00c6a6 removing default true value from auth 2022-03-29 18:16:18 -03:00
rodrigo.zamboni ef9cff601f Merge branch 'alpha' of https://github.com/dadosfera/maestro into alpha 2022-03-29 17:44:10 -03:00
rodrigo.zamboni a3ba9c743f Prepared Outputs and Transformations to use info params given by auth 2022-03-29 17:43:44 -03:00
Victor Radael 9dd3ec4579 FIX: Change Header Name 2022-03-29 15:44:53 -03:00
rodrigo.zamboni 8b19da9217 fixed merge conflicts 2022-03-29 15:22:42 -03:00
rodrigo.zamboni 238de920dc Prepared Inputs for info params added by auth 2022-03-29 15:18:35 -03:00
Victor Radael fe1a426e2b FIX: Add Cors Config 2022-03-29 15:16:37 -03:00
Victor Radael 1f85d3f502 FIX: Add Cors Config 2022-03-29 15:10:29 -03:00
rodrigo.zamboni c68207728b merge fix 2022-03-29 12:06:26 -03:00
rodrigo.zamboni c431867814 preparing info param 2022-03-29 12:04:10 -03:00
Victor Radael ca7f9855b1 Add body Info 2022-03-29 11:54:14 -03:00
Victor Radael 7a95ec537f REFACTOR: Add Auth Validators 2022-03-28 13:06:03 -03:00
Victor Radael 25d99089db CI: Fix 2022-03-24 14:44:26 -03:00
Victor Radael e336f31fda CI: Fix 2022-03-24 14:33:14 -03:00
Victor Radael 2bdb06494a CI: Fix 2022-03-24 13:54:53 -03:00
rodrigo.zamboni f94416200a CI: Deploy changes 2022-03-24 13:52:43 -03:00
rodrigo.zamboni 3bbd48894b CI: Fix deploy 2022-03-24 12:08:08 -03:00
Rodrigo Zamboni 1f1660cdf2 Merge pull request #42 from dadosfera/ft-int-duc
REFACTOR: Att Functions + Alpha Clients connection
2022-03-24 11:49:34 -03:00
Rodrigo Zamboni 3a7f392e8b Merge branch 'alpha' into ft-int-duc 2022-03-24 11:48:55 -03:00
rodrigo.zamboni 04584df44d REFACTOR: Att Functions + Alpha Clients connection 2022-03-24 11:41:55 -03:00
Victor Radael 26ec0b3dfb CI: Deploy 2022-03-21 10:56:46 -03:00
Victor Radael 44d4b8e08c CI: Deploy 2022-03-21 10:39:41 -03:00
Victor Radael 63fa756c9b Merge pull request #41 from dadosfera/ft-int-duc
Ft int duc
2022-03-21 10:24:05 -03:00
Victor Radael 91236dde86 CI: Deploy 2022-03-21 10:03:54 -03:00
rodrigo.zamboni a39ea091ee prepared tests to deploy 2022-03-18 15:20:32 -03:00
Rodrigo Zamboni 8787aa05c2 Merge pull request #40 from dadosfera/ft-int-duc
Ft int duc
2022-03-18 10:17:27 -03:00
rodrigo.zamboni 1523c6b4d1 maestro connection with microservices - clients and modules 2022-03-18 10:16:27 -03:00
Victor Radael e15a680b9f FIX: Health Check 2022-03-14 10:39:51 -03:00
Victor Radael 363bc8b350 REFACTOR: Project Pattern and AuthClient (DUC) 2022-03-14 10:35:03 -03:00
Victor Radael 2f8d9b9550 Merge pull request #39 from dadosfera/att-ci
CI: Att CI
2022-02-10 14:49:40 -03:00
Victor Radael e729e1f0f4 CI: Att CI 2022-02-10 14:47:06 -03:00
Victor Radael a20ff87b27 Merge pull request #38 from dadosfera/ft-att-doc
docs: Att docs
2022-02-10 14:37:58 -03:00
Victor Radael abf5c71c08 docs: Att docs 2022-02-10 14:32:42 -03:00
Alexsander Pereira 6b991bce7d Merge pull request #37 from dadosfera/ci
CI: Adding @semantic-release/npm to modify package.json on deploy
2022-02-10 12:21:00 -03:00
alexsanderp e007c2c49c CI: Adding @semantic-release/npm to modify package.json on deploy 2022-02-10 12:14:46 -03:00
Alexsander Pereira 99b7d9a8d7 Merge pull request #36 from dadosfera/beta
Beta to Main
2022-02-10 11:46:50 -03:00
Alexsander Pereira 688fe81500 Merge pull request #35 from dadosfera/alpha
CI: Adding @saithodev/semantic-release-backmerge to merge main into a…
2022-02-10 11:29:13 -03:00
alexsanderp f785253ece CI: Adding @saithodev/semantic-release-backmerge to merge main into alpha and beta after new release published 2022-02-10 11:16:04 -03:00
Alexsander Pereira afb4d957bc Merge pull request #34 from dadosfera/beta
Beta to Main
2022-02-10 00:00:32 -03:00
Alexsander Pereira b65acd208c Merge pull request #33 from dadosfera/alpha
Alpha to Beta
2022-02-09 23:43:21 -03:00
alexsanderp 6118fa207c CI: Removing update package.json and CHANGELOG.md to resolve merge conflicts 2022-02-09 23:31:24 -03:00
semantic-release-bot 747c34f3d8 chore(release): 1.0.1-alpha.1 [skip ci]
## [1.0.1-alpha.1](https://github.com/dadosfera/maestro/compare/v1.0.0...v1.0.1-alpha.1) (2022-02-10)

### CI

* Adding @newbish/changelog in actions extra_plugins ([b86849a](https://github.com/dadosfera/maestro/commit/b86849aef4672263ea14a61a6ad490ea5acb65c0))
* Adding CHANGELOG.md in @semantic-release/git asserts ([0b1ff0f](https://github.com/dadosfera/maestro/commit/0b1ff0f1307ec7fc70dae9709c5301fab4a11e79))
* Creating CHANGELOG.md with only production versions ([4499ec9](https://github.com/dadosfera/maestro/commit/4499ec941ab86c3e5cd431e23ee809d9163f1a95))
* Update @semantic-release/commit-analyzer releaseRules ([4e92080](https://github.com/dadosfera/maestro/commit/4e92080790f0ddbebb710d8312430a04f9b8e33a))
2022-02-10 02:17:36 +00:00
Alexsander Pereira 899c8cb050 Merge pull request #32 from dadosfera/main
Merge Main to Beta
2022-02-09 23:17:06 -03:00
Alexsander Pereira d1ea8eaccc Merge pull request #31 from dadosfera/main
Merge Main to Alpha
2022-02-09 23:16:35 -03:00
Alexsander Pereira 7261080054 Merge branch 'beta' into alpha 2022-02-09 23:11:15 -03:00
semantic-release-bot dbe65f859f chore(release): 1.0.0-alpha.3 [skip ci]
# [1.0.0-alpha.3](https://github.com/dadosfera/maestro/compare/v1.0.0-alpha.2...v1.0.0-alpha.3) (2022-02-09)

### CI

* Adding @newbish/changelog in actions extra_plugins ([b86849a](https://github.com/dadosfera/maestro/commit/b86849aef4672263ea14a61a6ad490ea5acb65c0))
* Adding CHANGELOG.md in @semantic-release/git asserts ([0b1ff0f](https://github.com/dadosfera/maestro/commit/0b1ff0f1307ec7fc70dae9709c5301fab4a11e79))
* Creating CHANGELOG.md with only production versions ([4499ec9](https://github.com/dadosfera/maestro/commit/4499ec941ab86c3e5cd431e23ee809d9163f1a95))
* Update @semantic-release/commit-analyzer releaseRules ([4e92080](https://github.com/dadosfera/maestro/commit/4e92080790f0ddbebb710d8312430a04f9b8e33a))
2022-02-09 23:36:48 +00:00
Alexsander Pereira 54f15b88c9 Merge pull request #29 from dadosfera/ci_cd
CI: Adding @newbish/changelog in actions extra_plugins
2022-02-09 20:34:37 -03:00
alexsanderp b86849aef4 CI: Adding @newbish/changelog in actions extra_plugins 2022-02-09 20:34:15 -03:00
Alexsander Pereira 44fbb3ffa9 Merge pull request #28 from dadosfera/ci_cd
CI: Update @semantic-release/commit-analyzer releaseRules
2022-02-09 20:29:38 -03:00
alexsanderp 4e92080790 CI: Update @semantic-release/commit-analyzer releaseRules 2022-02-09 20:29:04 -03:00
Alexsander Pereira b8d4fb3799 Merge pull request #27 from dadosfera/ci_cd
CI: Creating CHANGELOG.md with only production versions
2022-02-09 20:23:28 -03:00
alexsanderp 0b1ff0f130 CI: Adding CHANGELOG.md in @semantic-release/git asserts 2022-02-09 20:20:27 -03:00
alexsanderp 4499ec941a CI: Creating CHANGELOG.md with only production versions 2022-02-09 20:17:38 -03:00
239 changed files with 23467 additions and 6785 deletions
+3
View File
@@ -0,0 +1,3 @@
container_commands:
01setup_swap:
command: "bash .ebextensions/setup_swap.sh"
+7
View File
@@ -0,0 +1,7 @@
Resources:
AWSEBAutoScalingLaunchConfiguration:
Properties:
MetadataOptions:
HttpEndpoint: enabled
HttpPutResponseHopLimit: 2
HttpTokens: required
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
SWAPFILE=/var/swapfile
SWAP_MEGABYTES=1024
if [ -f $SWAPFILE ]; then
echo "Swapfile $SWAPFILE found, assuming already setup"
exit;
fi
/bin/dd if=/dev/zero of=$SWAPFILE bs=1M count=$SWAP_MEGABYTES
/bin/chmod 600 $SWAPFILE
/sbin/mkswap $SWAPFILE
/sbin/swapon $SWAPFILE
+19
View File
@@ -0,0 +1,19 @@
branch-defaults:
alpha:
environment: null
global:
application_name: maestro
branch: null
default_ec2_keyname: null
default_platform: Docker running on 64bit Amazon Linux 2
default_region: us-east-1
include_git_submodules: true
instance_profile: null
platform_name: null
platform_version: null
profile: null
repository: null
sc: git
workspace_type: Application
deploy:
artifact: deploy.zip
+1
View File
@@ -6,6 +6,7 @@ module.exports = {
},
plugins: ['@typescript-eslint/eslint-plugin'],
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'plugin:prettier/recommended',
],
+136
View File
@@ -0,0 +1,136 @@
name: Deploy
on:
push:
branches:
- main
workflow_dispatch:
inputs:
environment:
description: 'Deployment environment'
required: true
type: choice
options:
- dev
- stg
- prd
jobs:
extract_environment:
runs-on: ubuntu-latest
outputs:
environment: ${{ steps.extract_environment.outputs.environment }}
env:
EVENT: ${{ github.event_name }}
DEPLOY_ENV: ${{ github.event.inputs.environment }}
steps:
- name: Extract Environment
run: |
if [ ${EVENT} == "workflow_dispatch" ]; then
echo "##[set-output name=environment;]$(echo ${DEPLOY_ENV})"
elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "##[set-output name=environment;]$(echo "prd")"
fi
id: extract_environment
semantic_release:
runs-on: ubuntu-latest
outputs:
new_release_published: ${{ steps.semantic.outputs.new_release_published }}
new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
steps:
- name: Checkout
uses: actions/checkout@v2
- if: github.event_name != 'workflow_dispatch'
name: Semantic Release
uses: cycjimmy/semantic-release-action@v2
id: semantic
with:
semantic_version: 16
extra_plugins: |
conventional-changelog-eslint
branches: |
[
'main'
]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
deploy:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, '${{ needs.extract_environment.outputs.environment }}']
steps:
- name: Printing stats
env:
EVENT: ${{ github.event_name }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ENV: ${{ needs.extract_environment.outputs.environment }}
run: echo ${GITHUB_REF#refs/heads/}
- name: Checkout
uses: actions/checkout@v2
- name: Update Pip
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
run: |
python3 -m pip install awscli --upgrade
- name: Install AWS Elastic Beanstalk CLI
run: |
python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region
uses: aws-actions/configure-aws-credentials@v1
id: aws
with:
aws-region: us-east-1
- name: Login to AWS ECR
id: login_ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Build, Tag, and Push Image to AWS ECR
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker-compose -f build.docker-compose.yml build
docker-compose -f build.docker-compose.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
AWS_REGION: us-east-1
APP_NAME: ${{ github.event.repository.name }}
run: |
eb use $APP_NAME-$ENV
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
eb deploy
- name: Remove Docker's Trash
if: always()
run: |
docker system prune --volumes -a -f
docker system df
-105
View File
@@ -1,105 +0,0 @@
name: Deploy
on:
push:
branches:
- alpha
- beta
- main
jobs:
deploy:
runs-on: self-hosted
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Semantic Release
uses: cycjimmy/semantic-release-action@v2
id: semantic
with:
semantic_version: 16
extra_plugins: |
@semantic-release/git
@semantic-release/npm
conventional-changelog-eslint
branches: |
[
'main',
{
name: 'alpha',
prerelease: true
},
{
name: 'beta',
prerelease: true
}
]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Update Pip
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
run: |
python3 -m pip install awscli --upgrade
- name: Install AWS Elastic Beanstalk CLI
run: |
python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region
uses: aws-actions/configure-aws-credentials@v1
with:
aws-region: us-east-1
- name: Extract Environment
shell: bash
run: |
if [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "##[set-output name=result;]$(echo "production")"
else
echo "##[set-output name=result;]$(echo ${GITHUB_REF#refs/heads/})"
fi
id: extract_environment
- name: Login to AWS ECR
id: login_ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Build, Tag, and Push Image to AWS ECR
if: steps.semantic.outputs.new_release_published == 'true'
env:
ENV: ${{ steps.extract_environment.outputs.result }}
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
run: |
docker-compose -f build.docker-compose.yml build
docker-compose -f build.docker-compose.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
if: steps.semantic.outputs.new_release_published == 'true'
env:
ENV: ${{ steps.extract_environment.outputs.result }}
IMAGE_TAG: ${{ steps.semantic.outputs.new_release_version }}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
zip deploy.zip docker-compose.yml
- name: Deploy AWS Beanstalk
if: steps.semantic.outputs.new_release_published == 'true'
env:
ENV: ${{ steps.extract_environment.outputs.result }}
AWS_REGION: us-east-1
APP_NAME: maestro
run: |
[[ -d .elasticbeanstalk/ ]] && rm -R .elasticbeanstalk/
printf "1\nn" | eb init $APP_NAME --region $AWS_REGION
eb use $APP_NAME-$ENV
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
eb deploy
+10 -13
View File
@@ -2,8 +2,6 @@ name: Test
on:
pull_request:
branches:
- alpha
- beta
- main
jobs:
@@ -13,22 +11,21 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Build
env:
ENV: test
IMAGE_TAG: test
APP_NAME: ${{ github.event.repository.name }}
run: |
export ENV=test
docker-compose -f build.docker-compose.yml build
docker build -t $APP_NAME --target test .
- name: Run Test
env:
ENV: test
IMAGE_TAG: test
APP_NAME: ${{ github.event.repository.name }}
run: |
export ENV=test
docker-compose run --rm --entrypoint="npm run test" maestro
docker run --rm --entrypoint="npm" $APP_NAME run test
- name: Remove Docker's Trash
if: always()
run: |
docker system prune --volumes -a -f
docker system df
+2 -5
View File
@@ -29,7 +29,7 @@ lerna-debug.log*
# IDE - VSCode
.vscode/*
!.vscode/settings.json
#!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
@@ -37,7 +37,4 @@ lerna-debug.log*
# Ignore .env files
*.env
# Elastic Beanstalk Files
.elasticbeanstalk/*
!.elasticbeanstalk/*.cfg.yml
!.elasticbeanstalk/*.global.yml
.env
+1
View File
@@ -0,0 +1 @@
18.3.0
+13 -20
View File
@@ -5,18 +5,18 @@
{
"preset": "eslint",
"releaseRules": [
{ "tag": "Fix", "release": "patch" },
{ "tag": "Feat", "release": "minor" },
{ "tag": "Update", "release": "minor" },
{ "tag": "Upgrade", "release": "minor" },
{ "tag": "Breaking", "release": "major" },
{ "tag": "Docs", "release": "minor" },
{ "tag": "Chore", "release": "minor" },
{ "tag": "Build", "release": "minor" },
{ "tag": "Refactor", "release": "minor" },
{ "tag": "Test", "release": "minor" },
{ "tag": "CI", "release": "minor" },
{ "tag": "Perf", "release": "minor" }
{ "tag": "BREAKING", "release": "major" },
{ "tag": "FEAT", "release": "minor" },
{ "tag": "UPDATE", "release": "minor" },
{ "tag": "UPGRADE", "release": "minor" },
{ "tag": "CHORE", "release": "minor" },
{ "tag": "REFACTOR", "release": "minor" },
{ "tag": "PERF", "release": "minor" },
{ "tag": "FIX", "release": "patch" },
{ "tag": "DOCS", "release": "patch" },
{ "tag": "BUILD", "release": "patch" },
{ "tag": "TEST", "release": "patch" },
{ "tag": "CI", "release": "patch" }
]
}
],
@@ -26,13 +26,6 @@
"preset": "eslint"
}
],
"@semantic-release/npm",
"@semantic-release/github",
[
"@semantic-release/git", {
"assets": ["package.json"],
"message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
}
]
"@semantic-release/github"
]
}
+15 -6
View File
@@ -1,12 +1,21 @@
FROM node:14 as packages
WORKDIR /packages
COPY package.json .
RUN npm install
# install all dependencies
FROM node:18.3.0-alpine3.15 as packages
WORKDIR /app
COPY package.json package-lock.json ./
RUN apk add --no-cache aws-cli \
&& aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
&& npm install
FROM node:alpine
# unit test specific build
FROM node:18.3.0-alpine3.15 as test
WORKDIR /app
COPY --from=packages /app/node_modules ./node_modules
COPY . ./
FROM node:18.3.0-alpine3.15
WORKDIR /app
COPY . /app/
COPY --from=packages /packages/node_modules /app/node_modules
COPY --from=packages /app/node_modules ./node_modules
RUN npm run build
EXPOSE 3333
ENTRYPOINT npm run start
Binary file not shown.
+277 -49
View File
@@ -1,73 +1,301 @@
<p align="center">
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a>
</p>
<image src="./assets/maestro.svg" style="width:10rem">
</p>
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
[circleci-url]: https://circleci.com/gh/nestjs/nest
# Maestro
Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
<p align="center">
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
</p>
<!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
[![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
## 🚀 Starting
These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
## Description
### 📋 Requirements
- [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
- Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine
- Request access to the dev, stg and prd VPNs
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
### 🔧 Installation<a id="installation"></a>
- Clone the repository
- SSH
```
git clone git@github.com:dadosfera/maestro.git
```
or
## Installation
- HTTPS
```
git clone https://github.com/dadosfera/maestro.git
```
```bash
$ npm install
- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
```sh
nvm use
```
- Install the project dependencies:
```sh
npm i
```
- Setup the following enviroment variables:
```
ENV=
DUC_URL=
INFACTORY_URL=
OTFACTORY_URL=
PIFACTORY_URL=
SM_OAUTH_PATH=
```
- Start the server:
```sh
# dev mode
npm run start:dev
# or in debug mode
npm run start:debug
```
The service should start successfully.
## Authentication decorators
Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
### `@Authenticated`
If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
```ts
import { Authenticated } from '../../authentication/authentication.decorator';
@Controller('foo')
@Authenticated()
class FooController {
/* ... */
@Get('bar')
async getBar() {
this.logger.info('user is authenticated!');
return { authenticated: true };
}
}
```
## Running the app
```ts
import { Authenticated } from '../../authentication/authentication.decorator';
```bash
# development
$ npm run start
@Controller('foo')
class FooController {
/* ... */
# watch mode
$ npm run start:dev
@Get('bar')
@Authenticated()
async getBar() {
this.logger.info('user is authenticated!');
# production mode
$ npm run start:prod
return { authenticated: true };
}
@Post('bar')
async postBar() {
this.logger.info('user is NOT authenticated!');
return { authenticated: false };
}
}
```
## Test
### `@RequireAllPermissions`
This decorator requires that **all permissions** listed are granted to the requesting user.
```bash
# unit tests
$ npm run test
```ts
import { RequireAllPermissions } from '../../authentication/authentication.decorator';
import { Permissions } from '../../authentication/permissions.enum';
# e2e tests
$ npm run test:e2e
@Controller('foo')
class FooController {
/* ... */
# test coverage
$ npm run test:cov
@Get('bar')
@RequireAllPermissions(Permissions.BAR.MANAGE, Permissions.BAR.CREATE)
async getBar() {
/* ... */
}
}
```
## Support
### `@RequireSomePermission`
In the following case, the user is required to have **at least one** listed permission.
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
```ts
import { RequireSomePermission } from '../../authentication/authentication.decorator';
import { Permissions } from '../../authentication/permissions.enum';
## Stay in touch
@Controller('foo')
@RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.CREATE)
class FooController {
/* ... */
- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
- Website - [https://nestjs.com](https://nestjs.com/)
- Twitter - [@nestframework](https://twitter.com/nestframework)
@Get('bar')
async getBar() {
/* ... */
}
}
```
## License
### `@AuthenticateCondition`
If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
Nest is [MIT licensed](LICENSE).
In the following example:
- all routes on the `FooController` controller can only be requested from localhost
- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
```ts
import { AuthenticateCondition } from '../../authentication/authentication.decorator';
@Controller('foo')
// allow requests only from localhost
@AuthenticateCondition((request: Request) => request.ip === '::ffff:127.0.0.1')
class FooController {
/* ... */
@Get('bar')
async getBar() {
/* ... */
}
@Post('bar')
// allow requests only from a specific customer
@AuthenticateCondition((request: Request, user: RequestUser) => user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef')
async postBar() {
/* ... */
}
}
```
### Note on authentication decorators
- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (*and wouldn't like to...*) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
```ts
import {
RequireAllPermissions,
RequireSomePermission,
AuthenticateCondition,
} from '../../authentication/authentication.decorator';
import { Permissions } from '../../authentication/permissions.enum';
import { Waa, Baz } from './authenticationFunctions'
@Controller('foo')
@RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
@RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.ADMIN)
@AuthenticateCondition(Waa)
@AuthenticateCondition(Baz)
class FooController {
/* ... */
@Get('bar')
async getBar() {
/* ... */
}
@Post('bar')
@RequireAllPermissions(Permissions.BAR.MANAGE, Permissions.BAR.USE, Permissions.BAR.REQUEST)
async postBar() {
/* ... */
}
}
```
- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
```ts
// same behavior
@RequireAllPermissions(Permissions.BAR.MANAGE)
@RequireSomePermission(Permissions.BAR.MANAGE)
```
## `@User` parameter decorator
The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
```ts
import { User, RequestUser } from '../../authentication/user.decorator';
@Controller('foo')
class FooController {
/* ... */
@Get('bar')
async getBar(@User() user: RequestUser) {
this.logger.info(user);
return { user };
}
}
```
If the user is required to be logged in, set `required` to `true`, as you would want in the `change-password` operation:
```ts
import { User, RequestUser } from '../../authentication/user.decorator';
@Controller('user')
class UserController {
/* ... */
@Post('change-password')
@HttpCode(HttpStatus.OK)
async changePassword(
@User({ required: true }) user: RequestUser,
@Body() body: AuthChangePasswordRequest,
) {
const { oldPassword, newPassword } = body;
return this.authClient.changePassword({
userId: user.user_id,
oldPassword,
newPassword,
});
}
}
```
## 📦 Development
### ⌨️ Coding Style
By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
### Commits pattern
Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
Format: `<type>[optional scope]: <description>`
Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
### Branching naming convention
- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
**It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (_) could be used to separate the ID and description.**
- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
### Making a Pull Request
1. Commit your changes
2. Open the Pull Request on GitHub
3. Send Pull Request link in microsfera Google Chat Group for review and possible approval
## 🛠️ Built with
Some technologies used in this project:
- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
## ⚙️ Back-end Architecture
The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 13 KiB

+1 -1
View File
@@ -2,4 +2,4 @@ version: "3.8"
services:
maestro:
build: .
image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+3 -1
View File
@@ -1,11 +1,13 @@
version: "3.8"
services:
maestro:
image: 468720548566.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
container_name: maestro
restart: always
ports:
- 3333:3333
env_file:
- .env
node_exporter:
image: prom/node-exporter:v1.3.1
+11 -2
View File
@@ -2,7 +2,16 @@
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"assets": ["**/*.proto"],
"assets": [
"**/*.proto",
{
"include": "i18n/**/*",
"watchAssets": true
}
],
"plugins": [
"@nestjs/swagger"
],
"watchAssets": true
}
}
}
+14475 -3146
View File
File diff suppressed because it is too large Load Diff
+63 -31
View File
@@ -1,10 +1,13 @@
{
"name": "grpc-server-and-client-nestjs-micro-pattern",
"version": "1.0.0",
"name": "maestro",
"version": "0.0.0",
"description": "",
"author": "",
"private": true,
"license": "UNLICENSED",
"engines": {
"node": "18.3.0"
},
"scripts": {
"prebuild": "rimraf dist",
"build": "nest build",
@@ -21,48 +24,71 @@
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@grpc/grpc-js": "^1.5.0",
"@grpc/proto-loader": "^0.6.9",
"@nestjs/common": "^8.0.0",
"@nestjs/core": "^8.0.0",
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
"@nestjs/config": "^1.2.1",
"@nestjs/core": "^8.4.7",
"@nestjs/mapped-types": "*",
"@nestjs/microservices": "^8.2.4",
"@nestjs/platform-express": "^8.0.0",
"@nestjs/swagger": "^5.1.5",
"@nestjs/microservices": "^8.4.7",
"@nestjs/passport": "^8.2.2",
"@nestjs/platform-express": "^8.4.7",
"@nestjs/schedule": "^1.1.0",
"@nestjs/swagger": "^5.2.1",
"axios": "^0.25.0",
"dotenv": "^14.2.0",
"grpc": "^1.24.11",
"cron-parser": "^4.4.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.36.0",
"helmet": "^5.1.0",
"jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"protospack": "2.5.2",
"@dadosfera/protospack-v2": "3.10.0",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0",
"swagger-ui-express": "^4.3.0"
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.4.0"
},
"overrides": {
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@nestjs/cli": "^8.0.0",
"@nestjs/schematics": "^8.0.0",
"@nestjs/testing": "^8.0.0",
"@nestjs/cli": "^8.2.8",
"@nestjs/schematics": "^8.0.11",
"@nestjs/testing": "^8.4.7",
"@types/express": "^4.17.13",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.8",
"@types/jwk-to-pem": "^2.0.1",
"@types/node": "^16.0.0",
"@types/supertest": "^2.0.11",
"@typescript-eslint/eslint-plugin": "^5.0.0",
"@typescript-eslint/parser": "^5.0.0",
"eslint": "^8.0.1",
"eslint-config-prettier": "^8.3.0",
"@types/multer": "^1.4.7",
"@types/node": "^16.11.41",
"@types/passport-facebook": "^2.1.11",
"@types/passport-google-oauth20": "^2.0.11",
"@types/passport-oauth2": "^1.4.11",
"@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^5.29.0",
"@typescript-eslint/parser": "^5.29.0",
"eslint": "^8.18.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.0.0",
"jest": "^27.2.5",
"prettier": "^2.3.2",
"jest": "^27.5.1",
"nock": "^13.2.7",
"prettier": "^2.7.1",
"source-map-support": "^0.5.20",
"supertest": "^6.1.3",
"ts-jest": "^27.0.3",
"ts-loader": "^9.2.3",
"ts-node": "^10.0.0",
"tsconfig-paths": "^3.10.1",
"typescript": "^4.3.5"
"supertest": "^6.2.3",
"ts-jest": "^27.1.5",
"ts-loader": "^9.3.1",
"ts-node": "^10.8.1",
"tsconfig-paths": "^3.14.1",
"typescript": "^4.6.3"
},
"jest": {
"moduleFileExtensions": [
@@ -70,7 +96,13 @@
"json",
"ts"
],
"rootDir": "src",
"rootDir": ".",
"modulePaths": [
"<rootDir>"
],
"moduleDirectories": [
"node_modules"
],
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
+113 -40
View File
@@ -1,44 +1,117 @@
import { MiddlewareConsumer, Module, NestModule, RequestMethod } from '@nestjs/common';
import { InputGrpcClientModule } from './inputs/input-grpc-client.module';
import { UsersGrpcClientModule } from './users-grpc-client/users-grpc-client.module';
import { OutputsModule } from './outputs/outputs.module';
import { EnrichmentModule } from './enrichment/enrichment.module';
import { TransformationsModule } from './transformations/transformations.module';
import { PipelinesModule } from './pipelines/pipelines.module';
import { RestInputsModule } from './rest/rest-inputs/rest-inputs.module';
import { RestOutputsModule } from './rest/rest-outputs/rest-outputs.module';
import { RestEnrichmentsModule } from './rest/rest-enrichments/rest-enrichments.module';
import { RestTransformationsModule } from './rest/rest-transformations/rest-transformations.module';
import { RestPipelinesModule } from './rest/rest-pipelines/rest-pipelines.module';
import { RestUserModule } from './rest/rest-user/rest-user.module';
import { RestFactoryModule } from './rest/rest-factory/rest-factory.module';
import { LoggerMiddleware } from './middlewares/authentication';
import { RestInputsController } from './rest/rest-inputs/rest-inputs.controller';
import { RestOutputsController } from './rest/rest-outputs/rest-outputs.controller';
import { RestTransformationsController } from './rest/rest-transformations/rest-transformations.controller';
import { RestPipelinesController } from './rest/rest-pipelines/rest-pipelines.controller';
import { RestEnrichmentsController } from './rest/rest-enrichments/rest-enrichments.controller';
import { RestFactoryController } from './rest/rest-factory/rest-factory.controller';
import { HealthModule } from './health/health.module';
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { ConfigModule } from '@nestjs/config';
import { InputsController } from './modules/inputs/inputs.controller';
import { TransformationsController } from './modules/transformations/transformations.controller';
import { OutputsController } from './modules/outputs/outputs.controllers';
import { PipelinesController } from './modules/pipelines/pipelines.controller';
import { HealthController } from './modules/health/health.controller';
import { InputsService } from './modules/inputs/inputs.service';
import { TransformationsService } from './modules/transformations/transformations.service';
import { OutputsService } from './modules/outputs/outputs.service';
import { PipelinesService } from './modules/pipelines/pipelines.service';
import { HealthService } from './modules/health/health.service';
import { InputsClientService } from './clients/inputs/client.service';
import { TransformationsClientService } from './clients/transformations/client.service';
import { OutputsClientService } from './clients/outputs/client.service';
import { PipelinesClientService } from './clients/pipelines/client.service';
import { OutputsClientConfiguration } from './clients/outputs/client.config';
import { TransformationsClientConfiguration } from './clients/transformations/client.config';
import { InputsClientConfiguration } from './clients/inputs/client.config';
import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
import { CatalogController } from './modules/catalog/catalog.controller';
import { CatalogService } from './modules/catalog/catalog.service';
import { OauthController } from './modules/oauth/oauth.controller';
import { getOauthSecrets } from './utils/OauthSecrets';
import { HubspotStrategy } from './modules/oauth/passport-strategies/hubspot-strategy';
import { FacebookStrategy } from './modules/oauth/passport-strategies/facebook-strategy';
import { AuthenticationGuard } from './authentication/authentication.guard';
import { APP_GUARD } from '@nestjs/core';
import { GoogleStrategy } from './modules/oauth/passport-strategies/google-strategy';
import { MailChimpStrategy } from './modules/oauth/passport-strategies/mailchimp-strategy';
import { SalesforceStrategy } from './modules/oauth/passport-strategies/salesforce-strategy';
import { UsersModule } from './modules/users/users.module';
import { RolesModule } from './modules/roles/roles.module';
import { ConnectorModule } from './modules/connector/connector.module';
import { AuthModule } from './modules/auth/auth.module';
import { PermissionsModule } from './modules/permissions/permissions.module';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { TermsOfUseModule } from './modules/termsOfUse/termsOfUse.module';
const inputClient = new InputsClientConfiguration();
const outputClient = new OutputsClientConfiguration();
const pipelineClient = new PipelinesClientConfiguration();
const transformationClient = new TransformationsClientConfiguration();
@Module({
controllers: [
InputsController,
TransformationsController,
OutputsController,
PipelinesController,
HealthController,
CatalogController,
OauthController,
],
providers: [
DadosferaLogger,
{ provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
InputsService,
TransformationsService,
OutputsService,
PipelinesService,
HealthService,
InputsClientService,
TransformationsClientService,
OutputsClientService,
PipelinesClientService,
CatalogService,
HubspotStrategy,
FacebookStrategy,
GoogleStrategy,
MailChimpStrategy,
SalesforceStrategy,
{
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
],
imports: [
RestInputsModule, RestOutputsModule, RestEnrichmentsModule, RestTransformationsModule, RestPipelinesModule, RestUserModule, RestFactoryModule, HealthModule,
// InputGrpcClientModule,
//OutputsModule,
//EnrichmentModule,
//TransformationsModule,
//PipelinesModule
ConfigModule.forRoot({
isGlobal: true,
}),
ConnectorModule,
AuthModule,
PermissionsModule,
TermsOfUseModule,
ClientsModule.register([
{
name: 'INPUTS_PACKAGE',
...inputClient.config(),
},
{
name: 'TRANSFORMATIONS_PACKAGE',
...transformationClient.config(),
},
{
name: 'OUTPUTS_PACKAGE',
...outputClient.config(),
},
{
name: 'PIPELINES_PACKAGE',
...pipelineClient.config(),
},
]),
UsersModule,
RolesModule,
],
controllers: [],
providers: [],
})
export class AppModule implements NestModule{
configure(consumer: MiddlewareConsumer) {
consumer
.apply(LoggerMiddleware)
.forRoutes(RestInputsController,RestOutputsController,RestTransformationsController,RestPipelinesController,RestEnrichmentsController,RestFactoryController)
}
}
export class AppModule {}
@@ -0,0 +1,56 @@
import { Request } from 'express';
import { CustomDecorator } from '@nestjs/common';
import { PermissionsObjectPermission } from './permissions.enum';
import { RequestUser } from './user.decorator';
export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
export type AuthenticationFunction = (req: Request, user: any) => boolean;
// implementation copied from SetMetadata, but tweaked to get existing values
// of the metadataKey and accumulate it with the new metadataValue
function SetMultipleMetadata(metadataKey, metadataValue): CustomDecorator {
const decoratorFactory = (target, key, descriptor) => {
// .start: tweak
// descriptor?.value = function decorator; target = class decorator
const accumulatedVal =
Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
accumulatedVal.push(metadataValue);
// .end: tweak
if (descriptor) {
Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
return descriptor;
}
Reflect.defineMetadata(metadataKey, accumulatedVal, target);
return target;
};
decoratorFactory.KEY = metadataKey;
return decoratorFactory as any;
}
export function RequireAllPermissions(
...permissions: PermissionsObjectPermission[]
) {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
permissions.every(({ seqid }) => user.permissions.includes(seqid)),
);
}
export function RequireSomePermission(
...permissions: PermissionsObjectPermission[]
) {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
permissions.some(({ seqid }) => user.permissions.includes(seqid)),
);
}
export function AuthenticateCondition(func: AuthenticationFunction) {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, func);
}
export function Authenticated() {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, () => true);
}
@@ -0,0 +1,457 @@
import request from 'supertest';
import { Test } from '@nestjs/testing';
import { HttpStatus, INestApplication } from '@nestjs/common';
import { APP_GUARD } from '@nestjs/core';
import jwt from 'jsonwebtoken';
import { Body, Controller, Get } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
AuthenticateCondition,
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from './authentication.decorator';
import { AuthenticationGuard } from './authentication.guard';
import { PERMISSIONS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
const logger = {
info: (...args) => args,
error: (...args) => args,
};
@Controller('no-class-auth')
class NoClassAuthController {
@Get('body')
async getBody(@Body() body) {
return { body };
}
@Get('authenticated')
@Authenticated()
async mustBeAuthenticated(@Body() body) {
return { body };
}
@Get('required-permission')
@RequireSomePermission(PERMISSIONS.ZENDESK.permissions.OPEN)
async requiredPermission(@Body() body) {
return { body };
}
@Get('has-all-permissions')
@RequireAllPermissions(
PERMISSIONS.ZENDESK.permissions.OPEN,
PERMISSIONS.METABASE.permissions.OPEN,
)
async hasAllPermissions(@Body() body) {
return { body };
}
@Get('has-some-permission')
@RequireSomePermission(
PERMISSIONS.ZENDESK.permissions.OPEN,
PERMISSIONS.METABASE.permissions.OPEN,
)
async hasSomePermission(@Body() body) {
return { body };
}
}
@Controller('class-auth-condition')
@AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
@RequireSomePermission(PERMISSIONS.METABASE.permissions.OPEN)
class ClassAuthConditionController {
@Get('body')
async getBody(@Body() body) {
return { body };
}
@Get('authenticated')
@Authenticated()
async mustBeAuthenticated(@Body() body) {
return { body };
}
@Get('required-permission')
@AuthenticateCondition((req) => req.get('x-on-route') === 'ok')
@RequireSomePermission(PERMISSIONS.ZENDESK.permissions.OPEN)
async requiredPermission(@Body() body) {
return { body };
}
}
describe('authentication.guard', () => {
let app: INestApplication;
const jwtSecretA = {
kid: 'token-a-testing-shared-key',
pem: '$tr0ng-SH4Red-secr3t!!!~gl0ba1~]',
};
const jwtSecretB = {
kid: 'token-b',
pem: 'another-shared-token',
};
const fakeUserPayload = {
user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
username: 'super.admin',
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
};
beforeAll(async () => {
const moduleRef = await Test.createTestingModule({
providers: [
{
provide: AuthClientService,
useValue: {
getPublicKeys: async () => ({
keys: [jwtSecretA, jwtSecretB],
}),
},
},
{
provide: DadosferaLogger,
useValue: { logger },
},
{
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
app = moduleRef.createNestApplication();
await app.init();
});
afterAll(async () => {
await app.close();
});
function AssertBodyNoAuth(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.OK);
expect(res.body).toStrictEqual({ body: {} });
}
function AssertBodyWithAuth(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.OK);
expect(res.body).toStrictEqual({
body: {
info: {
customer: 'dadosfera',
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_tier: 'BASIC',
user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
},
},
});
}
function AssertUnauthorized(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.UNAUTHORIZED);
expect(res.body).toStrictEqual({
code: ErrorCodes.AUTH.UNAUTHORIZED,
error: 'Não autenticado',
message: 'É necessário estar logado para realizar essa operação',
statusCode: HttpStatus.UNAUTHORIZED,
});
}
function AssertForbidden(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.FORBIDDEN);
expect(res.body).toStrictEqual({
code: ErrorCodes.AUTH.FORBIDDEN,
error: 'Não autorizado',
message:
'Você não tem permissões suficientes para realizar essa operação',
statusCode: HttpStatus.FORBIDDEN,
});
}
function NoClassAuthTest(accessToken: string, tokenName: string[]) {
const route = '/no-class-auth';
describe(`${route}, ${tokenName?.length ? tokenName : 'none'} auth`, () => {
it('should GET /body with user data (if available)', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/body`)
.set({ Authorization: accessToken });
if (tokenName?.length) {
AssertBodyWithAuth(res);
} else {
AssertBodyNoAuth(res);
}
});
it('should GET /authenticated if authenticated', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/authenticated`)
.set({ Authorization: accessToken });
if (tokenName?.length) {
AssertBodyWithAuth(res);
} else {
AssertUnauthorized(res);
}
});
it('should GET /required-permission if authenticated', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/required-permission`)
.set({ Authorization: accessToken });
// zendesk required
if (tokenName?.includes('zendesk')) {
AssertBodyWithAuth(res);
} else if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
it('should GET /has-all-permissions if authenticated', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/has-all-permissions`)
.set({ Authorization: accessToken });
// zendesk and metabase
if (tokenName?.includes('zendesk') && tokenName?.includes('metabase')) {
AssertBodyWithAuth(res);
} else if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
it('should GET /has-some-permission if authenticated', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/has-some-permission`)
.set({ Authorization: accessToken });
// zendesk or metabase
if (tokenName?.includes('zendesk') || tokenName?.includes('metabase')) {
AssertBodyWithAuth(res);
} else if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
});
}
function ClassAuthConditionTest(accessToken: string, tokenName: string[]) {
const route = '/class-auth-condition';
describe(`${route}, ${tokenName?.length ? tokenName : 'none'} auth`, () => {
describe('GET /body', () => {
it('should GET /body if authenticated and with correct headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/body`)
.set({
'x-on-class': 'ok',
Authorization: accessToken,
});
// metabase required
if (tokenName?.includes('metabase')) {
AssertBodyWithAuth(res);
} else if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
it('should not GET /body without correct headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/body`)
.set({ Authorization: accessToken });
if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
});
describe('GET /authenticated', () => {
it('should GET /authenticated if authenticated and with correct headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/authenticated`)
.set({
'x-on-class': 'ok',
Authorization: accessToken,
});
// metabase required
if (tokenName?.includes('metabase')) {
AssertBodyWithAuth(res);
} else if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
it('should not GET /authenticated without correct class headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/authenticated`)
.set({ Authorization: accessToken });
if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
});
describe('GET /required-permission', () => {
it('should GET /required-permission if authenticated and with correct headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/required-permission`)
.set({
'x-on-class': 'ok',
'x-on-route': 'ok',
Authorization: accessToken,
});
// zendesk and metabase
if (
tokenName?.includes('zendesk') &&
tokenName?.includes('metabase')
) {
AssertBodyWithAuth(res);
} else if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
it('should not GET /required-permission without correct route headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/required-permission`)
.set({
'x-on-class': 'ok',
Authorization: accessToken,
});
if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
it('should not GET /required-permission without correct class headers', async () => {
const res = await request(app.getHttpServer())
.get(`${route}/required-permission`)
.set({
'x-on-route': 'ok',
Authorization: accessToken,
});
if (tokenName?.length) {
AssertForbidden(res);
} else {
AssertUnauthorized(res);
}
});
});
});
}
describe('token validation', () => {
it('should accept token kid A', async () => {
const Authorization = CreateToken([], {
jwt: {
kid: jwtSecretA.kid,
pem: jwtSecretA.pem,
},
});
const res = await request(app.getHttpServer())
.get('/no-class-auth/authenticated')
.set({
Authorization,
});
AssertBodyWithAuth(res);
});
it('should accept token kid B', async () => {
const Authorization = CreateToken([], {
jwt: {
kid: jwtSecretB.kid,
pem: jwtSecretB.pem,
},
});
const res = await request(app.getHttpServer())
.get('/no-class-auth/authenticated')
.set({
Authorization,
});
AssertBodyWithAuth(res);
});
it('should not accept unknown kid', async () => {
const Authorization = CreateToken([], {
jwt: {
kid: 'unknown-key-id',
pem: jwtSecretA.pem,
},
});
const res = await request(app.getHttpServer())
.get('/no-class-auth/authenticated')
.set({
Authorization,
});
AssertUnauthorized(res);
});
});
function CreateToken(permissions, overrides?: { user?: any; jwt?: any }) {
const tokenPayload = {
...fakeUserPayload,
permissions: permissions.map(({ seqid }) => seqid),
token_use: 'access',
...overrides?.user,
};
return jwt.sign(tokenPayload, overrides?.jwt?.pem ?? jwtSecretA.pem, {
keyid: overrides?.jwt?.kid ?? jwtSecretA.kid,
});
}
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
const tokenZ = CreateToken([PERMISSIONS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS.METABASE.permissions.OPEN]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([
PERMISSIONS.ZENDESK.permissions.OPEN,
PERMISSIONS.METABASE.permissions.OPEN,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
+136
View File
@@ -0,0 +1,136 @@
import {
Injectable,
CanActivate,
OnApplicationBootstrap,
ExecutionContext,
Inject,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
import jwt from 'jsonwebtoken';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { AuthClientService } from '../modules/auth/auth.service';
import {
AuthenticationFunction,
AUTH_FUNCTION_KEY,
} from './authentication.decorator';
import { RequestUser } from './user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
@Injectable()
export class AuthenticationGuard
implements CanActivate, OnApplicationBootstrap
{
logger: DadosferaLogger;
pems: Map<string, string>;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private reflector: Reflector,
private authClient: AuthClientService,
) {
this.pems = new Map();
this.logger = dadosferaLogger.logger;
}
async onApplicationBootstrap() {
return this.loadDucJWKS();
}
private async loadDucJWKS() {
const { keys } = await this.authClient.getPublicKeys();
keys.forEach((key) => {
this.pems.set(key.kid, key.pem);
});
}
canActivate(ctx: ExecutionContext): boolean {
const authFunctions = this.reflector.getAllAndMerge<
AuthenticationFunction[]
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
const mustBeAuthenticated = authFunctions.length > 0;
const request = ctx.switchToHttp().getRequest();
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!mustBeAuthenticated) {
// no need to be authenticated
return true;
}
if (!accessToken) {
// couldn't load valid token
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
}
// every authentication function must return true to authenticate
if (!authFunctions.every((func) => func(request, accessToken))) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
return true;
}
private validateToken(
request,
mustBeAuthenticated: boolean,
): RequestUser | false {
const accessToken = request.get('Authorization');
let accessTokenPayload: RequestUser;
// If the user isn't authenticated, an error will occurr anywhere here.
// Fancy error avoidance isn't performed by purpose, such as avoiding to access null values.
try {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
/*assert(
accessTokenDecoded.payload.iss === 'duc',
'token should be issued by DUC',
);*/
assert(
accessTokenDecoded?.payload?.token_use === 'access',
'should be an access token',
);
const { kid } = accessTokenDecoded.header;
const pemValue: string = this.pems.get(kid);
jwt.verify(accessToken, pemValue);
accessTokenPayload = accessTokenDecoded.payload;
} catch (err) {
// log errors if authentication is required
if (mustBeAuthenticated) {
this.logger.error('Unable to verify duc access token\n' + err?.stack);
}
return false;
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
access_token: accessToken,
};
// TODO: for backwards compatibility. remove in the future
request.body.info = {
user_id: accessTokenPayload.user_id,
customer_id: accessTokenPayload.customer_id,
customer: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
};
return accessTokenPayload;
}
}
+465
View File
@@ -0,0 +1,465 @@
import { Logger } from '@nestjs/common';
import assert from 'assert';
import { LanguageEnum } from 'src/utils/languages.enum';
const logger = new Logger();
export enum PermissionUsages {
PUBLIC = 'public',
INTERNAL = 'internal',
}
export interface PermissionsObjectPermission {
seqid: number;
claim: string;
usage: PermissionUsages;
name?: Record<LanguageEnum, string>;
}
export interface PermissionsObjectPermissionTranslated {
seqid: number;
claim: string;
usage: PermissionUsages;
name?: string;
}
export interface PermissionsObjectGroup {
title: {
'pt-br': string;
'en-us': string;
};
permissions: Record<string, PermissionsObjectPermission>;
}
/*
- like field numbers in gRPC, avoid to change (or reuse previously used) seqids once its deployed to DUC
- it is possible to update claim and usage, not seqid
*/
/*export const Permissions: {
[P in keyof any]: { [Q in keyof any]: Permission };
} = {*/
export const PERMISSIONS = {
AUTH: {
title: {
'pt-br': 'Autenticação',
'en-us': 'Authentication',
},
permissions: {
CREATE: {
seqid: 18,
claim: 'POST /auth',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': '',
'en-us': '',
},
},
},
},
PIPELINE: {
title: {
'pt-br': 'Pipelines',
'en-us': 'Pipelines',
},
permissions: {
CREATE: {
seqid: 23,
claim: 'POST /pipelines',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Criar pipelines',
'en-us': 'Create pipelines',
},
},
GET: {
seqid: 13,
claim: 'GET /pipelines',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Buscar pipelines',
'en-us': 'Get pipelines',
},
},
UPDATE: {
seqid: 29,
claim: 'PUT /pipelines',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Atualizar pipeline',
'en-us': 'Update pipeline',
},
},
DELETE: {
seqid: 5,
claim: 'DELETE /pipelines',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Remover pipeline',
'en-us': 'Delete pipeline',
},
},
},
},
INPUT: {
title: {
'pt-br': 'Conexões',
'en-us': 'Connections',
},
permissions: {
CREATE: {
seqid: 21,
claim: 'POST /inputs',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Criar conexões',
'en-us': 'Create connections',
},
},
GET: {
seqid: 9,
claim: 'GET /inputs',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Buscar conexões',
'en-us': 'Get connections',
},
},
UPDATE: {
seqid: 27,
claim: 'PUT /inputs',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Atualizar conexões',
'en-us': 'Create connections',
},
},
DELETE: {
seqid: 3,
claim: 'DELETE /inputs',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Remover conexões',
'en-us': 'Delete connections',
},
},
GET_ENTITIES: {
seqid: 10,
claim: 'GET /inputs/available-entities',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Remover conexões',
'en-us': 'Delete connections',
},
},
},
},
OUTPUT: {
title: {
'pt-br': 'Output',
'en-us': 'Output',
},
permissions: {
CREATE: {
seqid: 22,
claim: 'POST /outputs',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Criar outputs',
'en-us': 'Create outputs',
},
},
GET: {
seqid: 12,
claim: 'GET /outputs',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Buscar outputs',
'en-us': 'Get outputs',
},
},
UPDATE: {
seqid: 28,
claim: 'PUT /outputs',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Editar outputs',
'en-us': 'Edit outputs',
},
},
DELETE: {
seqid: 4,
claim: 'DELETE /outputs',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Remover outputs',
'en-us': 'Remove outputs',
},
},
},
},
TRANSFORMATIONS: {
title: {
'pt-br': 'Micro-transformações',
'en-us': 'Micro-transformations',
},
permissions: {
CREATE: {
seqid: 24,
claim: 'POST /transformations',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Criar micro-transformações',
'en-us': 'Create micro-transformations',
},
},
GET: {
seqid: 15,
claim: 'GET /transformations',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Buscar micro-transformações',
'en-us': 'Get micro-transformations',
},
},
UPDATE: {
seqid: 30,
claim: 'PUT /transformations',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Editar micro-transformações',
'en-us': 'Edit micro-transformations',
},
},
DELETE: {
seqid: 6,
claim: 'DELETE /transformations',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Remover micro-transformações',
'en-us': 'Remove micro-transformations',
},
},
},
},
CATALOG: {
title: {
'pt-br': 'Catálogo',
'en-us': 'Catalog',
},
permissions: {
CREATE: {
seqid: 19,
claim: 'POST /catalog',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Criar atributos no catálogo',
'en-us': 'Create catalog attributes',
},
},
GET: {
seqid: 7,
claim: 'GET /catalog',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Buscar catálogos',
'en-us': 'Get catalogs',
},
},
UPDATE: {
seqid: 25,
claim: 'PUT /catalog',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Editar catálogo',
'en-us': 'Edit catalogs',
},
},
DELETE: {
seqid: 1,
claim: 'DELETE /catalog',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Remover catálogo',
'en-us': 'Remove catalogs',
},
},
},
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
'en-us': 'Connectors',
},
permissions: {
CREATE: {
seqid: 20,
claim: 'POST /connectors',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Criar conectores',
'en-us': 'Create connectors',
},
},
GET: {
seqid: 8,
claim: 'GET /connectors',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Buscar conectores',
'en-us': 'Get connectors',
},
},
UPDATE: {
seqid: 26,
claim: 'PUT /connectors',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Editar conectores',
'en-us': 'Edit connectors',
},
},
DELETE: {
seqid: 2,
claim: 'DELETE /connectors',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Remover conectores',
'en-us': 'Remove connectors',
},
},
},
},
SNOWFLAKE: {
title: {
'pt-br': 'Snowflake',
'en-us': 'Snowflake',
},
permissions: {
OPEN: {
seqid: 14,
claim: 'GET /snowflake',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Snowflake',
'en-us': 'Access Snowflake',
},
},
},
},
ZENDESK: {
title: {
'pt-br': 'Zendesk',
'en-us': 'Zendesk',
},
permissions: {
OPEN: {
seqid: 16,
claim: 'GET /zendesk',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Zendesk',
'en-us': 'Access Zendesk',
},
},
},
},
METABASE: {
title: {
'pt-br': 'Metabase',
'en-us': 'Metabase',
},
permissions: {
OPEN: {
seqid: 11,
claim: 'GET /metabase',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Metabase',
'en-us': 'Access Metabase',
},
},
},
},
ORCHEST: {
title: {
'pt-br': 'Orchest',
'en-us': 'Orchest',
},
permissions: {
INTELLIGENCE: {
seqid: 31,
claim: 'intelligence:open',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar inteligência',
'en-us': 'Access intelligence',
},
},
APP_BUILDER: {
seqid: 32,
claim: 'app-builder:open',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar App Builder',
'en-us': 'Access App Builder',
},
},
MACHINE_LEARNING: {
seqid: 33,
claim: 'machine-learning:open',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Machine Learning',
'en-us': 'Access Machine Learning',
},
},
},
},
USERS: {
title: {
'pt-br': 'Usuários',
'en-us': 'Users',
},
permissions: {
ADMIN: {
seqid: 34,
claim: 'users:admin',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Administrar usuários e grupos',
'en-us': 'Manage user and roles',
},
},
},
},
};
// traverses the object searching for duplicate seqids or claims (executes at runtime)
let nextAvailableSeqid = 0;
const seqids = Object.values(PERMISSIONS).flatMap((namespace) =>
Object.values(namespace.permissions).map(({ seqid }) => seqid),
);
const claims = Object.values(PERMISSIONS).flatMap((namespace) =>
Object.values(namespace.permissions).map(({ claim }) => claim),
);
Object.values(PERMISSIONS).map((namespace) =>
Object.values(namespace.permissions).map(({ seqid, claim }) => {
const seqidsCount = seqids.filter((x) => x === seqid).length;
assert(seqidsCount === 1, `seqid ${seqid} count is not 1`);
const claimsCount = claims.filter((x) => x === claim).length;
assert(claimsCount === 1, `claim '${claim}' count is not 1`);
nextAvailableSeqid = Math.max(nextAvailableSeqid, seqid);
}),
);
logger.log(`next available seqid ${nextAvailableSeqid + 1}`);
+182
View File
@@ -0,0 +1,182 @@
import request from 'supertest';
import { Test } from '@nestjs/testing';
import { HttpStatus, INestApplication } from '@nestjs/common';
import { APP_GUARD } from '@nestjs/core';
import jwt from 'jsonwebtoken';
import { Controller, Get } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { AuthenticationGuard } from './authentication.guard';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { PERMISSIONS } from './permissions.enum';
const logger = {
info: (...args) => args,
error: (...args) => args,
};
@Controller('user')
class UserController {
@Get()
public getUser(@User() user) {
return { user };
}
@Get('options')
public getUserWithOptions(@User({}) user) {
return { user };
}
@Get('not-required')
public getUserNotRequired(@User({ required: false }) user) {
return { user };
}
@Get('required')
public getUserRequired(@User({ required: true }) user) {
return { user };
}
}
describe('user.decorator', () => {
let app: INestApplication;
const jwt_secret = {
kid: 'token-testing-shared-key',
pem: '$tr0ng-SH4Red-secr3t!!!~gl0ba1~]',
};
const fakeUserPayload = {
user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
username: 'super.admin',
permissions: [PERMISSIONS.METABASE.permissions.OPEN].map(
({ seqid }) => seqid,
),
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
access_token: '',
};
beforeAll(async () => {
const moduleRef = await Test.createTestingModule({
providers: [
{
provide: AuthClientService,
useValue: {
getPublicKeys: async () => ({
keys: [jwt_secret],
}),
},
},
{
provide: DadosferaLogger,
useValue: { logger },
},
{
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
],
controllers: [UserController],
}).compile();
app = moduleRef.createNestApplication();
await app.init();
});
afterAll(async () => {
await app.close();
});
function AssertNoAuth(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.OK);
expect(res.body).toStrictEqual({});
}
function AssertWithAuth(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.OK);
expect(res.body).toStrictEqual({ user: fakeUserPayload });
}
function AssertRequiredNoAuth(res: request.Response) {
expect(res.statusCode).toBe(HttpStatus.UNAUTHORIZED);
expect(res.body).toStrictEqual({
code: ErrorCodes.AUTH.UNAUTHORIZED,
error: 'Não autenticado',
message: 'É necessário estar logado para realizar essa operação',
statusCode: HttpStatus.UNAUTHORIZED,
});
}
function UserTest(accessToken: string) {
describe(`with${accessToken ? '' : 'out'} token`, () => {
it('should GET /user', async () => {
const res = await request(app.getHttpServer())
.get('/user')
.set({ Authorization: accessToken });
if (accessToken) {
AssertWithAuth(res);
} else {
AssertNoAuth(res);
}
});
it('should GET /user/options', async () => {
const res = await request(app.getHttpServer())
.get('/user/options')
.set({ Authorization: accessToken });
if (accessToken) {
AssertWithAuth(res);
} else {
AssertNoAuth(res);
}
});
it('should GET /user/not-required', async () => {
const res = await request(app.getHttpServer())
.get('/user/not-required')
.set({ Authorization: accessToken });
if (accessToken) {
AssertWithAuth(res);
} else {
AssertNoAuth(res);
}
});
it('should GET /user/required if logged', async () => {
const res = await request(app.getHttpServer())
.get('/user/required')
.set({ Authorization: accessToken });
if (accessToken) {
AssertWithAuth(res);
} else {
AssertRequiredNoAuth(res);
}
});
});
}
function CreateToken(overrides?) {
const tokenPayload = {
...fakeUserPayload,
token_use: 'access',
...overrides?.user,
};
return jwt.sign(tokenPayload, jwt_secret.pem, {
keyid: jwt_secret.kid,
...overrides?.jwt,
});
}
UserTest(null);
const token = CreateToken();
fakeUserPayload.access_token = token;
UserTest(token);
});
+25
View File
@@ -0,0 +1,25 @@
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
export interface RequestUser {
user_id: string;
username: string;
permissions: number[];
customer_id: string;
customer_name: string;
customer_tier: string;
access_token: string;
}
export const User: (options?: { required?: boolean }) => ParameterDecorator =
createParamDecorator((options: any, ctx: ExecutionContext) => {
const request = ctx.switchToHttp().getRequest();
if (!request.user && options?.required) {
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
}
return request.user;
});
+26
View File
@@ -0,0 +1,26 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { ProtoPackages, ProtoPaths } from '@dadosfera/protospack-v2/dist/lib/Duc';
export class DucClient {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.DUC_URL,
package: ProtoPackages.WritePackage,
// credentials: credentials.createSsl(),
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: ProtoPaths.WriteFilePath,
loader: {
keepCase: true,
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
+25
View File
@@ -0,0 +1,25 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { InputPackages, InputProtoFilePath } from 'protospack';
export class InputsClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: InputPackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: InputProtoFilePath,
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
+310
View File
@@ -0,0 +1,310 @@
import {
OnModuleInit,
Inject,
HttpException,
HttpStatus,
NotFoundException,
InternalServerErrorException,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
DecodeGrpcStruct,
EncodeJsonToGrpcStruct,
InputCreateS3Request,
InputNewCreateRequest,
InputService,
TestConnectionGetColumnsRequest,
TestConnectionRequest,
} from 'protospack';
import axios, { AxiosRequestConfig } from 'axios';
import { lastValueFrom } from 'rxjs';
import { InputModel } from 'src/modules/inputs/dtos/input.model';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import { mustache } from 'src/utils/mustache';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { OauthSecrets } from 'src/utils/OauthSecrets';
import { URLSearchParams } from 'url';
import { IIdRequest, UpdateInputRequest } from './interfaces';
export class InputsClientService implements OnModuleInit {
private inputService: InputService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('INPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
@Inject('OAUTH_SECRETS') private readonly oauthSecrets: OauthSecrets,
) {
this.logger = dadosferaLogger.logger;
}
async onModuleInit() {
this.inputService =
this.grpcClient.getService<InputService>('InputService');
}
async newCreate(createInputDto: InputNewCreateRequest) {
this.logger.info('InputClientService - Create');
const input = await lastValueFrom(
this.inputService.NewCreate(objectSnakeToCamel(createInputDto)),
);
objectCamelToSnake(input);
return input;
}
async createGeneric(createInputGeneric) {
this.logger.info('InputClientService - Create');
const { info, ...data } = createInputGeneric;
// data.credentials = await this.getInputTokens(data);
if (data.options) delete data.options;
if (data.credentials.oauth_code) delete data.credentials.oauth_code;
const grpcPayload = {
input: EncodeJsonToGrpcStruct(data),
info,
};
objectSnakeToCamel(grpcPayload);
const structReturn = await lastValueFrom(
this.inputService.Create(grpcPayload),
).catch((err: { details: string }) => {
if (err.details === 'Item Not found!') {
throw new NotFoundException('Input not found');
}
throw new InternalServerErrorException(err.details);
});
objectCamelToSnake(structReturn);
const inputCreated = DecodeGrpcStruct(structReturn.input);
return { input: inputCreated };
}
async createS3Inputs(createInputDto: InputCreateS3Request) {
this.logger.info('InputClientService - Create');
objectSnakeToCamel(createInputDto);
const createInputResponse = await lastValueFrom(
this.inputService.CreateS3(createInputDto),
).catch((e) => {
throw new HttpException(e.details, 500);
});
this.logger.info('Done');
objectCamelToSnake(createInputResponse);
return createInputResponse;
}
async findOne(data: IIdRequest) {
this.logger.info('InputClientService - FindOne');
const findOneInputResponse = await lastValueFrom(
this.inputService.FindOne(objectSnakeToCamel(data)),
).catch((e) => {
this.logger.error(e.details);
throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
});
objectCamelToSnake(findOneInputResponse);
return findOneInputResponse;
}
async findAll(data) {
this.logger.info('InputClientService - FindAll');
const findAllInputResponse = await lastValueFrom(
this.inputService.FindAll(objectSnakeToCamel(data)),
).catch((e) => {
this.logger.error(e.details);
throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
});
findAllInputResponse.inputs.forEach((input) => objectCamelToSnake(input));
return findAllInputResponse;
}
async update(updateInputDTO: UpdateInputRequest) {
this.logger.info('InputClientService - Update');
const updateInputResponse = await new Promise((resolve, reject) => {
this.inputService.Update(objectSnakeToCamel(updateInputDTO)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return updateInputResponse;
}
async remove(idRequest: IIdRequest) {
this.logger.info('InputClientService - Remove');
const removeInputResponse = await new Promise((resolve, reject) => {
this.inputService.Remove(objectSnakeToCamel(idRequest)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return removeInputResponse;
}
async testConnection(data: TestConnectionRequest) {
this.logger.info('InputClientService - TestConnection');
const testConnectionResponse = await new Promise((resolve, reject) => {
this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return testConnectionResponse;
}
async getColumns(data: TestConnectionGetColumnsRequest) {
this.logger.info('InputClientService - TestConnection/Get-Columns');
const getColumnsResponse = await new Promise((resolve, reject) => {
this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return getColumnsResponse;
}
async getAuthSecrets(plugin) {
switch (plugin) {
case 'hubspot':
return {
client_id: this.oauthSecrets.hubspot.client_id || '',
client_secret: this.oauthSecrets.hubspot.client_secret || '',
};
}
return {};
}
async getInputTokens(input: InputModel) {
const { credentials } = input;
let credentialsTokens = {};
switch (credentials.connection_type) {
case 'oauth':
credentialsTokens = await this.getOauthTokens(input);
break;
}
return { ...credentials, ...credentialsTokens };
}
async getOauthTokens(input: InputModel) {
const tokens: Record<string, any> = {};
const { options, credentials } = input;
const { oauth } = options;
const { get_tokens_url_params, get_tokens_set_response } = oauth;
const secrets = await this.getAuthSecrets(input.plugin);
const get_tokens_url_params_string = mustache(get_tokens_url_params, {
...secrets,
...credentials,
});
let get_tokens_url_params_obj: Record<string, any> = {};
try {
get_tokens_url_params_obj = JSON.parse(get_tokens_url_params_string);
} catch (error) {
throw new HttpException('Erro transformando get_tokens_url_params', 400);
}
let params = {};
switch (oauth.content_type) {
case 'application/x-www-form-urlencoded':
params = new URLSearchParams(get_tokens_url_params_obj);
params = params.toString();
break;
default:
break;
}
const axiosRequestConfig: AxiosRequestConfig = {
url: oauth.get_tokens_url,
method: oauth.get_tokens_method || 'POST',
data: params,
headers: { 'content-type': oauth.content_type || 'application/json' },
};
const { data } = await axios(axiosRequestConfig).catch((err) => {
this.logger.error(err.response.data);
throw new HttpException(err.response.data.message, err.response.status);
});
for (const key in get_tokens_set_response) {
const responseKey = get_tokens_set_response[key];
tokens[key] = data[responseKey];
}
const { redirect_uri } = get_tokens_url_params_obj;
tokens.redirect_uri = redirect_uri;
return tokens;
}
async getAvailableEntities(data) {
this.logger.info('InputClientService - GetAvailableEntities');
objectSnakeToCamel(data);
const response = await lastValueFrom(
this.inputService.GetAvailableEntities(data),
).catch((err) => {
this.logger.info(err.details);
if (err.details && err.details.includes('400'))
throw new HttpException('Plugin inválido', HttpStatus.BAD_REQUEST);
throw new HttpException(err.details, HttpStatus.INTERNAL_SERVER_ERROR);
});
objectCamelToSnake(response);
return response;
}
}
+64
View File
@@ -0,0 +1,64 @@
interface Values {
jdbc_user: string;
jdbc_password: string;
database: string;
endpoint: string;
tables: string[];
port: string;
engine: string;
schema: string;
}
interface Cron {
hours: string[];
hour_interval: boolean;
hour_resourse: boolean;
hour_resourse_value: number;
month_day: string[];
month_day_interval: boolean;
month_day_resourse: boolean;
month_day_resourse_value: number;
month: string[];
month_interval: boolean;
month_resourse: boolean;
month_resourse_value: number;
week_days: string[];
week_days_interval: boolean;
week_days_resourse: boolean;
week_days_resourse_value: number;
}
export interface Info {
customer_id: string;
user_id: string;
customer: string;
customer_tier: string;
}
export interface ICreateInputRequest {
cron: string;
name: string;
plugin: string;
values: Values;
operation: string;
info: Info;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface UpdateInputRequest {
id: string;
cron: string;
name: string;
plugin: string;
values: Values;
operation: string;
info: Info;
}
export interface ITestConnectionRequest {
plugin: string;
values: Values;
}
+24
View File
@@ -0,0 +1,24 @@
import { OutputPackages, OutputProtoFilePath } from 'protospack';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { credentials } from '@grpc/grpc-js';
export class OutputsClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.OTFACTORY_URL,
package: OutputPackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: OutputProtoFilePath,
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
+178
View File
@@ -0,0 +1,178 @@
import { Controller, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc, Payload } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { OutputServicesNames, OutputsServiceInterface } from 'protospack';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import {
ICreateOutputRequest,
IIdRequest,
IOutputUpdateRequest,
} from './interfaces';
@Controller('output')
export class OutputsClientService implements OnModuleInit {
private outputService: OutputsServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('OUTPUTS_PACKAGE') private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.outputService = this.grpcClient.getService<OutputsServiceInterface>(
OutputServicesNames.OutputService,
);
}
async create(@Payload() createOutputDto: ICreateOutputRequest) {
this.logger.info('OutputClientService - Create');
const createOutputResponse = await new Promise((resolve, reject) => {
this.outputService.Create(objectSnakeToCamel(createOutputDto)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
const convertedItem = objectCamelToSnake(createOutputResponse['output']);
createOutputResponse['output'] = convertedItem;
return createOutputResponse;
}
async findAll(data) {
this.logger.info('OutputClientService - FindAll');
const findAllOutputResponse = await new Promise((resolve, reject) => {
this.outputService.FindAll(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
const convertedOutputs = findAllOutputResponse['outputs'].map((ot) => {
return objectCamelToSnake(ot);
});
return { outputs: convertedOutputs };
}
async findOne(data: IIdRequest) {
this.logger.info('OutputClientService - FindOne');
const findOneOutputResponse = await new Promise((resolve, reject) => {
this.outputService.FindOne(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return objectCamelToSnake(findOneOutputResponse);
}
async update(updateOutPutDTO: IOutputUpdateRequest) {
this.logger.info('OutputClientService - Update');
const updateOutputResponse = await new Promise((resolve, reject) => {
this.outputService.Update(objectSnakeToCamel(updateOutPutDTO)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return objectCamelToSnake(updateOutputResponse);
}
async remove(data: IIdRequest) {
this.logger.info('OutputClientService - Remove');
const removeOutputResponse = await new Promise((resolve, reject) => {
this.outputService.Remove(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return removeOutputResponse;
}
}
+29
View File
@@ -0,0 +1,29 @@
export interface ICreateOutputRequest {
operation: string;
name: string;
plugin: string;
values: Values;
}
export interface Values {
prefix: string;
}
interface Info {
user_id: string;
customer_id: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IOutputUpdateRequest {
id: string;
operation: string;
values: Values;
plugin: string;
name: string;
info: Info;
}
+24
View File
@@ -0,0 +1,24 @@
import { ClientOptions, Transport } from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from 'protospack';
import { credentials } from '@grpc/grpc-js';
export class PipelinesClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: PipelinePackages,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: PipelineProtoFilePath,
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
+273
View File
@@ -0,0 +1,273 @@
import {
BadRequestException,
Inject,
InternalServerErrorException,
OnModuleInit,
} from '@nestjs/common';
import { ClientGrpc, Payload } from '@nestjs/microservices';
import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack';
import { lastValueFrom } from 'rxjs';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import {
ICreatePipelineDto,
IGetPipelineLogsRequest,
IIdRequest,
IUpdatePipelineRequest,
} from './interfaces';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('PIPELINES_PACKAGE') private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.pipelineService =
this.grpcClient.getService<PipelinesServiceInterface>(
PipelineServicesNames.PipelineService,
);
}
async create(@Payload() createPipelineDto: ICreatePipelineDto) {
this.logger.info('PipelinesClientService - Create');
const createPipelineResponse = await lastValueFrom(
this.pipelineService.Create(objectSnakeToCamel(createPipelineDto)),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((error: { details: string }) => {
if (error.details.includes('INVALID_REQUEST')) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [errorType, message] = error.details.split('|');
throw new BadRequestException(message);
}
throw new InternalServerErrorException(error.details);
});
return createPipelineResponse;
}
async findAll(data: IIdRequest) {
this.logger.info('PipelinesClientService - FindAll');
const findAllPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.FindAll(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x.pipelines);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return findAllPipelineResponse;
}
async findOne(data: IIdRequest) {
this.logger.info('PipelinesClientService - FindOne');
const findOnePipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.FindOne(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return findOnePipelineResponse;
}
async update(UpdatePipelineRequest: IUpdatePipelineRequest) {
this.logger.info('PipelinesClientService - Update');
const updatePipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService
.Update(objectSnakeToCamel(UpdatePipelineRequest))
.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return updatePipelineResponse;
}
async remove(data: IIdRequest) {
this.logger.info('PipelinesClientService - Remove');
const removePipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.remove(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// this.logger.info('Done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return removePipelineResponse;
}
async getPipelineLogsMessages(data: IGetPipelineLogsRequest) {
this.logger.info('PipelinesClientService - GetPipelineLogsMessages');
const logsPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService.getPipelineLogs(data).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// this.logger.info('Done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return logsPipelineResponse;
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService
.getPipelineStatus(objectSnakeToCamel(data))
.subscribe({
next(x) {
const statusArray = x.status.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
});
resolve({ status: statusArray });
},
error(err) {
reject(err);
},
complete() {
// this.logger.info('Done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await new Promise((resolve, reject) => {
this.pipelineService
.triggerPipeline(objectSnakeToCamel({ id, info }))
.subscribe({
next(x) {
if (x.status == false) {
reject(
'This pipeline is not ready yet to execute, Try again later!',
);
}
resolve(x);
},
error(err) {
this.logger.error('Observable Error');
reject(err);
},
complete() {
//this.logger.info('Done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return statusPipelineResponse;
}
}
+39
View File
@@ -0,0 +1,39 @@
export interface ICreatePipelineDto {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
info: Info;
}
export interface Info {
customer_id: sting;
user_id: string;
}
export interface IdRequest {
id: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
id: string;
info: Info;
}
export interface IGetPipelineLogsRequest {
id: string;
details: string;
}
@@ -0,0 +1,25 @@
import { ClientOptions, Transport } from '@nestjs/microservices';
import {
TransformationPackages,
TransformationProtoFilePath,
} from 'protospack';
import { credentials } from '@grpc/grpc-js';
export class TransformationsClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: TransformationPackages,
credentials: credentials.createSsl(),
protoPath: TransformationProtoFilePath,
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
@@ -0,0 +1,194 @@
import { Controller, Inject, OnModuleInit, Post } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { TransformationsServiceInterface } from 'protospack';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
objectCamelToSnake,
objectSnakeToCamel,
} from 'src/utils/CaseConverter';
import { ICreateTransformationsRequest, IIdRequest } from './interfaces';
@Controller('transformation')
export class TransformationsClientService implements OnModuleInit {
private transformationService: TransformationsServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('TRANSFORMATIONS_PACKAGE') private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.transformationService =
this.grpcClient.getService<TransformationsServiceInterface>(
'TransformationService',
);
}
@Post()
async create(createTransformationsDto: ICreateTransformationsRequest) {
this.logger.info('TransformationClientService - Create');
const createTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService
.Create(objectSnakeToCamel(createTransformationsDto))
.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
},
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
const convertedTransforms = createTransformationResponse[
'transformations'
].map((tr) => {
return objectCamelToSnake(tr);
});
return { transformations: convertedTransforms };
}
async findAll(data) {
this.logger.info('TransformationClientService - FindAll');
const findAllTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService.FindAll(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
},
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
const convertedTransforms = findAllTransformationResponse[
'transformations'
].map((tr) => {
return objectCamelToSnake(tr);
});
return { transformations: convertedTransforms };
}
async findOne(data: IIdRequest) {
this.logger.info('TransformationClientService - FindOne');
const findOneTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService.FindOne(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
},
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return objectCamelToSnake(findOneTransformationResponse);
}
async update(updateTransformationDTO) {
this.logger.info('TransformationClientService - Update');
const updateTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService
.Update(objectSnakeToCamel(updateTransformationDTO))
.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
},
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return objectCamelToSnake(updateTransformationResponse);
}
async remove(data: IIdRequest) {
this.logger.info('TransformationClientService - Remove');
const removeTransformationResponse = await new Promise(
(resolve, reject) => {
this.transformationService.Remove(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
},
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return objectCamelToSnake(removeTransformationResponse);
}
}
+37
View File
@@ -0,0 +1,37 @@
export interface ICreateTransformationsRequest {
transformations: Transformation[];
info: Info;
}
interface Param {
base_column: string;
column_name: string;
n_digits: number;
start_index: number;
randbelow: number;
const: string;
}
export interface Info {
customer_id: string;
user_id: string;
}
interface Transformation {
created_at: string;
id: string;
client_id: string;
type: string;
params: Param[];
input_source: string;
table: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IUpdateTransformationRequest {
transformation: Transformation;
info: Info;
}
-17
View File
@@ -1,17 +0,0 @@
import { join } from 'path';
import { ClientOptions, Transport } from '@nestjs/microservices';
export const GrpcClientConfiguration: ClientOptions = {
transport: Transport.GRPC,
options: {
//url: `${process.env.USERS_SVC_URL}:${process.env.USERS_SVC_PORT}`,
url: `127.0.0.1:50053`,
package: 'enrichment',
protoPath: join(__dirname, '..', 'proto', 'enrichment.proto'),
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
@@ -1,6 +0,0 @@
export class CreateEnrichmentDto {
type:string;
columns:string[]
}
@@ -1,6 +0,0 @@
import { PartialType } from '@nestjs/mapped-types';
import { CreateEnrichmentDto } from './create-enrichment.dto';
export class UpdateEnrichmentDto extends PartialType(CreateEnrichmentDto) {
}
@@ -1,20 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { EnrichmentController } from './enrichment.controller';
import { EnrichmentsService } from './enrichment.service';
describe('EnrichmentController', () => {
let controller: EnrichmentController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [EnrichmentController],
providers: [EnrichmentsService],
}).compile();
controller = module.get<EnrichmentController>(EnrichmentController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});
-60
View File
@@ -1,60 +0,0 @@
import {
Controller,
Delete,
Get,
OnModuleInit,
Param,
Post,
Put,
} from '@nestjs/common';
import { Client, ClientGrpc, Payload } from '@nestjs/microservices';
import { EnrichmentsService } from './enrichment.service';
import { CreateEnrichmentDto } from './dto/create-enrichment.dto';
import { UpdateEnrichmentDto } from './dto/update-enrichment.dto';
import { ApiTags } from '@nestjs/swagger';
import { GrpcClientConfiguration } from './config/grpc-client';
@ApiTags('enrichments')
@Controller('enrichment')
export class EnrichmentController implements OnModuleInit {
@Client(GrpcClientConfiguration)
private enrichmentServiceClient: ClientGrpc;
private enrichmentService: EnrichmentsService;
onModuleInit() {
this.enrichmentService =
this.enrichmentServiceClient.getService<EnrichmentsService>(
'EnrichmentService',
);
}
@Post()
create(@Payload() createEnrichmentDto: CreateEnrichmentDto) {
return this.enrichmentService.create(createEnrichmentDto);
}
@Get()
list() {
return this.enrichmentService.list();
}
@Get(':id')
show(@Param() id: string) {
return this.enrichmentService.show(id);
}
@Put(':id')
update(
@Param() id: string,
@Payload() updateEnrichmentDto: UpdateEnrichmentDto,
) {
return this.enrichmentService.update(id, updateEnrichmentDto);
}
@Delete(':id')
remove(@Param() id: string) {
return this.enrichmentService.remove(id);
}
}
-9
View File
@@ -1,9 +0,0 @@
import { Module } from '@nestjs/common';
import { EnrichmentsService } from './enrichment.service';
import { EnrichmentController } from './enrichment.controller';
@Module({
controllers: [EnrichmentController],
providers: [EnrichmentsService],
})
export class EnrichmentModule {}
-18
View File
@@ -1,18 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { EnrichmentsService } from './enrichment.service';
describe('EnrichmentService', () => {
let service: EnrichmentsService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [EnrichmentsService],
}).compile();
service = module.get<EnrichmentsService>(EnrichmentsService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
-26
View File
@@ -1,26 +0,0 @@
import { Injectable } from '@nestjs/common';
import { CreateEnrichmentDto } from './dto/create-enrichment.dto';
import { UpdateEnrichmentDto } from './dto/update-enrichment.dto';
@Injectable()
export class EnrichmentsService {
create(createEnrichmentDto: CreateEnrichmentDto) {
return 'This action adds a new enrichment';
}
list() {
return `This action returns all enrichment`;
}
show(id: string) {
return `This action returns a #${id} enrichment`;
}
update(id: string, updateEnrichmentDto: UpdateEnrichmentDto) {
return `This action updates a #${id} enrichment`;
}
remove(id: string) {
return `This action removes a #${id} enrichment`;
}
}
@@ -1,8 +0,0 @@
export class Enrichment {
id:string;
type:string;
columns:string[]
}
-43
View File
@@ -1,43 +0,0 @@
syntax = "proto3";
package enrichment;
import "google/protobuf/timestamp.proto";
service EnrichmentService {
rpc Create(CreateEnrichmentRequest) returns (Enrichment) {}
rpc Show(IdRequest) returns (Enrichment) {}
rpc List(Empty) returns (ListResponse){}
rpc Update(UpdateEnrichmentRequest) returns (Enrichment){}
rpc Remove(IdRequest) returns (Empty){}
}
message Empty {}
message IdRequest{
string id = 1;
}
message UpdateEnrichmentRequest{
string id = 1;
string type = 2;
repeated string columns = 3;
}
message Enrichment {
string id = 1;
string type = 2;
repeated string columns = 3;
}
message CreateEnrichmentRequest {
string type = 1;
repeated string columns = 2;
}
message ListResponse {
repeated Enrichment enrichments = 2;
}
@@ -0,0 +1,22 @@
import { ExceptionFilter, Catch, ArgumentsHost } from '@nestjs/common';
import { Response } from 'express';
import ErrorBuilder from '../utils/ErrorBuilder';
@Catch(Error)
export class GrpcToHttpExceptionFilter implements ExceptionFilter {
catch(exception: any, host: ArgumentsHost) {
const ctx = host.switchToHttp();
const response = ctx.getResponse<Response>();
// return this exception directly if is already of type ErrorBuilder,
// else transform it using the ErrorBuilder
const err =
exception.constructor.name === ErrorBuilder.name
? exception
: new ErrorBuilder(exception.details);
const { statusCode } = err.response;
return response.status(statusCode).json(err.response);
}
}
+24
View File
@@ -0,0 +1,24 @@
import {
ExceptionFilter,
Catch,
ArgumentsHost,
HttpException,
} from '@nestjs/common';
import { Request, Response } from 'express';
@Catch(HttpException)
export class HttpExceptionFilter implements ExceptionFilter {
catch(exception: HttpException, host: ArgumentsHost) {
const ctx = host.switchToHttp();
const response = ctx.getResponse<Response>();
const request = ctx.getRequest<Request>();
const status = exception.getStatus();
response.status(status).json({
status_code: status,
timestamp: new Date().toISOString(),
path: request.url,
method: request.method,
});
}
}
-1
View File
@@ -1 +0,0 @@
export class Health {}
-12
View File
@@ -1,12 +0,0 @@
import { Controller, Get } from '@nestjs/common';
import { HealthService } from './health.service';
@Controller('health')
export class HealthController {
constructor(private readonly healthService: HealthService) {}
@Get()
check() {
return this.healthService.check();
}
}
-9
View File
@@ -1,9 +0,0 @@
import { Module } from '@nestjs/common';
import { HealthService } from './health.service';
import { HealthController } from './health.controller';
@Module({
controllers: [HealthController],
providers: [HealthService],
})
export class HealthModule {}
-8
View File
@@ -1,8 +0,0 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class HealthService {
check() {
return { message: 'Ok' };
}
}
-17
View File
@@ -1,17 +0,0 @@
import { join } from 'path';
import { ClientOptions, Transport } from '@nestjs/microservices';
export const GrpcClientConfiguration: ClientOptions = {
transport: Transport.GRPC,
options: {
//url: `${process.env.USERS_SVC_URL}:${process.env.USERS_SVC_PORT}`,
url: `127.0.0.1:50052`,
package: 'input',
protoPath: join(__dirname, '..', 'proto', 'input.proto'),
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
@@ -1,13 +0,0 @@
export class CreateInputGrpcClientDto {
cron:string;
name:string;
plugin:string;
values:string;
operation:string;
}
@@ -1,5 +0,0 @@
export class TestConnectionDTO{
values:string;
plugin:string;
}
@@ -1,6 +0,0 @@
import { PartialType } from '@nestjs/mapped-types';
import { CreateInputGrpcClientDto } from './create-input-grpc-client.dto';
export class UpdateInputGrpcClientDto extends PartialType(
CreateInputGrpcClientDto,
) {}
@@ -1,14 +0,0 @@
export class InputGrpcClient {
id:string;
cron:string;
name:string;
plugin:string;
values:string;
operation:string;
}
@@ -1,63 +0,0 @@
import {
Get,
Post,
Controller,
Delete,
OnModuleInit,
Put,
Param
} from '@nestjs/common';
import { Client, ClientGrpc, Payload } from '@nestjs/microservices';
import { CreateInputGrpcClientDto } from './dto/create-input-grpc-client.dto';
import { UpdateInputGrpcClientDto } from './dto/update-input-grpc-client.dto';
import { InputGrpcClientService } from './input-grpc-client.service';
import { GrpcClientConfiguration } from './config/grpc-client';
import { TestConnectionDTO } from './dto/test-connection-grpc-client.dto';
import { ApiTags } from '@nestjs/swagger';
@Controller('grpc/input')
export class InputGrpcClientController implements OnModuleInit {
@Client(GrpcClientConfiguration)
private inputServiceClient: ClientGrpc;
private inputService: InputGrpcClientService;
onModuleInit() {
this.inputService =
this.inputServiceClient.getService<InputGrpcClientService>('InputService');
}
@Post()
async create(@Payload() createInputDto: CreateInputGrpcClientDto) {
return this.inputService.create(createInputDto);
}
@Get()
list() {
return this.inputService.list();
}
@Get(':id')
show(@Param() id: string) {
return this.inputService.show(id);
}
@Put(':id')
update(
@Payload() updateInputDto: UpdateInputGrpcClientDto,
@Param() id: string,
) {
return this.inputService.update(id, updateInputDto);
}
@Delete(':id')
remove(@Param() id: string) {
return this.inputService.remove(id);
}
@Get('/test-connection')
testConnection(@Payload() testConnectionDTO:TestConnectionDTO){
return this.inputService.testConnection(testConnectionDTO)
}
}
-13
View File
@@ -1,13 +0,0 @@
import { Module } from '@nestjs/common';
import { InputGrpcClientService } from './input-grpc-client.service';
import { InputGrpcClientController } from './input-grpc-client.controller';
// import { InputRestClientController } from './input-rest-client.controller';
@Module({
controllers: [
InputGrpcClientController,
// InputRestClientController
],
providers: [InputGrpcClientService],
})
export class InputGrpcClientModule {}
@@ -1,18 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { InputGrpcClientService } from './input-grpc-client.service';
describe('InputGrcpClientService', () => {
let service: InputGrpcClientService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [InputGrpcClientService],
}).compile();
service = module.get<InputGrpcClientService>(InputGrpcClientService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
-31
View File
@@ -1,31 +0,0 @@
import { Injectable } from '@nestjs/common';
import { CreateInputGrpcClientDto } from './dto/create-input-grpc-client.dto';
import { TestConnectionDTO } from './dto/test-connection-grpc-client.dto';
import { UpdateInputGrpcClientDto } from './dto/update-input-grpc-client.dto';
@Injectable()
export class InputGrpcClientService {
create(createInputGrcpClientDto: CreateInputGrpcClientDto) {
return 'This action adds a new InputGrcpClient';
}
list() {
return `This action returns all InputGrcpClient`;
}
show(id: string) {
return `This action returns a #${id} InputGrcpClient`;
}
update(id: string, updateInputGrcpClientDto: UpdateInputGrpcClientDto) {
return `This action updates a #${id} InputGrcpClient`;
}
remove(id: string) {
return `This action removes a #${id} InputGrcpClient`;
}
testConnection(testConnectionDTO:TestConnectionDTO){
return `This action tests a given connection`;
}
}
@@ -1,71 +0,0 @@
import { Body, Controller, Delete, Get, Param, Post, Put, Redirect } from '@nestjs/common';
import { Payload } from '@nestjs/microservices';
import axios from 'axios';
import { Console } from 'console';
import { CreateInputGrpcClientDto } from './dto/create-input-grpc-client.dto';
import { UpdateInputGrpcClientDto } from './dto/update-input-grpc-client.dto';
@Controller('input')
export class InputRestClientController {
@Get('/test-connection/')
async testConnection(){
console.log(process.env.DEV_URL+`/test-connection`, "ON TEST CONNECTION ROUTE")
await axios.get(process.env.DEV_URL+`/test-connection`).then( (response) => {
return response
})
}
@Post()
async create(@Body() createInputDto: CreateInputGrpcClientDto){
console.log(createInputDto)
console.log(process.env.DEV_URL+`/input`, "ON CREATE ROUTE")
await axios.post(process.env.DEV_URL+'/input',createInputDto).then( (response) => {
return response
})
}
@Get()
async list(){
console.log(process.env.DEV_URL+`/input`, "ON GET ROUTE")
await axios.get(process.env.DEV_URL+`/input`).then( (response) => {
return response
})
}
@Get('/:id')
async show(@Param() id: string){
console.log(process.env.DEV_URL+`/input/${id}`, "ON SHOW ROUTE")
await axios.get(process.env.DEV_URL+`/input/${id}`).then( (response) => {
return response
})
}
@Put(':id')
async update(
@Payload() updateInputDto: UpdateInputGrpcClientDto,
@Param() id: string,
){
console.log(process.env.DEV_URL+`/input/${id}`, "ON UPDATE ROUTE")
await axios.put(process.env.DEV_URL+`input/${id}`,updateInputDto).then( (response) => {
return response
})
}
@Delete(':id')
async delete(@Param() id: string){
console.log(process.env.DEV_URL+`/input/${id}`, "ON DELETE ROUTE")
await axios.delete(process.env.DEV_URL+`/input/${id}`).then( (response) => {
return response
})
}
}
-65
View File
@@ -1,65 +0,0 @@
syntax = "proto3";
package input;
import "google/protobuf/timestamp.proto";
service InputService {
rpc Create(CreateInputRequest) returns (Input) {}
rpc Show(IdRequest) returns (Input) {}
rpc List(Empty) returns (ListResponse){}
rpc Update(UpdateInputRequest) returns (Input){}
rpc Remove(IdRequest) returns (Empty){}
rpc TestConnection(TestConnectionRequest) returns (TestConnectionResponse){}
}
message Empty {}
message IdRequest{
string id = 1;
}
message UpdateInputRequest{
string id = 1;
string cron = 2;
string name = 3;
string plugin = 4;
string values = 5;
string operation = 6;
}
message Input {
string id = 1;
string cronFormated = 2;
string name = 3;
string plugin = 4;
string values = 5;
string operation = 6;
}
message CreateInputRequest {
string cron = 1;
string name = 2;
string plugin = 3;
string values = 4;
string operation = 5;
}
message ListResponse {
repeated Input inputs = 1;
}
message TestConnectionRequest{
string values = 1;
string plugin = 2;
}
message TestConnectionResponse{
// nao sei como vai ser a resposta
bool respose = 1;
}
+34 -7
View File
@@ -1,23 +1,50 @@
if (process.env.ENV !== 'local') require('elastic-apm-node/start');
import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import { writeFileSync } from 'fs';
import helmet from 'helmet';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import documentEmpty from '../swagger_empty.json';
import { AppModule } from './app.module';
import { config } from 'dotenv';
config();
async function bootstrap() {
const app = await NestFactory.create(AppModule);
DadosferaLogger.setupLogger({
serviceName: 'maestro',
serviceEnvironment: process.env.ENV,
});
const logger = new DadosferaLogger();
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: '*',
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
},
});
app.use(helmet());
const config = new DocumentBuilder()
.setTitle('Maestro Grpc Documentation')
.setTitle('Maestro')
.setDescription('Documentation for Maestro gateway')
.setVersion('1.0')
.addBearerAuth()
.addServer(
process.env.ENV === 'local'
? 'http://127.0.0.1:3333'
: 'https://maestro.dev.dadosfera.ai',
)
.build();
const document = SwaggerModule.createDocument(app, config);
writeFileSync('./swagger.json', JSON.stringify(document));
SwaggerModule.setup('api', app, document);
if (process.env.ENV != 'stg' && process.env.ENV != 'prd')
SwaggerModule.setup('api', app, document);
const swaggerDoc = process.env.ENV === 'dev' ? document : documentEmpty;
SwaggerModule.setup('api', app, swaggerDoc);
await app.listen(3333);
}
-30
View File
@@ -1,30 +0,0 @@
import { Request, Response, NextFunction } from 'express';
import axios from 'axios';
import { NestMiddleware } from '@nestjs/common';
export class LoggerMiddleware implements NestMiddleware{
use = async (
request: Request,
response: Response,
next: NextFunction
) => {
try {
const verify_response = await axios.post(
`${process.env.DEV_URL}/user/verify`,
{ "token":request.get('Authorization')}
).then(response =>{
if(response.data.message == "Success"){
next()
}
})
} catch (error) {
response.status(401).send({ message: 'Authentication Failed!' });
}
};
}
+162
View File
@@ -0,0 +1,162 @@
import {
Body,
Controller,
Headers,
Post,
HttpCode,
HttpStatus,
Inject,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
AuthSignInRequest,
AuthRefreshAccessTokenRequest,
AuthChangePasswordRequest,
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { AuthClientService } from './auth.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
@ApiTags('Auth')
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('auth')
export class AuthController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
@Post('sign-in')
@HttpCode(HttpStatus.OK)
async signIn(@Body() { username, password, totp }: AuthSignInRequest) {
this.logger.info('/auth - SignIn');
return this.authClient.signIn({ username, password, totp });
}
@Post('refresh-access-token')
@HttpCode(HttpStatus.OK)
async refreshAccessToken(
@Body() { refreshToken }: AuthRefreshAccessTokenRequest,
) {
this.logger.info('/auth - RefreshAccessToken');
return this.authClient.refreshAccessToken({ refreshToken });
}
@Post('change-password')
@HttpCode(HttpStatus.OK)
async changePassword(
@Body() body: AuthChangePasswordRequest,
@Headers() headers,
) {
this.logger.info('/auth - change-password');
const { oldPassword, newPassword } = body;
const { authorization: accessToken } = headers;
return this.authClient.changePassword({
accessToken,
oldPassword,
newPassword,
});
}
@Post('reset-password')
@HttpCode(HttpStatus.OK)
async resetPassword(@Body() body: AuthResetPasswordRequest) {
this.logger.info('/auth - reset-password');
const { username } = body;
return this.authClient.resetPassword({ username });
}
@Post('verify-reset-password-code')
@HttpCode(HttpStatus.OK)
async verifyResetPasswordCode(
@Body() body: AuthVerifyResetPasswordCodeRequest,
) {
this.logger.info('/auth - verify-reset-password-code');
const { username, code } = body;
return this.authClient.verifyResetPasswordCode({ username, code });
}
@Post('confirm-reset-password')
@HttpCode(HttpStatus.OK)
async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
this.logger.info('/auth - confirm-reset-password');
const { username, code, newPassword } = body;
return this.authClient.confirmResetPassword({
username,
code,
newPassword,
});
}
@Post('enable-totp')
@HttpCode(HttpStatus.OK)
async enableTotpMFA(
@Body() body: AuthEnableTotpMfaRequest,
@Headers() headers,
) {
this.logger.info('/auth - enable-totp');
const { password } = body;
const { authorization: accessToken } = headers;
return this.authClient.enableTotpMFA({ accessToken, password });
}
@Post('disable-totp')
@HttpCode(HttpStatus.OK)
async disableTotpMFA(
@Body() body: AuthDisableTotpMfaRequest,
@Headers() headers,
) {
this.logger.info('/auth - disable-totp');
const { password } = body;
const { authorization: accessToken } = headers;
return this.authClient.disableTotpMFA({ accessToken, password });
}
@Post('dismiss-totp')
@HttpCode(HttpStatus.OK)
async dismissTotpMFA(@Headers() headers) {
this.logger.info('/auth - disable-totp');
const { authorization: accessToken } = headers;
return this.authClient.dismissTotpMFA({ accessToken });
}
@Post('verify-totp')
@HttpCode(HttpStatus.OK)
async verifyTotp(@Body() body: AuthVerifyTotpMfaRequest, @Headers() headers) {
this.logger.info('/auth - enable-totp');
const { totp } = body;
const { authorization: accessToken } = headers;
return this.authClient.verifyTotp({ accessToken, totp });
}
}
+24
View File
@@ -0,0 +1,24 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { AuthController } from './auth.controller';
import { AuthClientService } from './auth.service';
import { DucClient } from '../../clients/duc/client.config';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([
{
name: 'DUC_PACKAGE',
...ducClient.config(),
},
]),
],
controllers: [AuthController],
providers: [AuthClientService, DadosferaLogger],
exports: [AuthClientService],
})
export class AuthModule {}
+140
View File
@@ -0,0 +1,140 @@
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSignInRequest,
AuthRefreshAccessTokenRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthDismissTotpMfaRequest,
AuthVerifyTotpMfaRequest,
AuthChangePasswordRequest,
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('DUC_PACKAGE') private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
}
async getPublicKeys() {
this.logger.info('GetPublicKeys');
return lastValueFrom(this.authService.AuthGetPublicKeys({}));
}
async signIn({ username, password, totp }: AuthSignInRequest) {
this.logger.info('SignIn');
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }),
);
}
async refreshAccessToken({ refreshToken }: AuthRefreshAccessTokenRequest) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken }),
);
}
async changePassword({
accessToken,
oldPassword,
newPassword,
}: AuthChangePasswordRequest) {
this.logger.info('ChangePassword');
return lastValueFrom(
this.authService.AuthChangePassword({
accessToken,
oldPassword,
newPassword,
}),
);
}
async resetPassword({ username }: AuthResetPasswordRequest) {
this.logger.info('resetPassword');
return lastValueFrom(this.authService.AuthResetPassword({ username }));
}
async verifyResetPasswordCode({
username,
code,
}: AuthVerifyResetPasswordCodeRequest) {
this.logger.info('verifyResetPasswordCode');
return lastValueFrom(
this.authService.AuthVerifyResetPasswordCode({ username, code }),
);
}
async confirmResetPassword({
username,
code,
newPassword,
}: AuthConfirmResetPasswordRequest) {
this.logger.info('confirmResetPassword');
return lastValueFrom(
this.authService.AuthConfirmResetPassword({
username,
code,
newPassword,
}),
);
}
async enableTotpMFA({ accessToken, password }: AuthEnableTotpMfaRequest) {
this.logger.info('enableTotpMFA');
return lastValueFrom(
this.authService.AuthEnableTotpMfa({ accessToken, password }),
);
}
async disableTotpMFA({ accessToken, password }: AuthDisableTotpMfaRequest) {
this.logger.info('disableTotpMFA');
return lastValueFrom(
this.authService.AuthDisableTotpMfa({ accessToken, password }),
);
}
async dismissTotpMFA({ accessToken }: AuthDismissTotpMfaRequest) {
this.logger.info('dismissTotpMFA');
return lastValueFrom(this.authService.AuthDismissTotpMfa({ accessToken }));
}
async verifyTotp({ accessToken, totp }: AuthVerifyTotpMfaRequest) {
this.logger.info('disableTotpMFA');
return lastValueFrom(
this.authService.AuthVerifyTotpMfa({ accessToken, totp }),
);
}
}
+12
View File
@@ -0,0 +1,12 @@
import { ApiProperty } from '@nestjs/swagger';
export class LoginDto {
@ApiProperty()
username: string;
@ApiProperty()
password: string;
@ApiProperty({ required: false })
totp: string;
}
+422
View File
@@ -0,0 +1,422 @@
import {
Body,
Controller,
Delete,
Get,
Inject,
Param,
Post,
Query,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { AuthenticateCondition } from '../../authentication/authentication.decorator';
import { PERMISSIONS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@ApiTags('Catalog')
@Controller('catalog')
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS.CATALOG.permissions[action].seqid,
);
})
export class CatalogController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogService: CatalogService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('all')
async catalogAll(@Body() body) {
this.logger.info(`/catalog - ON CATALOG ALL ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.catalogAll(body);
return res;
}
@Get('data_apps')
async dataAppsAll(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL DATA APPS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.dataAppsAll(body);
return res;
}
@Delete('data_apps/:id')
async dataAppsOne(@Param() params, @Body() body) {
this.logger.info(`/catalog - ON FIND ONE DATA APP ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { id } = params;
const res = await this.catalogService.dataAppsOne(id, body);
return res;
}
@Get('dashboard-metabase')
async getAllDashboardMetabase(@Body() body) {
this.logger.info(`/catalog - ON GET ALL DASHBOARDS METABASE ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getAllDashboardMetabase(body);
return res;
}
@Get('dashboard-metabase/:id')
async getOneDashboardMetabase(@Body() body, @Param() params) {
const { id } = params;
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getOneDashboardMetabase(id, body);
return res;
}
@Get('table-metadata')
async getAllTableMetadata(@Body() body, @Query() query) {
this.logger.info(`/catalog - ON GET ALL TABLES METADATA ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
if (!query) {
const res = await this.catalogService.getAllTableMetadata(body);
return res;
} else {
const res = await this.catalogService.getOneTableMetadata(body, query);
return res;
}
}
@Delete('table-metadata/:id')
async deleteOneTableMetadata(@Param() params, @Body() body) {
const { id } = params;
this.logger.info(`/catalog - ON DELETE ONE TABLE METADATA ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.deleteOneTableMetadata(id, body);
return res;
}
@Get('column-metadata')
async getOneColumnMetadata(@Body() body, @Query() params) {
this.logger.info(`/catalog - ON GET ONE COLUMN METADATA ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getOneColumnMetadata(body, params);
return res;
}
@Delete('column-metadata/:id')
async deleteOneColumnMetadata(@Param() params, @Body() body) {
this.logger.info(`/catalog - ON DELETE ONE COLUMN METADATA ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { id } = params;
const res = await this.catalogService.deleteOneColumnMetadata(id, body);
return res;
}
@Get('data-preview')
async getOneDataPreview(@Body() body, @Query() params) {
this.logger.info(`/catalog - ON GET ONE DATAPREVIEW ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getOneDataPreview(body, params);
return res;
}
@Get('data-status')
async getDataStatus(@Body() body) {
this.logger.info(`/catalog - ON GET DATA STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getDataStatus(body);
return res;
}
@Get('data-status')
async createDataStatus(@Body() body) {
this.logger.info(`/catalog - ON CREATE DATA STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createDataStatus(body);
return res;
}
@Get('data-description')
async getDataDescription(@Body() body) {
this.logger.info(`/catalog - ON GET DATA DESCRIPTION ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getDataDescription(body);
return res;
}
@Post('data-description')
async createDataDescription(@Body() body) {
this.logger.info(`/catalog - ON CREATE DATA DESCRIPTION ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createDataDescription(body);
return res;
}
@Get('data-docs')
async getDataDocs(@Body() body) {
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getDataDocs(body);
return res;
}
@Post('data-docs')
async createDataDocs(@Body() body) {
this.logger.info(`/catalog - ON CREATE DATA DOCS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createDataDocs(body);
return res;
}
@Get('data-rating')
async getDataRating(@Body() body) {
this.logger.info(`/catalog - ON GET DATA RATING ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getDataRating(body);
return res;
}
@Post('data-rating')
async createDataRating(@Body() body) {
this.logger.info(`/catalog - ON GET DATA RATING ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createDataRating(body);
return res;
}
@Get('summary-rating/:id')
async getSummaryRating() {
this.logger.info(`/catalog - ON GET SUMMARY RATING ROUTE`);
//const { id } = params;
//
//const res = await this.catalogService.getSummaryRating(id, body);
return {
avg_rating: 0,
rating_count: 0,
};
}
@Get('data-comment')
async getDataComment(@Body() body) {
this.logger.info(`/catalog - ON GET DATA COMMENT ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getDataComment(body);
return res;
}
@Post('data-comment')
async createDataComment(@Body() body) {
this.logger.info(`/catalog - ON GET DATA COMMENT ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createDataComment(body);
return res;
}
@Get('data-review')
async getDataReview(@Body() body, @Query() params) {
this.logger.info(`/catalog - ON GET DATA REVIEW ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getDataReview(body, params);
return res;
}
@Post('tags')
async createTags(@Body() body) {
this.logger.info(`/catalog - ON CREATE TAG ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createTags(body);
return res;
}
@Get('tags')
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.findAllTags(body);
return res;
}
@Delete('tags/:id')
async deleteTags(@Param() params, @Body() body) {
this.logger.info(`/catalog - ON DELETE TAG ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { id } = params;
const res = await this.catalogService.deleteTags(id, body);
return res;
}
@Post('table-tags')
async createTableTags(@Body() body) {
this.logger.info(`/catalog - ON CREATE TABLE TAG ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.createTableTags(body);
return res;
}
@Get('table-tags')
async getAllTableTags(@Body() body) {
this.logger.info(`/catalog - ON GET ALL TABLE TAGS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.findAllTableTags(body);
return res;
}
@Delete('table-tags/:id')
async deleteTableTags(@Param() params, @Body() body) {
this.logger.info(`/catalog - ON CREATE TABLE TAGS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { id } = params;
const res = await this.catalogService.deleteTableTags(id, body);
return res;
}
@Get('table-rules')
async getTableRules(@Body() body) {
this.logger.info(`/catalog - ON GET TABLE RULES ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.getTableRules(body);
return res;
}
@Delete('table-rules/:id')
async deleteTableRules(@Param() params, @Body() body) {
this.logger.info(`/catalog - ON DELETE TABLE RULES ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { id } = params;
const res = await this.catalogService.deleteTableRules(id, body);
return res;
}
}
+262
View File
@@ -0,0 +1,262 @@
import axios from 'axios';
class CatalogService {
_getNimbusUrl(body) {
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
return `https://nimbus-${customer}.dadosfera.ai`;
}
return `https://nimbus-${customer}.${process.env.ENV}.dadosfera.ai`;
}
async catalogAll(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/all/`);
return data;
}
async dataAppsAll(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data_apps/`);
return data;
}
async dataAppsOne(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/data_apps/${id}/`,
);
return data;
}
async getAllDashboardMetabase(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/dashboard-metabase/`,
);
return data;
}
async getOneDashboardMetabase(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/dashboard-metabase/${id}`,
);
// const [, path] = data.iframe_url.split('.dadosfera');
// const host = `httpss://metabase-${body.info.customer.toLowerCase()}.dadosfera.ai`;
// data.iframe_url = host + path;
return data;
}
async getAllTableMetadata(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-metadata/`,
);
return data;
}
async getOneTableMetadata(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-metadata/`,
{ params: params },
);
return data;
}
async deleteOneTableMetadata(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-metadata/${id}`,
);
return data;
}
async getOneColumnMetadata(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/column-metadata/`,
{ params: params },
);
return data;
}
async deleteOneColumnMetadata(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/column-metadata/${id}/`,
);
return data;
}
async getOneDataPreview(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-preview/`, {
params: params,
});
return data;
}
async getDataStatus(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-status/`);
return data;
}
async createDataStatus(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-status/`,
body,
);
return data;
}
async getDataDescription(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/data-description/`,
);
return data;
}
async createDataDescription(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-description/`,
body,
);
return data;
}
async getDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-docs/`);
return data;
}
async createDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-docs/`,
body,
);
return data;
}
async getDataRating(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-rating/`);
return data;
}
async createDataRating(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-rating/`,
body,
);
return data;
}
async getSummaryRating(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/summary-rating/${id}`,
);
return data;
}
async getDataComment(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-comment/`);
return data;
}
async createDataComment(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-comment/`,
body,
);
return data;
}
async getDataReview(body, params) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/data-review/`, {
params: params,
});
return data;
}
async createTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(`${nimbusUrl}/api/catalog/tags/`, body);
return data;
}
async findAllTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(`${nimbusUrl}/api/catalog/tags/`, body);
return data;
}
async findAllTableTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-tags/`,
body,
);
return data;
}
async deleteTags(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(`${nimbusUrl}/api/catalog/tags/${id}`);
return data;
}
async createTableTags(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/table-tags/`,
body,
);
return data;
}
async deleteTableTags(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-tags/${id}`,
);
return data;
}
async getTableRules(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
`${nimbusUrl}/api/catalog/table-rules/`,
body,
);
return data;
}
async deleteTableRules(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.delete(
`${nimbusUrl}/api/catalog/table-rules/${id}`,
);
return data;
}
}
export { CatalogService };
+31
View File
@@ -0,0 +1,31 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { ConnectorManager } from '@dadosfera/protospack-v2';
export class ConnectorClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: [
ConnectorManager.ProtoPackages.WritePackage,
ConnectorManager.ProtoPackages.ReadPackage,
],
// credentials: undefined,
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: [
ConnectorManager.ProtoPaths.WriteFilePath,
ConnectorManager.ProtoPaths.ReadFilePath,
],
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
+193
View File
@@ -0,0 +1,193 @@
import { Metadata } from '@grpc/grpc-js';
import {
OnModuleInit,
Inject,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { ConnectorManager } from '@dadosfera/protospack-v2';
import { lastValueFrom } from 'rxjs';
export class ConnectorClientService implements OnModuleInit {
private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices;
private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices;
constructor(
@Inject('CONNECTOR_PACKAGE') private readonly grpcClient: ClientGrpc,
) {}
onModuleInit() {
this.connectorServiceWrite =
this.grpcClient.getService<ConnectorManager.WriteService.ConnectorManagerWriteServices>(
ConnectorManager.ProtoServices.ConnectorManagerWriteServices,
);
this.connectorServiceRead =
this.grpcClient.getService<ConnectorManager.ReadService.ConnectorManagerReadServices>(
ConnectorManager.ProtoServices.ConnectorManagerReadServices,
);
}
async uploadConnector(uploadConnector) {
const connector: ConnectorManager.Entities.ConnectorCreateRequest = {
file: {
buffer: uploadConnector.file.buffer,
mimetypes: uploadConnector.file.mimetype,
},
connector: JSON.stringify(uploadConnector.connector),
};
const serviceBody: ConnectorManager.Messages.RegisterConnectorRequest = {
connector,
};
return lastValueFrom(
this.connectorServiceWrite.RegisterConnector(serviceBody),
).catch((err) => {
console.log(err.details);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async uploadFile(uploadFile) {
const body: ConnectorManager.Messages.UploadFileRequest = {
file: {
buffer: uploadFile.file.buffer,
mimetypes: uploadFile.file.mimetype,
},
name: uploadFile.name,
};
return lastValueFrom(this.connectorServiceWrite.UploadFile(body)).catch(
(err) => {
console.log(err.details);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
},
);
}
async uploadConnectorsFile(uploadFile) {
const connectors = JSON.parse(uploadFile.file.buffer.toString('utf8'));
const body = connectors.map((connector) => {
return JSON.stringify(connector);
});
const request = {
connectors: body,
};
return lastValueFrom(
this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage(
request,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async getAllConnectors(body) {
const meta = new Metadata();
meta.add('Language', body.language);
return lastValueFrom(
this.connectorServiceRead.GetAllConnectors(
{
search: body.search,
filters: JSON.stringify(body.filters),
language: body.language,
page: body.page,
size: body.size,
},
meta,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async getConnectorDetails(plugin: string, language: string) {
const meta = new Metadata();
meta.add('Language', language);
return lastValueFrom(
this.connectorServiceRead.GetConnectorDetails(
{
plugin,
},
meta,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async getConnector(plugin: string, language: string) {
const meta = new Metadata();
meta.add('Language', language);
return lastValueFrom(
this.connectorServiceRead.GetConnector(
{
plugin,
},
meta,
),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async getConnectorsTags() {
return lastValueFrom(this.connectorServiceRead.GetConnectorsTags({})).catch(
(err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
},
);
}
async deleteConnector(plugin: string, version: string) {
return lastValueFrom(
this.connectorServiceWrite.DeleteConnector({
version,
plugin,
}),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async updateConnector({ plugin, changes }) {
return lastValueFrom(
this.connectorServiceWrite.UpdateConnector({
plugin,
changes,
}),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
}
@@ -0,0 +1,268 @@
import {
Body,
Controller,
Delete,
Get,
HttpException,
Param,
Put,
Post,
Query,
UploadedFile,
UseInterceptors,
Inject,
Headers,
UploadedFiles,
} from '@nestjs/common';
import { FileInterceptor, FilesInterceptor } from '@nestjs/platform-express';
import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ConnectorClientService } from './client.service';
import { AddTagDto } from './dtos/add-tag';
import { CreateConnectorDto } from './dtos/create-connector';
import { UploadConnectorsFileDto } from './dtos/create-multiple-connector';
import { DeleteConnectorDto } from './dtos/delete-connector';
import { GetAllDto } from './dtos/get-all';
import { RemoveTagDto } from './dtos/remove-tag';
import { UpdateDto } from './dtos/update';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
import { PERMISSIONS } from 'src/authentication/permissions.enum';
@ApiTags('connectors')
@ApiBearerAuth()
@Authenticated()
@Controller('connectors')
export class ConnectorController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private connectorClientService: ConnectorClientService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.CREATE)
@ApiConsumes('multipart/form-data')
@UseInterceptors(FileInterceptor('file'))
async uploadConnector(
@UploadedFile() file,
@Body() body: CreateConnectorDto,
) {
this.logger.info('/upload - Upload Connector Route');
const response = await this.connectorClientService.uploadConnector({
file,
connector: JSON.parse(body.connector),
});
return response;
}
@Post('/uploads')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.CREATE)
@ApiConsumes('multipart/form-data')
@UseInterceptors(FilesInterceptor('files'))
async uploadFile(@UploadedFiles() files) {
this.logger.info('/upload - Upload Connector Route');
const promises = files.map(async (file) => {
return await this.connectorClientService.uploadFile({
file,
name: file.originalname.split('.')[0],
});
});
const response = await Promise.all(promises);
return response;
}
@Post('/multiple-connectors-upload')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.CREATE)
@ApiConsumes('multipart/form-data')
@UseInterceptors(FileInterceptor('file'))
async uploadConnectorsFile(@UploadedFile() file: UploadConnectorsFileDto) {
console.log(`/upload`, 'Upload Connector Route');
const response = await this.connectorClientService.uploadConnectorsFile({
file,
});
return response;
}
@Get()
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
async getAllConnectors(
@Headers('dadosfera-lang') language,
@Query() queries: GetAllDto,
) {
this.logger.info('/upload - Upload Connector Route');
const { search, size, page, ...filters } = queries;
const response: any = await this.connectorClientService.getAllConnectors({
filters: filters || {},
search: search || '',
language: language || 'pt-br',
size: size || 10,
page: page || 1,
});
const connectors = JSON.parse(response.connectors).connectors;
return {
message: response.message,
connectors,
};
}
@Get('/tags')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
async getConnectorsTags() {
return await this.connectorClientService.getConnectorsTags();
}
@Get('/:plugin')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
async getConnector(
@Headers('dadosfera-lang') language,
@Param('plugin') plugin: string,
@Query('version') version: string,
) {
this.logger.info('/upload - Upload Connector Route');
const pluginId = `${plugin}-${version}`;
const response: any = await this.connectorClientService.getConnector(
pluginId,
language,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Get('/:plugin/details')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.GET)
async getConnectorDetails(
@Headers('dadosfera-lang') language,
@Param('plugin') plugin: string,
@Query('version') version: string,
) {
this.logger.info('/upload - Upload Connector Route');
const pluginId = `${plugin}-${version}`;
const response: any = await this.connectorClientService.getConnectorDetails(
pluginId,
language || 'pt-br',
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Put('/:plugin')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.UPDATE)
@ApiConsumes('multipart/form-data')
async updateConnector(
@Param('plugin') plugin: string,
@Body() body: UpdateDto,
) {
this.logger.info('/upload - Upload Connector Route');
const changes = body;
const response: any = await this.connectorClientService.updateConnector({
plugin,
changes: JSON.stringify(changes),
});
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Put('/:plugin/add-tag')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.UPDATE)
async addTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: AddTagDto,
) {
this.logger.info('/upload - Upload Connector Route');
const { tags } = body;
if (!tags || !(typeof tags === 'object') || !tags.length) {
throw new HttpException('Not found tags attributes', 400);
}
const changes = { plugin, tags };
const response: any = await this.connectorClientService.updateConnector({
plugin,
changes: JSON.stringify(changes),
});
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Put('/:plugin/remove-tag')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.UPDATE)
async removeTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: RemoveTagDto,
) {
this.logger.info('/upload - Upload Connector Route');
const { tags } = body;
const remove = tags;
if (!remove || !(typeof remove === 'object') || !remove.length) {
throw new HttpException('Not found remove attribute', 400);
}
const changes = { plugin, remove };
const response: any = await this.connectorClientService.updateConnector({
plugin,
changes: JSON.stringify(changes),
});
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Delete('/:plugin')
@RequireAllPermissions(PERMISSIONS.CONNECTORS.permissions.DELETE)
async deleteConnector(
@Param('plugin') plugin: string,
@Body() { version }: DeleteConnectorDto,
) {
this.logger.info('/upload - Upload Connector Route');
const response: any = await this.connectorClientService.deleteConnector(
plugin,
version,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
}
+22
View File
@@ -0,0 +1,22 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { ConnectorClientService } from './client.service';
import { ConnectorClientConfiguration } from './client.config';
import { ConnectorController } from './connector.controller';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
const connectorClient = new ConnectorClientConfiguration();
@Module({
controllers: [ConnectorController],
providers: [ConnectorClientService, DadosferaLogger],
imports: [
ClientsModule.register([
{
name: 'CONNECTOR_PACKAGE',
...connectorClient.config(),
},
]),
],
})
export class ConnectorModule {}
+6
View File
@@ -0,0 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
export class AddTagDto {
@ApiProperty()
tags: string[];
}
@@ -0,0 +1,9 @@
import { ApiProperty } from '@nestjs/swagger';
export class CreateConnectorDto {
@ApiProperty({ format: 'binary' })
file: string;
@ApiProperty()
connector: string;
}
@@ -0,0 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
export class UploadConnectorsFileDto {
@ApiProperty({ format: 'binary' })
file: string;
}
@@ -0,0 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
export class DeleteConnectorDto {
@ApiProperty()
version: string;
}
+18
View File
@@ -0,0 +1,18 @@
import { ApiProperty } from '@nestjs/swagger';
export class GetAllDto {
@ApiProperty()
search: string;
@ApiProperty()
language: string;
@ApiProperty()
size: string;
@ApiProperty()
page: string;
@ApiProperty()
filters: any;
}
+6
View File
@@ -0,0 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
export class RemoveTagDto {
@ApiProperty()
tags: string[];
}
+42
View File
@@ -0,0 +1,42 @@
import { ApiProperty } from '@nestjs/swagger';
export class UpdateDto {
@ApiProperty({ format: 'binary', required: false })
file?: string;
@ApiProperty()
version: string;
@ApiProperty({ required: false })
category?: string;
@ApiProperty({ required: false })
plugin?: string;
@ApiProperty({ required: false })
image?: string;
@ApiProperty({ required: false })
docs?: string;
@ApiProperty({ required: false })
name?: string;
@ApiProperty({
required: false,
description: 'skip_select_entities: boolean skip_select_columns: boolean;',
})
options?: {
skip_select_entities: boolean;
skip_select_columns: boolean;
};
@ApiProperty({
required: false,
type: 'array',
items: {
properties: {},
},
})
connection_controls?: object[];
}
@@ -0,0 +1,9 @@
import { ApiProperty } from '@nestjs/swagger';
export class UploadFileDto {
@ApiProperty({ format: 'binary' })
file: string;
@ApiProperty()
name: string;
}
+22
View File
@@ -0,0 +1,22 @@
import { Controller, Get, Inject } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { HealthService } from './health.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@ApiTags('Health')
@Controller('health')
export class HealthController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly healthService: HealthService,
) {
this.logger = dadosferaLogger.logger;
}
@Get()
check() {
return this.healthService.check();
}
}
+16
View File
@@ -0,0 +1,16 @@
import { Inject, Injectable } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@Injectable()
export class HealthService {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
check() {
return { message: 'Ok' };
}
}
+215
View File
@@ -0,0 +1,215 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { Info } from 'src/clients/inputs/interfaces';
import { Method } from 'axios';
export class InputModel {
category: string;
plugin: string;
name: string;
cron?: string;
credentials?: {
connection_type: string;
oauth: string;
oauth_code: string;
start_date: string;
};
options?: {
oauth?: {
get_tokens_url: string;
get_tokens_method?: Method;
get_tokens_url_params: string;
get_tokens_set_response: Record<string, string>;
content_type: string;
};
};
}
class GoogleAnalyticsClientSecrets {
@ApiProperty()
type: string;
@ApiProperty()
project_id: string;
@ApiProperty()
private_key_id: string;
@ApiProperty()
private_key: string;
@ApiProperty()
client_email: string;
@ApiProperty()
client_id: string;
@ApiProperty()
auth_uri: string;
@ApiProperty()
token_uri: string;
@ApiProperty()
auth_provider_x509_cert_url: string;
@ApiProperty()
client_x509_cert_url: string;
}
class OauthObject {
@ApiPropertyOptional()
get_tokens_url: string;
@ApiPropertyOptional()
get_tokens_url_params: string;
@ApiPropertyOptional()
get_tokens_set_response: Record<string, any>;
@ApiPropertyOptional()
content_type: string;
}
class InputOptions {
@ApiPropertyOptional()
oauth: OauthObject;
@ApiPropertyOptional()
skip_select_columns: true;
@ApiPropertyOptional()
skip_select_entities: false;
@ApiPropertyOptional()
skip_transformation: true;
}
class CredentialsJdbc {
@ApiPropertyOptional()
jdbc_user: string;
@ApiPropertyOptional()
jdbc_password: string;
@ApiPropertyOptional()
database: string;
@ApiPropertyOptional()
endpoint: string;
@ApiPropertyOptional()
port: string;
@ApiPropertyOptional()
engine: string;
@ApiPropertyOptional()
schema: string;
}
class FileFormatParams {
@ApiPropertyOptional()
file_format: string;
@ApiPropertyOptional()
encoding: string;
}
class Credentials extends CredentialsJdbc {
@ApiProperty()
connection_type: string;
@ApiPropertyOptional()
client_aws_access_key_id: string;
@ApiPropertyOptional()
client_aws_secret_access_key: string;
@ApiPropertyOptional()
client_bucket: string;
@ApiPropertyOptional()
file_to_extract: string;
@ApiPropertyOptional()
file_format_params: FileFormatParams;
@ApiPropertyOptional()
view_id: string;
@ApiPropertyOptional()
client_secrets: GoogleAnalyticsClientSecrets;
@ApiPropertyOptional()
start_date: string;
@ApiPropertyOptional()
end_date: string;
@ApiPropertyOptional()
oauth_code: string;
}
class Column {
@ApiProperty()
name: string;
@ApiProperty()
type: string;
}
class TableColumns {
@ApiProperty()
name: string;
@ApiProperty()
columns: Column[];
@ApiProperty()
references: Column[];
}
export class GetAvailableEntitiesReq {
@ApiProperty()
plugin: string;
}
export class GetAvailableEntitiesRes {
@ApiProperty()
entities: string[];
}
export class TestConnectionGetColumnsReq {
@ApiProperty()
plugin: string;
@ApiProperty()
tables: string[];
@ApiPropertyOptional()
credentials: CredentialsJdbc;
@ApiPropertyOptional()
id: string;
@ApiPropertyOptional()
info: Info;
}
export class TestConnectionGetColumnsRes {
@ApiProperty()
tables: TableColumns[];
}
export class TestConnectionReq {
@ApiProperty()
plugin: string;
@ApiProperty()
credentials: CredentialsJdbc;
@ApiPropertyOptional()
info: Info;
}
export class TestConnectionRes {
@ApiProperty()
connection_state: boolean;
@ApiProperty()
total_entities: number;
@ApiProperty()
database_tables: string[];
}
export class CreateInputReq {
@ApiPropertyOptional()
id: string;
@ApiProperty()
plugin: string;
@ApiProperty()
category: string;
@ApiProperty()
credentials: Credentials;
@ApiProperty()
name: string;
@ApiPropertyOptional()
options: InputOptions;
@ApiPropertyOptional()
info: Info;
}
export class Input {
@ApiProperty()
id: string;
@ApiProperty()
category: string;
@ApiProperty()
plugin: string;
@ApiProperty()
name: string;
@ApiProperty()
cron: string;
@ApiProperty()
credentials: Credentials;
@ApiProperty()
client_id: string;
@ApiProperty()
created_at: string;
@ApiProperty()
updated_at: string;
@ApiPropertyOptional()
info: Info;
}
+189
View File
@@ -0,0 +1,189 @@
import {
Body,
Controller,
Delete,
Get,
Inject,
Param,
Patch,
Post,
} from '@nestjs/common';
import { InputsService } from './inputs.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { InputsClientService } from 'src/clients/inputs/client.service';
import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
import { PERMISSIONS } from '../../authentication/permissions.enum';
import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import {
CreateInputReq,
GetAvailableEntitiesReq,
GetAvailableEntitiesRes,
Input,
TestConnectionGetColumnsReq,
TestConnectionGetColumnsRes,
TestConnectionReq,
TestConnectionRes,
} from './dtos/input.model';
@ApiTags('inputs')
@Controller('inputs')
@AuthenticateCondition((req, user) => {
let action: any;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
case 'PATCH':
action = 'UPDATE';
break;
default:
action = req.method;
}
if (
`${req.method} ${req.route.path}` ===
'GET /inputs/available-entities/:plugin'
) {
action = 'GET_ENTITIES';
}
return user.permissions.includes(PERMISSIONS.INPUT.permissions[action].seqid);
})
export class InputsController {
inputService: InputsService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private inputsClientService: InputsClientService,
) {
this.logger = dadosferaLogger.logger;
this.inputService = new InputsService(this.inputsClientService);
}
@Get('available-entities/:plugin')
@ApiOkResponse({ type: GetAvailableEntitiesRes })
async getAvailableEntities(
@Body() body,
@Param() params: GetAvailableEntitiesReq,
) {
const { info } = body;
const { plugin } = params;
return await this.inputService.getAvailableEntities({ info, plugin });
}
@Post('/test-connection')
@ApiOkResponse({ type: TestConnectionRes })
async testConnection(@Body() data: TestConnectionReq) {
this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
});
const response = await this.inputService.testConnection(data);
return response;
}
@Post('/test-connection/get-columns')
@ApiOkResponse({ type: TestConnectionGetColumnsRes })
async getColumns(@Body() data: TestConnectionGetColumnsReq) {
this.logger.info(
`/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
{
user: data.info.user_id,
customer: data.info.customer,
},
);
const response = await this.inputService.getColumns(data);
return response;
}
@Post()
@ApiOkResponse({ type: Input })
async create(@Body() createInputDto: CreateInputReq) {
this.logger.info(`/input - ON CREATE ROUTE`, {
user: createInputDto.info.user_id,
customer: createInputDto.info.customer,
});
const response = await this.inputService.create(createInputDto);
return response;
}
@Post(':id')
@ApiOkResponse({ type: Input })
async reCreate(@Param('id') id: string, @Body() input: CreateInputReq) {
if (!input.id) input.id = id;
this.logger.info(`POST /${id}`, 'ON RECREATE ROUTE');
const response = await this.inputService.reCreate(input);
return response;
}
@Get()
async findAll(@Body() body) {
this.logger.info(`/input - ON FIND ALL ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const response = await this.inputService.findAll(body);
return response;
}
@Get('/:id')
async findOne(@Body() body, @Param() params) {
const { id } = params;
this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const response = await this.inputService.findOne({ id, ...body });
return response;
}
@Patch(':id')
async update(@Body() updateInputDto: UpdateInputRequest, @Param() params) {
const { id } = params;
const { info } = updateInputDto;
this.logger.info(`/input/${id} - ON UPDATE ROUTE`, {
user: info.user_id,
customer: info.customer_id,
});
delete updateInputDto.info;
const response = await this.inputService.update(id, updateInputDto, info);
return response;
}
@Delete(':id')
async delete(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(`/input/${id} - ON DELETE ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
});
const response = await this.inputService.remove({ id, ...data });
return response;
}
}
+195
View File
@@ -0,0 +1,195 @@
import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { Timeout } from '@nestjs/schedule';
import { DecodeGrpcStruct } from 'protospack';
import CronParser, { CronExpression } from 'cron-parser';
import { InputsClientService } from 'src/clients/inputs/client.service';
import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
@Injectable()
export class InputsService {
constructor(private inputClient: InputsClientService) {}
secondsInADay = 60 * 60 * 24;
secondsInAnHour = 60 * 60;
adjustInputPayload(payload) {
if (payload?.input_generic) return DecodeGrpcStruct(payload.input_generic);
if (!payload.input_s3 && !payload.input_jdbc) return payload;
return payload?.input_s3 || payload?.input_jdbc;
}
getDifferenceInSeconds(date1: Date, date2: Date) {
const diffInMs = Math.abs(date2.getTime() - date1.getTime());
return diffInMs / 1000;
}
validateCron(data) {
const { info, cron } = data;
const { customer_tier } = info;
if (!cron) return;
let interval: CronExpression;
try {
interval = CronParser.parseExpression(cron);
} catch (error) {
throw new HttpException(
'Intervalo de tempo inválido',
HttpStatus.BAD_REQUEST,
);
}
const nextDate = interval.next().toDate();
const afterNextDate = interval.next().toDate();
const secondsApart = this.getDifferenceInSeconds(nextDate, afterNextDate);
if (customer_tier === 'BASIC' && secondsApart < this.secondsInADay) {
throw new HttpException(
'Intervalo de tempo não pode ser inferior a um dia.',
HttpStatus.FORBIDDEN,
);
} else if (secondsApart < this.secondsInAnHour) {
throw new HttpException(
'Intervalo de tempo não pode ser inferior a uma hora.',
HttpStatus.FORBIDDEN,
);
}
}
async create(@Body() data) {
this.validateCron(data);
let response;
switch (data.plugin.toLowerCase()) {
case 'csv':
case 'json':
case 'parquet': {
const { info, ...input } = data;
const inputPayload = this.generateInputS3Payload(input);
response = await this.inputClient.createS3Inputs({
input: inputPayload,
info,
});
break;
}
case 'oracle':
case 'mysql':
case 'postgresql':
case 'sqlserver':
response = await this.inputClient.newCreate(data);
break;
default:
response = await this.inputClient.createGeneric(data);
break;
}
const adjustedInput = this.adjustInputPayload(response.input);
return { ...response, input: adjustedInput };
}
async reCreate(data) {
const { info, cron } = data;
if (cron) this.validateCron({ info, cron });
const response = await this.inputClient.createGeneric(data);
const adjustedInput = this.adjustInputPayload(response.input);
return { ...response, input: adjustedInput };
}
async getAvailableEntities(data): Promise<{ entities: string[] }> {
return await this.inputClient.getAvailableEntities(data);
}
async findAll(body) {
try {
const findAllInputResponse: any = await this.inputClient.findAll(body);
if (findAllInputResponse?.inputs?.length) {
findAllInputResponse.inputs = findAllInputResponse.inputs.map((input) =>
this.adjustInputPayload(input),
);
}
return findAllInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async findOne(idRequest: IIdRequest) {
try {
const findOneInputResponse: any = await this.inputClient.findOne(
idRequest,
);
findOneInputResponse.input = this.adjustInputPayload(
findOneInputResponse.input,
);
return findOneInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async update(id: string, data, info: Info) {
this.validateCron({ ...data, info });
try {
const updateInputResponse: any = await this.inputClient.update({
id,
info,
...data,
});
updateInputResponse.input = this.adjustInputPayload(
updateInputResponse?.input,
);
return updateInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async remove(idRequest: IIdRequest) {
try {
const removeInputResponse = await this.inputClient.remove(idRequest);
return removeInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
@Timeout(60000 * 10) // Timeout set for 10 minutes
async testConnection(data) {
try {
const testConnectionInputResponse = await this.inputClient.testConnection(
data,
);
return testConnectionInputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async getColumns(data) {
try {
const testConnectionGetColumnsResponse =
await this.inputClient.getColumns(data);
return testConnectionGetColumnsResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
generateInputS3Payload(payload) {
const { credentials, plugin, cron } = payload;
if (!credentials) return payload;
const {
client_aws_access_key_id,
client_aws_secret_access_key,
file_format_params,
format_file_params,
client_bucket,
file_to_extract,
} = credentials;
const formatedPayload = {
plugin,
cron,
source_bucket: client_bucket,
source_prefix: file_to_extract,
auth_parameters: {
aws_access_key_id: client_aws_access_key_id,
aws_secret_access_key: client_aws_secret_access_key,
},
file_format_params: file_format_params || format_file_params,
};
return formatedPayload;
}
}
+130
View File
@@ -0,0 +1,130 @@
import { Controller, Get, Redirect, Req, UseGuards } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { AuthGuard } from '@nestjs/passport';
import { InputsClientService } from 'src/clients/inputs/client.service';
import { InputsService } from '../inputs/inputs.service';
import { Request } from 'express';
@ApiTags('oauth')
@Controller('oauth')
export class OauthController {
inputService: InputsService;
frontendRedirectUri = '';
constructor(private inputsClientService: InputsClientService) {
this.inputService = new InputsService(this.inputsClientService);
switch (process.env.ENV) {
case 'dev':
case 'stg':
this.frontendRedirectUri = `https://app.${process.env.ENV}.dadosfera.ai/coletix/auth-callback`;
break;
case 'prd':
this.frontendRedirectUri = `https://app.dadosfera.ai/coletix/auth-callback`;
break;
default:
this.frontendRedirectUri = `http://localhost:4200/coletix/auth-callback`;
}
}
@Get('hubspot')
@UseGuards(AuthGuard('hubspot'))
async oauthHubspot() {
return true;
}
@Get('hubspot/callback')
@UseGuards(AuthGuard('hubspot'))
@Redirect()
async oauthHubspotCallback(@Req() req) {
return await this.callback(req, 'hubspot');
}
@Get('googleads')
@UseGuards(AuthGuard('google'))
async oauthGoogleAds() {
return true;
}
@Get('googleads/callback')
@UseGuards(AuthGuard('google'))
@Redirect()
async oauthGoogleAdsCallback(@Req() req) {
return await this.callback(req, 'googleads');
}
@Get('google-sheets')
@UseGuards(AuthGuard('google'))
async oauthGoogleSheets() {
return true;
}
@Get('google-sheets/callback')
@UseGuards(AuthGuard('google'))
@Redirect()
async oauthGoogleSheetsCallback(@Req() req) {
return await this.callback(req, 'google-sheets');
}
@Get('facebook')
@UseGuards(AuthGuard('facebook'))
async oauthFacebook() {
return true;
}
@Get('facebook/callback')
@UseGuards(AuthGuard('facebook'))
@Redirect()
async oauthFacebookCallback(@Req() req) {
return await this.callback(req, 'facebook');
}
@Get('mailchimp')
@UseGuards(AuthGuard('mailchimp'))
async oauthMailchimp() {
return true;
}
@Get('mailchimp/callback')
@UseGuards(AuthGuard('mailchimp'))
@Redirect()
async oauthMailchimpCallback(@Req() req) {
return await this.callback(req, 'mailchimp');
}
@Get('salesforce')
@UseGuards(AuthGuard('salesforce'))
async oauthSalesforce() {
return true;
}
@Get('salesforce/callback')
@UseGuards(AuthGuard('salesforce'))
@Redirect()
async oauthSalesforceCallback(@Req() req) {
return await this.callback(req, 'salesforce');
}
async callback(@Req() req: Request, plugin) {
const { error, state: customer_id } = req.query;
if (error)
return {
url: `${this.frontendRedirectUri}?error=${error}`,
};
const request: any = req;
const { authInfo } = request;
if (!authInfo)
return {
url: `${this.frontendRedirectUri}?error=invalid_credentials`,
};
const { accessToken: access_token, refreshToken: refresh_token } = authInfo;
const response = await this.inputService.create({
info: { customer_id, customer: 'customer', user_id: 'empty' },
plugin,
credentials: { access_token, refresh_token },
});
return { url: `${this.frontendRedirectUri}?input_id=${response.input.id}` };
}
}
@@ -0,0 +1,45 @@
import {
Profile,
Strategy,
StrategyOption,
VerifyFunction,
} from 'passport-facebook';
import { PassportStrategy } from '@nestjs/passport';
import { Inject, Injectable } from '@nestjs/common';
import { OauthSecrets } from 'src/utils/OauthSecrets';
import { Request } from 'express';
import { ParamsDictionary } from 'express-serve-static-core';
import { ParsedQs } from 'qs';
@Injectable()
export class FacebookStrategy extends PassportStrategy(Strategy) {
constructor(
@Inject('OAUTH_SECRETS')
private readonly oauthSecrets: OauthSecrets,
) {
const options: StrategyOption = {
clientID: oauthSecrets.facebook.client_id,
clientSecret: oauthSecrets.facebook.client_secret,
callbackURL: oauthSecrets.facebook.redirect_uri,
};
const verify: VerifyFunction = (
accessToken: string,
refreshToken: string,
profile: Profile,
done: (error: any, user?: any, info?: any) => void,
) => {
return done(null, profile, { accessToken, refreshToken });
};
super(options, verify);
}
authenticate(
req: Request<ParamsDictionary, any, any, ParsedQs, Record<string, any>>,
options?: any,
): void {
const { customer_id } = req.query;
options.state = customer_id;
options.scope = 'ads_read ads_management';
super.authenticate(req, options);
}
}
@@ -0,0 +1,68 @@
import { Profile, Strategy, StrategyOptions } from 'passport-google-oauth20';
import { PassportStrategy } from '@nestjs/passport';
import { Inject, Injectable } from '@nestjs/common';
import { OauthSecrets } from 'src/utils/OauthSecrets';
@Injectable()
export class GoogleStrategy extends PassportStrategy(Strategy, 'google') {
redirect_uri: string;
constructor(
@Inject('OAUTH_SECRETS')
private readonly oauthSecrets: OauthSecrets,
) {
const options: StrategyOptions = {
clientID: oauthSecrets.google.client_id,
clientSecret: oauthSecrets.google.client_secret,
callbackURL: '/oauth/googleads/callback',
};
const verify = (
accessToken: string,
refreshToken: string,
profile: Profile,
done,
) => {
console.log('verify');
return done(null, profile, { accessToken, refreshToken });
};
super(options, verify);
switch (process.env.ENV) {
case 'dev':
case 'stg':
this.redirect_uri = `https://maestro.${process.env.ENV}.dadosfera.ai/oauth`;
break;
case 'prd':
this.redirect_uri = `https://maestro.dadosfera.ai/oauth`;
break;
default:
this.redirect_uri = `/oauth`;
break;
}
}
authenticate(req, options) {
const plugin: string = req.route.path
.replace('/oauth/', '')
.replace('/callback', '');
const { customer_id } = req.query;
options.scope = ['https://www.googleapis.com/auth/userinfo.email'];
options.callbackURL = `${this.redirect_uri}/${plugin}/callback`;
switch (plugin) {
case 'googleads':
options.scope.push('https://www.googleapis.com/auth/adwords');
break;
case 'google-sheets':
options.scope.push(
'https://www.googleapis.com/auth/drive.metadata.readonly',
'https://www.googleapis.com/auth/spreadsheets.readonly',
);
break;
}
options.state = customer_id;
options.accessType = 'offline';
options.prompt = 'consent';
super.authenticate(req, options);
}
}
@@ -0,0 +1,32 @@
import { Strategy } from 'passport-hubspot-oauth2';
import { PassportStrategy } from '@nestjs/passport';
import { Inject, Injectable } from '@nestjs/common';
import { OauthSecrets } from 'src/utils/OauthSecrets';
@Injectable()
export class HubspotStrategy extends PassportStrategy(Strategy) {
constructor(
@Inject('OAUTH_SECRETS')
private readonly oauthSecrets: OauthSecrets,
) {
super(
{
clientID: oauthSecrets.hubspot.client_id,
clientSecret: oauthSecrets.hubspot.client_secret,
callbackURL: oauthSecrets.hubspot.redirect_uri,
scope:
'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
passReqToCallback: true,
},
(accessToken, refreshToken, tokenInfo, profile, done) => {
return done(null, profile, { accessToken, refreshToken, tokenInfo });
},
);
}
authenticate(req, options) {
const { customer_id } = req.query;
options.state = customer_id;
super.authenticate(req, options);
}
}
@@ -0,0 +1,35 @@
import { Strategy } from 'passport-mailchimp';
import { PassportStrategy } from '@nestjs/passport';
import { Inject, Injectable, Req } from '@nestjs/common';
import { OauthSecrets } from 'src/utils/OauthSecrets';
@Injectable()
export class MailChimpStrategy extends PassportStrategy(Strategy, 'mailchimp') {
constructor(
@Inject('OAUTH_SECRETS')
private readonly oauthSecrets: OauthSecrets,
) {
const options = {
clientID: oauthSecrets.mailchimp.client_id,
clientSecret: oauthSecrets.mailchimp.client_secret,
callbackURL: oauthSecrets.mailchimp.redirect_uri,
scope: '',
};
const verify = (
accessToken: string,
refreshToken: string,
profile: any,
done,
) => {
return done(null, profile, { accessToken, refreshToken });
};
super(options, verify);
}
authenticate(@Req() req, options) {
const { customer_id } = req.query;
options.state = customer_id;
super.authenticate(req, options);
}
}
@@ -0,0 +1,47 @@
import { Strategy } from 'passport-forcedotcom';
import { PassportStrategy } from '@nestjs/passport';
import { Inject, Injectable } from '@nestjs/common';
import { OauthSecrets } from 'src/utils/OauthSecrets';
@Injectable()
export class SalesforceStrategy extends PassportStrategy(
Strategy,
'salesforce',
) {
constructor(
@Inject('OAUTH_SECRETS')
private readonly oauthSecrets: OauthSecrets,
) {
const options = {
clientID: oauthSecrets.salesforce.client_id,
clientSecret: oauthSecrets.salesforce.client_secret,
callbackURL: oauthSecrets.salesforce.redirect_uri,
scope: [
'wave_api',
'chatter_api',
'chatbot_api',
'cdp_ingest_api',
'cdp_profile_api',
'api',
'pardot_api',
'cdp_segment_api',
'cdp_query_api',
'refresh_token',
'offline_access',
],
};
const verify = ({ params }, refreshToken: string, profile: any, done) => {
return done(null, profile, {
accessToken: params.access_token,
refreshToken,
});
};
super(options, verify);
}
authenticate(req, options) {
const { customer_id } = req.query;
options.state = customer_id;
super.authenticate(req, options);
}
}
+123
View File
@@ -0,0 +1,123 @@
import {
Body,
Controller,
Delete,
Get,
Inject,
Param,
Post,
Put,
} from '@nestjs/common';
import { Payload } from '@nestjs/microservices';
import { ApiTags } from '@nestjs/swagger';
import { OutputsClientService } from 'src/clients/outputs/client.service';
import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
import { PERMISSIONS } from '../../authentication/permissions.enum';
import { OutputsService } from './outputs.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@ApiTags('Outputs')
@Controller('outputs')
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS.OUTPUT.permissions[action].seqid,
);
})
export class OutputsController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private outputsClientService: OutputsClientService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
async create(@Body() createOutputDto) {
this.logger.info(process.env.DEV_URL + `/output - ON CREATE ROUTE`, {
user: createOutputDto.info.user_id,
customer: createOutputDto.info.customer,
});
const outputService = new OutputsService(this.outputsClientService);
const response = await outputService.create(createOutputDto);
return response;
}
@Get()
async findAll(@Body() data) {
this.logger.info(process.env.DEV_URL + `/output - ON Find All ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
});
const outputService = new OutputsService(this.outputsClientService);
const response = await outputService.findAll(data);
return response;
}
@Get('/:id')
async findOne(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(
process.env.DEV_URL + `/output/${id} - ON Find One ROUTE`,
{
user: data.info.user_id,
customer: data.info.customer,
},
);
const outputService = new OutputsService(this.outputsClientService);
const response = await outputService.findOne({ id, ...data });
return response;
}
@Put(':id')
async update(@Payload() updateOutputDto, @Param() params) {
const { id } = params;
const { info } = updateOutputDto;
this.logger.info(process.env.DEV_URL + `/output/${id} - ON UPDATE ROUTE`, {
user: updateOutputDto.info.user_id,
customer: updateOutputDto.info.customer,
});
delete updateOutputDto.info;
const outputService = new OutputsService(this.outputsClientService);
const response = await outputService.update(id, updateOutputDto, info);
return response;
}
@Delete(':id')
async delete(@Body() data, @Param() params) {
const { id } = params;
this.logger.info(process.env.DEV_URL + `/output/${id} - ON DELETE ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
});
const outputService = new OutputsService(this.outputsClientService);
const response = await outputService.remove({ id, ...data });
return response;
}
}
+63
View File
@@ -0,0 +1,63 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { IIdRequest, Info } from 'src/clients/outputs/interfaces';
import { OutputsClientService } from 'src/clients/outputs/client.service';
@Injectable()
export class OutputsService {
constructor(private outputClient: OutputsClientService) {}
async create(createOutputDto) {
try {
const createOutputResponse = await this.outputClient.create(
createOutputDto,
);
return createOutputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async findOne(data: IIdRequest) {
try {
const findOneOutputResponse = await this.outputClient.findOne(data);
return findOneOutputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async findAll(data) {
try {
const findAllOutputResponse = await this.outputClient.findAll(data);
return findAllOutputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async update(id: string, data, info: Info) {
try {
const updateOutputResponse = await this.outputClient.update({
id,
info,
...data,
});
return updateOutputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async remove(data: IIdRequest) {
try {
const removeOutputResponse = await this.outputClient.remove(data);
return removeOutputResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
}

Some files were not shown because too many files have changed in this diff Show More