Compare commits

...
43 Commits
Author SHA1 Message Date
Gabriel Amorim d7062efc60 Merge pull request #183 from dadosfera/feat/rest-api
Feat/rest api
2022-12-05 15:22:05 -03:00
Gabriel Rosa 4a3dd98977 Merge branch 'main' into feat/rest-api 2022-12-05 13:58:22 -03:00
Gabriel Amorim 06f4260b61 Merge pull request #181 from dadosfera/stg2
Stg2
2022-11-29 19:16:48 -03:00
Gabriel Rosa 64717c8d92 Merge branch 'main' into stg2 2022-11-29 14:41:11 -03:00
Anderson Feitosa e6e66c77e1 Merge pull request #180 from dadosfera/feature/update-pipeline-cron
feature/update-pipeline-cron
2022-11-29 14:04:31 -03:00
Gabriel Rosa 131e82b4c5 FIX: connection details improvements 2022-11-29 09:47:19 -03:00
Gabriel Rosa 32d37ecadf Removed dev deploy environment 2022-11-29 08:42:26 -03:00
Gabriel Rosa 4d667acb86 removed unused import 2022-11-25 18:21:58 -03:00
Gabriel Rosa 9d417ff61b FIX: treating new connectors upload reponse 2022-11-25 18:21:43 -03:00
Anderson Feitosa b01c4c6c25 UPGRADE: change protospack version 2022-11-25 13:00:09 -03:00
Anderson Feitosa 449a1abb63 FEAT: update auth guard to patch method 2022-11-25 12:11:51 -03:00
Anderson Feitosa 04ca17a69d FEAT: update protospack 2022-11-25 11:37:34 -03:00
Anderson Feitosa 34a0eb40a7 DOCS: docsfera patch 2022-11-23 14:58:17 -03:00
Anderson Feitosa 2df69fba84 FEAT: add log on update 2022-11-23 13:35:02 -03:00
Anderson Feitosa dcbcc5ff36 FEAT: add patch method on pipeline controller 2022-11-23 13:02:32 -03:00
Gabriel Rosa 703b5f36b3 Merge branch 'main' into feat/rest-api 2022-11-21 13:36:13 -03:00
Gabriel Rosa 54d10cb873 FIX: try to delete pipeline even if it does not exists in opensearch 2022-11-17 09:25:05 -03:00
Gabriel Amorim 569c3cf573 Merge pull request #179 from dadosfera/feat/incremental-table
Feat/incremental table
2022-11-16 16:52:35 -03:00
Gabriel Rosa c420ef62ed new protospack version 2022-11-16 16:46:38 -03:00
Alexsander Pereira 966ba00fec Merge pull request #178 from dadosfera/stg2
FEAT: Adding stg2
2022-11-16 15:47:49 -03:00
Alexsander Pereira 0aadf7fc40 FEAT: Adding stg2 2022-11-16 15:18:18 -03:00
Gabriel Rosa 3e3268cb26 FIX: New getAvailableEntities payload 2022-11-16 10:37:21 -03:00
Anderson Feitosa 09bd623b62 Merge pull request #177 from dadosfera/catalog-permissions
FIX: create data asset permission public
2022-11-14 16:19:09 -03:00
Gabriel Rosa 4bf9a5831a FIX: create data asset permission public 2022-11-14 16:15:57 -03:00
Anderson Feitosa 264077289b Merge pull request #176 from dadosfera/catalog-permissions
FIX: renamed catalog create permission
2022-11-14 15:52:04 -03:00
Gabriel Rosa 14c686cff4 FIX: renamed catalog create permission
- Catalog controller using permission decorator on every route
2022-11-14 15:40:38 -03:00
Gabriel Amorim 0e57667419 Merge pull request #175 from dadosfera/feat/modules-permissions
Feat/modules permissions
2022-11-11 12:38:04 -03:00
Gabriel Rosa ac7dc4905e FIX: changed PROCESS permisison group title 2022-11-11 12:35:04 -03:00
Gabriel Rosa 94282718ad Changed permission group name 2022-11-11 12:27:57 -03:00
Gabriel Rosa 7c228c5c53 FIX: set new module permissions public 2022-11-11 12:26:14 -03:00
Gabriel Amorim 14e0f3d2d3 Merge pull request #174 from dadosfera/fix/event-tracker
Fix/event tracker
2022-11-11 10:18:42 -03:00
Victor Radael 85ef0d7c74 Merge remote-tracking branch 'origin' into fix/event-tracker 2022-11-11 10:16:07 -03:00
Victor Radael e00b6481d0 FIX: Tracker 2022-11-11 10:15:49 -03:00
Victor Radael 8906c73d84 Revert "FIX: Event Tracker"
This reverts commit 3bc37db891.
2022-11-11 10:13:24 -03:00
Victor Radael 3bc37db891 FIX: Event Tracker 2022-11-11 10:12:20 -03:00
Gabriel Rosa 3c877b8b8d Merge branch 'main' into feat/rest-api 2022-11-10 15:14:45 -03:00
Gabriel Rosa e6b0034805 Merge branch 'main' into feat/rest-api 2022-11-09 18:29:04 -03:00
Gabriel Rosa b9c1f784d3 FIX: treatingconnector upload response 2022-11-09 18:25:57 -03:00
Gabriel Rosa 3edf5c2416 Merge branch 'main' into feat/rest-api 2022-11-09 15:03:09 -03:00
Gabriel Rosa f3e00a15c8 Remove unused imports 2022-11-09 14:06:30 -03:00
Gabriel Rosa 0422c7a369 Set branch name on swagger version 2022-11-09 14:06:12 -03:00
Gabriel Rosa f22d9370db test new upload documentation 2022-11-09 11:27:54 -03:00
Gabriel Rosa a9ee74724f FIX: better error handling 2022-11-08 12:11:05 -03:00
22 changed files with 322 additions and 1060 deletions
+6 -6
View File
@@ -10,8 +10,8 @@ on:
required: true required: true
type: choice type: choice
options: options:
- dev
- stg - stg
- stg2
- prd - prd
jobs: jobs:
@@ -152,17 +152,17 @@ jobs:
- name: Extract Docs BlockId and PageId - name: Extract Docs BlockId and PageId
env: env:
ENV: ${{ needs.extract_environment.outputs.environment }} ENV: ${{ needs.extract_environment.outputs.environment }}
DEV_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }} STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
DEV_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }} STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }} STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }} STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }} PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }} PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
shell: bash shell: bash
run: | run: |
if [ $ENV == "dev" ]; then if [ $ENV == "stg2" ]; then
echo "block_id=$DEV_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$DEV_DOCS_PAGE_ID" >> $GITHUB_OUTPUT echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "stg" ]; then elif [ $ENV == "stg" ]; then
echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
+54 -496
View File
@@ -505,18 +505,16 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
} },
], {
"requestBody": { "name": "version",
"required": true, "required": true,
"content": { "in": "query",
"application/json": { "schema": {
"schema": { "type": "string"
"$ref": "#/components/schemas/DeleteConnectorDto"
}
} }
} }
}, ],
"responses": { "responses": {
"200": { "200": {
"description": "" "description": ""
@@ -948,7 +946,7 @@
], ],
"responses": { "responses": {
"200": { "200": {
"description": "List of users of a customer", "description": "",
"content": { "content": {
"application/json": { "application/json": {
"schema": { "schema": {
@@ -2351,6 +2349,25 @@
"PipelinesV2" "PipelinesV2"
] ]
}, },
"patch": {
"operationId": "PipelinesController_updateByPatch",
"parameters": [],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
]
},
"delete": { "delete": {
"operationId": "PipelinesController_delete", "operationId": "PipelinesController_delete",
"parameters": [], "parameters": [],
@@ -2394,88 +2411,6 @@
] ]
} }
}, },
"/inputs/test-connection": {
"post": {
"operationId": "InputsController_testConnection",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestConnectionReq"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestConnectionRes"
}
}
}
},
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Inputs"
]
}
},
"/inputs/test-connection/get-columns": {
"post": {
"operationId": "InputsController_getColumns",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestConnectionGetColumnsReq"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestConnectionGetColumnsRes"
}
}
}
},
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Inputs"
]
}
},
"/inputs": { "/inputs": {
"post": { "post": {
"operationId": "InputsController_create", "operationId": "InputsController_create",
@@ -3298,9 +3233,9 @@
} }
}, },
"info": { "info": {
"title": "Maestro - fix/modules-permissions", "title": "Maestro - stg2",
"description": "Documentation for Maestro gateway", "description": "Documentation for Maestro gateway",
"version": "10.11.2022 - 14:43", "version": "0.0.0",
"contact": {} "contact": {}
}, },
"tags": [], "tags": [],
@@ -3568,17 +3503,6 @@
"tags" "tags"
] ]
}, },
"DeleteConnectorDto": {
"type": "object",
"properties": {
"version": {
"type": "string"
}
},
"required": [
"version"
]
},
"PermissionGroupPermission": { "PermissionGroupPermission": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -5174,13 +5098,31 @@
"updated_at" "updated_at"
] ]
}, },
"AvailableEntity": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"replication_methods": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"name",
"replication_methods"
]
},
"GetAvailableEntitiesRes": { "GetAvailableEntitiesRes": {
"type": "object", "type": "object",
"properties": { "properties": {
"entities": { "entities": {
"type": "array", "type": "array",
"items": { "items": {
"type": "string" "$ref": "#/components/schemas/AvailableEntity"
} }
} }
}, },
@@ -5188,99 +5130,6 @@
"entities" "entities"
] ]
}, },
"CredentialsJdbc": {
"type": "object",
"properties": {
"jdbc_user": {
"type": "string"
},
"jdbc_password": {
"type": "string"
},
"database": {
"type": "string"
},
"endpoint": {
"type": "string"
},
"port": {
"type": "string"
},
"engine": {
"type": "string"
},
"schema": {
"type": "string"
}
}
},
"TestConnectionReq": {
"type": "object",
"properties": {
"plugin": {
"type": "string"
},
"credentials": {
"$ref": "#/components/schemas/CredentialsJdbc"
},
"info": {
"type": "object"
}
},
"required": [
"plugin",
"credentials"
]
},
"TestConnectionRes": {
"type": "object",
"properties": {
"connection_state": {
"type": "boolean"
},
"total_entities": {
"type": "number"
},
"database_tables": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"connection_state",
"total_entities",
"database_tables"
]
},
"TestConnectionGetColumnsReq": {
"type": "object",
"properties": {
"plugin": {
"type": "string"
},
"tables": {
"type": "array",
"items": {
"type": "string"
}
},
"credentials": {
"$ref": "#/components/schemas/CredentialsJdbc"
},
"id": {
"type": "string"
},
"info": {
"type": "object"
}
},
"required": [
"plugin",
"tables"
]
},
"TableColumns": { "TableColumns": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -5298,203 +5147,16 @@
"items": { "items": {
"type": "string" "type": "string"
} }
},
"type": {
"type": "string"
} }
}, },
"required": [ "required": [
"name", "name",
"columns", "columns",
"references" "references",
] "type"
},
"TestConnectionGetColumnsRes": {
"type": "object",
"properties": {
"tables": {
"type": "array",
"items": {
"$ref": "#/components/schemas/TableColumns"
}
}
},
"required": [
"tables"
]
},
"FileFormatParams": {
"type": "object",
"properties": {
"sep": {
"type": "string"
},
"header": {
"type": "boolean"
},
"encoding": {
"type": "string"
}
},
"required": [
"sep",
"header",
"encoding"
]
},
"GoogleAnalyticsClientSecrets": {
"type": "object",
"properties": {
"type": {
"type": "string"
},
"project_id": {
"type": "string"
},
"private_key_id": {
"type": "string"
},
"private_key": {
"type": "string"
},
"client_email": {
"type": "string"
},
"client_id": {
"type": "string"
},
"auth_uri": {
"type": "string"
},
"token_uri": {
"type": "string"
},
"auth_provider_x509_cert_url": {
"type": "string"
},
"client_x509_cert_url": {
"type": "string"
}
},
"required": [
"type",
"project_id",
"private_key_id",
"private_key",
"client_email",
"client_id",
"auth_uri",
"token_uri",
"auth_provider_x509_cert_url",
"client_x509_cert_url"
]
},
"Credentials": {
"type": "object",
"properties": {
"jdbc_user": {
"type": "string"
},
"jdbc_password": {
"type": "string"
},
"database": {
"type": "string"
},
"endpoint": {
"type": "string"
},
"port": {
"type": "string"
},
"engine": {
"type": "string"
},
"schema": {
"type": "string"
},
"connection_type": {
"type": "string"
},
"client_aws_access_key_id": {
"type": "string"
},
"client_aws_secret_access_key": {
"type": "string"
},
"client_bucket": {
"type": "string"
},
"file_to_extract": {
"type": "string"
},
"file_format_params": {
"$ref": "#/components/schemas/FileFormatParams"
},
"view_id": {
"type": "string"
},
"client_secrets": {
"$ref": "#/components/schemas/GoogleAnalyticsClientSecrets"
},
"start_date": {
"type": "string"
},
"end_date": {
"type": "string"
},
"oauth_code": {
"type": "string"
}
},
"required": [
"connection_type"
]
},
"OauthObject": {
"type": "object",
"properties": {
"get_tokens_url": {
"type": "string"
},
"get_tokens_url_params": {
"type": "string"
},
"get_tokens_set_response": {
"type": "object"
},
"content_type": {
"type": "string"
}
}
},
"InputOptions": {
"type": "object",
"properties": {
"oauth": {
"$ref": "#/components/schemas/OauthObject"
},
"skip_select_columns": {
"type": "boolean"
},
"skip_select_entities": {
"type": "boolean"
},
"skip_transformation": {
"type": "boolean"
}
}
},
"AuthParameters": {
"type": "object",
"properties": {
"aws_access_key_id": {
"type": "string"
},
"aws_secret_access_key": {
"type": "string"
}
},
"required": [
"aws_access_key_id",
"aws_secret_access_key"
] ]
}, },
"CreateInputReq": { "CreateInputReq": {
@@ -5517,58 +5179,6 @@
}, },
"type": { "type": {
"type": "string" "type": "string"
},
"category": {
"type": "string",
"deprecated": true
},
"credentials": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/Credentials"
}
]
},
"options": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/InputOptions"
}
]
},
"info": {
"type": "object",
"deprecated": true
},
"cron": {
"type": "string",
"deprecated": true
},
"source_bucket": {
"type": "string",
"deprecated": true
},
"source_prefix": {
"type": "string",
"deprecated": true
},
"auth_parameters": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/AuthParameters"
}
]
},
"file_format_params": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/FileFormatParams"
}
]
} }
}, },
"required": [ "required": [
@@ -5607,58 +5217,6 @@
}, },
"updated_at": { "updated_at": {
"type": "string" "type": "string"
},
"category": {
"type": "string",
"deprecated": true
},
"credentials": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/Credentials"
}
]
},
"options": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/InputOptions"
}
]
},
"info": {
"type": "object",
"deprecated": true
},
"cron": {
"type": "string",
"deprecated": true
},
"source_bucket": {
"type": "string",
"deprecated": true
},
"source_prefix": {
"type": "string",
"deprecated": true
},
"auth_parameters": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/AuthParameters"
}
]
},
"file_format_params": {
"deprecated": true,
"allOf": [
{
"$ref": "#/components/schemas/FileFormatParams"
}
]
} }
}, },
"required": [ "required": [
+7 -7
View File
@@ -12,7 +12,7 @@
"dependencies": { "dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0", "@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.19.4", "@dadosfera/protospack-v2": "3.21.0",
"@grpc/grpc-js": "^1.6.7", "@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13", "@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7", "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
} }
}, },
"node_modules/@dadosfera/protospack-v2": { "node_modules/@dadosfera/protospack-v2": {
"version": "3.19.4", "version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.4.tgz", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-Vb6bdFgQKYHf0kDK9kuMtN+nt3nkaMhJZ3wNVaUmQNxsnpGyARWBL29R2n+YnxS8g23tRwuj331sjg+DY0Fnhg==", "integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"dependencies": { "dependencies": {
"@grpc/grpc-js": "^1.6.7", "@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5", "rxjs": "^7.5.5",
@@ -12330,9 +12330,9 @@
} }
}, },
"@dadosfera/protospack-v2": { "@dadosfera/protospack-v2": {
"version": "3.19.4", "version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.4.tgz", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-Vb6bdFgQKYHf0kDK9kuMtN+nt3nkaMhJZ3wNVaUmQNxsnpGyARWBL29R2n+YnxS8g23tRwuj331sjg+DY0Fnhg==", "integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"requires": { "requires": {
"@grpc/grpc-js": "^1.6.7", "@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5", "rxjs": "^7.5.5",
+1 -1
View File
@@ -28,7 +28,7 @@
"dependencies": { "dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0", "@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.19.4", "@dadosfera/protospack-v2": "3.21.0",
"@grpc/grpc-js": "^1.6.7", "@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13", "@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7", "@nestjs/common": "^8.4.7",
@@ -44,7 +44,7 @@ class NoClassAuthController {
@Get('has-all-permissions') @Get('has-all-permissions')
@RequireAllPermissions( @RequireAllPermissions(
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN, PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.METABASE.permissions.OPEN, PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
) )
async hasAllPermissions(@Body() body) { async hasAllPermissions(@Body() body) {
return { body }; return { body };
@@ -53,7 +53,7 @@ class NoClassAuthController {
@Get('has-some-permission') @Get('has-some-permission')
@RequireSomePermission( @RequireSomePermission(
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN, PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.METABASE.permissions.OPEN, PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
) )
async hasSomePermission(@Body() body) { async hasSomePermission(@Body() body) {
return { body }; return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
@Controller('class-auth-condition') @Controller('class-auth-condition')
@AuthenticateCondition((req) => req.get('x-on-class') === 'ok') @AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
@RequireSomePermission(PERMISSIONS_GROUPS.METABASE.permissions.OPEN) @RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
class ClassAuthConditionController { class ClassAuthConditionController {
@Get('body') @Get('body')
async getBody(@Body() body) { async getBody(@Body() body) {
@@ -444,13 +444,13 @@ describe('authentication.guard', () => {
NoClassAuthTest(tokenZ, ['zendesk']); NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']); ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.METABASE.permissions.OPEN]); const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']); NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']); ClassAuthConditionTest(tokenM, ['metabase']);
const tokenZM = CreateToken([ const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN, PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.METABASE.permissions.OPEN, PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
]); ]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']); NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']); ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
+36 -27
View File
@@ -297,12 +297,12 @@ export const PERMISSIONS_GROUPS = {
}, },
CREATE: { CREATE: {
seqid: 19, seqid: 19,
claim: 'POST /catalog', claim: 'catalog:create',
usage: PermissionUsages.INTERNAL, usage: PermissionUsages.PUBLIC,
name: { name: {
'pt-br': 'Criar atributos no catálogo', 'pt-br': 'Criar Ativos',
'en-us': 'Create attributes', 'en-us': 'Create Assets',
'es-es': 'Crear atributos', 'es-es': 'Crear Activos',
}, },
}, },
UPDATE: { UPDATE: {
@@ -429,14 +429,14 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
METABASE: { ANALYZE: {
title: { title: {
'pt-br': 'Analisar | Visualização', 'pt-br': 'Analisar | Visualização',
'en-us': 'Analyze | Visualization', 'en-us': 'Analyze | Visualization',
'es-es': 'Analizar | Visualización', 'es-es': 'Analizar | Visualización',
}, },
permissions: { permissions: {
OPEN: { METABASE: {
seqid: 11, seqid: 11,
claim: 'GET /metabase', claim: 'GET /metabase',
usage: PermissionUsages.PUBLIC, usage: PermissionUsages.PUBLIC,
@@ -446,6 +446,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Acceso Metabase', 'es-es': 'Acceso Metabase',
}, },
}, },
INTELLIGENCE: {
seqid: 31,
claim: 'intelligence:open',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Inteligência',
'en-us': 'Access Intelligence Module',
'es-s': 'Acceder Módulo de Inteligencia',
},
},
}, },
}, },
@@ -456,16 +466,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Módulos de la Dadosfera', 'es-es': 'Módulos de la Dadosfera',
}, },
permissions: { permissions: {
INTELLIGENCE: {
seqid: 31,
claim: 'intelligence:open',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Inteligência',
'en-us': 'Access Intelligence Module',
'es-s': 'Acceder Módulo de Inteligencia',
},
},
APP_BUILDER: { APP_BUILDER: {
seqid: 32, seqid: 32,
claim: 'app-builder:open', claim: 'app-builder:open',
@@ -486,16 +486,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Acceder Machine Learning', 'es-es': 'Acceder Machine Learning',
}, },
}, },
PROCESS: {
seqid: 43,
claim: 'process:open',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Processamento',
'en-us': 'Access Processing Module',
'es-es': 'Acceder Módulo de Tramitación',
},
},
}, },
}, },
USERS: { USERS: {
@@ -517,6 +507,25 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
PROCESS: {
title: {
'pt-br': 'Processar',
'en-us': 'Process',
'es-es': 'Procesar',
},
permissions: {
TRANSFORMATION: {
seqid: 43,
claim: 'process:open',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Transformação',
'en-us': 'Access Transformation Module',
'es-es': 'Acceder Módulo de transformación',
},
},
},
},
DADOSFERA: { DADOSFERA: {
title: { title: {
'pt-br': 'Usuários', 'pt-br': 'Usuários',
+1 -1
View File
@@ -49,7 +49,7 @@ describe('user.decorator', () => {
const fakeUserPayload = { const fakeUserPayload = {
user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125', user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
username: 'super.admin', username: 'super.admin',
permissions: [PERMISSIONS_GROUPS.METABASE.permissions.OPEN].map( permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
({ seqid }) => seqid, ({ seqid }) => seqid,
), ),
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11', customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
+1 -7
View File
@@ -30,12 +30,8 @@ async function bootstrap() {
await app.listen(3333); await app.listen(3333);
} }
function configureSwagger(app: INestApplication) {
const date = new Date();
const swaggerVersion = `${date.getDate()}.${
date.getMonth() + 1
}.${date.getFullYear()} - ${date.getHours()}:${date.getMinutes()}`;
function configureSwagger(app: INestApplication) {
let branchName = ''; let branchName = '';
try { try {
branchName = execSync('git rev-parse --abbrev-ref HEAD') branchName = execSync('git rev-parse --abbrev-ref HEAD')
@@ -50,7 +46,6 @@ function configureSwagger(app: INestApplication) {
const config = new DocumentBuilder() const config = new DocumentBuilder()
.setTitle(swaggerTitle) .setTitle(swaggerTitle)
.setDescription('Documentation for Maestro gateway') .setDescription('Documentation for Maestro gateway')
.setVersion(swaggerVersion)
.addBearerAuth( .addBearerAuth(
{ type: 'http', scheme: 'bearer', bearerFormat: 'JWT' }, { type: 'http', scheme: 'bearer', bearerFormat: 'JWT' },
'Authorization', 'Authorization',
@@ -60,7 +55,6 @@ function configureSwagger(app: INestApplication) {
const document = SwaggerModule.createDocument(app, config); const document = SwaggerModule.createDocument(app, config);
if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document); if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
writeFileSync( writeFileSync(
'docsfera.json', 'docsfera.json',
JSON.stringify( JSON.stringify(
+53 -28
View File
@@ -14,8 +14,8 @@ import {
} from '@nestjs/common'; } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger'; import { ApiTags } from '@nestjs/swagger';
import { import {
AuthenticateCondition,
Authenticated, Authenticated,
RequireSomePermission,
} from '../../authentication/authentication.decorator'; } from '../../authentication/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service'; import { CatalogService } from './catalog.service';
@@ -34,36 +34,9 @@ import {
IUpdateDataRequest, IUpdateDataRequest,
} from './dtos'; } from './dtos';
const catalogPermissions = PERMISSIONS_GROUPS.CATALOG.permissions;
@ApiTags('Catalog') @ApiTags('Catalog')
@Controller('catalog') @Controller('catalog')
@Authenticated() @Authenticated()
@AuthenticateCondition((req, user: RequestUser) => {
if (user.permissions.includes(catalogPermissions.DATA_MANAGER.seqid))
return true;
if (req.path.includes('/manage-permissions')) return true;
let action: keyof typeof catalogPermissions;
switch (req.method) {
case 'POST':
case 'PUT':
action = 'UPDATE';
break;
case 'DELETE':
if (req.path.includes('/comment')) action = 'UPDATE';
else action = req.method;
break;
default:
action = req.method as 'GET';
}
return user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions[action].seqid,
);
})
export class CatalogController { export class CatalogController {
logger: DadosferaLogger; logger: DadosferaLogger;
constructor( constructor(
@@ -75,6 +48,10 @@ export class CatalogController {
} }
@Get() @Get()
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async catalogAll( async catalogAll(
@User() user: RequestUser, @User() user: RequestUser,
@Query() query, @Query() query,
@@ -110,6 +87,10 @@ export class CatalogController {
} }
@Get('tags') @Get('tags')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async findAllTags(@Body() body) { async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, { this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id, user: body.info.user_id,
@@ -129,6 +110,10 @@ export class CatalogController {
} }
@Get('data-asset/:id') @Get('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAsset( async getDataAsset(
@User() user: RequestUser, @User() user: RequestUser,
@Headers() headers, @Headers() headers,
@@ -183,6 +168,10 @@ export class CatalogController {
} }
@Get('data-asset/rls/:id') @Get('data-asset/rls/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetRls( async getDataAssetRls(
@User() user: RequestUser, @User() user: RequestUser,
@Headers() headers, @Headers() headers,
@@ -237,6 +226,10 @@ export class CatalogController {
} }
@Get('data-asset/:id/columns-metadata') @Get('data-asset/:id/columns-metadata')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetColumnsMetadata( async getDataAssetColumnsMetadata(
@User() user: RequestUser, @User() user: RequestUser,
@Headers('Dadosfera-Lang') language, @Headers('Dadosfera-Lang') language,
@@ -273,6 +266,10 @@ export class CatalogController {
} }
@Get('data-asset/:id/preview') @Get('data-asset/:id/preview')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetPreview( async getDataAssetPreview(
@User() user: RequestUser, @User() user: RequestUser,
@Headers('Dadosfera-lang') language, @Headers('Dadosfera-lang') language,
@@ -308,6 +305,10 @@ export class CatalogController {
} }
@Get('data-asset/:id/docs') @Get('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getDataAssetDocs( async getDataAssetDocs(
@User() user: RequestUser, @User() user: RequestUser,
@Headers('Dadosfera-Lang') language, @Headers('Dadosfera-Lang') language,
@@ -343,6 +344,10 @@ export class CatalogController {
} }
@Put('data-asset/:id') @Put('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async updateDataAsset( async updateDataAsset(
@User() user: RequestUser, @User() user: RequestUser,
@Headers('Dadosfera-Lang') language, @Headers('Dadosfera-Lang') language,
@@ -372,6 +377,10 @@ export class CatalogController {
} }
@Post('data-asset/:id/docs') @Post('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async manageDataAssetDocs( async manageDataAssetDocs(
@User() user: RequestUser, @User() user: RequestUser,
@Headers() headers, @Headers() headers,
@@ -440,6 +449,10 @@ export class CatalogController {
} }
@Post() @Post()
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async createDataAsset( async createDataAsset(
@User() user: RequestUser, @User() user: RequestUser,
@Body() body: ICreateDataAsset, @Body() body: ICreateDataAsset,
@@ -460,6 +473,10 @@ export class CatalogController {
} }
@Post('data-asset/:id/comment') @Post('data-asset/:id/comment')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async commentOnDataAsset( async commentOnDataAsset(
@Param('id') id: string, @Param('id') id: string,
@User() user: RequestUser, @User() user: RequestUser,
@@ -482,6 +499,10 @@ export class CatalogController {
} }
@Delete('data-asset/:id') @Delete('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async deleteDataAsset( async deleteDataAsset(
@Param('id') data_asset_id: string, @Param('id') data_asset_id: string,
@User() user: RequestUser, @User() user: RequestUser,
@@ -504,6 +525,10 @@ export class CatalogController {
} }
@Delete('data-asset/:id/comment') @Delete('data-asset/:id/comment')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async deleteComment( async deleteComment(
@Param('id') id: string, @Param('id') id: string,
@User() user: RequestUser, @User() user: RequestUser,
+6 -36
View File
@@ -1,9 +1,4 @@
import { import { OnModuleInit, Inject } from '@nestjs/common';
OnModuleInit,
Inject,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices'; import { ClientGrpc } from '@nestjs/microservices';
import { ConnectionManager } from '@dadosfera/protospack-v2'; import { ConnectionManager } from '@dadosfera/protospack-v2';
import { lastValueFrom } from 'rxjs'; import { lastValueFrom } from 'rxjs';
@@ -86,12 +81,7 @@ export class ConnectionClientService implements OnModuleInit {
}, },
metadata, metadata,
), ),
).catch((err) => { );
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
} }
async updateConnection( async updateConnection(
@@ -112,12 +102,7 @@ export class ConnectionClientService implements OnModuleInit {
}, },
metadata, metadata,
), ),
).catch((err) => { );
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
} }
async deleteConnection({ body, metadata }): Promise<ConnectionRes> { async deleteConnection({ body, metadata }): Promise<ConnectionRes> {
@@ -130,12 +115,7 @@ export class ConnectionClientService implements OnModuleInit {
}, },
metadata, metadata,
), ),
).catch((err) => { );
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
} }
async getAllConnections({ body, metadata }): Promise<ConnectionsRes> { async getAllConnections({ body, metadata }): Promise<ConnectionsRes> {
@@ -146,12 +126,7 @@ export class ConnectionClientService implements OnModuleInit {
{ search, filters: JSON.stringify(filters), size, page }, { search, filters: JSON.stringify(filters), size, page },
metadata, metadata,
), ),
).catch((err) => { );
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
} }
async getConnectionDetails({ async getConnectionDetails({
@@ -162,12 +137,7 @@ export class ConnectionClientService implements OnModuleInit {
return lastValueFrom( return lastValueFrom(
this.connectionServiceRead.GetConnectionDetails({ id }, metadata), this.connectionServiceRead.GetConnectionDetails({ id }, metadata),
).catch((err) => { );
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
} }
async INTERNAL_getAllConnections({ async INTERNAL_getAllConnections({
@@ -9,6 +9,7 @@ import {
Get, Get,
Headers, Headers,
Query, Query,
UseFilters,
} from '@nestjs/common'; } from '@nestjs/common';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger'; import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { ConnectionClientService } from './client.service'; import { ConnectionClientService } from './client.service';
@@ -28,8 +29,10 @@ import {
} from './dtos/connection'; } from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection'; import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions; const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@UseFilters(new GrpcToHttpExceptionFilter())
@ApiTags('connections') @ApiTags('connections')
@ApiBearerAuth() @ApiBearerAuth()
@Authenticated() @Authenticated()
+22 -160
View File
@@ -83,163 +83,30 @@ export class ConnectorClientService implements OnModuleInit {
); );
}); });
const responseParsed = JSON.parse(res.message); const responseParsed = JSON.parse(res.message);
const connectorErrors = responseParsed.items const connectorsResponse = responseParsed.items.filter(
.filter((i) => i.update._index === 'connectors' && i.update.error) (i) => i.update._index === process.env.CONNECTORS_INDEX,
);
const connectorErrors = connectorsResponse
.filter((i) => i.update.error)
.map((i) => ({ .map((i) => ({
id: i.update._id, id: i.update?._id,
reason: i.update.error, reason: i.update?.error,
status: i.error.status, status: i.error?.status,
}));
const connectionControlsErrors = responseParsed.items
.filter(
(i) => i.update._index === 'connection_controls' && i.update.error,
)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const connectionStepsErrors = responseParsed.items
.filter((i) => i.update._index === 'connection_steps' && i.update.error)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const configControlsErrors = responseParsed.items
.filter((i) => i.update._index === 'config_controls' && i.update.error)
.map((i) => ({
id: i.update._id,
reason: i.update.error,
status: i.error.status,
}));
const connectorCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connectors' && i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const connectionControlsCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_controls' &&
i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const connectionStepsCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_steps' &&
i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const configControlsCreated = responseParsed.items
.filter(
(i) =>
i.update._index === 'config_controls' &&
i.update.result === 'created',
)
.map((i) => ({
id: i.update._id,
}));
const connectorUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connectors' && i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const connectionControlsUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_controls' &&
i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const connectionStepsUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_steps' &&
i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const configControlsUpdated = responseParsed.items
.filter(
(i) =>
i.update._index === 'config_controls' &&
i.update.result === 'updated',
)
.map((i) => ({
id: i.update._id,
}));
const connectorNoop = responseParsed.items
.filter(
(i) => i.update._index === 'connectors' && i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
}));
const connectionControlsNoop = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_controls' &&
i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
}));
const connectionStepsNoop = responseParsed.items
.filter(
(i) =>
i.update._index === 'connection_steps' && i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
}));
const configControlsNoop = responseParsed.items
.filter(
(i) =>
i.update._index === 'config_controls' && i.update.result === 'noop',
)
.map((i) => ({
id: i.update._id,
})); }));
const connectorCreated = connectorsResponse
.filter((i) => i.update.result === 'created')
.map((i) => i.update._id);
const connectorUpdated = connectorsResponse
.filter((i) => i.update.result === 'updated')
.map((i) => i.update._id);
const connectorNoop = connectorsResponse
.filter((i) => i.update.result === 'noop')
.map((i) => i.update._id);
return { return {
errors: { errors: connectorErrors,
connectors: connectorErrors, created: connectorCreated,
connection_controls: connectionControlsErrors, updated: connectorUpdated,
connection_steps: connectionStepsErrors, noop: connectorNoop,
config_controls: configControlsErrors,
},
created: {
connectors: connectorCreated,
connection_controls: connectionControlsCreated,
connection_steps: connectionStepsCreated,
config_controls: configControlsCreated,
},
updated: {
connectors: connectorUpdated,
connection_controls: connectionControlsUpdated,
connection_steps: connectionStepsUpdated,
config_controls: configControlsUpdated,
},
noop: {
connectors: connectorNoop,
connection_controls: connectionControlsNoop,
connection_steps: connectionStepsNoop,
config_controls: configControlsNoop,
},
}; };
} }
@@ -318,12 +185,7 @@ export class ConnectorClientService implements OnModuleInit {
version, version,
plugin, plugin,
}), }),
).catch((err) => { );
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
} }
async updateConnector({ plugin, changes }) { async updateConnector({ plugin, changes }) {
@@ -19,7 +19,6 @@ import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ConnectorClientService } from './client.service'; import { ConnectorClientService } from './client.service';
import { AddTagDto } from './dtos/add-tag'; import { AddTagDto } from './dtos/add-tag';
import { CreateConnectorDto } from './dtos/create-connector'; import { CreateConnectorDto } from './dtos/create-connector';
import { DeleteConnectorDto } from './dtos/delete-connector';
import { GetAllDto } from './dtos/get-all'; import { GetAllDto } from './dtos/get-all';
import { RemoveTagDto } from './dtos/remove-tag'; import { RemoveTagDto } from './dtos/remove-tag';
import { UpdateDto } from './dtos/update'; import { UpdateDto } from './dtos/update';
@@ -274,7 +273,7 @@ export class ConnectorController {
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE) @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
async deleteConnector( async deleteConnector(
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@Body() { version }: DeleteConnectorDto, @Query('version') version: string,
) { ) {
this.logger.info('/upload - Upload Connector Route'); this.logger.info('/upload - Upload Connector Route');
+10 -156
View File
@@ -1,106 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from 'protospack/dist/lib/interfaces';
export class GoogleAnalyticsClientSecrets {
@ApiProperty()
type: string;
@ApiProperty()
project_id: string;
@ApiProperty()
private_key_id: string;
@ApiProperty()
private_key: string;
@ApiProperty()
client_email: string;
@ApiProperty()
client_id: string;
@ApiProperty()
auth_uri: string;
@ApiProperty()
token_uri: string;
@ApiProperty()
auth_provider_x509_cert_url: string;
@ApiProperty()
client_x509_cert_url: string;
}
export class OauthObject {
@ApiPropertyOptional()
get_tokens_url: string;
@ApiPropertyOptional()
get_tokens_url_params: string;
@ApiPropertyOptional()
get_tokens_set_response: Record<string, any>;
@ApiPropertyOptional()
content_type: string;
}
export class InputOptions {
@ApiPropertyOptional()
oauth: OauthObject;
@ApiPropertyOptional()
skip_select_columns: true;
@ApiPropertyOptional()
skip_select_entities: false;
@ApiPropertyOptional()
skip_transformation: true;
}
export class FileFormatParams {
@ApiProperty()
sep: string;
@ApiProperty()
header: boolean;
@ApiProperty()
encoding: string;
}
export class AuthParameters {
@ApiProperty()
aws_access_key_id: string;
@ApiProperty()
aws_secret_access_key: string;
}
export class CredentialsJdbc {
@ApiPropertyOptional()
jdbc_user: string;
@ApiPropertyOptional()
jdbc_password: string;
@ApiPropertyOptional()
database: string;
@ApiPropertyOptional()
endpoint: string;
@ApiPropertyOptional()
port: string;
@ApiPropertyOptional()
engine: string;
@ApiPropertyOptional()
schema: string;
}
export class Credentials extends CredentialsJdbc {
@ApiProperty()
connection_type: string;
@ApiPropertyOptional()
client_aws_access_key_id: string;
@ApiPropertyOptional()
client_aws_secret_access_key: string;
@ApiPropertyOptional()
client_bucket: string;
@ApiPropertyOptional()
file_to_extract: string;
@ApiPropertyOptional()
file_format_params: FileFormatParams;
@ApiPropertyOptional()
view_id: string;
@ApiPropertyOptional()
client_secrets: GoogleAnalyticsClientSecrets;
@ApiPropertyOptional()
start_date: string;
@ApiPropertyOptional()
end_date: string;
@ApiPropertyOptional()
oauth_code: string;
}
export class Column { export class Column {
@ApiProperty() @ApiProperty()
name: string; name: string;
@@ -114,6 +13,14 @@ export class TableColumns {
columns: string[]; columns: string[];
@ApiProperty() @ApiProperty()
references: Column[]; references: Column[];
@ApiProperty()
type: string;
}
export class AvailableEntity {
@ApiProperty()
name: string;
@ApiProperty()
replication_methods: string[];
} }
export class Input { export class Input {
@@ -133,26 +40,6 @@ export class Input {
created_at: string; created_at: string;
@ApiProperty() @ApiProperty()
updated_at: string; updated_at: string;
//deprecated attributes (for retro compatibility)
@ApiPropertyOptional({ deprecated: true })
category: string;
@ApiPropertyOptional({ deprecated: true })
credentials: Credentials;
@ApiPropertyOptional({ deprecated: true })
options: InputOptions;
@ApiPropertyOptional({ deprecated: true })
info: Info;
@ApiPropertyOptional({ deprecated: true })
cron: string;
@ApiPropertyOptional({ deprecated: true })
source_bucket: string;
@ApiPropertyOptional({ deprecated: true })
source_prefix: string;
@ApiPropertyOptional({ deprecated: true })
auth_parameters: AuthParameters;
@ApiPropertyOptional({ deprecated: true })
file_format_params: FileFormatParams;
} }
export class GetAvailableEntitiesReq { export class GetAvailableEntitiesReq {
@ApiProperty() @ApiProperty()
@@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq {
} }
export class GetAvailableEntitiesRes { export class GetAvailableEntitiesRes {
@ApiProperty() @ApiProperty({ type: [AvailableEntity] })
entities: string[]; entities: AvailableEntity[];
} }
export class TestConnectionGetColumnsReq {
@ApiProperty()
plugin: string;
@ApiProperty()
tables: string[];
@ApiPropertyOptional()
credentials: CredentialsJdbc;
@ApiPropertyOptional()
id: string;
@ApiPropertyOptional()
info: Info;
}
export class TestConnectionGetColumnsRes {
@ApiProperty({ type: [TableColumns] })
tables: TableColumns[];
}
export class TestConnectionReq {
@ApiProperty()
plugin: string;
@ApiProperty()
credentials: CredentialsJdbc;
@ApiPropertyOptional()
info: Info;
}
export class TestConnectionRes {
@ApiProperty()
connection_state: boolean;
@ApiProperty()
total_entities: number;
@ApiProperty()
database_tables: string[];
}
export class CreateInputReq extends OmitType(Input, [ export class CreateInputReq extends OmitType(Input, [
'id', 'id',
'created_at', 'created_at',
+9 -38
View File
@@ -19,16 +19,10 @@ import {
GetAvailableEntitiesReq, GetAvailableEntitiesReq,
GetAvailableEntitiesRes, GetAvailableEntitiesRes,
Input, Input,
TestConnectionGetColumnsReq,
TestConnectionGetColumnsRes,
TestConnectionReq,
TestConnectionRes,
} from './dtos/input.model'; } from './dtos/input.model';
import { UpdateInputRequest } from './dtos/old_interfaces'; import { UpdateInputRequest } from './dtos/old_interfaces';
import { RequestUser, User } from 'src/authentication/user.decorator'; import { RequestUser, User } from 'src/authentication/user.decorator';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@ApiTags('Inputs') @ApiTags('Inputs')
@@ -75,42 +69,19 @@ export class InputsController {
@Get('available-entities/:plugin') @Get('available-entities/:plugin')
@ApiOkResponse({ type: GetAvailableEntitiesRes }) @ApiOkResponse({ type: GetAvailableEntitiesRes })
async getAvailableEntities( async getAvailableEntities(
@Body() body,
@Param() params: GetAvailableEntitiesReq, @Param() params: GetAvailableEntitiesReq,
@User() user: RequestUser,
) { ) {
const { info } = body;
const { plugin } = params; const { plugin } = params;
return await this.inputService.getAvailableEntities({ info, plugin }); const availableEntities = await this.inputService.getAvailableEntities({
} info: {
customer: user.customer_name,
@Post('/test-connection') customer_id: user.customer_id,
@ApiOkResponse({ type: TestConnectionRes }) user_id: user.user_id,
async testConnection(@Body() data: TestConnectionReq) {
this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
});
const response = await this.inputService.testConnection(data);
return response;
}
@Post('/test-connection/get-columns')
@ApiOkResponse({ type: TestConnectionGetColumnsRes })
async getColumns(@Body() data: TestConnectionGetColumnsReq) {
throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED);
this.logger.info(
`/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
{
user: data.info.user_id,
customer: data.info.customer,
}, },
); plugin,
});
const response = await this.inputService.getColumns(data); return availableEntities;
return response;
} }
@Post() @Post()
+4 -44
View File
@@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces'; import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
import { Input } from '@dadosfera/protospack-v2'; import { Input } from '@dadosfera/protospack-v2';
import { import {
GetAvailableEntitiesRequest,
InputCreateGenericRequest, InputCreateGenericRequest,
InputCreateS3Request, InputCreateS3Request,
InputNewCreateRequest, InputNewCreateRequest,
TestConnectionGetColumnsRequest,
TestConnectionRequest, TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -88,32 +88,6 @@ export class InputsService {
return testConnectionResponse; return testConnectionResponse;
}, },
getColumns: async (data: TestConnectionGetColumnsRequest) => {
this.logger.info('InputClientService - TestConnection/Get-Columns');
const getColumnsResponse = await new Promise((resolve, reject) => {
this.inputReadService.GetColumns(data).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
reject(err);
},
complete() {
// console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return getColumnsResponse;
},
getAvailableEntities: async (data) => { getAvailableEntities: async (data) => {
this.logger.info('InputClientService - GetAvailableEntities'); this.logger.info('InputClientService - GetAvailableEntities');
data; data;
@@ -188,12 +162,9 @@ export class InputsService {
// this.validateCron(data); // this.validateCron(data);
const { body, info } = data; const { body, info } = data;
if (body.info) delete body.info; const inputCreateGenericRequest: InputCreateGenericRequest = {
const inputCreateGenericRequest: any | InputCreateGenericRequest = {
input: { input: {
...body, ...body,
credentials: body.credentials && JSON.stringify(body.credentials),
}, },
info, info,
}; };
@@ -205,8 +176,8 @@ export class InputsService {
return { input: adjustedInput }; return { input: adjustedInput };
} }
async getAvailableEntities(data): Promise<{ entities: string[] }> { async getAvailableEntities(data: GetAvailableEntitiesRequest) {
return await this.OLD_inputClient.getAvailableEntities(data); return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
} }
async findAll(info: Info) { async findAll(info: Info) {
@@ -263,17 +234,6 @@ export class InputsService {
} }
} }
async getColumns(data) {
try {
const testConnectionGetColumnsResponse =
await this.OLD_inputClient.getColumns(data);
return testConnectionGetColumnsResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
generateInputS3Payload(payload) { generateInputS3Payload(payload) {
const { credentials, plugin, cron } = payload; const { credentials, plugin, cron } = payload;
if (!credentials) return payload; if (!credentials) return payload;
@@ -1,7 +1,9 @@
import { Body, Controller, Inject, Param, Post } from '@nestjs/common'; import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
import { init } from 'mixpanel'; import { init } from 'mixpanel';
import { Authenticated } from 'src/authentication/authentication.decorator';
import { RequestUser, User } from 'src/authentication/user.decorator'; import { RequestUser, User } from 'src/authentication/user.decorator';
@Authenticated()
@Controller('trackEvent') @Controller('trackEvent')
export class MixpanelController { export class MixpanelController {
constructor( constructor(
@@ -12,6 +12,7 @@ import {
UseFilters, UseFilters,
HttpCode, HttpCode,
HttpStatus, HttpStatus,
Patch,
} from '@nestjs/common'; } from '@nestjs/common';
import { import {
ApiCreatedResponse, ApiCreatedResponse,
@@ -45,6 +46,10 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
action = 'UPDATE'; action = 'UPDATE';
break; break;
case 'PATCH':
action = 'UPDATE';
break;
default: default:
action = req.method; action = req.method;
} }
@@ -175,20 +180,49 @@ export class PipelinesController {
@Put('/:id') @Put('/:id')
async update( async update(
@Headers('Dadosfera-Lang') language,
@Body() updatePipelineDto, @Body() updatePipelineDto,
@Param('id') id, @Param('id') id,
@User() user: RequestUser, @User() user: RequestUser,
) { ) {
this.logger.info('PipelinesController - findOne', { user }); this.logger.info('PipelinesController - update', { user });
const { info } = updatePipelineDto; const { info } = updatePipelineDto;
delete updatePipelineDto.info; delete updatePipelineDto.info;
const response = await this.pipelinesClientService.update({ const { customer_id, customer_name, user_id, username } = user;
...updatePipelineDto, if (!language) language = 'en-us';
info,
id, const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
}); });
const response = await this.pipelinesClientService.update(
{
...updatePipelineDto,
info,
id,
},
metadata,
);
this.logger.info('PipelinesController - update: OK', { user });
return response;
}
@Patch('/:id')
async updateByPatch(
@Headers('Dadosfera-Lang') language,
@Body() updatePipelineDto,
@Param('id') id,
@User() user: RequestUser,
) {
this.logger.info('PipelinesController - patch', { user });
const response = await this.update(language, updatePipelineDto, id, user);
this.logger.info('PipelinesController - patch: OK', { user });
return response; return response;
} }
+47 -36
View File
@@ -1,9 +1,4 @@
import { import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common';
BadRequestException,
Inject,
InternalServerErrorException,
OnModuleInit,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices'; import { ClientGrpc } from '@nestjs/microservices';
import { import {
Messages, Messages,
@@ -23,6 +18,8 @@ import { InputsService } from '../inputs/inputs.service';
import { RequestUser } from 'src/authentication/user.decorator'; import { RequestUser } from 'src/authentication/user.decorator';
import { TransformationsService } from '../transformations/transformations.service'; import { TransformationsService } from '../transformations/transformations.service';
import { getObjValueFromPath } from 'src/utils/ObjValueFromPath'; import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
export class PipelinesService implements OnModuleInit { export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger; logger: DadosferaLogger;
@@ -99,7 +96,7 @@ export class PipelinesService implements OnModuleInit {
const [errorType, message] = error.details.split('|'); const [errorType, message] = error.details.split('|');
throw new BadRequestException(message); throw new BadRequestException(message);
} }
throw new InternalServerErrorException(error.details); throw new ErrorBuilder(error.details);
}); });
this.logger.info('Done'); this.logger.info('Done');
@@ -140,11 +137,17 @@ export class PipelinesService implements OnModuleInit {
return findOnePipelineResponse; return findOnePipelineResponse;
} }
async update(UpdatePipelineRequest: Messages.PipelineV2UpdateRequest) { async update(
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
metadata,
) {
this.logger.info('PipelinesClientService - Update'); this.logger.info('PipelinesClientService - Update');
const updatePipelineResponse = await lastValueFrom( const updatePipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2Update(UpdatePipelineRequest), this.pipelineWriteService.PipelineV2Update(
UpdatePipelineRequest,
metadata,
),
); );
this.logger.info('Done'); this.logger.info('Done');
@@ -158,45 +161,53 @@ export class PipelinesService implements OnModuleInit {
customer_id: user.customer_id, customer_id: user.customer_id,
customer: user.customer_name, customer: user.customer_name,
}; };
let existsInProduct = true;
const { pipeline } = await lastValueFrom( const { pipeline } = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne({ id }, metadata), this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
); ).catch((e) => {
if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false;
else throw new ErrorBuilder(e.details);
return { pipeline: undefined };
});
await lastValueFrom( await lastValueFrom(
this.pipelineWriteService.PipelineV2Remove({ id }, metadata), this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
); );
//{pipeline:{tables: {tables: [], input_id: ''}}} if (existsInProduct) {
const input = pipeline.config.tables //{pipeline:{tables: {tables: [], input_id: ''}}}
? JSON.parse(pipeline.config.tables) const input = pipeline.config.tables
: null; ? JSON.parse(pipeline.config.tables)
if (input) : null;
await this.inputsService if (input)
.remove({ await this.inputsService
id: input.input_id,
info,
})
.catch((error) =>
this.logger.error('Could not delete input', {
data: { input, error },
}),
);
const transformations: { id: string }[] = pipeline.transformations
? JSON.parse(pipeline.transformations)
: null;
if (transformations && transformations.length)
for (const transformation of transformations) {
await this.transformationsService
.remove({ .remove({
id: transformation.id, id: input.input_id,
info, info,
}) })
.catch((error) => .catch((error) =>
this.logger.error('Could not delete transformation', { this.logger.error('Could not delete input', {
data: { transformation, error }, data: { input, error },
}), }),
); );
}
const transformations: { id: string }[] = pipeline.transformations
? JSON.parse(pipeline.transformations)
: null;
if (transformations && transformations.length)
for (const transformation of transformations) {
await this.transformationsService
.remove({
id: transformation.id,
info,
})
.catch((error) =>
this.logger.error('Could not delete transformation', {
data: { transformation, error },
}),
);
}
}
} }
async findOneProperties(id: string, metadata: Metadata) { async findOneProperties(id: string, metadata: Metadata) {
-5
View File
@@ -52,7 +52,6 @@ import { Metadata } from '@grpc/grpc-js';
import ErrorBuilder from 'src/utils/ErrorBuilder'; import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes'; import ErrorCodes from 'src/utils/errorCodes';
// TODO GET de hierarquias e do PATCH em usuário
@ApiTags('Users') @ApiTags('Users')
@Controller('users') @Controller('users')
@ApiHeader({ @ApiHeader({
@@ -75,10 +74,6 @@ export class UsersController {
} }
@Get() @Get()
@ApiOkResponse({
type: GetAllUsersByCustomerIdRes,
description: 'List of users of a customer',
})
async getAllUsersByCustomerId( async getAllUsersByCustomerId(
@User() user: RequestUser, @User() user: RequestUser,
@Headers('dadosfera-lang') language, @Headers('dadosfera-lang') language,
+10
View File
@@ -214,6 +214,16 @@ export function EnrichErrorCode(code: string) {
message: 'Id encontra-se num formato inválido', message: 'Id encontra-se num formato inválido',
code, code,
}; };
case ErrorCodes.CONNECTION.NOT_FOUND:
case ErrorCodes.CONNECTION.PLATFORM_NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Conexão não encontrada',
message:
'Não encontramos a conexão solicitada. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.CONNECTION_TEST.ERROR: case ErrorCodes.CONNECTION_TEST.ERROR:
return { return {
statusCode: HttpStatus.BAD_REQUEST, statusCode: HttpStatus.BAD_REQUEST,
+5
View File
@@ -60,6 +60,10 @@ const PIPELINE = {
const INPUT = { const INPUT = {
NOT_FOUND: 'INPUT.NOT_FOUND', NOT_FOUND: 'INPUT.NOT_FOUND',
}; };
const CONNECTION = {
NOT_FOUND: 'CONNECTION.NOT_FOUND',
PLATFORM_NOT_FOUND: 'CONNECTION.PLATFORM_NOT_FOUND',
};
const TRANSFORMATION = { const TRANSFORMATION = {
NOT_FOUND: 'TRANSFORMATION.NOT_FOUND', NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
}; };
@@ -77,6 +81,7 @@ const ErrorCodes = {
PIPELINE, PIPELINE,
INPUT, INPUT,
TRANSFORMATION, TRANSFORMATION,
CONNECTION,
}; };
export default ErrorCodes; export default ErrorCodes;