mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-04 05:34:48 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d7062efc60 | ||
|
|
4a3dd98977 | ||
|
|
06f4260b61 | ||
|
|
64717c8d92 | ||
|
|
e6e66c77e1 | ||
|
|
131e82b4c5 | ||
|
|
32d37ecadf | ||
|
|
4d667acb86 | ||
|
|
9d417ff61b | ||
|
|
b01c4c6c25 | ||
|
|
449a1abb63 | ||
|
|
04ca17a69d | ||
|
|
34a0eb40a7 | ||
|
|
2df69fba84 | ||
|
|
dcbcc5ff36 | ||
|
|
703b5f36b3 | ||
|
|
54d10cb873 | ||
|
|
569c3cf573 | ||
|
|
c420ef62ed | ||
|
|
966ba00fec | ||
|
|
0aadf7fc40 | ||
|
|
3e3268cb26 | ||
|
|
09bd623b62 | ||
|
|
4bf9a5831a | ||
|
|
264077289b | ||
|
|
14c686cff4 | ||
|
|
3c877b8b8d | ||
|
|
e6b0034805 | ||
|
|
b9c1f784d3 | ||
|
|
3edf5c2416 | ||
|
|
f3e00a15c8 | ||
|
|
0422c7a369 | ||
|
|
f22d9370db | ||
|
|
a9ee74724f |
@@ -10,8 +10,8 @@ on:
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- dev
|
||||
- stg
|
||||
- stg2
|
||||
- prd
|
||||
|
||||
jobs:
|
||||
@@ -152,17 +152,17 @@ jobs:
|
||||
- name: Extract Docs BlockId and PageId
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
DEV_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
|
||||
DEV_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
|
||||
STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
|
||||
STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
|
||||
STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
|
||||
STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
|
||||
PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
|
||||
PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
|
||||
shell: bash
|
||||
run: |
|
||||
if [ $ENV == "dev" ]; then
|
||||
echo "block_id=$DEV_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
|
||||
echo "page_id=$DEV_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
|
||||
if [ $ENV == "stg2" ]; then
|
||||
echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
|
||||
echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
|
||||
elif [ $ENV == "stg" ]; then
|
||||
echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
|
||||
echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
|
||||
|
||||
+54
-496
@@ -505,18 +505,16 @@
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/DeleteConnectorDto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "version",
|
||||
"required": true,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": ""
|
||||
@@ -948,7 +946,7 @@
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of users of a customer",
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
@@ -2351,6 +2349,25 @@
|
||||
"PipelinesV2"
|
||||
]
|
||||
},
|
||||
"patch": {
|
||||
"operationId": "PipelinesController_updateByPatch",
|
||||
"parameters": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"PipelinesV2"
|
||||
]
|
||||
},
|
||||
"delete": {
|
||||
"operationId": "PipelinesController_delete",
|
||||
"parameters": [],
|
||||
@@ -2394,88 +2411,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/inputs/test-connection": {
|
||||
"post": {
|
||||
"operationId": "InputsController_testConnection",
|
||||
"parameters": [],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/TestConnectionReq"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/TestConnectionRes"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"201": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Inputs"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/inputs/test-connection/get-columns": {
|
||||
"post": {
|
||||
"operationId": "InputsController_getColumns",
|
||||
"parameters": [],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/TestConnectionGetColumnsReq"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/TestConnectionGetColumnsRes"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"201": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Inputs"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/inputs": {
|
||||
"post": {
|
||||
"operationId": "InputsController_create",
|
||||
@@ -3298,9 +3233,9 @@
|
||||
}
|
||||
},
|
||||
"info": {
|
||||
"title": "Maestro - main",
|
||||
"title": "Maestro - stg2",
|
||||
"description": "Documentation for Maestro gateway",
|
||||
"version": "11.11.2022 - 12:24",
|
||||
"version": "0.0.0",
|
||||
"contact": {}
|
||||
},
|
||||
"tags": [],
|
||||
@@ -3568,17 +3503,6 @@
|
||||
"tags"
|
||||
]
|
||||
},
|
||||
"DeleteConnectorDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"version"
|
||||
]
|
||||
},
|
||||
"PermissionGroupPermission": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -5174,13 +5098,31 @@
|
||||
"updated_at"
|
||||
]
|
||||
},
|
||||
"AvailableEntity": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"replication_methods": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"replication_methods"
|
||||
]
|
||||
},
|
||||
"GetAvailableEntitiesRes": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"entities": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
"$ref": "#/components/schemas/AvailableEntity"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5188,99 +5130,6 @@
|
||||
"entities"
|
||||
]
|
||||
},
|
||||
"CredentialsJdbc": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"jdbc_user": {
|
||||
"type": "string"
|
||||
},
|
||||
"jdbc_password": {
|
||||
"type": "string"
|
||||
},
|
||||
"database": {
|
||||
"type": "string"
|
||||
},
|
||||
"endpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"port": {
|
||||
"type": "string"
|
||||
},
|
||||
"engine": {
|
||||
"type": "string"
|
||||
},
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TestConnectionReq": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"plugin": {
|
||||
"type": "string"
|
||||
},
|
||||
"credentials": {
|
||||
"$ref": "#/components/schemas/CredentialsJdbc"
|
||||
},
|
||||
"info": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"plugin",
|
||||
"credentials"
|
||||
]
|
||||
},
|
||||
"TestConnectionRes": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"connection_state": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"total_entities": {
|
||||
"type": "number"
|
||||
},
|
||||
"database_tables": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"connection_state",
|
||||
"total_entities",
|
||||
"database_tables"
|
||||
]
|
||||
},
|
||||
"TestConnectionGetColumnsReq": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"plugin": {
|
||||
"type": "string"
|
||||
},
|
||||
"tables": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"credentials": {
|
||||
"$ref": "#/components/schemas/CredentialsJdbc"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"info": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"plugin",
|
||||
"tables"
|
||||
]
|
||||
},
|
||||
"TableColumns": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -5298,203 +5147,16 @@
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"columns",
|
||||
"references"
|
||||
]
|
||||
},
|
||||
"TestConnectionGetColumnsRes": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tables": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/TableColumns"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"tables"
|
||||
]
|
||||
},
|
||||
"FileFormatParams": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sep": {
|
||||
"type": "string"
|
||||
},
|
||||
"header": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"encoding": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sep",
|
||||
"header",
|
||||
"encoding"
|
||||
]
|
||||
},
|
||||
"GoogleAnalyticsClientSecrets": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"project_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"private_key_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"private_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_email": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"auth_uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"token_uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"auth_provider_x509_cert_url": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_x509_cert_url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"project_id",
|
||||
"private_key_id",
|
||||
"private_key",
|
||||
"client_email",
|
||||
"client_id",
|
||||
"auth_uri",
|
||||
"token_uri",
|
||||
"auth_provider_x509_cert_url",
|
||||
"client_x509_cert_url"
|
||||
]
|
||||
},
|
||||
"Credentials": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"jdbc_user": {
|
||||
"type": "string"
|
||||
},
|
||||
"jdbc_password": {
|
||||
"type": "string"
|
||||
},
|
||||
"database": {
|
||||
"type": "string"
|
||||
},
|
||||
"endpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"port": {
|
||||
"type": "string"
|
||||
},
|
||||
"engine": {
|
||||
"type": "string"
|
||||
},
|
||||
"schema": {
|
||||
"type": "string"
|
||||
},
|
||||
"connection_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_aws_access_key_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_aws_secret_access_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_bucket": {
|
||||
"type": "string"
|
||||
},
|
||||
"file_to_extract": {
|
||||
"type": "string"
|
||||
},
|
||||
"file_format_params": {
|
||||
"$ref": "#/components/schemas/FileFormatParams"
|
||||
},
|
||||
"view_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"client_secrets": {
|
||||
"$ref": "#/components/schemas/GoogleAnalyticsClientSecrets"
|
||||
},
|
||||
"start_date": {
|
||||
"type": "string"
|
||||
},
|
||||
"end_date": {
|
||||
"type": "string"
|
||||
},
|
||||
"oauth_code": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"connection_type"
|
||||
]
|
||||
},
|
||||
"OauthObject": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"get_tokens_url": {
|
||||
"type": "string"
|
||||
},
|
||||
"get_tokens_url_params": {
|
||||
"type": "string"
|
||||
},
|
||||
"get_tokens_set_response": {
|
||||
"type": "object"
|
||||
},
|
||||
"content_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"InputOptions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"oauth": {
|
||||
"$ref": "#/components/schemas/OauthObject"
|
||||
},
|
||||
"skip_select_columns": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"skip_select_entities": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"skip_transformation": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"AuthParameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"aws_access_key_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"aws_secret_access_key": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"aws_access_key_id",
|
||||
"aws_secret_access_key"
|
||||
"references",
|
||||
"type"
|
||||
]
|
||||
},
|
||||
"CreateInputReq": {
|
||||
@@ -5517,58 +5179,6 @@
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"category": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"credentials": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/Credentials"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/InputOptions"
|
||||
}
|
||||
]
|
||||
},
|
||||
"info": {
|
||||
"type": "object",
|
||||
"deprecated": true
|
||||
},
|
||||
"cron": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"source_bucket": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"source_prefix": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"auth_parameters": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/AuthParameters"
|
||||
}
|
||||
]
|
||||
},
|
||||
"file_format_params": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/FileFormatParams"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -5607,58 +5217,6 @@
|
||||
},
|
||||
"updated_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"category": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"credentials": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/Credentials"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/InputOptions"
|
||||
}
|
||||
]
|
||||
},
|
||||
"info": {
|
||||
"type": "object",
|
||||
"deprecated": true
|
||||
},
|
||||
"cron": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"source_bucket": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"source_prefix": {
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"auth_parameters": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/AuthParameters"
|
||||
}
|
||||
]
|
||||
},
|
||||
"file_format_params": {
|
||||
"deprecated": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/FileFormatParams"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
Generated
+7
-7
@@ -12,7 +12,7 @@
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-secrets-manager": "^3.112.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack-v2": "3.19.4",
|
||||
"@dadosfera/protospack-v2": "3.21.0",
|
||||
"@grpc/grpc-js": "^1.6.7",
|
||||
"@grpc/proto-loader": "^0.6.13",
|
||||
"@nestjs/common": "^8.4.7",
|
||||
@@ -1727,9 +1727,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.19.4",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.4.tgz",
|
||||
"integrity": "sha512-Vb6bdFgQKYHf0kDK9kuMtN+nt3nkaMhJZ3wNVaUmQNxsnpGyARWBL29R2n+YnxS8g23tRwuj331sjg+DY0Fnhg==",
|
||||
"version": "3.21.0",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
|
||||
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.6.7",
|
||||
"rxjs": "^7.5.5",
|
||||
@@ -12330,9 +12330,9 @@
|
||||
}
|
||||
},
|
||||
"@dadosfera/protospack-v2": {
|
||||
"version": "3.19.4",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.4.tgz",
|
||||
"integrity": "sha512-Vb6bdFgQKYHf0kDK9kuMtN+nt3nkaMhJZ3wNVaUmQNxsnpGyARWBL29R2n+YnxS8g23tRwuj331sjg+DY0Fnhg==",
|
||||
"version": "3.21.0",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
|
||||
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
|
||||
"requires": {
|
||||
"@grpc/grpc-js": "^1.6.7",
|
||||
"rxjs": "^7.5.5",
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-secrets-manager": "^3.112.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack-v2": "3.19.4",
|
||||
"@dadosfera/protospack-v2": "3.21.0",
|
||||
"@grpc/grpc-js": "^1.6.7",
|
||||
"@grpc/proto-loader": "^0.6.13",
|
||||
"@nestjs/common": "^8.4.7",
|
||||
|
||||
@@ -297,12 +297,12 @@ export const PERMISSIONS_GROUPS = {
|
||||
},
|
||||
CREATE: {
|
||||
seqid: 19,
|
||||
claim: 'POST /catalog',
|
||||
usage: PermissionUsages.INTERNAL,
|
||||
claim: 'catalog:create',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Criar atributos no catálogo',
|
||||
'en-us': 'Create attributes',
|
||||
'es-es': 'Crear atributos',
|
||||
'pt-br': 'Criar Ativos',
|
||||
'en-us': 'Create Assets',
|
||||
'es-es': 'Crear Activos',
|
||||
},
|
||||
},
|
||||
UPDATE: {
|
||||
|
||||
+1
-7
@@ -30,12 +30,8 @@ async function bootstrap() {
|
||||
|
||||
await app.listen(3333);
|
||||
}
|
||||
function configureSwagger(app: INestApplication) {
|
||||
const date = new Date();
|
||||
const swaggerVersion = `${date.getDate()}.${
|
||||
date.getMonth() + 1
|
||||
}.${date.getFullYear()} - ${date.getHours()}:${date.getMinutes()}`;
|
||||
|
||||
function configureSwagger(app: INestApplication) {
|
||||
let branchName = '';
|
||||
try {
|
||||
branchName = execSync('git rev-parse --abbrev-ref HEAD')
|
||||
@@ -50,7 +46,6 @@ function configureSwagger(app: INestApplication) {
|
||||
const config = new DocumentBuilder()
|
||||
.setTitle(swaggerTitle)
|
||||
.setDescription('Documentation for Maestro gateway')
|
||||
.setVersion(swaggerVersion)
|
||||
.addBearerAuth(
|
||||
{ type: 'http', scheme: 'bearer', bearerFormat: 'JWT' },
|
||||
'Authorization',
|
||||
@@ -60,7 +55,6 @@ function configureSwagger(app: INestApplication) {
|
||||
const document = SwaggerModule.createDocument(app, config);
|
||||
|
||||
if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
|
||||
|
||||
writeFileSync(
|
||||
'docsfera.json',
|
||||
JSON.stringify(
|
||||
|
||||
@@ -14,8 +14,8 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import {
|
||||
AuthenticateCondition,
|
||||
Authenticated,
|
||||
RequireSomePermission,
|
||||
} from '../../authentication/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { CatalogService } from './catalog.service';
|
||||
@@ -34,36 +34,9 @@ import {
|
||||
IUpdateDataRequest,
|
||||
} from './dtos';
|
||||
|
||||
const catalogPermissions = PERMISSIONS_GROUPS.CATALOG.permissions;
|
||||
@ApiTags('Catalog')
|
||||
@Controller('catalog')
|
||||
@Authenticated()
|
||||
@AuthenticateCondition((req, user: RequestUser) => {
|
||||
if (user.permissions.includes(catalogPermissions.DATA_MANAGER.seqid))
|
||||
return true;
|
||||
|
||||
if (req.path.includes('/manage-permissions')) return true;
|
||||
let action: keyof typeof catalogPermissions;
|
||||
|
||||
switch (req.method) {
|
||||
case 'POST':
|
||||
case 'PUT':
|
||||
action = 'UPDATE';
|
||||
break;
|
||||
|
||||
case 'DELETE':
|
||||
if (req.path.includes('/comment')) action = 'UPDATE';
|
||||
else action = req.method;
|
||||
break;
|
||||
|
||||
default:
|
||||
action = req.method as 'GET';
|
||||
}
|
||||
|
||||
return user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions[action].seqid,
|
||||
);
|
||||
})
|
||||
export class CatalogController {
|
||||
logger: DadosferaLogger;
|
||||
constructor(
|
||||
@@ -75,6 +48,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get()
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async catalogAll(
|
||||
@User() user: RequestUser,
|
||||
@Query() query,
|
||||
@@ -110,6 +87,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get('tags')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async findAllTags(@Body() body) {
|
||||
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
|
||||
user: body.info.user_id,
|
||||
@@ -129,6 +110,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get('data-asset/:id')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async getDataAsset(
|
||||
@User() user: RequestUser,
|
||||
@Headers() headers,
|
||||
@@ -183,6 +168,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get('data-asset/rls/:id')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async getDataAssetRls(
|
||||
@User() user: RequestUser,
|
||||
@Headers() headers,
|
||||
@@ -237,6 +226,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get('data-asset/:id/columns-metadata')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async getDataAssetColumnsMetadata(
|
||||
@User() user: RequestUser,
|
||||
@Headers('Dadosfera-Lang') language,
|
||||
@@ -273,6 +266,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get('data-asset/:id/preview')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async getDataAssetPreview(
|
||||
@User() user: RequestUser,
|
||||
@Headers('Dadosfera-lang') language,
|
||||
@@ -308,6 +305,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Get('data-asset/:id/docs')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async getDataAssetDocs(
|
||||
@User() user: RequestUser,
|
||||
@Headers('Dadosfera-Lang') language,
|
||||
@@ -343,6 +344,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Put('data-asset/:id')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async updateDataAsset(
|
||||
@User() user: RequestUser,
|
||||
@Headers('Dadosfera-Lang') language,
|
||||
@@ -372,6 +377,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Post('data-asset/:id/docs')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async manageDataAssetDocs(
|
||||
@User() user: RequestUser,
|
||||
@Headers() headers,
|
||||
@@ -440,6 +449,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Post()
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async createDataAsset(
|
||||
@User() user: RequestUser,
|
||||
@Body() body: ICreateDataAsset,
|
||||
@@ -460,6 +473,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Post('data-asset/:id/comment')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async commentOnDataAsset(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
@@ -482,6 +499,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Delete('data-asset/:id')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async deleteDataAsset(
|
||||
@Param('id') data_asset_id: string,
|
||||
@User() user: RequestUser,
|
||||
@@ -504,6 +525,10 @@ export class CatalogController {
|
||||
}
|
||||
|
||||
@Delete('data-asset/:id/comment')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async deleteComment(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
import {
|
||||
OnModuleInit,
|
||||
Inject,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import { OnModuleInit, Inject } from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { ConnectionManager } from '@dadosfera/protospack-v2';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
@@ -86,12 +81,7 @@ export class ConnectionClientService implements OnModuleInit {
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
).catch((err) => {
|
||||
throw new HttpException(
|
||||
err.details,
|
||||
err.code === 6 ? HttpStatus.CONFLICT : 400,
|
||||
);
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
async updateConnection(
|
||||
@@ -112,12 +102,7 @@ export class ConnectionClientService implements OnModuleInit {
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
).catch((err) => {
|
||||
throw new HttpException(
|
||||
err.details,
|
||||
err.code === 6 ? HttpStatus.CONFLICT : 400,
|
||||
);
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
async deleteConnection({ body, metadata }): Promise<ConnectionRes> {
|
||||
@@ -130,12 +115,7 @@ export class ConnectionClientService implements OnModuleInit {
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
).catch((err) => {
|
||||
throw new HttpException(
|
||||
err.details,
|
||||
err.code === 6 ? HttpStatus.CONFLICT : 400,
|
||||
);
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
async getAllConnections({ body, metadata }): Promise<ConnectionsRes> {
|
||||
@@ -146,12 +126,7 @@ export class ConnectionClientService implements OnModuleInit {
|
||||
{ search, filters: JSON.stringify(filters), size, page },
|
||||
metadata,
|
||||
),
|
||||
).catch((err) => {
|
||||
throw new HttpException(
|
||||
err.details,
|
||||
err.code === 6 ? HttpStatus.CONFLICT : 400,
|
||||
);
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
async getConnectionDetails({
|
||||
@@ -162,12 +137,7 @@ export class ConnectionClientService implements OnModuleInit {
|
||||
|
||||
return lastValueFrom(
|
||||
this.connectionServiceRead.GetConnectionDetails({ id }, metadata),
|
||||
).catch((err) => {
|
||||
throw new HttpException(
|
||||
err.details,
|
||||
err.code === 6 ? HttpStatus.CONFLICT : 400,
|
||||
);
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
async INTERNAL_getAllConnections({
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
Get,
|
||||
Headers,
|
||||
Query,
|
||||
UseFilters,
|
||||
} from '@nestjs/common';
|
||||
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
|
||||
import { ConnectionClientService } from './client.service';
|
||||
@@ -28,8 +29,10 @@ import {
|
||||
} from './dtos/connection';
|
||||
import { CreateConnectionDto } from './dtos/connection';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
|
||||
const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
|
||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||
@ApiTags('connections')
|
||||
@ApiBearerAuth()
|
||||
@Authenticated()
|
||||
|
||||
@@ -83,163 +83,30 @@ export class ConnectorClientService implements OnModuleInit {
|
||||
);
|
||||
});
|
||||
const responseParsed = JSON.parse(res.message);
|
||||
const connectorErrors = responseParsed.items
|
||||
.filter((i) => i.update._index === 'connectors' && i.update.error)
|
||||
const connectorsResponse = responseParsed.items.filter(
|
||||
(i) => i.update._index === process.env.CONNECTORS_INDEX,
|
||||
);
|
||||
const connectorErrors = connectorsResponse
|
||||
.filter((i) => i.update.error)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
reason: i.update.error,
|
||||
status: i.error.status,
|
||||
}));
|
||||
const connectionControlsErrors = responseParsed.items
|
||||
.filter(
|
||||
(i) => i.update._index === 'connection_controls' && i.update.error,
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
reason: i.update.error,
|
||||
status: i.error.status,
|
||||
}));
|
||||
const connectionStepsErrors = responseParsed.items
|
||||
.filter((i) => i.update._index === 'connection_steps' && i.update.error)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
reason: i.update.error,
|
||||
status: i.error.status,
|
||||
}));
|
||||
const configControlsErrors = responseParsed.items
|
||||
.filter((i) => i.update._index === 'config_controls' && i.update.error)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
reason: i.update.error,
|
||||
status: i.error.status,
|
||||
}));
|
||||
const connectorCreated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connectors' && i.update.result === 'created',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectionControlsCreated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connection_controls' &&
|
||||
i.update.result === 'created',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectionStepsCreated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connection_steps' &&
|
||||
i.update.result === 'created',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const configControlsCreated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'config_controls' &&
|
||||
i.update.result === 'created',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectorUpdated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connectors' && i.update.result === 'updated',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectionControlsUpdated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connection_controls' &&
|
||||
i.update.result === 'updated',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectionStepsUpdated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connection_steps' &&
|
||||
i.update.result === 'updated',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const configControlsUpdated = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'config_controls' &&
|
||||
i.update.result === 'updated',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectorNoop = responseParsed.items
|
||||
.filter(
|
||||
(i) => i.update._index === 'connectors' && i.update.result === 'noop',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectionControlsNoop = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connection_controls' &&
|
||||
i.update.result === 'noop',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const connectionStepsNoop = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'connection_steps' && i.update.result === 'noop',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
}));
|
||||
const configControlsNoop = responseParsed.items
|
||||
.filter(
|
||||
(i) =>
|
||||
i.update._index === 'config_controls' && i.update.result === 'noop',
|
||||
)
|
||||
.map((i) => ({
|
||||
id: i.update._id,
|
||||
id: i.update?._id,
|
||||
reason: i.update?.error,
|
||||
status: i.error?.status,
|
||||
}));
|
||||
const connectorCreated = connectorsResponse
|
||||
.filter((i) => i.update.result === 'created')
|
||||
.map((i) => i.update._id);
|
||||
const connectorUpdated = connectorsResponse
|
||||
.filter((i) => i.update.result === 'updated')
|
||||
.map((i) => i.update._id);
|
||||
const connectorNoop = connectorsResponse
|
||||
.filter((i) => i.update.result === 'noop')
|
||||
.map((i) => i.update._id);
|
||||
return {
|
||||
errors: {
|
||||
connectors: connectorErrors,
|
||||
connection_controls: connectionControlsErrors,
|
||||
connection_steps: connectionStepsErrors,
|
||||
config_controls: configControlsErrors,
|
||||
},
|
||||
created: {
|
||||
connectors: connectorCreated,
|
||||
connection_controls: connectionControlsCreated,
|
||||
connection_steps: connectionStepsCreated,
|
||||
config_controls: configControlsCreated,
|
||||
},
|
||||
updated: {
|
||||
connectors: connectorUpdated,
|
||||
connection_controls: connectionControlsUpdated,
|
||||
connection_steps: connectionStepsUpdated,
|
||||
config_controls: configControlsUpdated,
|
||||
},
|
||||
noop: {
|
||||
connectors: connectorNoop,
|
||||
connection_controls: connectionControlsNoop,
|
||||
connection_steps: connectionStepsNoop,
|
||||
config_controls: configControlsNoop,
|
||||
},
|
||||
errors: connectorErrors,
|
||||
created: connectorCreated,
|
||||
updated: connectorUpdated,
|
||||
noop: connectorNoop,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -318,12 +185,7 @@ export class ConnectorClientService implements OnModuleInit {
|
||||
version,
|
||||
plugin,
|
||||
}),
|
||||
).catch((err) => {
|
||||
throw new HttpException(
|
||||
err.details,
|
||||
err.code === 6 ? HttpStatus.CONFLICT : 400,
|
||||
);
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
async updateConnector({ plugin, changes }) {
|
||||
|
||||
@@ -19,7 +19,6 @@ import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
|
||||
import { ConnectorClientService } from './client.service';
|
||||
import { AddTagDto } from './dtos/add-tag';
|
||||
import { CreateConnectorDto } from './dtos/create-connector';
|
||||
import { DeleteConnectorDto } from './dtos/delete-connector';
|
||||
import { GetAllDto } from './dtos/get-all';
|
||||
import { RemoveTagDto } from './dtos/remove-tag';
|
||||
import { UpdateDto } from './dtos/update';
|
||||
@@ -274,7 +273,7 @@ export class ConnectorController {
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
|
||||
async deleteConnector(
|
||||
@Param('plugin') plugin: string,
|
||||
@Body() { version }: DeleteConnectorDto,
|
||||
@Query('version') version: string,
|
||||
) {
|
||||
this.logger.info('/upload - Upload Connector Route');
|
||||
|
||||
|
||||
@@ -1,106 +1,5 @@
|
||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||
import { Info } from 'protospack/dist/lib/interfaces';
|
||||
|
||||
export class GoogleAnalyticsClientSecrets {
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
@ApiProperty()
|
||||
project_id: string;
|
||||
@ApiProperty()
|
||||
private_key_id: string;
|
||||
@ApiProperty()
|
||||
private_key: string;
|
||||
@ApiProperty()
|
||||
client_email: string;
|
||||
@ApiProperty()
|
||||
client_id: string;
|
||||
@ApiProperty()
|
||||
auth_uri: string;
|
||||
@ApiProperty()
|
||||
token_uri: string;
|
||||
@ApiProperty()
|
||||
auth_provider_x509_cert_url: string;
|
||||
@ApiProperty()
|
||||
client_x509_cert_url: string;
|
||||
}
|
||||
export class OauthObject {
|
||||
@ApiPropertyOptional()
|
||||
get_tokens_url: string;
|
||||
@ApiPropertyOptional()
|
||||
get_tokens_url_params: string;
|
||||
@ApiPropertyOptional()
|
||||
get_tokens_set_response: Record<string, any>;
|
||||
@ApiPropertyOptional()
|
||||
content_type: string;
|
||||
}
|
||||
export class InputOptions {
|
||||
@ApiPropertyOptional()
|
||||
oauth: OauthObject;
|
||||
@ApiPropertyOptional()
|
||||
skip_select_columns: true;
|
||||
@ApiPropertyOptional()
|
||||
skip_select_entities: false;
|
||||
@ApiPropertyOptional()
|
||||
skip_transformation: true;
|
||||
}
|
||||
export class FileFormatParams {
|
||||
@ApiProperty()
|
||||
sep: string;
|
||||
@ApiProperty()
|
||||
header: boolean;
|
||||
@ApiProperty()
|
||||
encoding: string;
|
||||
}
|
||||
|
||||
export class AuthParameters {
|
||||
@ApiProperty()
|
||||
aws_access_key_id: string;
|
||||
@ApiProperty()
|
||||
aws_secret_access_key: string;
|
||||
}
|
||||
export class CredentialsJdbc {
|
||||
@ApiPropertyOptional()
|
||||
jdbc_user: string;
|
||||
@ApiPropertyOptional()
|
||||
jdbc_password: string;
|
||||
@ApiPropertyOptional()
|
||||
database: string;
|
||||
@ApiPropertyOptional()
|
||||
endpoint: string;
|
||||
@ApiPropertyOptional()
|
||||
port: string;
|
||||
@ApiPropertyOptional()
|
||||
engine: string;
|
||||
@ApiPropertyOptional()
|
||||
schema: string;
|
||||
}
|
||||
export class Credentials extends CredentialsJdbc {
|
||||
@ApiProperty()
|
||||
connection_type: string;
|
||||
|
||||
@ApiPropertyOptional()
|
||||
client_aws_access_key_id: string;
|
||||
@ApiPropertyOptional()
|
||||
client_aws_secret_access_key: string;
|
||||
@ApiPropertyOptional()
|
||||
client_bucket: string;
|
||||
@ApiPropertyOptional()
|
||||
file_to_extract: string;
|
||||
@ApiPropertyOptional()
|
||||
file_format_params: FileFormatParams;
|
||||
|
||||
@ApiPropertyOptional()
|
||||
view_id: string;
|
||||
@ApiPropertyOptional()
|
||||
client_secrets: GoogleAnalyticsClientSecrets;
|
||||
@ApiPropertyOptional()
|
||||
start_date: string;
|
||||
@ApiPropertyOptional()
|
||||
end_date: string;
|
||||
|
||||
@ApiPropertyOptional()
|
||||
oauth_code: string;
|
||||
}
|
||||
export class Column {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@@ -114,6 +13,14 @@ export class TableColumns {
|
||||
columns: string[];
|
||||
@ApiProperty()
|
||||
references: Column[];
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
}
|
||||
export class AvailableEntity {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
replication_methods: string[];
|
||||
}
|
||||
|
||||
export class Input {
|
||||
@@ -133,26 +40,6 @@ export class Input {
|
||||
created_at: string;
|
||||
@ApiProperty()
|
||||
updated_at: string;
|
||||
|
||||
//deprecated attributes (for retro compatibility)
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
category: string;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
credentials: Credentials;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
options: InputOptions;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
info: Info;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
cron: string;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
source_bucket: string;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
source_prefix: string;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
auth_parameters: AuthParameters;
|
||||
@ApiPropertyOptional({ deprecated: true })
|
||||
file_format_params: FileFormatParams;
|
||||
}
|
||||
export class GetAvailableEntitiesReq {
|
||||
@ApiProperty()
|
||||
@@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq {
|
||||
}
|
||||
|
||||
export class GetAvailableEntitiesRes {
|
||||
@ApiProperty()
|
||||
entities: string[];
|
||||
@ApiProperty({ type: [AvailableEntity] })
|
||||
entities: AvailableEntity[];
|
||||
}
|
||||
|
||||
export class TestConnectionGetColumnsReq {
|
||||
@ApiProperty()
|
||||
plugin: string;
|
||||
@ApiProperty()
|
||||
tables: string[];
|
||||
@ApiPropertyOptional()
|
||||
credentials: CredentialsJdbc;
|
||||
@ApiPropertyOptional()
|
||||
id: string;
|
||||
@ApiPropertyOptional()
|
||||
info: Info;
|
||||
}
|
||||
export class TestConnectionGetColumnsRes {
|
||||
@ApiProperty({ type: [TableColumns] })
|
||||
tables: TableColumns[];
|
||||
}
|
||||
|
||||
export class TestConnectionReq {
|
||||
@ApiProperty()
|
||||
plugin: string;
|
||||
@ApiProperty()
|
||||
credentials: CredentialsJdbc;
|
||||
@ApiPropertyOptional()
|
||||
info: Info;
|
||||
}
|
||||
export class TestConnectionRes {
|
||||
@ApiProperty()
|
||||
connection_state: boolean;
|
||||
@ApiProperty()
|
||||
total_entities: number;
|
||||
@ApiProperty()
|
||||
database_tables: string[];
|
||||
}
|
||||
export class CreateInputReq extends OmitType(Input, [
|
||||
'id',
|
||||
'created_at',
|
||||
|
||||
@@ -19,16 +19,10 @@ import {
|
||||
GetAvailableEntitiesReq,
|
||||
GetAvailableEntitiesRes,
|
||||
Input,
|
||||
TestConnectionGetColumnsReq,
|
||||
TestConnectionGetColumnsRes,
|
||||
TestConnectionReq,
|
||||
TestConnectionRes,
|
||||
} from './dtos/input.model';
|
||||
import { UpdateInputRequest } from './dtos/old_interfaces';
|
||||
import { RequestUser, User } from 'src/authentication/user.decorator';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
|
||||
@ApiTags('Inputs')
|
||||
@@ -75,42 +69,19 @@ export class InputsController {
|
||||
@Get('available-entities/:plugin')
|
||||
@ApiOkResponse({ type: GetAvailableEntitiesRes })
|
||||
async getAvailableEntities(
|
||||
@Body() body,
|
||||
@Param() params: GetAvailableEntitiesReq,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const { info } = body;
|
||||
const { plugin } = params;
|
||||
return await this.inputService.getAvailableEntities({ info, plugin });
|
||||
}
|
||||
|
||||
@Post('/test-connection')
|
||||
@ApiOkResponse({ type: TestConnectionRes })
|
||||
async testConnection(@Body() data: TestConnectionReq) {
|
||||
this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
|
||||
user: data.info.user_id,
|
||||
customer: data.info.customer,
|
||||
});
|
||||
|
||||
const response = await this.inputService.testConnection(data);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
@Post('/test-connection/get-columns')
|
||||
@ApiOkResponse({ type: TestConnectionGetColumnsRes })
|
||||
async getColumns(@Body() data: TestConnectionGetColumnsReq) {
|
||||
throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED);
|
||||
this.logger.info(
|
||||
`/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
|
||||
{
|
||||
user: data.info.user_id,
|
||||
customer: data.info.customer,
|
||||
const availableEntities = await this.inputService.getAvailableEntities({
|
||||
info: {
|
||||
customer: user.customer_name,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.inputService.getColumns(data);
|
||||
|
||||
return response;
|
||||
plugin,
|
||||
});
|
||||
return availableEntities;
|
||||
}
|
||||
|
||||
@Post()
|
||||
|
||||
@@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
|
||||
import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
|
||||
import { Input } from '@dadosfera/protospack-v2';
|
||||
import {
|
||||
GetAvailableEntitiesRequest,
|
||||
InputCreateGenericRequest,
|
||||
InputCreateS3Request,
|
||||
InputNewCreateRequest,
|
||||
TestConnectionGetColumnsRequest,
|
||||
TestConnectionRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
@@ -88,32 +88,6 @@ export class InputsService {
|
||||
|
||||
return testConnectionResponse;
|
||||
},
|
||||
getColumns: async (data: TestConnectionGetColumnsRequest) => {
|
||||
this.logger.info('InputClientService - TestConnection/Get-Columns');
|
||||
const getColumnsResponse = await new Promise((resolve, reject) => {
|
||||
this.inputReadService.GetColumns(data).subscribe({
|
||||
next(x) {
|
||||
resolve(objectCamelToSnake(x));
|
||||
},
|
||||
error(err) {
|
||||
reject(err);
|
||||
},
|
||||
complete() {
|
||||
// console.log('done');
|
||||
},
|
||||
});
|
||||
})
|
||||
.then((res) => {
|
||||
this.logger.info('Done');
|
||||
return res;
|
||||
})
|
||||
.catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
|
||||
return getColumnsResponse;
|
||||
},
|
||||
getAvailableEntities: async (data) => {
|
||||
this.logger.info('InputClientService - GetAvailableEntities');
|
||||
data;
|
||||
@@ -188,12 +162,9 @@ export class InputsService {
|
||||
// this.validateCron(data);
|
||||
const { body, info } = data;
|
||||
|
||||
if (body.info) delete body.info;
|
||||
|
||||
const inputCreateGenericRequest: any | InputCreateGenericRequest = {
|
||||
const inputCreateGenericRequest: InputCreateGenericRequest = {
|
||||
input: {
|
||||
...body,
|
||||
credentials: body.credentials && JSON.stringify(body.credentials),
|
||||
},
|
||||
info,
|
||||
};
|
||||
@@ -205,8 +176,8 @@ export class InputsService {
|
||||
return { input: adjustedInput };
|
||||
}
|
||||
|
||||
async getAvailableEntities(data): Promise<{ entities: string[] }> {
|
||||
return await this.OLD_inputClient.getAvailableEntities(data);
|
||||
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
|
||||
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
|
||||
}
|
||||
|
||||
async findAll(info: Info) {
|
||||
@@ -263,17 +234,6 @@ export class InputsService {
|
||||
}
|
||||
}
|
||||
|
||||
async getColumns(data) {
|
||||
try {
|
||||
const testConnectionGetColumnsResponse =
|
||||
await this.OLD_inputClient.getColumns(data);
|
||||
|
||||
return testConnectionGetColumnsResponse;
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
generateInputS3Payload(payload) {
|
||||
const { credentials, plugin, cron } = payload;
|
||||
if (!credentials) return payload;
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
UseFilters,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Patch,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -45,6 +46,10 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
|
||||
action = 'UPDATE';
|
||||
break;
|
||||
|
||||
case 'PATCH':
|
||||
action = 'UPDATE';
|
||||
break;
|
||||
|
||||
default:
|
||||
action = req.method;
|
||||
}
|
||||
@@ -175,20 +180,49 @@ export class PipelinesController {
|
||||
|
||||
@Put('/:id')
|
||||
async update(
|
||||
@Headers('Dadosfera-Lang') language,
|
||||
@Body() updatePipelineDto,
|
||||
@Param('id') id,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
this.logger.info('PipelinesController - findOne', { user });
|
||||
this.logger.info('PipelinesController - update', { user });
|
||||
const { info } = updatePipelineDto;
|
||||
delete updatePipelineDto.info;
|
||||
|
||||
const response = await this.pipelinesClientService.update({
|
||||
...updatePipelineDto,
|
||||
info,
|
||||
id,
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
if (!language) language = 'en-us';
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
});
|
||||
|
||||
const response = await this.pipelinesClientService.update(
|
||||
{
|
||||
...updatePipelineDto,
|
||||
info,
|
||||
id,
|
||||
},
|
||||
metadata,
|
||||
);
|
||||
|
||||
this.logger.info('PipelinesController - update: OK', { user });
|
||||
return response;
|
||||
}
|
||||
|
||||
@Patch('/:id')
|
||||
async updateByPatch(
|
||||
@Headers('Dadosfera-Lang') language,
|
||||
@Body() updatePipelineDto,
|
||||
@Param('id') id,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
this.logger.info('PipelinesController - patch', { user });
|
||||
const response = await this.update(language, updatePipelineDto, id, user);
|
||||
this.logger.info('PipelinesController - patch: OK', { user });
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Inject,
|
||||
InternalServerErrorException,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import {
|
||||
Messages,
|
||||
@@ -23,6 +18,8 @@ import { InputsService } from '../inputs/inputs.service';
|
||||
import { RequestUser } from 'src/authentication/user.decorator';
|
||||
import { TransformationsService } from '../transformations/transformations.service';
|
||||
import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
|
||||
export class PipelinesService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
@@ -99,7 +96,7 @@ export class PipelinesService implements OnModuleInit {
|
||||
const [errorType, message] = error.details.split('|');
|
||||
throw new BadRequestException(message);
|
||||
}
|
||||
throw new InternalServerErrorException(error.details);
|
||||
throw new ErrorBuilder(error.details);
|
||||
});
|
||||
this.logger.info('Done');
|
||||
|
||||
@@ -140,11 +137,17 @@ export class PipelinesService implements OnModuleInit {
|
||||
return findOnePipelineResponse;
|
||||
}
|
||||
|
||||
async update(UpdatePipelineRequest: Messages.PipelineV2UpdateRequest) {
|
||||
async update(
|
||||
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
|
||||
metadata,
|
||||
) {
|
||||
this.logger.info('PipelinesClientService - Update');
|
||||
|
||||
const updatePipelineResponse = await lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2Update(UpdatePipelineRequest),
|
||||
this.pipelineWriteService.PipelineV2Update(
|
||||
UpdatePipelineRequest,
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
this.logger.info('Done');
|
||||
|
||||
@@ -158,45 +161,53 @@ export class PipelinesService implements OnModuleInit {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
};
|
||||
let existsInProduct = true;
|
||||
const { pipeline } = await lastValueFrom(
|
||||
this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
|
||||
);
|
||||
).catch((e) => {
|
||||
if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false;
|
||||
else throw new ErrorBuilder(e.details);
|
||||
|
||||
return { pipeline: undefined };
|
||||
});
|
||||
await lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
|
||||
);
|
||||
|
||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
||||
const input = pipeline.config.tables
|
||||
? JSON.parse(pipeline.config.tables)
|
||||
: null;
|
||||
if (input)
|
||||
await this.inputsService
|
||||
.remove({
|
||||
id: input.input_id,
|
||||
info,
|
||||
})
|
||||
.catch((error) =>
|
||||
this.logger.error('Could not delete input', {
|
||||
data: { input, error },
|
||||
}),
|
||||
);
|
||||
|
||||
const transformations: { id: string }[] = pipeline.transformations
|
||||
? JSON.parse(pipeline.transformations)
|
||||
: null;
|
||||
if (transformations && transformations.length)
|
||||
for (const transformation of transformations) {
|
||||
await this.transformationsService
|
||||
if (existsInProduct) {
|
||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
||||
const input = pipeline.config.tables
|
||||
? JSON.parse(pipeline.config.tables)
|
||||
: null;
|
||||
if (input)
|
||||
await this.inputsService
|
||||
.remove({
|
||||
id: transformation.id,
|
||||
id: input.input_id,
|
||||
info,
|
||||
})
|
||||
.catch((error) =>
|
||||
this.logger.error('Could not delete transformation', {
|
||||
data: { transformation, error },
|
||||
this.logger.error('Could not delete input', {
|
||||
data: { input, error },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const transformations: { id: string }[] = pipeline.transformations
|
||||
? JSON.parse(pipeline.transformations)
|
||||
: null;
|
||||
if (transformations && transformations.length)
|
||||
for (const transformation of transformations) {
|
||||
await this.transformationsService
|
||||
.remove({
|
||||
id: transformation.id,
|
||||
info,
|
||||
})
|
||||
.catch((error) =>
|
||||
this.logger.error('Could not delete transformation', {
|
||||
data: { transformation, error },
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async findOneProperties(id: string, metadata: Metadata) {
|
||||
|
||||
@@ -52,7 +52,6 @@ import { Metadata } from '@grpc/grpc-js';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
|
||||
// TODO GET de hierarquias e do PATCH em usuário
|
||||
@ApiTags('Users')
|
||||
@Controller('users')
|
||||
@ApiHeader({
|
||||
@@ -75,10 +74,6 @@ export class UsersController {
|
||||
}
|
||||
|
||||
@Get()
|
||||
@ApiOkResponse({
|
||||
type: GetAllUsersByCustomerIdRes,
|
||||
description: 'List of users of a customer',
|
||||
})
|
||||
async getAllUsersByCustomerId(
|
||||
@User() user: RequestUser,
|
||||
@Headers('dadosfera-lang') language,
|
||||
|
||||
@@ -214,6 +214,16 @@ export function EnrichErrorCode(code: string) {
|
||||
message: 'Id encontra-se num formato inválido',
|
||||
code,
|
||||
};
|
||||
case ErrorCodes.CONNECTION.NOT_FOUND:
|
||||
case ErrorCodes.CONNECTION.PLATFORM_NOT_FOUND:
|
||||
return {
|
||||
statusCode: HttpStatus.NOT_FOUND,
|
||||
error: 'Conexão não encontrada',
|
||||
message:
|
||||
'Não encontramos a conexão solicitada. Caso o erro persista, entre em contato com o suporte',
|
||||
code,
|
||||
};
|
||||
|
||||
case ErrorCodes.CONNECTION_TEST.ERROR:
|
||||
return {
|
||||
statusCode: HttpStatus.BAD_REQUEST,
|
||||
|
||||
@@ -60,6 +60,10 @@ const PIPELINE = {
|
||||
const INPUT = {
|
||||
NOT_FOUND: 'INPUT.NOT_FOUND',
|
||||
};
|
||||
const CONNECTION = {
|
||||
NOT_FOUND: 'CONNECTION.NOT_FOUND',
|
||||
PLATFORM_NOT_FOUND: 'CONNECTION.PLATFORM_NOT_FOUND',
|
||||
};
|
||||
const TRANSFORMATION = {
|
||||
NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
|
||||
};
|
||||
@@ -77,6 +81,7 @@ const ErrorCodes = {
|
||||
PIPELINE,
|
||||
INPUT,
|
||||
TRANSFORMATION,
|
||||
CONNECTION,
|
||||
};
|
||||
|
||||
export default ErrorCodes;
|
||||
|
||||
Reference in New Issue
Block a user