Compare commits

...
Author SHA1 Message Date
marcos.rodrigues 00cbadbb45 Merge branch 'beta' into bugfix/catalog-module 2026-08-07 12:26:11 -03:00
marcos.rodrigues 47ad527d38 FIX: skip nimbus update when customer haven't catalog module 2026-08-07 12:18:37 -03:00
iruy-fr 51044a23b3 FIX: trigger cache connections rollout 2026-08-03 09:19:27 -03:00
yuri.rodrigues bb29d126c1 Merge pull request #503 from dadosfera/feat/cache-connections-rollout
feat(connection-test): refresh connection catalog cache
2026-07-31 20:56:11 -03:00
iruy-fr 9d0f449eeb feat(connection-test): refresh connection catalog cache 2026-07-31 16:33:15 -03:00
iruy-fr 0eafa67e6f FIX: trigger cache connections deployment 2026-07-31 09:56:05 -03:00
yuri.rodrigues c3937472ec Merge pull request #502 from dadosfera/feat/cache-connections-rollout
feat: read connection metadata from catalog cache
2026-07-31 09:46:57 -03:00
iruy-fr 84d64424ca feat: read connection metadata from catalog cache 2026-07-30 10:17:28 -03:00
vinicius gadea f0bfc5c94b Merge pull request #500 from dadosfera/feat/custom-properties
FEAT: add color and emoji properties to CustomPropertyDto
2026-07-28 12:21:19 -03:00
viniciusgadea ad86a6a698 FEAT: simplify color and emoji property definitions in docsfera.json 2026-07-28 12:01:50 -03:00
viniciusgadea af3b11ad54 FEAT: add color and emoji properties to CustomPropertyDto 2026-07-28 07:42:18 -03:00
marcos.rodrigues d54f381998 Merge pull request #499 from dadosfera/release/2026-07-27
Release/2026 07 27
2026-07-27 17:59:56 -03:00
viniciusgadea 3fd586753e FEAT: add endpoint and logic to update data asset certification status 2026-07-27 17:07:19 -03:00
viniciusgadea 9b3894f9e4 FIX: pin npm version to 10.8.2 in Dockerfile for consistency 2026-07-27 17:07:10 -03:00
viniciusgadea 4a5f8f679a FEAT: rename documentation_status to certification_status in docs and update package.json for protospack versioning 2026-07-27 17:06:58 -03:00
viniciusgadea a444f5e5ec FEAT: add documentation status enum and property to data asset 2026-07-27 17:06:25 -03:00
marcos.rodrigues 0d30c1cf83 FEAT: remove deprecated protospack lib 2026-07-27 17:06:10 -03:00
vinicius gadea 15860519e1 Merge pull request #498 from dadosfera/feat/custom-properties
FEAT: add custom properties endpoints and DTOs for catalog management
2026-07-22 11:57:51 -03:00
viniciusgadea cffda86eab FEAT: add CustomPropertyDto and update IUpdateDataRequest to use custom properties array 2026-07-22 10:51:53 -03:00
viniciusgadea a93fbfbbd9 FEAT: remove custom properties endpoints and DTOs from catalog management 2026-07-21 16:32:38 -03:00
viniciusgadea e1cfc6e1a8 FEAT: add custom properties endpoints and DTOs for catalog management 2026-07-21 11:21:22 -03:00
vinicius gadea 1e63df6536 Merge pull request #497 from dadosfera/feat/documentation-status
FEAT: add endpoint and logic to update data asset certification status
2026-07-20 13:44:43 -03:00
viniciusgadea 8eb7fd0169 FEAT: add endpoint and logic to update data asset certification status 2026-07-17 09:55:17 -03:00
vinicius gadea f8c6a8b747 Merge pull request #496 from dadosfera/feat/documentation-status
FIX: pin npm version to 10.8.2 in Dockerfile for consistency
2026-07-16 17:06:04 -03:00
viniciusgadea 8a9d6c2f9c FIX: pin npm version to 10.8.2 in Dockerfile for consistency 2026-07-16 17:05:10 -03:00
vinicius gadea bcb2a0b7cb Merge pull request #495 from dadosfera/feat/documentation-status
Feat/documentation status
2026-07-16 16:47:52 -03:00
viniciusgadea 2e181e70af FEAT: rename documentation_status to certification_status in docs and update package.json for protospack versioning 2026-07-16 09:08:11 -03:00
vinicius gadea ef615adb8e Merge branch 'beta' into feat/documentation-status 2026-07-13 08:42:37 -03:00
viniciusgadea e03900811b FEAT: add documentation status enum and property to data asset 2026-07-13 08:33:35 -03:00
yuri.rodrigues 5bc5fb0977 Merge pull request #494 from dadosfera/feat/pipeline-run-jobs
feat: add endpoint to retrieve pipeline run jobs
2026-06-23 13:38:55 -03:00
iruy-fr 011032e3d4 FEAT: update API title in docsfera.json to reflect project name 2026-06-23 11:58:28 -03:00
iruy-fr 17363e74f4 FEAT: simplify pipeline run jobs handling and normalize run ID usage 2026-06-23 11:54:59 -03:00
iruy-fr 63efff6adf FEAT: enhance pipeline run jobs endpoint with error handling and response structure 2026-06-22 20:25:16 -03:00
iruy-fr b5f569e522 Merge branch 'beta' into feat/pipeline-run-jobs
# Conflicts:
#	docsfera.json
2026-06-19 17:08:34 -03:00
iruy-fr 5c77577992 feat: add endpoint to retrieve pipeline run jobs 2026-06-19 17:04:42 -03:00
marcos.rodrigues 4b9e113185 Merge pull request #493 from dadosfera/chore/remove-deprecated-lib
FEAT: remove deprecated protospack lib
2026-06-14 15:10:22 -03:00
marcos.rodrigues 06d505c50a FEAT: remove deprecated protospack lib 2026-06-13 21:07:14 -03:00
Marcos Rodrigues Silva c70abfc826 Merge pull request #492 from dadosfera/release/25-05
Release/25 05
2026-05-25 17:48:12 -03:00
marcos-silva-rodrigues db2c7d6c02 FIX: tests 2026-05-25 17:43:38 -03:00
marcos-silva-rodrigues 0a99ce1aa4 FEAT: update deployment to include firebase base url 2026-05-25 14:51:40 -03:00
marcos-silva-rodrigues 39c66f030a FIX: release note endpoint 2026-05-25 14:51:32 -03:00
marcos-silva-rodrigues 1223e21ac4 FEAT: pipeline upgrade route 2026-05-25 14:51:20 -03:00
Marcos Rodrigues Silva 2177f6725c Merge pull request #491 from dadosfera/hotfix/release-notes
Hotfix/release notes
2026-05-25 14:10:46 -03:00
marcos-silva-rodrigues 65ab16236f FEAT: update deployment to include firebase base url 2026-05-25 14:09:16 -03:00
marcos-silva-rodrigues b4cc8151d7 FIX: release note endpoint 2026-05-25 14:01:00 -03:00
yuri.rodrigues e8b982998f Merge pull request #490 from dadosfera/feat/platform-job-rout
Feat/platform job rout
2026-05-21 16:48:38 -03:00
iruy-fr ccd4159c59 fix: validate workflow 2026-05-21 16:43:37 -03:00
iruy-fr 9e49abb40d fix: validate workflow 2026-05-21 16:37:47 -03:00
iruy-fr 21a82b64f3 fix: validate workflow 2026-05-21 16:31:45 -03:00
iruy-fr 2414fcf21e fix: validate workflow 2026-05-21 16:23:45 -03:00
iruy-fr 94fbdb2226 fix: validate workflow 2026-05-21 16:21:30 -03:00
iruy-fr 985170d7ae chore: exposure from route pipeline run jobs to maestro 2026-05-21 16:12:35 -03:00
iruy-fr b38b8f51e2 FEAT: Add endpoint to fetch pipeline run jobs 2026-05-20 15:34:49 -03:00
Marcos Rodrigues Silva ea16e62d6b Merge pull request #489 from dadosfera/feature/pipeline-upgrade
FEAT: pipeline upgrade route
2026-05-11 12:58:06 -03:00
marcos-silva-rodrigues 6182705410 FEAT: pipeline upgrade route 2026-05-11 12:52:07 -03:00
Marcos Rodrigues Silva a5d78a97ae Merge pull request #486 from dadosfera/beta
Beta
2026-05-04 18:00:02 -03:00
Marcos Rodrigues Silva 2b33c22149 Merge pull request #488 from dadosfera/feature/table-schema-filter
FIX: roles
2026-04-29 18:02:04 -03:00
marcos-silva-rodrigues e32787baff FIX: roles 2026-04-29 17:06:04 -03:00
Marcos Rodrigues Silva f449d8ebd9 Merge pull request #487 from dadosfera/feature/table-schema-filter
FEAT: list assets by pipeline and multiple ids
2026-04-29 16:50:52 -03:00
marcos-silva-rodrigues 02627d023a FEAT: list assets by pipeline and multiple ids 2026-04-29 16:26:43 -03:00
Marcos Rodrigues Silva cf94c73648 Merge pull request #485 from dadosfera/feature/table-schema-filter
FIX: update package lock
2026-04-23 15:36:18 -03:00
marcos-silva-rodrigues 6b3241281b FIX: update package lock 2026-04-23 15:33:18 -03:00
Marcos Rodrigues Silva fe1caa003e Merge pull request #484 from dadosfera/feature/table-schema-filter
Feature/table schema filter
2026-04-23 15:16:11 -03:00
marcos-silva-rodrigues e980c58507 FEAT: list schema and filter assets by schema 2026-04-23 15:15:14 -03:00
Marcos Rodrigues Silva ff5f8672e2 Merge pull request #483 from dadosfera/beta
Beta
2026-04-20 10:46:37 -03:00
Marcos Rodrigues Silva d51456ecf7 Merge pull request #482 from dadosfera/hotfix/disable-edit-pipeline
FIX: block cancel first pipeline
2026-04-16 15:02:20 -03:00
marcos-silva-rodrigues 0cb9da13d0 FIX: block cancel first pipeline 2026-04-16 14:58:19 -03:00
Marcos Rodrigues Silva f198449c16 Merge pull request #481 from dadosfera/hotfix/disable-edit-pipeline
Hotfix/disable edit pipeline
2026-04-13 14:29:58 -03:00
marcos-silva-rodrigues 0d627b0451 FEAT: guard to prevent pipeline update when pipeline is running 2026-04-13 14:23:44 -03:00
48 changed files with 1592 additions and 785 deletions
+6
View File
@@ -71,6 +71,11 @@ jobs:
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff plugin
run: |
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
helm diff version
- name: Debug Helm env
run: |
helm env
@@ -102,4 +107,5 @@ jobs:
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+1 -1
View File
@@ -1,5 +1,5 @@
FROM node:20-alpine AS base_image
RUN npm install -g npm@latest
RUN npm install -g npm@10.8.2
FROM base_image AS build_base
WORKDIR /app
@@ -111,8 +111,12 @@ spec:
value: "{{ .Values.maestro.redis_tls }}"
- name: PLATFORM_API_URL
value: {{ .Values.maestro.platform_api_url }}
- name: CONNECTIONS_API_URL
value: {{ .Values.maestro.connections_api_url | default "" | quote }}
- name: STORAGE_EXPLORER_API_URL
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
- name: FIREBASE_BASE_URL
value: {{ .Values.maestro.firebase_base_url }}
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
+2
View File
@@ -9,7 +9,9 @@ maestro:
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
hostname: maestro.stg.dadosfera.ai
+1
View File
@@ -55,6 +55,7 @@ maestro:
redis_database: "0"
redis_tls: "true"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
autoscaling:
enabled: false
minReplicas: 1
+388 -320
View File
@@ -3343,14 +3343,7 @@
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
"description": ""
}
},
"tags": [
@@ -3541,6 +3534,64 @@
]
}
},
"/pipelinesV2/{id}/data-assets": {
"get": {
"operationId": "PipelinesController_findAllDataAssetByPipeline",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "object",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object"
}
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2/{pipelineId}/inputs/{id}": {
"patch": {
"operationId": "PipelinesController_updatePipelineInput",
@@ -3589,6 +3640,46 @@
]
}
},
"/pipelinesV2/{id}/upgrade": {
"patch": {
"operationId": "PipelinesController_upgradeConnector",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"204": {
"description": ""
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2/init-upload": {
"post": {
"operationId": "PipelinesController_initUploadFile",
@@ -3776,88 +3867,6 @@
]
}
},
"/pipelines/start/{id}": {
"post": {
"operationId": "PipelinesController_activate",
"summary": "",
"deprecated": true,
"description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Pipelines"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/pipelines/{id}/status": {
"get": {
"operationId": "PipelinesController_getPipelineStatus",
"summary": "",
"deprecated": true,
"description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Pipelines"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/transformations": {
"post": {
"operationId": "TransformationsController_create",
@@ -4001,7 +4010,7 @@
]
}
},
"/platform/pipeline": {
"/platform/pipelines": {
"post": {
"operationId": "PlatformApiController_createPipeline",
"summary": "Create a new pipeline",
@@ -4026,9 +4035,7 @@
"access-token": []
}
]
}
},
"/platform/pipelines": {
},
"get": {
"operationId": "PlatformApiController_getPipelines",
"summary": "List all pipelines for customer",
@@ -4055,7 +4062,7 @@
]
}
},
"/platform/pipeline/{pipelineId}": {
"/platform/pipelines/{pipelineId}": {
"get": {
"operationId": "PlatformApiController_getPipeline",
"summary": "Get pipeline by ID",
@@ -4159,7 +4166,7 @@
]
}
},
"/platform/pipeline/execute": {
"/platform/pipelines/execute": {
"post": {
"operationId": "PlatformApiController_executePipeline",
"summary": "Execute a pipeline",
@@ -4186,7 +4193,7 @@
]
}
},
"/platform/pipeline/pause": {
"/platform/pipelines/pause": {
"post": {
"operationId": "PlatformApiController_pausePipeline",
"summary": "Pause a pipeline",
@@ -4213,7 +4220,7 @@
]
}
},
"/platform/pipeline/unpause": {
"/platform/pipelines/unpause": {
"post": {
"operationId": "PlatformApiController_unpausePipeline",
"summary": "Unpause a pipeline",
@@ -4240,7 +4247,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/memory": {
"/platform/pipelines/{pipelineId}/memory": {
"put": {
"operationId": "PlatformApiController_updatePipelineMemory",
"summary": "Update pipeline memory configuration",
@@ -4276,7 +4283,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/metadata": {
"/platform/pipelines/{pipelineId}/metadata": {
"put": {
"operationId": "PlatformApiController_updatePipelineMetadata",
"summary": "Update pipeline metadata",
@@ -4403,7 +4410,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run": {
"/platform/pipelines/{pipelineId}/pipeline_run": {
"get": {
"operationId": "PlatformApiController_getPipelineRuns",
"summary": "Get pipeline runs for a pipeline",
@@ -4439,7 +4446,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run/{runId}": {
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}": {
"get": {
"operationId": "PlatformApiController_getPipelineRun",
"summary": "Get specific pipeline run",
@@ -4483,7 +4490,7 @@
]
}
},
"/platform/pipeline/pipeline_run/{runId}/logs": {
"/platform/pipelines/pipeline_run/{runId}/logs": {
"get": {
"operationId": "PlatformApiController_getPipelineRunLogs",
"summary": "Get pipeline run logs",
@@ -4519,7 +4526,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run/{runId}/cancel": {
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/cancel": {
"post": {
"operationId": "PlatformApiController_cancelPipelineRun",
"summary": "Cancel a running pipeline run",
@@ -4563,6 +4570,62 @@
]
}
},
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
"get": {
"operationId": "PlatformApiController_getPipelineRunJobs",
"summary": "Get pipeline run jobs",
"description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.",
"parameters": [
{
"name": "pipelineId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "runId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "DB-backed job runs for the selected pipeline run.",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"jobs": {
"type": "array",
"items": {
"type": "object"
}
}
},
"required": [
"jobs"
]
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/{jobId}/input": {
"put": {
"operationId": "PlatformApiController_updateJobInput",
@@ -4742,114 +4805,6 @@
]
}
},
"/platform/jobs/jdbc/{jobId}": {
"get": {
"operationId": "PlatformApiController_getJdbcJob",
"summary": "Get JDBC job details",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/jdbc/{jobId}/sync-mode": {
"post": {
"operationId": "PlatformApiController_updateJdbcSyncMode",
"summary": "Update JDBC job sync mode",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/{jobId}/rename-tables": {
"post": {
"operationId": "PlatformApiController_renameJobTables",
"summary": "Rename job output tables and sync to catalog",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/jdbc/configs/allowed_datatypes": {
"get": {
"operationId": "PlatformApiController_getJdbcAllowedDatatypes",
@@ -4877,114 +4832,6 @@
]
}
},
"/platform/jobs/singer/{jobId}": {
"get": {
"operationId": "PlatformApiController_getSingerJob",
"summary": "Get Singer job details",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/singer/{jobId}/sync-mode": {
"post": {
"operationId": "PlatformApiController_updateSingerSyncMode",
"summary": "Update Singer job sync mode",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/s3/{jobId}": {
"get": {
"operationId": "PlatformApiController_getS3Job",
"summary": "Get S3 job details",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/health": {
"get": {
"operationId": "PlatformApiController_healthCheck",
@@ -5649,6 +5496,90 @@
]
}
},
"/catalog/schemas": {
"get": {
"operationId": "CatalogController_findSchemas",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/custom-properties": {
"get": {
"operationId": "CatalogController_getCustomPropertyDefinitions",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/data-asset/{id}": {
"get": {
"operationId": "CatalogController_getDataAsset",
@@ -6063,6 +5994,66 @@
]
}
},
"/catalog/data-asset/{id}/certification-status": {
"put": {
"operationId": "CatalogController_updateDataAssetCertificationStatus",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/data-asset/{id}/manage-permissions": {
"put": {
"operationId": "CatalogController_manageDataAssetPermissions",
@@ -8714,6 +8705,29 @@
"Health"
]
}
},
"/release_note": {
"get": {
"operationId": "ReleaseNoteController_getLatestReleaseNote",
"parameters": [],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"security": [
{
"access-token": []
}
]
}
}
},
"info": {
@@ -11073,6 +11087,37 @@
"docs"
]
},
"CustomPropertyDto": {
"type": "object",
"properties": {
"key": {
"type": "string"
},
"value": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"text",
"number",
"date",
"boolean"
]
},
"color": {
"type": "string"
},
"emoji": {
"type": "string"
}
},
"required": [
"key",
"value",
"type"
]
},
"IUpdateDataRequest": {
"type": "object",
"properties": {
@@ -11101,6 +11146,12 @@
},
"docs": {
"type": "string"
},
"custom_properties": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomPropertyDto"
}
}
},
"required": [
@@ -11120,6 +11171,23 @@
"data_asset"
]
},
"IUpdateCertificationStatusRequest": {
"type": "object",
"properties": {
"certification_status": {
"type": "string",
"enum": [
"draft",
"in_review",
"approved",
"deprecated"
]
}
},
"required": [
"certification_status"
]
},
"ICreateDataAsset": {
"type": "object",
"properties": {
+4 -14
View File
@@ -16,8 +16,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.40.0-beta.5",
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1735,19 +1734,10 @@
"winston-log2gelf": "^2.4.0"
}
},
"node_modules/@dadosfera/protospack": {
"version": "2.5.3",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
"license": "ISC",
"dependencies": {
"rxjs": "^7.5.5"
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.5",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.5.tgz",
"integrity": "sha512-iocKv/XXp2jKAasO5ONgm31cKfLgNsU4pEKZMr6YnR7nQaH11WcW7rnuagNxWoik++wLUqbYyf0bZWRDzMlCPA==",
"version": "3.40.0-beta.14",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
+1 -2
View File
@@ -34,8 +34,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.40.0-beta.5",
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+3 -2
View File
@@ -17,7 +17,6 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
import { NetworkConfigModule } from './modules/network-config/network-config.module';
import { InputsModule } from './modules/inputs/inputs.module';
import { OauthModule } from './modules/oauth/oauth.module';
import { PipelinesModule } from './modules/pipelines/pipelines.module';
import { TransformationsModule } from './modules/transformations/transformations.module';
import { HealthModule } from './modules/health/health.module';
import { CatalogModule } from './modules/catalog/catalog.module';
@@ -35,6 +34,8 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod
import { ApiKeyModule } from './modules/api-key/api-key.module';
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
import { ReleaseNoteModule } from './modules/release_note/release_note.module';
@Module({
providers: [
@@ -58,7 +59,6 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor
PermissionsModule,
TermsOfUseModule,
ConnectionTestModule,
PipelinesModule,
TransformationsModule,
UsersModule,
RolesModule,
@@ -79,6 +79,7 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor
StorageExplorerModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
ReleaseNoteModule,
],
})
export class AppModule {}
@@ -153,6 +153,7 @@ export class AuthenticationGuard
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
roles: accessTokenPayload.roles,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
+1
View File
@@ -11,6 +11,7 @@ export function extractUserFrom(aRawJwt: string) {
user_id: payload.user_id,
username: payload.username,
permissions: payload.permissions,
roles: payload.roles,
customer_id: payload.customer_id,
customer_name: payload.customer_name,
customer_tier: payload.customer_tier,
+10
View File
@@ -357,6 +357,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Crear y editar atributos en el catálogo',
},
},
CERTIFY: {
seqid: 53,
claim: 'catalog:certify',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Alterar o status de certificação dos Ativos',
'en-us': "Change Assets' certification status",
'es-es': 'Cambiar el estado de certificación de los Activos',
},
},
DELETE: {
seqid: 1,
claim: 'catalog:delete',
+1
View File
@@ -12,6 +12,7 @@ export interface RequestUser {
customer_tier: string;
access_token: string;
customer_modules: string[];
roles: string[];
}
export const User: (options?: { required?: boolean }) => ParameterDecorator =
+65
View File
@@ -0,0 +1,65 @@
import {
BadRequestException,
CanActivate,
ExecutionContext,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { map, Observable } from 'rxjs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import {
ReadService,
ProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { PipelinesClientConfiguration } from 'src/modules/pipelinesV2/pipelines-client';
import { PlatformApiService } from 'src/modules/platform-api/platform-api.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class PipelineExecutionGuard implements CanActivate {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly platformApiService: PlatformApiService,
) {
this.logger = dadosferaLogger.logger;
}
async canActivate(context: ExecutionContext): Promise<boolean> {
try {
this.logger.info(
'PipelineExecutionGuard: Checking if pipeline can be executed...',
);
const request = context.switchToHttp().getRequest();
const pipelineId = request.params.pipelineId;
const user = request.user;
const idRegex = /[^0-9a-zA-Z_$]+/g;
const convertedId = pipelineId.replace(idRegex, '_');
const status = await this.platformApiService.proxy(
'GET',
`/pipeline/${convertedId}/pipeline_run`,
user,
);
const currentStatus = status[status.length - 1]
this.logger.info('Pipeline current status response:' + JSON.stringify(currentStatus));
if (currentStatus.last_status.toLowerCase() === 'running') {
this.logger.error('Pipeline is running, cannot update input now');
throw new BadRequestException('Pipeline is running, cannot update input now');
} else {
return true;
}
} catch (error) {
this.logger.error('Error in PipelineExecutionGuard: ' + error.message);
throw new BadRequestException('Error checking pipeline status: ' + error.message);
}
}
}
+77 -1
View File
@@ -17,6 +17,7 @@ import {
HttpStatus,
Res,
} from '@nestjs/common';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
ApiCreatedResponse,
ApiHeaders,
@@ -46,6 +47,7 @@ import {
IMakeAComment,
IOneDataAsset,
IPreviewResponse,
IUpdateCertificationStatusRequest,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
@@ -241,6 +243,51 @@ export class CatalogController {
return res;
}
@Get('schemas')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async findSchemas(@User() user: RequestUser) {
const { username, user_id, customer_id, customer_name } = user;
this.logger.info(`/catalog - ON FIND SCHEMAS ROUTE`, {
username,
customer_name,
});
const metadata = PackTheMetadata({
username,
user_id,
customer_id,
customer_name,
});
try {
const res = await this.catalogService.findSchemas(metadata);
return res;
} catch (error) {
throw new HttpException(error.message, HttpStatus.NOT_FOUND);
}
}
@Get('custom-properties')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getCustomPropertyDefinitions(@User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
return this.catalogService.getCustomPropertyDefinitions(metadata);
}
@Get('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
@@ -465,6 +512,8 @@ export class CatalogController {
language,
});
delete (body as any).certification_status;
const result = await this.catalogService.updateOneDataAsset({
body,
data_asset_id,
@@ -478,6 +527,33 @@ export class CatalogController {
return result;
}
@Put('data-asset/:id/certification-status')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async updateDataAssetCertificationStatus(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') data_asset_id: string,
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
): Promise<IUpdateCertificationStatusRequest> {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
return this.catalogService.updateCertificationStatus({
body,
data_asset_id,
metadata,
});
}
@Post('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
@@ -943,4 +1019,4 @@ export class CatalogController {
this.logger.error(error.message);
}
}
}
}
-3
View File
@@ -5,20 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
+43 -18
View File
@@ -29,6 +29,7 @@ import {
AssetReporter,
BatchRemoveRlsRulesRequest,
CreateDataDocsDTO,
IUpdateCertificationStatusRequest,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
@@ -119,6 +120,10 @@ class CatalogService implements OnModuleInit {
}
}
async getCustomPropertyDefinitions(metadata: Metadata) {
return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
@@ -214,15 +219,6 @@ class CatalogService implements OnModuleInit {
this.logger.debug('Extracted filters:', { filters });
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (
filters.manually !== undefined &&
filters.manually !== null &&
@@ -233,15 +229,6 @@ class CatalogService implements OnModuleInit {
delete filters.manually;
}
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (filters.owner) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
@@ -402,6 +389,28 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] };
}
async updateCertificationStatus(data: {
data_asset_id: string;
body: IUpdateCertificationStatusRequest;
metadata: Metadata;
}) {
const { body, data_asset_id, metadata } = data;
await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{
id: data_asset_id,
changes: JSON.stringify({
certification_status: body.certification_status,
}),
},
metadata,
),
);
return { certification_status: body.certification_status };
}
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
@@ -517,6 +526,22 @@ class CatalogService implements OnModuleInit {
return response;
}
async findSchemas(metadata: Metadata) {
this.logger.info('CatalogService - findSchemas');
try {
const response = await lastValueFrom(
this.catalogReadService.GetSchemas({}, metadata),
);
return response;
} catch (error) {
this.logger.error('Error fetching schemas:', error);
throw error;
}
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
+43 -1
View File
@@ -1,4 +1,10 @@
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
import {
IsEnum,
IsNotEmpty,
IsOptional,
IsString,
} from 'class-validator';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export enum DataAssetShareType {
@@ -6,6 +12,12 @@ export enum DataAssetShareType {
public = 'public',
private = 'private',
}
export enum CertificationStatus {
draft = 'draft',
in_review = 'in_review',
approved = 'approved',
deprecated = 'deprecated',
}
export enum OrderEnum {
asc = 'asc',
desc = 'desc',
@@ -191,6 +203,27 @@ export class IData {
day_opening: number;
}
export enum CustomPropertyType {
TEXT = 'text',
NUMBER = 'number',
DATE = 'date',
BOOLEAN = 'boolean',
}
export class CustomPropertyDto {
@ApiProperty()
key: string;
@ApiProperty()
value: string;
@ApiProperty({ enum: CustomPropertyType })
type: CustomPropertyType;
@ApiPropertyOptional()
color?: string;
@ApiPropertyOptional()
emoji?: string;
}
export class IUpdateDataRequest {
@ApiProperty()
name: string;
@@ -204,7 +237,16 @@ export class IUpdateDataRequest {
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
@ApiPropertyOptional({ type: [CustomPropertyDto] })
custom_properties?: CustomPropertyDto[];
}
export class IUpdateCertificationStatusRequest {
@ApiProperty({ enum: CertificationStatus })
@IsEnum(CertificationStatus)
certification_status: CertificationStatus;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
@@ -358,4 +400,4 @@ export type CreateDataDocsDTO = {
docs: string;
asset_type: string;
}
}
@@ -262,6 +262,7 @@ export class ShareService implements OnModuleInit {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
roles: accessTokenPayload.roles,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
@@ -22,6 +22,9 @@ import {
ConnectionTestListTablesRes,
GetTableMetadataRes,
GetTableMetadataReq,
RefreshCatalogReq,
RefreshCatalogRes,
RefreshCatalogStatusReq,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated } from 'src/decorators/authentication.decorator';
@@ -84,7 +87,7 @@ export class ConnectionTestController {
});
return this.connectionTestService.connectionTestListSchemas(
body,
user.customer_name,
user,
);
}
@@ -101,7 +104,7 @@ export class ConnectionTestController {
});
return this.connectionTestService.connectionTestListTables(
body,
user.customer_name,
user,
);
}
@@ -118,7 +121,38 @@ export class ConnectionTestController {
});
return this.connectionTestService.getTableMetadata(
body,
user.customer_name,
user,
);
}
@Post('refresh-catalog')
@ApiOkResponse({ type: RefreshCatalogRes })
@HttpCode(HttpStatus.ACCEPTED)
async refreshCatalog(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: RefreshCatalogReq,
) {
this.logger.info('/connection-test/refresh-catalog', {
user: user.user_id,
customer: user.customer_name,
connection: body.connection_id,
});
return this.connectionTestService.refreshCatalog(body, user);
}
@Post('refresh-catalog/status')
@ApiOkResponse({ type: RefreshCatalogRes })
@HttpCode(HttpStatus.OK)
async refreshCatalogStatus(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: RefreshCatalogStatusReq,
) {
this.logger.info('/connection-test/refresh-catalog/status', {
user: user.user_id,
customer: user.customer_name,
connection: body.connection_id,
session: body.session_id,
});
return this.connectionTestService.refreshCatalogStatus(body, user);
}
}
@@ -5,10 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { ConnectionTestClientConfiguration } from './connection-test-client.config';
import { ConnectionModule } from '../connection/connection.module';
import { ConnectionsApiModule } from '../connections-api/connections-api.module';
import { PlatformApiModule } from '../platform-api/platform-api.module';
const client = new ConnectionTestClientConfiguration();
@Module({
controllers: [ConnectionTestController],
providers: [ConnectionTestService, DadosferaLogger],
imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
imports: [
ClientsModule.register([client.providerOptions]),
ConnectionModule,
ConnectionsApiModule,
PlatformApiModule,
],
})
export class ConnectionTestModule {}
@@ -0,0 +1,205 @@
import { ConnectionTestService } from './connection-test.service';
import { RequestUser } from 'src/decorators/user.decorator';
describe('ConnectionTestService catalog cache', () => {
const user: RequestUser = {
user_id: 'user-id',
username: 'user@example.com',
permissions: [],
customer_id: 'customer-id',
customer_name: 'customer-name',
customer_tier: 'standard',
access_token: 'token',
customer_modules: [],
roles: [],
};
const grpcClient = { getService: jest.fn().mockReturnValue({}) };
const connectionsService = {};
const connectionsApiService = { proxy: jest.fn() };
const platformApiService = { proxy: jest.fn() };
let service: ConnectionTestService;
beforeEach(() => {
jest.clearAllMocks();
service = new ConnectionTestService(
grpcClient as any,
connectionsService as any,
connectionsApiService as any,
platformApiService as any,
);
});
it('keeps the existing schemas response contract', async () => {
connectionsApiService.proxy.mockResolvedValue({
schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }],
});
await expect(
service.connectionTestListSchemas(
{ connection_id: 'config-id', plugin: 'postgresql' },
user,
),
).resolves.toEqual({
operation_result: true,
schema_list: ['analytics', 'public'],
});
});
it('keeps the existing tables response contract', async () => {
connectionsApiService.proxy.mockResolvedValue({
tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
});
await expect(
service.connectionTestListTables(
{
connection_id: 'config-id',
plugin: 'postgresql',
schema: 'public',
},
user,
),
).resolves.toEqual({
operation_result: true,
table_list: ['customers', 'orders'],
});
});
it('maps cached columns to the existing table metadata contract', async () => {
connectionsApiService.proxy.mockResolvedValue({
columns: [
{
column_name: 'id',
data_type: 'bigint',
is_primary_key: true,
},
],
});
await expect(
service.getTableMetadata(
{
connection_id: 'config-id',
plugin: 'postgresql',
schema: 'public',
table_list: ['customers'],
},
user,
),
).resolves.toEqual({
operation_result: true,
tables_metadata: [
{
table_name: 'customers',
columns: [
{
name: 'id',
type: 'bigint',
is_primary_key: true,
},
],
references: [],
},
],
});
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
'GET',
'/connection_catalog/config-id/schemas/public/tables/customers/columns',
user,
);
});
it('submits a catalog refresh without holding the request open', async () => {
platformApiService.proxy.mockResolvedValue({
session_id: 'session-id',
date: '20260731',
});
await expect(
service.refreshCatalog(
{ connection_id: 'config-id', plugin: 'postgresql' },
user,
),
).resolves.toEqual({
operation_result: true,
status: 'PENDING',
session_id: 'session-id',
date: '20260731',
});
expect(platformApiService.proxy).toHaveBeenCalledWith(
'POST',
'/connection_test',
user,
{
customer_id: user.customer_name,
plugin: 'postgresql',
task: {
task_type: 'refresh_catalog',
connection: {
provider: 'connection_manager',
config_id: 'config-id',
},
},
},
);
});
it('keeps polling without changing the catalog pointer while pending', async () => {
platformApiService.proxy.mockResolvedValue({ status: 'PENDING' });
await expect(
service.refreshCatalogStatus(
{
connection_id: 'config-id',
plugin: 'postgresql',
session_id: 'session-id',
date: '20260731',
},
user,
),
).resolves.toEqual({
operation_result: false,
status: 'PENDING',
session_id: 'session-id',
date: '20260731',
});
expect(connectionsApiService.proxy).not.toHaveBeenCalled();
});
it('publishes the catalog pointer after the refresh finishes', async () => {
platformApiService.proxy.mockResolvedValue({ status: 'DONE' });
connectionsApiService.proxy.mockResolvedValue({
last_catalog_refresh_status: 'SUCCESS',
});
await expect(
service.refreshCatalogStatus(
{
connection_id: 'config/id',
plugin: 'postgresql',
session_id: 'session-id',
date: '20260731',
},
user,
),
).resolves.toEqual({
operation_result: true,
status: 'DONE',
session_id: 'session-id',
date: '20260731',
});
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
'PUT',
'/connection_config/config%2Fid/catalog_metadata',
user,
{
last_catalog_refresh_status: 'SUCCESS',
last_catalog_connection_test_date: '20260731',
last_catalog_connection_test_session_id: 'session-id',
},
);
});
});
@@ -1,4 +1,4 @@
import { Inject, Injectable } from '@nestjs/common';
import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { ConnectionTest } from '@dadosfera/protospack-v2';
import { lastValueFrom } from 'rxjs';
@@ -13,6 +13,9 @@ import {
ConnectionTestPingRes,
GetTableMetadataReq,
GetTableMetadataRes,
RefreshCatalogReq,
RefreshCatalogRes,
RefreshCatalogStatusReq,
} from './dto/connection-test';
import { ConnectionClientService } from '../connection/client.service';
import {
@@ -21,6 +24,8 @@ import {
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ConnectionsApiService } from '../connections-api/connections-api.service';
import { PlatformApiService } from '../platform-api/platform-api.service';
@Injectable()
export class ConnectionTestService {
@@ -28,6 +33,8 @@ export class ConnectionTestService {
constructor(
@Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
private connectionsService: ConnectionClientService,
private connectionsApiService: ConnectionsApiService,
private platformApiService: PlatformApiService,
) {
this.connectionTestReadClient =
grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
@@ -147,45 +154,137 @@ export class ConnectionTestService {
}
async connectionTestListSchemas(
body: ConnectionTestListSchemasReq,
customer_name: string,
user: RequestUser,
): Promise<ConnectionTestListSchemasRes> {
const { connection_id, plugin } = body;
return lastValueFrom(
this.connectionTestReadClient.ListSchemas({
connection_id,
customer_name,
plugin,
}),
const result = await this.connectionsApiService.proxy(
'GET',
`/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`,
user,
);
return {
operation_result: true,
schema_list: result.schemas.map((schema) => schema.schema_name),
};
}
async connectionTestListTables(
body: ConnectionTestListTablesReq,
customer_name: string,
user: RequestUser,
): Promise<ConnectionTestListTablesRes> {
const { connection_id, plugin, schema } = body;
return lastValueFrom(
this.connectionTestReadClient.ListTables({
connection_id,
customer_name,
plugin,
schema,
}),
const result = await this.connectionsApiService.proxy(
'GET',
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
`/schemas/${encodeURIComponent(body.schema)}/tables`,
user,
);
return {
operation_result: true,
table_list: result.tables.map((table) => table.table_name),
};
}
async getTableMetadata(
body: GetTableMetadataReq,
customer_name: string,
user: RequestUser,
): Promise<GetTableMetadataRes> {
const { schema, plugin, table_list, connection_id } = body;
return lastValueFrom(
this.connectionTestReadClient.GetTableMetadata({
connection_id,
customer_name,
plugin,
schema,
table_list,
const tables_metadata = await Promise.all(
body.table_list.map(async (table_name) => {
const result = await this.connectionsApiService.proxy(
'GET',
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
`/schemas/${encodeURIComponent(body.schema)}` +
`/tables/${encodeURIComponent(table_name)}/columns`,
user,
);
return {
table_name,
columns: result.columns.map((column) => ({
name: column.column_name,
type: column.data_type,
is_primary_key: column.is_primary_key,
})),
references: [],
};
}),
);
return { operation_result: true, tables_metadata };
}
async refreshCatalog(
body: RefreshCatalogReq,
user: RequestUser,
): Promise<RefreshCatalogRes> {
const task = await this.platformApiService.proxy(
'POST',
'/connection_test',
user,
{
customer_id: user.customer_name,
plugin: body.plugin,
task: {
task_type: 'refresh_catalog',
connection: {
provider: 'connection_manager',
config_id: body.connection_id,
},
},
},
);
if (!task.session_id || !task.date) {
throw new HttpException(
'Platform API did not return a catalog refresh task identifier',
HttpStatus.BAD_GATEWAY,
);
}
return {
operation_result: true,
status: 'PENDING',
session_id: task.session_id,
date: task.date,
};
}
async refreshCatalogStatus(
body: RefreshCatalogStatusReq,
user: RequestUser,
): Promise<RefreshCatalogRes> {
const result = await this.platformApiService.proxy(
'POST',
'/connection_test/status',
user,
{
session_id: body.session_id,
date: body.date,
},
);
if (result.status === 'DONE') {
await this.connectionsApiService.proxy(
'PUT',
`/connection_config/${encodeURIComponent(
body.connection_id,
)}/catalog_metadata`,
user,
{
last_catalog_refresh_status: 'SUCCESS',
last_catalog_connection_test_date: body.date,
last_catalog_connection_test_session_id: body.session_id,
},
);
} else if (result.status === 'ERROR' || result.status === 'EXPIRED') {
throw new HttpException(
`Catalog refresh finished with status ${result.status}`,
HttpStatus.BAD_GATEWAY,
);
}
return {
operation_result: result.status === 'DONE',
status: result.status,
session_id: body.session_id,
date: body.date,
};
}
}
@@ -1,5 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { IsString, IsOptional } from 'class-validator';
import { IsIn, IsString, IsOptional } from 'class-validator';
import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
export class ColumnDto {
@@ -7,6 +7,8 @@ export class ColumnDto {
name: string;
@ApiProperty()
type: string;
@ApiProperty()
is_primary_key: boolean;
}
export class TableMetadataDto {
@ApiProperty()
@@ -131,3 +133,37 @@ export class GetTableMetadataRes {
@ApiProperty({ type: [TableMetadataDto] })
tables_metadata: TableMetadataDto[];
}
export class RefreshCatalogReq {
@ApiProperty()
@IsString()
connection_id: string;
@ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] })
@IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver'])
plugin: string;
}
export class RefreshCatalogStatusReq extends RefreshCatalogReq {
@ApiProperty()
@IsString()
session_id: string;
@ApiProperty()
@IsString()
date: string;
}
export class RefreshCatalogRes {
@ApiProperty()
operation_result: boolean;
@ApiProperty()
status: string;
@ApiProperty()
session_id: string;
@ApiProperty()
date: string;
}
@@ -0,0 +1,11 @@
export const CONNECTIONS_API_CONFIG = {
getUrl: (): string => {
const url = process.env.CONNECTIONS_API_URL;
if (!url) {
throw new Error('CONNECTIONS_API_URL environment variable is not set');
}
return url;
},
region: process.env.AWS_REGION || 'us-east-1',
timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10),
};
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ConnectionsApiService } from './connections-api.service';
@Module({
providers: [ConnectionsApiService, DadosferaLogger],
exports: [ConnectionsApiService],
})
export class ConnectionsApiModule {}
@@ -0,0 +1,99 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import { SignatureV4 } from '@aws-sdk/signature-v4';
import { Sha256 } from '@aws-crypto/sha256-js';
import { defaultProvider } from '@aws-sdk/credential-provider-node';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { CONNECTIONS_API_CONFIG } from './connections-api.config';
@Injectable()
export class ConnectionsApiService {
private signer: SignatureV4;
private logger: any;
constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) {
this.logger = dadosferaLogger.logger;
this.signer = new SignatureV4({
service: 'execute-api',
region: CONNECTIONS_API_CONFIG.region,
credentials: defaultProvider(),
sha256: Sha256,
});
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, string>,
): Promise<any> {
const baseUrl = CONNECTIONS_API_CONFIG.getUrl();
const url = new URL(`${baseUrl}${path}`);
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
host: url.hostname,
'content-type': 'application/json',
customer_name: user.customer_name || '',
customer_id: user.customer_id || '',
'x-user-id': user.user_id || '',
'x-username': user.username || '',
'x-customer-tier': user.customer_tier || '',
'x-customer-id': user.customer_id || '',
};
const requestToSign = {
method: method.toUpperCase(),
protocol: url.protocol,
hostname: url.hostname,
port: url.port ? parseInt(url.port, 10) : undefined,
path: url.pathname + url.search,
headers,
body: body ? JSON.stringify(body) : undefined,
};
try {
const signedRequest = await this.signer.sign(requestToSign);
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers: signedRequest.headers as Record<string, string>,
data: body,
timeout: CONNECTIONS_API_CONFIG.timeout,
validateStatus: () => true,
});
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Connections API proxy error', {
error: error.message,
path,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Connections API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Connections API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
}
+5 -1
View File
@@ -1,4 +1,8 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
interface Values {
jdbc_user: string;
+5 -4
View File
@@ -23,6 +23,7 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { CreateInputReq } from './dtos/input.model';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
@@ -73,10 +74,10 @@ export class InputsService {
objectCamelToSnake(createInputResponse);
return createInputResponse;
},
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
const updateInputResponse = await lastValueFrom(
this.inputWriteService.InputUpdate(updateInputDTO),
this.inputWriteService.InputUpdate(updateInputDTO, metadata),
);
return updateInputResponse;
@@ -206,7 +207,7 @@ export class InputsService {
return findOneInputResponse;
}
async update(id: string, data, info: Info) {
async update(id: string, data, info: Info, metadata?: Metadata) {
// this.validateCron({ ...data, info });
try {
const {
@@ -217,7 +218,7 @@ export class InputsService {
id,
...data,
info,
});
}, metadata);
const updateInputResponse = this.adjustInputPayload(
input,
-78
View File
@@ -1,78 +0,0 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.pipelineService =
this.grpcClient.getService<PipelinesServiceInterface>(
PipelineServicesNames.PipelineService,
);
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.getPipelineStatus(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.triggerPipeline({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
-36
View File
@@ -1,36 +0,0 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
info: Info;
}
export interface IdRequest {
id: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
id: string;
info: Info;
}
export interface IGetPipelineLogsRequest {
id: string;
details: string;
}
-33
View File
@@ -1,33 +0,0 @@
import {
ClientsProviderAsyncOptions,
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js';
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
export class PipelinesClientConfiguration {
public name = 'PipelinesClientConfiguration';
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: PipelinePackages,
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: PipelineProtoFilePath,
loader: {
keepCase: true,
enums: String,
defaults: false,
},
},
};
providerOptions: ClientsProviderAsyncOptions = {
name: this.name,
...this.config,
};
}
@@ -1,72 +0,0 @@
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelineService: PipelinesService,
) {
this.logger = dadosferaLogger.logger;
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
})
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
this.logger.info(
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.runPipeline({ id, info });
return response;
}
@Get(':id/status')
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.getPipelineStatus(body);
return response;
}
}
-19
View File
@@ -1,19 +0,0 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesController } from './pipelines.controller';
import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesClientService } from './client.service';
const client = new PipelinesClientConfiguration();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [PipelinesController],
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
exports: [PipelinesService],
})
export class PipelinesModule {}
@@ -1,33 +0,0 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { PipelinesClientService } from './client.service';
import { IIdRequest } from './interfaces';
import { objectCamelToSnake } from 'src/utils/CaseConverter';
@Injectable()
export class PipelinesService {
constructor(private pipelineClient: PipelinesClientService) {}
async getPipelineStatus(data: IIdRequest) {
try {
const pipelineStatusResponse =
await this.pipelineClient.getPipelineStatus(data);
return objectCamelToSnake(pipelineStatusResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async runPipeline({ id, info }: IIdRequest) {
try {
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
id,
info,
});
return objectCamelToSnake(triggerPipelineResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
}
+6 -1
View File
@@ -1,5 +1,4 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class PipelineInputsDTO {
@ApiProperty()
@@ -62,6 +61,12 @@ export interface IIdRequest {
info: Info;
}
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
+47 -17
View File
@@ -14,7 +14,7 @@ import {
Patch,
HttpException,
BadRequestException,
CacheTTL,
UseGuards,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -24,7 +24,6 @@ import {
ApiTags,
} from '@nestjs/swagger';
import {
AuthenticateCondition,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
@@ -35,7 +34,6 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
ICompleteUploadCSVFile,
ICreatePipelineCSVFile,
@@ -49,9 +47,8 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { TableColumns } from '../inputs/dtos/input.model';
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any };
type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
@@ -65,9 +62,7 @@ export class PipelinesController {
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelinesClientService: PipelinesService,
private oldPipelinesService: OldPipelineService,
) {
this.logger = dadosferaLogger.logger;
}
@@ -195,6 +190,7 @@ export class PipelinesController {
@Get(':id/status')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
@@ -202,7 +198,7 @@ export class PipelinesController {
customer: body.info.customer,
});
const response = await this.oldPipelinesService.getPipelineStatus(body);
const response = await this.pipelinesClientService.getPipelineStatus(body);
return response;
}
@@ -249,6 +245,31 @@ export class PipelinesController {
return pipelineRes;
}
@Get("/:id/data-assets")
@RequireSomePermission(
PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
PERMISSIONS_GROUPS.CATALOG.permissions.GET
)
async findAllDataAssetByPipeline(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser,
@Query('object') object: string
) {
const payload = {
pipeline: id,
object: object,
};
this.logger.info(`GET pipelinesV2/:id/data-assets` + JSON.stringify(payload));
const result =
await this.pipelinesClientService.findAllDataAssetByPipeline(payload, user);
return result;
}
@Patch('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async update(
@@ -286,6 +307,7 @@ export class PipelinesController {
@Patch('/:pipelineId/inputs/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@UseGuards(PipelineExecutionGuard)
async updatePipelineInput(
@Language() language: LanguageEnum,
@Body() pipelineInputDTO: UpdatePlatformInputRequest,
@@ -295,7 +317,6 @@ export class PipelinesController {
) {
this.logger.info('PipelinesController - update', { user });
const { customer_id, customer_name, user_id, username } = user;
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
@@ -303,13 +324,7 @@ export class PipelinesController {
pipeline_id: pipelineId
};
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const metadata = PackTheMetadata(user);
const response = await this.pipelinesClientService.updatePipelineInput(
pipelineId,
@@ -343,6 +358,21 @@ export class PipelinesController {
return response;
}
@Patch('/:id/upgrade')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@HttpCode(HttpStatus.NO_CONTENT)
async upgradeConnector(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
) {
this.logger.info('PipelinesController - upgrade connector');
const metadata = PackTheMetadata(user);
await this.pipelinesClientService.upgrade(id, metadata);
}
@Delete(':id')
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
@@ -472,7 +502,7 @@ export class PipelinesController {
},
);
const response = await this.oldPipelinesService.runPipeline({ id, info });
const response = await this.pipelinesClientService.runPipeline({ id, info });
return response;
}
+3 -3
View File
@@ -7,25 +7,25 @@ import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module';
import { PlatformApiModule } from '../platform-api/platform-api.module';
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
import { CatalogModule } from '../catalog/catalog.module';
const client = new PipelinesClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
ConnectorModule,
InputsModule,
TransformationsModule,
PlatformApiModule,
NimbusServicesModule
NimbusServicesModule,
CatalogModule
],
controllers: [PipelinesController],
providers: [PipelinesService, DadosferaLogger, NimbusService],
+134 -30
View File
@@ -1,6 +1,7 @@
/* eslint-disable no-async-promise-executor */
import {
BadRequestException,
ConflictException,
HttpException,
HttpStatus,
Inject,
@@ -17,7 +18,7 @@ import { lastValueFrom } from 'rxjs';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
@@ -32,6 +33,10 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { AxiosError } from 'axios';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { IDataAsset } from '../catalog/dtos';
import { CatalogService } from '../catalog/catalog.service';
type RollbackPromise = () => Promise<any>;
@@ -48,7 +53,8 @@ export class PipelinesService implements OnModuleInit {
private readonly inputsService: InputsService,
private readonly transformationsService: TransformationsService,
private readonly platformAPI: PlatformApiService,
private readonly nimbusService: NimbusService
private readonly nimbusService: NimbusService,
private readonly catalogService: CatalogService
) {
this.logger = dadosferaLogger.logger;
}
@@ -171,6 +177,15 @@ export class PipelinesService implements OnModuleInit {
return updatePipelineResponse;
}
async upgrade(id: string, metadata: Metadata) {
await lastValueFrom(
this.pipelineWriteService.Upgrade(
{ id },
metadata,
),
);
}
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
const { id, metadata, user } = data;
const info = {
@@ -368,7 +383,8 @@ export class PipelinesService implements OnModuleInit {
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info
info,
metadata
);
const inputRollback = () => {
@@ -389,34 +405,36 @@ export class PipelinesService implements OnModuleInit {
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
if (user.customer_modules.includes('catalog')) {
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
}
try {
@@ -577,4 +595,90 @@ export class PipelinesService implements OnModuleInit {
this.logger.info('Platform api response: ' + JSON.stringify(response));
}
async findAllDataAssetByPipeline(data: {
pipeline: string,
object?: string
}, user: RequestUser) {
const metadata = PackTheMetadata(user);
const isDataAdmin = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const {
data_assets: resultString
} = await lastValueFrom(
this.pipelineReadService.FindAllDataAssetByPipeline(data, metadata)
);
const result = JSON.parse(resultString) as any;
const data_assets: IDataAsset[] = []
result.forEach(data_asset => {
if (data_asset?.owner === user.username) has_permission = true;
for (const role of user.roles) {
if (data_asset.roles.includes(role)) has_permission = true;
}
if (data_asset.users.includes(user.user_id)) has_permission = true;
if (isDataAdmin || has_permission) {
delete data_asset.p_roles;
delete data_asset.p_users;
data_assets.push(data_asset as IDataAsset);
}
});
const assets = await this.catalogService.getAssetsUsersAndRoles(data_assets, user.customer_id);
return assets;
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
@@ -12,8 +12,9 @@ import {
BadRequestException,
HttpException,
NotFoundException,
UseGuards,
} from '@nestjs/common';
import { ApiTags, ApiOperation } from '@nestjs/swagger';
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
@@ -31,6 +32,7 @@ import { CatalogService } from '../catalog/catalog.service';
import { PackTheMetadata } from '../../utils/PackTheMetadata';
import { ValidationTableDTO } from './platform-api.dto';
import { InputsService } from '../inputs/inputs.service';
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
type ValidateTablesDTO = {
@@ -70,6 +72,10 @@ export class PlatformApiController {
return id?.replace(/-/g, '_') || '';
}
private decodePathParam(value: string): string {
return value ? decodeURIComponent(value) : '';
}
/**
* Denormalize ID back to UUID format (replace _ with -).
* Used when we receive a normalized ID but need the original UUID.
@@ -132,7 +138,7 @@ export class PlatformApiController {
}
private readonly VALID_CONNECTORS = ['jdbc', 'singer', 's3'];
private readonly MAX_MEMORY_MB = 12000; // 12GB maximum memory per pipeline/job
private readonly MAX_MEMORY_MB = 12000; // 12GB maximum memory per pipelines/job
/**
* Validate that connector is provided and is a valid type.
@@ -403,55 +409,9 @@ export class PlatformApiController {
}
}
/**
* Sync sync-mode changes to DynamoDB for JDBC connectors.
* Always passes both target_load_type and incremental_column_name to ensure proper sync.
*/
private async syncJdbcSyncModeToDynamoDB(
jobId: string,
body: any,
user: RequestUser,
): Promise<void> {
// JDBC sync mode uses target_load_type field
const changes: any = {};
if ('target_load_type' in body) {
changes.target_load_type = body.target_load_type;
}
// Handle incremental_column_name:
// - If provided in body, use that value
// - If changing to full_load, explicitly clear it
if ('incremental_column_name' in body) {
changes.incremental_column_name = body.incremental_column_name;
changes.incremental_column_type = body.incremental_column_type;
} else if (body.target_load_type === 'full_load') {
// Changing to full_load without specifying incremental_column - clear it
changes.incremental_column_name = null;
}
await this.syncJobInputToDynamoDB(jobId, changes, user, 'jdbc');
}
/**
* Sync sync-mode changes to DynamoDB for Singer connectors.
*/
private async syncSingerSyncModeToDynamoDB(
jobId: string,
body: any,
user: RequestUser,
): Promise<void> {
// Singer sync mode uses replication_method field
// Map to DynamoDB type: FULL_TABLE -> full_load, INCREMENTAL -> incremental
if ('replication_method' in body) {
const type = body.replication_method === 'INCREMENTAL' ? 'incremental' : 'full_load';
await this.syncJobInputToDynamoDB(jobId, { load_type: type }, user, 'singer');
}
}
// ==================== PIPELINE ROUTES ====================
@Post('pipeline')
@Post('pipelines')
@ApiOperation({ summary: 'Create a new pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async createPipeline(@Body() body: any, @User() user: RequestUser) {
@@ -572,7 +532,7 @@ export class PlatformApiController {
);
}
@Get('pipeline/:pipelineId')
@Get('pipelines/:pipelineId')
@ApiOperation({ summary: 'Get pipeline by ID' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipeline(
@@ -583,7 +543,7 @@ export class PlatformApiController {
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
}
@Patch('pipeline/:pipelineId')
@Patch('pipelines/:pipelineId')
@ApiOperation({ summary: 'Update pipeline by ID' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updatePipeline(
@@ -634,7 +594,7 @@ export class PlatformApiController {
return result;
}
@Delete('pipeline/:pipelineId')
@Delete('pipelines/:pipelineId')
@ApiOperation({ summary: 'Delete pipeline by ID' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
async deletePipeline(
@@ -664,7 +624,7 @@ export class PlatformApiController {
return result;
}
@Post('pipeline/execute')
@Post('pipelines/execute')
@ApiOperation({ summary: 'Execute a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async executePipeline(@Body() body: any, @User() user: RequestUser) {
@@ -674,10 +634,10 @@ export class PlatformApiController {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
return this.platformApiService.proxy('POST', '/pipelines/execute', user, enrichedBody);
}
@Post('pipeline/pause')
@Post('pipelines/pause')
@ApiOperation({ summary: 'Pause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pausePipeline(@Body() body: any, @User() user: RequestUser) {
@@ -690,7 +650,7 @@ export class PlatformApiController {
return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
}
@Post('pipeline/unpause')
@Post('pipelines/unpause')
@ApiOperation({ summary: 'Unpause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
@@ -703,7 +663,7 @@ export class PlatformApiController {
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
}
@Put('pipeline/:pipelineId/memory')
@Put('pipelines/:pipelineId/memory')
@ApiOperation({ summary: 'Update pipeline memory configuration' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updatePipelineMemory(
@@ -726,7 +686,7 @@ export class PlatformApiController {
// ==================== PIPELINE METADATA ROUTES ====================
@Put('pipeline/:pipelineId/metadata')
@Put('pipelines/:pipelineId/metadata')
@ApiOperation({ summary: 'Update pipeline metadata' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updatePipelineMetadata(
@@ -795,7 +755,7 @@ export class PlatformApiController {
// ==================== PIPELINE RUN ROUTES ====================
@Get('pipeline/:pipelineId/pipeline_run')
@Get('pipelines/:pipelineId/pipeline_run')
@ApiOperation({ summary: 'Get pipeline runs for a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRuns(
@@ -813,7 +773,7 @@ export class PlatformApiController {
);
}
@Get('pipeline/:pipelineId/pipeline_run/:runId')
@Get('pipelines/:pipelineId/pipeline_run/:runId')
@ApiOperation({ summary: 'Get specific pipeline run' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRun(
@@ -830,7 +790,7 @@ export class PlatformApiController {
);
}
@Get('pipeline/pipeline_run/:runId/logs')
@Get('pipelines/pipeline_run/:runId/logs')
@ApiOperation({ summary: 'Get pipeline run logs' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRunLogs(
@@ -848,7 +808,7 @@ export class PlatformApiController {
);
}
@Post('pipeline/:pipelineId/pipeline_run/:runId/cancel')
@Post('pipelines/:pipelineId/pipeline_run/:runId/cancel')
@ApiOperation({ summary: 'Cancel a running pipeline run' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async cancelPipelineRun(
@@ -859,6 +819,16 @@ export class PlatformApiController {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const normalizedRunId = this.normalizePipelineId(runId);
const status = await this.platformApiService.proxy(
'GET',
`/pipeline/${normalizedPipelineId}/pipeline_run`,
user,
);
if (status.length === 1) {
throw new BadRequestException('The first pipeline cannot be canceled');
}
return this.platformApiService.proxy(
'POST',
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
@@ -866,6 +836,40 @@ export class PlatformApiController {
);
}
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
@ApiOperation({
summary: 'Get pipeline run jobs',
description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
})
@ApiOkResponse({
description: 'DB-backed job runs for the selected pipeline run.',
schema: {
type: 'object',
properties: {
jobs: {
type: 'array',
items: { type: 'object' },
},
},
required: ['jobs'],
},
})
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRunJobs(
@Param('pipelineId') pipelineId: string,
@Param('runId') runId: string,
@User() user: RequestUser,
) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const decodedRunId = this.decodePathParam(runId);
return this.platformApiService.proxy(
'GET',
`/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
user,
);
}
// ==================== JOBS - COLUMN EDITING ROUTES ====================
@Put('jobs/:jobId/input')
@@ -968,6 +972,7 @@ export class PlatformApiController {
@Delete('pipelines/:pipelineId/inputs/:inputId')
@ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
@UseGuards(PipelineExecutionGuard)
async deleteTable(
@Param('pipelineId') pipelineId: string,
@Param('inputId') inputId: string,
@@ -0,0 +1,14 @@
export type ReleaseNoteDTO = {
id: string;
date: string;
tag: string;
title: string;
visible: boolean;
expiryDate: string;
content: string;
showEmojis: boolean;
image?: string;
link?: string;
linkText?: string;
};
@@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteController } from './release_note.controller';
import { ReleaseNoteService } from './release_note.service';
describe('ReleaseNoteController', () => {
let controller: ReleaseNoteController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [ReleaseNoteController],
providers: [ReleaseNoteService],
}).compile();
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});
@@ -0,0 +1,26 @@
import { Controller, Get, Inject } from '@nestjs/common';
import { ReleaseNoteService } from './release_note.service';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Controller('release_note')
@Authenticated()
export class ReleaseNoteController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly releaseNoteService: ReleaseNoteService,
) {
this.logger = dadosferaLogger.logger;
}
@Get()
async getLatestReleaseNote(@Language() language: LanguageEnum) {
this.logger.info(`Fetching latest release note for language: ${language}`);
return await this.releaseNoteService.getLatestReleaseNote(language);
}
}
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { ReleaseNoteService } from './release_note.service';
import { ReleaseNoteController } from './release_note.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [ReleaseNoteController],
providers: [ReleaseNoteService, DadosferaLogger]
})
export class ReleaseNoteModule {}
@@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteService } from './release_note.service';
describe('ReleaseNoteService', () => {
let service: ReleaseNoteService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ReleaseNoteService],
}).compile();
service = module.get<ReleaseNoteService>(ReleaseNoteService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
@@ -0,0 +1,46 @@
import { Inject, Injectable } from '@nestjs/common';
import axios, { AxiosInstance } from 'axios';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ReleaseNoteDTO } from './dto/release_note.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class ReleaseNoteService {
client: AxiosInstance;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
this.client = axios.create({
baseURL: process.env.FIREBASE_BASE_URL,
});
}
async getLatestReleaseNote(lang: LanguageEnum) {
try {
const lng = lang.split('-');
const language = lng[0] + '-' + lng[1].toUpperCase();
const endpoint = `/release_note/${language}.json`;
const {
data,
status,
config
} = await this.client.get<ReleaseNoteDTO>(endpoint)
this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`);
this.logger.info(`Request URL: ${config.baseURL}/${config.url}`);
return data;
} catch (error) {
this.logger.error(`Error fetching release note: ${error.message}`);
if (axios.isAxiosError(error)) {
this.logger.error(`Axios error details: ${error.toJSON()}`);
}
}
}
}
+5 -2
View File
@@ -1,5 +1,8 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
export interface ICreateTransformationsRequest {
transformations: Transformation[];
info: Info;
+1
View File
@@ -9,6 +9,7 @@ interface IMetadata {
details?: string;
sensitive?: string;
roles?: string[];
customer_modules?: string[];
is_data_manager?: boolean;
access_token?: string;
host?: string;