Compare commits

..
Author SHA1 Message Date
marcos.rodrigues 00cbadbb45 Merge branch 'beta' into bugfix/catalog-module 2026-08-07 12:26:11 -03:00
marcos.rodrigues 47ad527d38 FIX: skip nimbus update when customer haven't catalog module 2026-08-07 12:18:37 -03:00
iruy-fr 51044a23b3 FIX: trigger cache connections rollout 2026-08-03 09:19:27 -03:00
yuri.rodrigues bb29d126c1 Merge pull request #503 from dadosfera/feat/cache-connections-rollout
feat(connection-test): refresh connection catalog cache
2026-07-31 20:56:11 -03:00
iruy-fr 9d0f449eeb feat(connection-test): refresh connection catalog cache 2026-07-31 16:33:15 -03:00
iruy-fr 0eafa67e6f FIX: trigger cache connections deployment 2026-07-31 09:56:05 -03:00
yuri.rodrigues c3937472ec Merge pull request #502 from dadosfera/feat/cache-connections-rollout
feat: read connection metadata from catalog cache
2026-07-31 09:46:57 -03:00
iruy-fr 84d64424ca feat: read connection metadata from catalog cache 2026-07-30 10:17:28 -03:00
vinicius gadea f0bfc5c94b Merge pull request #500 from dadosfera/feat/custom-properties
FEAT: add color and emoji properties to CustomPropertyDto
2026-07-28 12:21:19 -03:00
viniciusgadea ad86a6a698 FEAT: simplify color and emoji property definitions in docsfera.json 2026-07-28 12:01:50 -03:00
viniciusgadea af3b11ad54 FEAT: add color and emoji properties to CustomPropertyDto 2026-07-28 07:42:18 -03:00
vinicius gadea 15860519e1 Merge pull request #498 from dadosfera/feat/custom-properties
FEAT: add custom properties endpoints and DTOs for catalog management
2026-07-22 11:57:51 -03:00
viniciusgadea cffda86eab FEAT: add CustomPropertyDto and update IUpdateDataRequest to use custom properties array 2026-07-22 10:51:53 -03:00
viniciusgadea a93fbfbbd9 FEAT: remove custom properties endpoints and DTOs from catalog management 2026-07-21 16:32:38 -03:00
viniciusgadea e1cfc6e1a8 FEAT: add custom properties endpoints and DTOs for catalog management 2026-07-21 11:21:22 -03:00
vinicius gadea 1e63df6536 Merge pull request #497 from dadosfera/feat/documentation-status
FEAT: add endpoint and logic to update data asset certification status
2026-07-20 13:44:43 -03:00
viniciusgadea 8eb7fd0169 FEAT: add endpoint and logic to update data asset certification status 2026-07-17 09:55:17 -03:00
vinicius gadea f8c6a8b747 Merge pull request #496 from dadosfera/feat/documentation-status
FIX: pin npm version to 10.8.2 in Dockerfile for consistency
2026-07-16 17:06:04 -03:00
viniciusgadea 8a9d6c2f9c FIX: pin npm version to 10.8.2 in Dockerfile for consistency 2026-07-16 17:05:10 -03:00
vinicius gadea bcb2a0b7cb Merge pull request #495 from dadosfera/feat/documentation-status
Feat/documentation status
2026-07-16 16:47:52 -03:00
viniciusgadea 2e181e70af FEAT: rename documentation_status to certification_status in docs and update package.json for protospack versioning 2026-07-16 09:08:11 -03:00
vinicius gadea ef615adb8e Merge branch 'beta' into feat/documentation-status 2026-07-13 08:42:37 -03:00
viniciusgadea e03900811b FEAT: add documentation status enum and property to data asset 2026-07-13 08:33:35 -03:00
yuri.rodrigues 5bc5fb0977 Merge pull request #494 from dadosfera/feat/pipeline-run-jobs
feat: add endpoint to retrieve pipeline run jobs
2026-06-23 13:38:55 -03:00
iruy-fr 011032e3d4 FEAT: update API title in docsfera.json to reflect project name 2026-06-23 11:58:28 -03:00
iruy-fr 17363e74f4 FEAT: simplify pipeline run jobs handling and normalize run ID usage 2026-06-23 11:54:59 -03:00
iruy-fr 63efff6adf FEAT: enhance pipeline run jobs endpoint with error handling and response structure 2026-06-22 20:25:16 -03:00
iruy-fr b5f569e522 Merge branch 'beta' into feat/pipeline-run-jobs
# Conflicts:
#	docsfera.json
2026-06-19 17:08:34 -03:00
iruy-fr 5c77577992 feat: add endpoint to retrieve pipeline run jobs 2026-06-19 17:04:42 -03:00
marcos.rodrigues 4b9e113185 Merge pull request #493 from dadosfera/chore/remove-deprecated-lib
FEAT: remove deprecated protospack lib
2026-06-14 15:10:22 -03:00
marcos.rodrigues 06d505c50a FEAT: remove deprecated protospack lib 2026-06-13 21:07:14 -03:00
Marcos Rodrigues Silva 2177f6725c Merge pull request #491 from dadosfera/hotfix/release-notes
Hotfix/release notes
2026-05-25 14:10:46 -03:00
marcos-silva-rodrigues 65ab16236f FEAT: update deployment to include firebase base url 2026-05-25 14:09:16 -03:00
marcos-silva-rodrigues b4cc8151d7 FIX: release note endpoint 2026-05-25 14:01:00 -03:00
yuri.rodrigues e8b982998f Merge pull request #490 from dadosfera/feat/platform-job-rout
Feat/platform job rout
2026-05-21 16:48:38 -03:00
iruy-fr ccd4159c59 fix: validate workflow 2026-05-21 16:43:37 -03:00
iruy-fr 9e49abb40d fix: validate workflow 2026-05-21 16:37:47 -03:00
iruy-fr 21a82b64f3 fix: validate workflow 2026-05-21 16:31:45 -03:00
iruy-fr 2414fcf21e fix: validate workflow 2026-05-21 16:23:45 -03:00
iruy-fr 94fbdb2226 fix: validate workflow 2026-05-21 16:21:30 -03:00
iruy-fr 985170d7ae chore: exposure from route pipeline run jobs to maestro 2026-05-21 16:12:35 -03:00
iruy-fr b38b8f51e2 FEAT: Add endpoint to fetch pipeline run jobs 2026-05-20 15:34:49 -03:00
Marcos Rodrigues Silva ea16e62d6b Merge pull request #489 from dadosfera/feature/pipeline-upgrade
FEAT: pipeline upgrade route
2026-05-11 12:58:06 -03:00
marcos-silva-rodrigues 6182705410 FEAT: pipeline upgrade route 2026-05-11 12:52:07 -03:00
20 changed files with 641 additions and 368 deletions
+6
View File
@@ -71,6 +71,11 @@ jobs:
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff plugin
run: |
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
helm diff version
- name: Debug Helm env
run: |
helm env
@@ -102,4 +107,5 @@ jobs:
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
@@ -111,6 +111,8 @@ spec:
value: "{{ .Values.maestro.redis_tls }}"
- name: PLATFORM_API_URL
value: {{ .Values.maestro.platform_api_url }}
- name: CONNECTIONS_API_URL
value: {{ .Values.maestro.connections_api_url | default "" | quote }}
- name: STORAGE_EXPLORER_API_URL
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
- name: FIREBASE_BASE_URL
+1
View File
@@ -9,6 +9,7 @@ maestro:
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
+53 -231
View File
@@ -900,9 +900,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -952,9 +949,6 @@
"connections"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -1003,9 +997,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -1042,9 +1033,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -1078,9 +1066,6 @@
"connections"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -2418,9 +2403,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2504,9 +2486,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2546,9 +2525,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2588,9 +2564,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2628,9 +2601,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2670,9 +2640,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2714,9 +2681,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2758,9 +2722,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -3110,9 +3071,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3162,9 +3120,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3235,9 +3190,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3278,9 +3230,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3321,9 +3270,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3364,9 +3310,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3407,9 +3350,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3457,9 +3397,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3505,9 +3442,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3556,9 +3490,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3597,9 +3528,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3658,9 +3586,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3709,9 +3634,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3752,9 +3674,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3797,9 +3716,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3849,9 +3765,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3901,9 +3814,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3951,9 +3861,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4124,9 +4031,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4152,9 +4056,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4191,9 +4092,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4228,9 +4126,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4265,9 +4160,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4295,9 +4187,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4325,9 +4214,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4355,9 +4241,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4394,9 +4277,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4433,9 +4313,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4463,9 +4340,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4493,9 +4367,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4533,9 +4404,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4572,9 +4440,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4619,9 +4484,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4658,9 +4520,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4707,7 +4566,60 @@
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
"get": {
"operationId": "PlatformApiController_getPipelineRunJobs",
"summary": "Get pipeline run jobs",
"description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.",
"parameters": [
{
"name": "pipelineId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "runId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "DB-backed job runs for the selected pipeline run.",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"jobs": {
"type": "array",
"items": {
"type": "object"
}
}
},
"required": [
"jobs"
]
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
@@ -4744,9 +4656,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4781,9 +4690,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4820,9 +4726,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4859,9 +4762,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4899,9 +4799,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4929,9 +4826,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4959,9 +4853,6 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -5373,9 +5264,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5419,9 +5307,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5531,9 +5416,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5566,9 +5448,6 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -5611,9 +5490,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5749,9 +5625,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5810,9 +5683,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5861,9 +5731,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5957,9 +5824,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6010,9 +5874,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6071,9 +5932,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6130,9 +5988,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6193,9 +6048,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6236,9 +6088,6 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6279,9 +6128,6 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6342,9 +6188,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6403,9 +6246,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6782,9 +6622,6 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -7238,9 +7075,6 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7277,9 +7111,6 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7316,9 +7147,6 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7355,9 +7183,6 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7394,9 +7219,6 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
-2
View File
@@ -722,8 +722,6 @@ export const DADOSFERA_MODULES_KEYS = {
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
CATALOG: 'catalog',
COLLECT: 'collect',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
-54
View File
@@ -85,9 +85,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async searchCatalog(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@@ -127,9 +124,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@@ -173,9 +167,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Get('data-asset')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query;
@@ -234,9 +225,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
@@ -305,9 +293,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id: string,
@@ -419,9 +404,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -453,9 +435,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -487,9 +466,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -521,9 +497,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAsset(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -559,9 +532,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAssetCertificationStatus(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -589,9 +559,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetDocs(
@User() user: RequestUser,
@Headers() headers,
@@ -629,9 +596,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@@ -654,9 +618,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/revoke-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async revokeDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@@ -682,9 +643,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async createDataAsset(
@User() user: RequestUser,
@Body() body: ICreateDataAsset,
@@ -709,9 +667,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async commentOnDataAsset(
@Param('id') id: string,
@User() user: RequestUser,
@@ -738,9 +693,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
@@ -762,9 +714,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteComment(
@Param('id') id: string,
@User() user: RequestUser,
@@ -918,9 +867,6 @@ export class CatalogController {
@Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getNimbusDashboards(
@User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest,
@@ -22,21 +22,20 @@ import {
ConnectionTestListTablesRes,
GetTableMetadataRes,
GetTableMetadataReq,
RefreshCatalogReq,
RefreshCatalogRes,
RefreshCatalogStatusReq,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
@ApiInternalOnlyController()
@ApiTags('Connection Test')
@Controller('connection-test')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionTestController {
logger: any;
constructor(
@@ -88,7 +87,7 @@ export class ConnectionTestController {
});
return this.connectionTestService.connectionTestListSchemas(
body,
user.customer_name,
user,
);
}
@@ -105,7 +104,7 @@ export class ConnectionTestController {
});
return this.connectionTestService.connectionTestListTables(
body,
user.customer_name,
user,
);
}
@@ -122,7 +121,38 @@ export class ConnectionTestController {
});
return this.connectionTestService.getTableMetadata(
body,
user.customer_name,
user,
);
}
@Post('refresh-catalog')
@ApiOkResponse({ type: RefreshCatalogRes })
@HttpCode(HttpStatus.ACCEPTED)
async refreshCatalog(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: RefreshCatalogReq,
) {
this.logger.info('/connection-test/refresh-catalog', {
user: user.user_id,
customer: user.customer_name,
connection: body.connection_id,
});
return this.connectionTestService.refreshCatalog(body, user);
}
@Post('refresh-catalog/status')
@ApiOkResponse({ type: RefreshCatalogRes })
@HttpCode(HttpStatus.OK)
async refreshCatalogStatus(
@User() user: RequestUser,
@Body(new ValidationPipe()) body: RefreshCatalogStatusReq,
) {
this.logger.info('/connection-test/refresh-catalog/status', {
user: user.user_id,
customer: user.customer_name,
connection: body.connection_id,
session: body.session_id,
});
return this.connectionTestService.refreshCatalogStatus(body, user);
}
}
@@ -5,10 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { ConnectionTestClientConfiguration } from './connection-test-client.config';
import { ConnectionModule } from '../connection/connection.module';
import { ConnectionsApiModule } from '../connections-api/connections-api.module';
import { PlatformApiModule } from '../platform-api/platform-api.module';
const client = new ConnectionTestClientConfiguration();
@Module({
controllers: [ConnectionTestController],
providers: [ConnectionTestService, DadosferaLogger],
imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
imports: [
ClientsModule.register([client.providerOptions]),
ConnectionModule,
ConnectionsApiModule,
PlatformApiModule,
],
})
export class ConnectionTestModule {}
@@ -0,0 +1,205 @@
import { ConnectionTestService } from './connection-test.service';
import { RequestUser } from 'src/decorators/user.decorator';
describe('ConnectionTestService catalog cache', () => {
const user: RequestUser = {
user_id: 'user-id',
username: 'user@example.com',
permissions: [],
customer_id: 'customer-id',
customer_name: 'customer-name',
customer_tier: 'standard',
access_token: 'token',
customer_modules: [],
roles: [],
};
const grpcClient = { getService: jest.fn().mockReturnValue({}) };
const connectionsService = {};
const connectionsApiService = { proxy: jest.fn() };
const platformApiService = { proxy: jest.fn() };
let service: ConnectionTestService;
beforeEach(() => {
jest.clearAllMocks();
service = new ConnectionTestService(
grpcClient as any,
connectionsService as any,
connectionsApiService as any,
platformApiService as any,
);
});
it('keeps the existing schemas response contract', async () => {
connectionsApiService.proxy.mockResolvedValue({
schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }],
});
await expect(
service.connectionTestListSchemas(
{ connection_id: 'config-id', plugin: 'postgresql' },
user,
),
).resolves.toEqual({
operation_result: true,
schema_list: ['analytics', 'public'],
});
});
it('keeps the existing tables response contract', async () => {
connectionsApiService.proxy.mockResolvedValue({
tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
});
await expect(
service.connectionTestListTables(
{
connection_id: 'config-id',
plugin: 'postgresql',
schema: 'public',
},
user,
),
).resolves.toEqual({
operation_result: true,
table_list: ['customers', 'orders'],
});
});
it('maps cached columns to the existing table metadata contract', async () => {
connectionsApiService.proxy.mockResolvedValue({
columns: [
{
column_name: 'id',
data_type: 'bigint',
is_primary_key: true,
},
],
});
await expect(
service.getTableMetadata(
{
connection_id: 'config-id',
plugin: 'postgresql',
schema: 'public',
table_list: ['customers'],
},
user,
),
).resolves.toEqual({
operation_result: true,
tables_metadata: [
{
table_name: 'customers',
columns: [
{
name: 'id',
type: 'bigint',
is_primary_key: true,
},
],
references: [],
},
],
});
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
'GET',
'/connection_catalog/config-id/schemas/public/tables/customers/columns',
user,
);
});
it('submits a catalog refresh without holding the request open', async () => {
platformApiService.proxy.mockResolvedValue({
session_id: 'session-id',
date: '20260731',
});
await expect(
service.refreshCatalog(
{ connection_id: 'config-id', plugin: 'postgresql' },
user,
),
).resolves.toEqual({
operation_result: true,
status: 'PENDING',
session_id: 'session-id',
date: '20260731',
});
expect(platformApiService.proxy).toHaveBeenCalledWith(
'POST',
'/connection_test',
user,
{
customer_id: user.customer_name,
plugin: 'postgresql',
task: {
task_type: 'refresh_catalog',
connection: {
provider: 'connection_manager',
config_id: 'config-id',
},
},
},
);
});
it('keeps polling without changing the catalog pointer while pending', async () => {
platformApiService.proxy.mockResolvedValue({ status: 'PENDING' });
await expect(
service.refreshCatalogStatus(
{
connection_id: 'config-id',
plugin: 'postgresql',
session_id: 'session-id',
date: '20260731',
},
user,
),
).resolves.toEqual({
operation_result: false,
status: 'PENDING',
session_id: 'session-id',
date: '20260731',
});
expect(connectionsApiService.proxy).not.toHaveBeenCalled();
});
it('publishes the catalog pointer after the refresh finishes', async () => {
platformApiService.proxy.mockResolvedValue({ status: 'DONE' });
connectionsApiService.proxy.mockResolvedValue({
last_catalog_refresh_status: 'SUCCESS',
});
await expect(
service.refreshCatalogStatus(
{
connection_id: 'config/id',
plugin: 'postgresql',
session_id: 'session-id',
date: '20260731',
},
user,
),
).resolves.toEqual({
operation_result: true,
status: 'DONE',
session_id: 'session-id',
date: '20260731',
});
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
'PUT',
'/connection_config/config%2Fid/catalog_metadata',
user,
{
last_catalog_refresh_status: 'SUCCESS',
last_catalog_connection_test_date: '20260731',
last_catalog_connection_test_session_id: 'session-id',
},
);
});
});
@@ -1,4 +1,4 @@
import { Inject, Injectable } from '@nestjs/common';
import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { ConnectionTest } from '@dadosfera/protospack-v2';
import { lastValueFrom } from 'rxjs';
@@ -13,6 +13,9 @@ import {
ConnectionTestPingRes,
GetTableMetadataReq,
GetTableMetadataRes,
RefreshCatalogReq,
RefreshCatalogRes,
RefreshCatalogStatusReq,
} from './dto/connection-test';
import { ConnectionClientService } from '../connection/client.service';
import {
@@ -21,6 +24,8 @@ import {
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ConnectionsApiService } from '../connections-api/connections-api.service';
import { PlatformApiService } from '../platform-api/platform-api.service';
@Injectable()
export class ConnectionTestService {
@@ -28,6 +33,8 @@ export class ConnectionTestService {
constructor(
@Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
private connectionsService: ConnectionClientService,
private connectionsApiService: ConnectionsApiService,
private platformApiService: PlatformApiService,
) {
this.connectionTestReadClient =
grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
@@ -147,45 +154,137 @@ export class ConnectionTestService {
}
async connectionTestListSchemas(
body: ConnectionTestListSchemasReq,
customer_name: string,
user: RequestUser,
): Promise<ConnectionTestListSchemasRes> {
const { connection_id, plugin } = body;
return lastValueFrom(
this.connectionTestReadClient.ListSchemas({
connection_id,
customer_name,
plugin,
}),
const result = await this.connectionsApiService.proxy(
'GET',
`/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`,
user,
);
return {
operation_result: true,
schema_list: result.schemas.map((schema) => schema.schema_name),
};
}
async connectionTestListTables(
body: ConnectionTestListTablesReq,
customer_name: string,
user: RequestUser,
): Promise<ConnectionTestListTablesRes> {
const { connection_id, plugin, schema } = body;
return lastValueFrom(
this.connectionTestReadClient.ListTables({
connection_id,
customer_name,
plugin,
schema,
}),
const result = await this.connectionsApiService.proxy(
'GET',
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
`/schemas/${encodeURIComponent(body.schema)}/tables`,
user,
);
return {
operation_result: true,
table_list: result.tables.map((table) => table.table_name),
};
}
async getTableMetadata(
body: GetTableMetadataReq,
customer_name: string,
user: RequestUser,
): Promise<GetTableMetadataRes> {
const { schema, plugin, table_list, connection_id } = body;
return lastValueFrom(
this.connectionTestReadClient.GetTableMetadata({
connection_id,
customer_name,
plugin,
schema,
table_list,
const tables_metadata = await Promise.all(
body.table_list.map(async (table_name) => {
const result = await this.connectionsApiService.proxy(
'GET',
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
`/schemas/${encodeURIComponent(body.schema)}` +
`/tables/${encodeURIComponent(table_name)}/columns`,
user,
);
return {
table_name,
columns: result.columns.map((column) => ({
name: column.column_name,
type: column.data_type,
is_primary_key: column.is_primary_key,
})),
references: [],
};
}),
);
return { operation_result: true, tables_metadata };
}
async refreshCatalog(
body: RefreshCatalogReq,
user: RequestUser,
): Promise<RefreshCatalogRes> {
const task = await this.platformApiService.proxy(
'POST',
'/connection_test',
user,
{
customer_id: user.customer_name,
plugin: body.plugin,
task: {
task_type: 'refresh_catalog',
connection: {
provider: 'connection_manager',
config_id: body.connection_id,
},
},
},
);
if (!task.session_id || !task.date) {
throw new HttpException(
'Platform API did not return a catalog refresh task identifier',
HttpStatus.BAD_GATEWAY,
);
}
return {
operation_result: true,
status: 'PENDING',
session_id: task.session_id,
date: task.date,
};
}
async refreshCatalogStatus(
body: RefreshCatalogStatusReq,
user: RequestUser,
): Promise<RefreshCatalogRes> {
const result = await this.platformApiService.proxy(
'POST',
'/connection_test/status',
user,
{
session_id: body.session_id,
date: body.date,
},
);
if (result.status === 'DONE') {
await this.connectionsApiService.proxy(
'PUT',
`/connection_config/${encodeURIComponent(
body.connection_id,
)}/catalog_metadata`,
user,
{
last_catalog_refresh_status: 'SUCCESS',
last_catalog_connection_test_date: body.date,
last_catalog_connection_test_session_id: body.session_id,
},
);
} else if (result.status === 'ERROR' || result.status === 'EXPIRED') {
throw new HttpException(
`Catalog refresh finished with status ${result.status}`,
HttpStatus.BAD_GATEWAY,
);
}
return {
operation_result: result.status === 'DONE',
status: result.status,
session_id: body.session_id,
date: body.date,
};
}
}
@@ -1,5 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { IsString, IsOptional } from 'class-validator';
import { IsIn, IsString, IsOptional } from 'class-validator';
import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
export class ColumnDto {
@@ -7,6 +7,8 @@ export class ColumnDto {
name: string;
@ApiProperty()
type: string;
@ApiProperty()
is_primary_key: boolean;
}
export class TableMetadataDto {
@ApiProperty()
@@ -131,3 +133,37 @@ export class GetTableMetadataRes {
@ApiProperty({ type: [TableMetadataDto] })
tables_metadata: TableMetadataDto[];
}
export class RefreshCatalogReq {
@ApiProperty()
@IsString()
connection_id: string;
@ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] })
@IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver'])
plugin: string;
}
export class RefreshCatalogStatusReq extends RefreshCatalogReq {
@ApiProperty()
@IsString()
session_id: string;
@ApiProperty()
@IsString()
date: string;
}
export class RefreshCatalogRes {
@ApiProperty()
operation_result: boolean;
@ApiProperty()
status: string;
@ApiProperty()
session_id: string;
@ApiProperty()
date: string;
}
@@ -16,9 +16,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
} from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
@@ -40,9 +39,6 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@ApiTags('connections')
@Authenticated()
@Controller('connections')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionController {
logger: any;
constructor(
@@ -0,0 +1,11 @@
export const CONNECTIONS_API_CONFIG = {
getUrl: (): string => {
const url = process.env.CONNECTIONS_API_URL;
if (!url) {
throw new Error('CONNECTIONS_API_URL environment variable is not set');
}
return url;
},
region: process.env.AWS_REGION || 'us-east-1',
timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10),
};
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ConnectionsApiService } from './connections-api.service';
@Module({
providers: [ConnectionsApiService, DadosferaLogger],
exports: [ConnectionsApiService],
})
export class ConnectionsApiModule {}
@@ -0,0 +1,99 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import { SignatureV4 } from '@aws-sdk/signature-v4';
import { Sha256 } from '@aws-crypto/sha256-js';
import { defaultProvider } from '@aws-sdk/credential-provider-node';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { CONNECTIONS_API_CONFIG } from './connections-api.config';
@Injectable()
export class ConnectionsApiService {
private signer: SignatureV4;
private logger: any;
constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) {
this.logger = dadosferaLogger.logger;
this.signer = new SignatureV4({
service: 'execute-api',
region: CONNECTIONS_API_CONFIG.region,
credentials: defaultProvider(),
sha256: Sha256,
});
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, string>,
): Promise<any> {
const baseUrl = CONNECTIONS_API_CONFIG.getUrl();
const url = new URL(`${baseUrl}${path}`);
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
host: url.hostname,
'content-type': 'application/json',
customer_name: user.customer_name || '',
customer_id: user.customer_id || '',
'x-user-id': user.user_id || '',
'x-username': user.username || '',
'x-customer-tier': user.customer_tier || '',
'x-customer-id': user.customer_id || '',
};
const requestToSign = {
method: method.toUpperCase(),
protocol: url.protocol,
hostname: url.hostname,
port: url.port ? parseInt(url.port, 10) : undefined,
path: url.pathname + url.search,
headers,
body: body ? JSON.stringify(body) : undefined,
};
try {
const signedRequest = await this.signer.sign(requestToSign);
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers: signedRequest.headers as Record<string, string>,
data: body,
timeout: CONNECTIONS_API_CONFIG.timeout,
validateStatus: () => true,
});
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Connections API proxy error', {
error: error.message,
path,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Connections API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Connections API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
}
+1 -26
View File
@@ -25,10 +25,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -100,9 +99,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getAllConnectors(
@Language() language: LanguageEnum,
@Query() queries: GetAllDto,
@@ -135,9 +131,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorsTags() {
return await this.connectorClientService.getConnectorsTags();
}
@@ -150,9 +143,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnector(
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@@ -181,9 +171,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorDetails(
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@@ -206,9 +193,6 @@ export class ConnectorController {
@Put('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@ApiConsumes('multipart/form-data')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async updateConnector(
@Param('plugin') plugin: string,
@Body() body: UpdateDto,
@@ -230,9 +214,6 @@ export class ConnectorController {
@Put('/:plugin/add-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async addTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: AddTagDto,
@@ -260,9 +241,6 @@ export class ConnectorController {
@Put('/:plugin/remove-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async removeTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: RemoveTagDto,
@@ -291,9 +269,6 @@ export class ConnectorController {
@Delete('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async deleteConnector(
@Param('plugin') plugin: string,
@Query('version') version: string,
@@ -25,10 +25,9 @@ import {
} from '@nestjs/swagger';
import {
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
@@ -58,9 +57,6 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -14,17 +14,16 @@ import {
NotFoundException,
UseGuards,
} from '@nestjs/common';
import { ApiTags, ApiOperation } from '@nestjs/swagger';
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
} from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
@@ -50,7 +49,6 @@ type RenameTablesBody = {
@ApiTags('Platform API')
@Controller('platform')
@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
export class PlatformApiController {
private logger: any;
@@ -74,6 +72,10 @@ export class PlatformApiController {
return id?.replace(/-/g, '_') || '';
}
private decodePathParam(value: string): string {
return value ? decodeURIComponent(value) : '';
}
/**
* Denormalize ID back to UUID format (replace _ with -).
* Used when we receive a normalized ID but need the original UUID.
@@ -834,6 +836,40 @@ export class PlatformApiController {
);
}
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
@ApiOperation({
summary: 'Get pipeline run jobs',
description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
})
@ApiOkResponse({
description: 'DB-backed job runs for the selected pipeline run.',
schema: {
type: 'object',
properties: {
jobs: {
type: 'array',
items: { type: 'object' },
},
},
required: ['jobs'],
},
})
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRunJobs(
@Param('pipelineId') pipelineId: string,
@Param('runId') runId: string,
@User() user: RequestUser,
) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const decodedRunId = this.decodePathParam(runId);
return this.platformApiService.proxy(
'GET',
`/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
user,
);
}
// ==================== JOBS - COLUMN EDITING ROUTES ====================
@Put('jobs/:jobId/input')
@@ -1,7 +1,6 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteController } from './release_note.controller';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteController', () => {
let controller: ReleaseNoteController;
@@ -9,7 +8,7 @@ describe('ReleaseNoteController', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [ReleaseNoteController],
providers: [ReleaseNoteService, DadosferaLogger],
providers: [ReleaseNoteService],
}).compile();
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
@@ -1,13 +1,12 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteService', () => {
let service: ReleaseNoteService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ReleaseNoteService, DadosferaLogger],
providers: [ReleaseNoteService],
}).compile();
service = module.get<ReleaseNoteService>(ReleaseNoteService);