Compare commits

..
Author SHA1 Message Date
viniciusgadea c5cc482c77 FEAT: simplify color and emoji property definitions in docsfera.json 2026-08-12 12:14:01 -03:00
viniciusgadea 679c3056c2 FEAT: add color and emoji properties to CustomPropertyDto 2026-08-12 12:13:54 -03:00
viniciusgadea 8fa1a695e9 FEAT: add CustomPropertyDto and update IUpdateDataRequest to use custom properties array 2026-08-12 12:13:43 -03:00
viniciusgadea df62bac686 FEAT: remove custom properties endpoints and DTOs from catalog management 2026-08-12 12:13:35 -03:00
viniciusgadea d63d71831e FEAT: add custom properties endpoints and DTOs for catalog management 2026-08-12 12:13:24 -03:00
marcos.rodrigues a47519c431 FIX: require collect module in endpoints 2026-08-12 11:37:03 -03:00
marcos.rodrigues 8a92da470c Merge pull request #506 from dadosfera/bugfix/catalog-module-main
FIX: skip nimbus update when customer haven't catalog module
2026-08-07 16:27:19 -03:00
marcos.rodrigues b89909ad66 FIX: skip nimbus update when customer haven't catalog module 2026-08-07 14:48:46 -03:00
marcos.rodrigues d54f381998 Merge pull request #499 from dadosfera/release/2026-07-27
Release/2026 07 27
2026-07-27 17:59:56 -03:00
viniciusgadea 3fd586753e FEAT: add endpoint and logic to update data asset certification status 2026-07-27 17:07:19 -03:00
viniciusgadea 9b3894f9e4 FIX: pin npm version to 10.8.2 in Dockerfile for consistency 2026-07-27 17:07:10 -03:00
viniciusgadea 4a5f8f679a FEAT: rename documentation_status to certification_status in docs and update package.json for protospack versioning 2026-07-27 17:06:58 -03:00
viniciusgadea a444f5e5ec FEAT: add documentation status enum and property to data asset 2026-07-27 17:06:25 -03:00
marcos.rodrigues 0d30c1cf83 FEAT: remove deprecated protospack lib 2026-07-27 17:06:10 -03:00
18 changed files with 657 additions and 163 deletions
-6
View File
@@ -71,11 +71,6 @@ jobs:
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff plugin
run: |
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
helm diff version
- name: Debug Helm env
run: |
helm env
@@ -107,5 +102,4 @@ jobs:
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+386 -61
View File
@@ -900,6 +900,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -949,6 +952,9 @@
"connections"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -997,6 +1003,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -1033,6 +1042,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -1066,6 +1078,9 @@
"connections"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -2403,6 +2418,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2486,6 +2504,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2525,6 +2546,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2564,6 +2588,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2601,6 +2628,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2640,6 +2670,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2681,6 +2714,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -2722,6 +2758,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -3071,6 +3110,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3120,6 +3162,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3190,6 +3235,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3230,6 +3278,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3270,6 +3321,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3310,6 +3364,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3350,6 +3407,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3397,6 +3457,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3442,6 +3505,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3490,6 +3556,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3528,6 +3597,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3586,6 +3658,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3634,6 +3709,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3674,6 +3752,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3716,6 +3797,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3765,6 +3849,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3814,6 +3901,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3861,6 +3951,9 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4031,6 +4124,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4056,6 +4152,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4092,6 +4191,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4126,6 +4228,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4160,6 +4265,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4187,6 +4295,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4214,6 +4325,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4241,6 +4355,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4277,6 +4394,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4313,6 +4433,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4340,6 +4463,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4367,6 +4493,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4404,6 +4533,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4440,6 +4572,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4484,6 +4619,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4520,6 +4658,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4566,60 +4707,7 @@
"security": [
{
"access-token": []
}
]
}
},
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
"get": {
"operationId": "PlatformApiController_getPipelineRunJobs",
"summary": "Get pipeline run jobs",
"description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.",
"parameters": [
{
"name": "pipelineId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "runId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "DB-backed job runs for the selected pipeline run.",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"jobs": {
"type": "array",
"items": {
"type": "object"
}
}
},
"required": [
"jobs"
]
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
@@ -4656,6 +4744,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4690,6 +4781,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4726,6 +4820,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4762,6 +4859,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4799,6 +4899,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4826,6 +4929,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -4853,6 +4959,9 @@
"Platform API"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -5264,6 +5373,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5307,6 +5419,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5416,6 +5531,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5448,6 +5566,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -5490,6 +5611,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5538,6 +5662,48 @@
]
}
},
"/catalog/custom-properties": {
"get": {
"operationId": "CatalogController_getCustomPropertyDefinitions",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/data-asset/{id}": {
"get": {
"operationId": "CatalogController_getDataAsset",
@@ -5583,6 +5749,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5641,6 +5810,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5689,6 +5861,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5782,6 +5957,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5832,6 +6010,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5890,6 +6071,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -5943,6 +6127,72 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/data-asset/{id}/certification-status": {
"put": {
"operationId": "CatalogController_updateDataAssetCertificationStatus",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
},
@@ -5986,6 +6236,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6026,6 +6279,9 @@
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6086,6 +6342,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6144,6 +6403,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6520,6 +6782,9 @@
{
"access-token": []
},
{
"access-token": []
},
{
"access-token": []
}
@@ -6973,6 +7238,9 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7009,6 +7277,9 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7045,6 +7316,9 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7081,6 +7355,9 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -7117,6 +7394,9 @@
"Connection Test"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -10985,6 +11265,37 @@
"docs"
]
},
"CustomPropertyDto": {
"type": "object",
"properties": {
"key": {
"type": "string"
},
"value": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"text",
"number",
"date",
"boolean"
]
},
"color": {
"type": "string"
},
"emoji": {
"type": "string"
}
},
"required": [
"key",
"value",
"type"
]
},
"IUpdateDataRequest": {
"type": "object",
"properties": {
@@ -11014,14 +11325,11 @@
"docs": {
"type": "string"
},
"certification_status": {
"type": "string",
"enum": [
"draft",
"in_review",
"approved",
"deprecated"
]
"custom_properties": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomPropertyDto"
}
}
},
"required": [
@@ -11041,6 +11349,23 @@
"data_asset"
]
},
"IUpdateCertificationStatusRequest": {
"type": "object",
"properties": {
"certification_status": {
"type": "string",
"enum": [
"draft",
"in_review",
"approved",
"deprecated"
]
}
},
"required": [
"certification_status"
]
},
"ICreateDataAsset": {
"type": "object",
"properties": {
+4 -4
View File
@@ -16,7 +16,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.10",
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
"integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
"version": "3.40.0-beta.14",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
+1 -1
View File
@@ -34,7 +34,7 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.10",
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+12
View File
@@ -357,6 +357,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Crear y editar atributos en el catálogo',
},
},
CERTIFY: {
seqid: 53,
claim: 'catalog:certify',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Alterar o status de certificação dos Ativos',
'en-us': "Change Assets' certification status",
'es-es': 'Cambiar el estado de certificación de los Activos',
},
},
DELETE: {
seqid: 1,
claim: 'catalog:delete',
@@ -712,6 +722,8 @@ export const DADOSFERA_MODULES_KEYS = {
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
CATALOG: 'catalog',
COLLECT: 'collect',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+103 -1
View File
@@ -17,6 +17,7 @@ import {
HttpStatus,
Res,
} from '@nestjs/common';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
ApiCreatedResponse,
ApiHeaders,
@@ -46,6 +47,7 @@ import {
IMakeAComment,
IOneDataAsset,
IPreviewResponse,
IUpdateCertificationStatusRequest,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
@@ -83,6 +85,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async searchCatalog(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@@ -122,6 +127,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@@ -165,6 +173,9 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Get('data-asset')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query;
@@ -223,6 +234,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
@@ -269,11 +283,31 @@ export class CatalogController {
}
@Get('custom-properties')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getCustomPropertyDefinitions(@User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
return this.catalogService.getCustomPropertyDefinitions(metadata);
}
@Get('data-asset/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id: string,
@@ -385,6 +419,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -416,6 +453,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetPreview(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -447,6 +487,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetDocs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -478,6 +521,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAsset(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -493,6 +539,8 @@ export class CatalogController {
language,
});
delete (body as any).certification_status;
const result = await this.catalogService.updateOneDataAsset({
body,
data_asset_id,
@@ -506,11 +554,44 @@ export class CatalogController {
return result;
}
@Put('data-asset/:id/certification-status')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAssetCertificationStatus(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') data_asset_id: string,
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
): Promise<IUpdateCertificationStatusRequest> {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
return this.catalogService.updateCertificationStatus({
body,
data_asset_id,
metadata,
});
}
@Post('data-asset/:id/docs')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetDocs(
@User() user: RequestUser,
@Headers() headers,
@@ -548,6 +629,9 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@@ -570,6 +654,9 @@ export class CatalogController {
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/revoke-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async revokeDataAssetPermissions(
@Param('id') id: string,
@User() user: RequestUser,
@@ -595,6 +682,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async createDataAsset(
@User() user: RequestUser,
@Body() body: ICreateDataAsset,
@@ -619,6 +709,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async commentOnDataAsset(
@Param('id') id: string,
@User() user: RequestUser,
@@ -645,6 +738,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
@@ -666,6 +762,9 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteComment(
@Param('id') id: string,
@User() user: RequestUser,
@@ -819,6 +918,9 @@ export class CatalogController {
@Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getNimbusDashboards(
@User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest,
@@ -971,4 +1073,4 @@ export class CatalogController {
this.logger.error(error.message);
}
}
}
}
+27
View File
@@ -29,6 +29,7 @@ import {
AssetReporter,
BatchRemoveRlsRulesRequest,
CreateDataDocsDTO,
IUpdateCertificationStatusRequest,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
@@ -119,6 +120,10 @@ class CatalogService implements OnModuleInit {
}
}
async getCustomPropertyDefinitions(metadata: Metadata) {
return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
@@ -384,6 +389,28 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] };
}
async updateCertificationStatus(data: {
data_asset_id: string;
body: IUpdateCertificationStatusRequest;
metadata: Metadata;
}) {
const { body, data_asset_id, metadata } = data;
await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{
id: data_asset_id,
changes: JSON.stringify({
certification_status: body.certification_status,
}),
},
metadata,
),
);
return { certification_status: body.certification_status };
}
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
+37 -3
View File
@@ -1,4 +1,10 @@
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
import {
IsEnum,
IsNotEmpty,
IsOptional,
IsString,
} from 'class-validator';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export enum DataAssetShareType {
@@ -197,6 +203,27 @@ export class IData {
day_opening: number;
}
export enum CustomPropertyType {
TEXT = 'text',
NUMBER = 'number',
DATE = 'date',
BOOLEAN = 'boolean',
}
export class CustomPropertyDto {
@ApiProperty()
key: string;
@ApiProperty()
value: string;
@ApiProperty({ enum: CustomPropertyType })
type: CustomPropertyType;
@ApiPropertyOptional()
color?: string;
@ApiPropertyOptional()
emoji?: string;
}
export class IUpdateDataRequest {
@ApiProperty()
name: string;
@@ -210,9 +237,16 @@ export class IUpdateDataRequest {
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
@ApiPropertyOptional({ enum: CertificationStatus })
certification_status?: CertificationStatus;
@ApiPropertyOptional({ type: [CustomPropertyDto] })
custom_properties?: CustomPropertyDto[];
}
export class IUpdateCertificationStatusRequest {
@ApiProperty({ enum: CertificationStatus })
@IsEnum(CertificationStatus)
certification_status: CertificationStatus;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
@@ -366,4 +400,4 @@ export type CreateDataDocsDTO = {
docs: string;
asset_type: string;
}
}
@@ -24,15 +24,19 @@ import {
GetTableMetadataReq,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
@ApiInternalOnlyController()
@ApiTags('Connection Test')
@Controller('connection-test')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionTestController {
logger: any;
constructor(
@@ -16,8 +16,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
@@ -39,6 +40,9 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@ApiTags('connections')
@Authenticated()
@Controller('connections')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionController {
logger: any;
constructor(
+26 -1
View File
@@ -25,9 +25,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -99,6 +100,9 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getAllConnectors(
@Language() language: LanguageEnum,
@Query() queries: GetAllDto,
@@ -131,6 +135,9 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorsTags() {
return await this.connectorClientService.getConnectorsTags();
}
@@ -143,6 +150,9 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnector(
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@@ -171,6 +181,9 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorDetails(
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@@ -193,6 +206,9 @@ export class ConnectorController {
@Put('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@ApiConsumes('multipart/form-data')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async updateConnector(
@Param('plugin') plugin: string,
@Body() body: UpdateDto,
@@ -214,6 +230,9 @@ export class ConnectorController {
@Put('/:plugin/add-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async addTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: AddTagDto,
@@ -241,6 +260,9 @@ export class ConnectorController {
@Put('/:plugin/remove-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async removeTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: RemoveTagDto,
@@ -269,6 +291,9 @@ export class ConnectorController {
@Delete('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async deleteConnector(
@Param('plugin') plugin: string,
@Query('version') version: string,
+5 -4
View File
@@ -23,6 +23,7 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { CreateInputReq } from './dtos/input.model';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
@@ -73,10 +74,10 @@ export class InputsService {
objectCamelToSnake(createInputResponse);
return createInputResponse;
},
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
const updateInputResponse = await lastValueFrom(
this.inputWriteService.InputUpdate(updateInputDTO),
this.inputWriteService.InputUpdate(updateInputDTO, metadata),
);
return updateInputResponse;
@@ -206,7 +207,7 @@ export class InputsService {
return findOneInputResponse;
}
async update(id: string, data, info: Info) {
async update(id: string, data, info: Info, metadata?: Metadata) {
// this.validateCron({ ...data, info });
try {
const {
@@ -217,7 +218,7 @@ export class InputsService {
id,
...data,
info,
});
}, metadata);
const updateInputResponse = this.adjustInputPayload(
input,
@@ -15,7 +15,6 @@ import {
HttpException,
BadRequestException,
UseGuards,
Res,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -26,9 +25,10 @@ import {
} from '@nestjs/swagger';
import {
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
@@ -58,6 +58,9 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -318,7 +321,6 @@ export class PipelinesController {
) {
this.logger.info('PipelinesController - update', { user });
const { customer_id, customer_name, user_id, username } = user;
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
@@ -326,13 +328,7 @@ export class PipelinesController {
pipeline_id: pipelineId
};
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const metadata = PackTheMetadata(user);
const response = await this.pipelinesClientService.updatePipelineInput(
pipelineId,
+31 -28
View File
@@ -383,7 +383,8 @@ export class PipelinesService implements OnModuleInit {
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info
info,
metadata
);
const inputRollback = () => {
@@ -404,34 +405,36 @@ export class PipelinesService implements OnModuleInit {
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
if (user.customer_modules.includes('catalog')) {
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
}
try {
@@ -14,16 +14,17 @@ import {
NotFoundException,
UseGuards,
} from '@nestjs/common';
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
import { ApiTags, ApiOperation } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
} from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
@@ -49,6 +50,7 @@ type RenameTablesBody = {
@ApiTags('Platform API')
@Controller('platform')
@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
export class PlatformApiController {
private logger: any;
@@ -72,10 +74,6 @@ export class PlatformApiController {
return id?.replace(/-/g, '_') || '';
}
private decodePathParam(value: string): string {
return value ? decodeURIComponent(value) : '';
}
/**
* Denormalize ID back to UUID format (replace _ with -).
* Used when we receive a normalized ID but need the original UUID.
@@ -836,40 +834,6 @@ export class PlatformApiController {
);
}
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
@ApiOperation({
summary: 'Get pipeline run jobs',
description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
})
@ApiOkResponse({
description: 'DB-backed job runs for the selected pipeline run.',
schema: {
type: 'object',
properties: {
jobs: {
type: 'array',
items: { type: 'object' },
},
},
required: ['jobs'],
},
})
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRunJobs(
@Param('pipelineId') pipelineId: string,
@Param('runId') runId: string,
@User() user: RequestUser,
) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const decodedRunId = this.decodePathParam(runId);
return this.platformApiService.proxy(
'GET',
`/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
user,
);
}
// ==================== JOBS - COLUMN EDITING ROUTES ====================
@Put('jobs/:jobId/input')
@@ -1,6 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteController } from './release_note.controller';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteController', () => {
let controller: ReleaseNoteController;
@@ -8,7 +9,7 @@ describe('ReleaseNoteController', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [ReleaseNoteController],
providers: [ReleaseNoteService],
providers: [ReleaseNoteService, DadosferaLogger],
}).compile();
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
@@ -1,12 +1,13 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteService', () => {
let service: ReleaseNoteService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ReleaseNoteService],
providers: [ReleaseNoteService, DadosferaLogger],
}).compile();
service = module.get<ReleaseNoteService>(ReleaseNoteService);
+1
View File
@@ -9,6 +9,7 @@ interface IMetadata {
details?: string;
sensitive?: string;
roles?: string[];
customer_modules?: string[];
is_data_manager?: boolean;
access_token?: string;
host?: string;